From 5b9da3f4a8b7223bfcacc1b65fb194ab3080ee40 Mon Sep 17 00:00:00 2001 From: al8n Date: Tue, 23 Jun 2026 21:45:56 +1200 Subject: [PATCH 001/128] feat(serf-proto): scaffold on memberlist-proto 0.4; isolate old crates in legacy/ workspace --- .gitignore | 2 + Cargo.toml | 9 ++-- legacy/Cargo.toml | 45 +++++++++++++++++++ {serf-core => legacy}/README.md | 0 .../examples}/toyconsul/Cargo.toml | 0 .../examples}/toyconsul/README.md | 0 .../examples}/toyconsul/src/main.rs | 0 {fuzz => legacy/fuzz}/.gitignore | 0 {fuzz => legacy/fuzz}/Cargo.toml | 0 .../fuzz}/fuzz_targets/messages.rs | 0 {serf-core => legacy/serf-core}/Cargo.toml | 0 {serf => legacy/serf-core}/README.md | 0 .../serf-core}/src/broadcast.rs | 0 .../serf-core}/src/coalesce.rs | 0 .../serf-core}/src/coalesce/member.rs | 0 .../serf-core}/src/coalesce/user.rs | 0 .../serf-core}/src/delegate.rs | 0 .../serf-core}/src/delegate/composite.rs | 0 .../serf-core}/src/delegate/merge.rs | 0 .../serf-core}/src/delegate/reconnect.rs | 0 {serf-core => legacy/serf-core}/src/error.rs | 0 {serf-core => legacy/serf-core}/src/event.rs | 0 .../serf-core}/src/event/crate_event.rs | 0 .../serf-core}/src/key_manager.rs | 0 {serf-core => legacy/serf-core}/src/lib.rs | 0 .../serf-core}/src/options.rs | 0 {serf-core => legacy/serf-core}/src/serf.rs | 0 .../serf-core}/src/serf/api.rs | 0 .../serf-core}/src/serf/base.rs | 0 .../serf-core}/src/serf/base/tests.rs | 0 .../serf-core}/src/serf/base/tests/serf.rs | 0 .../src/serf/base/tests/serf/delegate.rs | 0 .../src/serf/base/tests/serf/event.rs | 0 .../src/serf/base/tests/serf/join.rs | 0 .../src/serf/base/tests/serf/leave.rs | 0 .../src/serf/base/tests/serf/reap.rs | 0 .../src/serf/base/tests/serf/reconnect.rs | 0 .../src/serf/base/tests/serf/remove.rs | 0 .../src/serf/base/tests/serf/snapshot.rs | 0 .../serf-core}/src/serf/delegate.rs | 0 .../serf-core}/src/serf/internal_query.rs | 0 .../serf-core}/src/serf/query.rs | 0 .../serf-core}/src/snapshot.rs | 0 {serf-core => legacy/serf-core}/src/types.rs | 0 .../serf-core}/src/types/arbitrary_impl.rs | 0 .../serf-core}/src/types/clock.rs | 0 .../serf-core}/src/types/conflict.rs | 0 .../serf-core}/src/types/coordinate.rs | 0 .../serf-core}/src/types/filter.rs | 0 .../serf-core}/src/types/filter/tag_filter.rs | 0 .../serf-core}/src/types/fuzzy.rs | 0 .../serf-core}/src/types/join.rs | 0 .../serf-core}/src/types/key.rs | 0 .../serf-core}/src/types/leave.rs | 0 .../serf-core}/src/types/member.rs | 0 .../serf-core}/src/types/message.rs | 0 .../serf-core}/src/types/push_pull.rs | 0 .../serf-core}/src/types/query.rs | 0 .../serf-core}/src/types/query/response.rs | 0 .../serf-core}/src/types/quickcheck_impl.rs | 0 .../serf-core}/src/types/tags.rs | 0 .../serf-core}/src/types/tests.rs | 0 .../serf-core}/src/types/user_event.rs | 0 .../src/types/user_event/message.rs | 0 .../src/types/user_event/user_events.rs | 0 .../serf-core}/src/types/version.rs | 0 {serf => legacy/serf}/Cargo.toml | 0 legacy/serf/README.md | 1 + {serf => legacy/serf}/src/lib.rs | 0 {serf => legacy/serf}/src/smol.rs | 0 {serf => legacy/serf}/src/tokio.rs | 0 {serf => legacy/serf}/test/main.rs | 0 {serf => legacy/serf}/test/main/net.rs | 0 .../serf}/test/main/net/coordinates.rs | 0 .../serf}/test/main/net/delegate.rs | 0 .../test/main/net/delegate/local_state.rs | 0 .../serf}/test/main/net/delegate/nodemeta.rs | 0 .../test/main/net/delegate/ping_delegate.rs | 0 .../test/main/net/delegate/remote_state.rs | 0 {serf => legacy/serf}/test/main/net/event.rs | 0 .../test/main/net/event/default_query.rs | 0 .../serf}/test/main/net/event/event_user.rs | 0 .../main/net/event/event_user_size_limit.rs | 0 .../test/main/net/event/events_failed.rs | 0 .../serf}/test/main/net/event/events_join.rs | 0 .../serf}/test/main/net/event/events_leave.rs | 0 ...events_leave_avoid_infinite_rebroadcast.rs | 0 .../serf}/test/main/net/event/query.rs | 0 .../test/main/net/event/query_deduplicate.rs | 0 .../serf}/test/main/net/event/query_filter.rs | 0 .../test/main/net/event/query_old_message.rs | 0 .../test/main/net/event/query_same_clock.rs | 0 .../test/main/net/event/query_size_limit.rs | 0 .../net/event/query_size_limit_increased.rs | 0 .../net/event/remove_failed_events_leave.rs | 0 .../test/main/net/event/should_process.rs | 0 .../main/net/event/user_event_old_message.rs | 0 .../main/net/event/user_event_same_clock.rs | 0 .../serf}/test/main/net/get_queue_max.rs | 0 {serf => legacy/serf}/test/main/net/join.rs | 0 .../test/main/net/join/intent_buffer_early.rs | 0 .../serf}/test/main/net/join/intent_newer.rs | 0 .../test/main/net/join/intent_old_message.rs | 0 .../main/net/join/intent_reset_leaving.rs | 0 .../serf}/test/main/net/join/join_cancel.rs | 0 .../test/main/net/join/join_ignore_old.rs | 0 .../serf}/test/main/net/join/join_leave.rs | 0 .../test/main/net/join/join_leave_join.rs | 0 .../serf}/test/main/net/join/leave_ltime.rs | 0 .../test/main/net/join/pending_intent.rs | 0 .../test/main/net/join/pending_intents.rs | 0 {serf => legacy/serf}/test/main/net/leave.rs | 0 .../test/main/net/leave/force_leave_failed.rs | 0 .../main/net/leave/force_leave_leaving.rs | 0 .../test/main/net/leave/force_leave_left.rs | 0 .../main/net/leave/intent_buffer_early.rs | 0 .../serf}/test/main/net/leave/intent_newer.rs | 0 .../test/main/net/leave/intent_old_message.rs | 0 .../main/net/leave/rejoin_different_role.rs | 0 .../test/main/net/leave/snapshot_recovery.rs | 0 .../serf}/test/main/net/local_member.rs | 0 .../serf}/test/main/net/name_resolution.rs | 0 .../serf}/test/main/net/num_nodes.rs | 0 {serf => legacy/serf}/test/main/net/reap.rs | 0 .../serf}/test/main/net/reap/handler.rs | 0 .../test/main/net/reap/handler_shutdown.rs | 0 .../serf}/test/main/net/reap/reap.rs | 0 .../serf}/test/main/net/reconnect.rs | 0 .../test/main/net/reconnect/reconnect.rs | 0 .../serf}/test/main/net/reconnect/same_ip.rs | 0 .../serf}/test/main/net/reconnect/timeout.rs | 0 {serf => legacy/serf}/test/main/net/remove.rs | 0 .../serf}/test/main/net/remove/failed_node.rs | 0 .../main/net/remove/failed_node_ourself.rs | 0 .../test/main/net/remove/failed_node_prune.rs | 0 {serf => legacy/serf}/test/main/net/role.rs | 0 .../serf}/test/main/net/set_tags.rs | 0 .../serf}/test/main/net/snapshot.rs | 0 .../test/main/net/snapshot/snapshoter.rs | 0 .../net/snapshot/snapshoter_force_compact.rs | 0 .../main/net/snapshot/snapshoter_leave.rs | 0 .../net/snapshot/snapshoter_leave_rejoin.rs | 0 .../main/net/snapshot/snapshoter_recovery.rs | 0 {serf => legacy/serf}/test/main/net/state.rs | 0 {serf => legacy/serf}/test/main/net/stats.rs | 0 {serf => legacy/serf}/test/main/net/update.rs | 0 .../serf}/test/main/net/write_keyring_file.rs | 0 serf-proto/Cargo.toml | 20 +++++++++ serf-proto/src/lib.rs | 5 +++ 149 files changed, 77 insertions(+), 5 deletions(-) create mode 100644 legacy/Cargo.toml rename {serf-core => legacy}/README.md (100%) rename {examples => legacy/examples}/toyconsul/Cargo.toml (100%) rename {examples => legacy/examples}/toyconsul/README.md (100%) rename {examples => legacy/examples}/toyconsul/src/main.rs (100%) rename {fuzz => legacy/fuzz}/.gitignore (100%) rename {fuzz => legacy/fuzz}/Cargo.toml (100%) rename {fuzz => legacy/fuzz}/fuzz_targets/messages.rs (100%) rename {serf-core => legacy/serf-core}/Cargo.toml (100%) rename {serf => legacy/serf-core}/README.md (100%) rename {serf-core => legacy/serf-core}/src/broadcast.rs (100%) rename {serf-core => legacy/serf-core}/src/coalesce.rs (100%) rename {serf-core => legacy/serf-core}/src/coalesce/member.rs (100%) rename {serf-core => legacy/serf-core}/src/coalesce/user.rs (100%) rename {serf-core => legacy/serf-core}/src/delegate.rs (100%) rename {serf-core => legacy/serf-core}/src/delegate/composite.rs (100%) rename {serf-core => legacy/serf-core}/src/delegate/merge.rs (100%) rename {serf-core => legacy/serf-core}/src/delegate/reconnect.rs (100%) rename {serf-core => legacy/serf-core}/src/error.rs (100%) rename {serf-core => legacy/serf-core}/src/event.rs (100%) rename {serf-core => legacy/serf-core}/src/event/crate_event.rs (100%) rename {serf-core => legacy/serf-core}/src/key_manager.rs (100%) rename {serf-core => legacy/serf-core}/src/lib.rs (100%) rename {serf-core => legacy/serf-core}/src/options.rs (100%) rename {serf-core => legacy/serf-core}/src/serf.rs (100%) rename {serf-core => legacy/serf-core}/src/serf/api.rs (100%) rename {serf-core => legacy/serf-core}/src/serf/base.rs (100%) rename {serf-core => legacy/serf-core}/src/serf/base/tests.rs (100%) rename {serf-core => legacy/serf-core}/src/serf/base/tests/serf.rs (100%) rename {serf-core => legacy/serf-core}/src/serf/base/tests/serf/delegate.rs (100%) rename {serf-core => legacy/serf-core}/src/serf/base/tests/serf/event.rs (100%) rename {serf-core => legacy/serf-core}/src/serf/base/tests/serf/join.rs (100%) rename {serf-core => legacy/serf-core}/src/serf/base/tests/serf/leave.rs (100%) rename {serf-core => legacy/serf-core}/src/serf/base/tests/serf/reap.rs (100%) rename {serf-core => legacy/serf-core}/src/serf/base/tests/serf/reconnect.rs (100%) rename {serf-core => legacy/serf-core}/src/serf/base/tests/serf/remove.rs (100%) rename {serf-core => legacy/serf-core}/src/serf/base/tests/serf/snapshot.rs (100%) rename {serf-core => legacy/serf-core}/src/serf/delegate.rs (100%) rename {serf-core => legacy/serf-core}/src/serf/internal_query.rs (100%) rename {serf-core => legacy/serf-core}/src/serf/query.rs (100%) rename {serf-core => legacy/serf-core}/src/snapshot.rs (100%) rename {serf-core => legacy/serf-core}/src/types.rs (100%) rename {serf-core => legacy/serf-core}/src/types/arbitrary_impl.rs (100%) rename {serf-core => legacy/serf-core}/src/types/clock.rs (100%) rename {serf-core => legacy/serf-core}/src/types/conflict.rs (100%) rename {serf-core => legacy/serf-core}/src/types/coordinate.rs (100%) rename {serf-core => legacy/serf-core}/src/types/filter.rs (100%) rename {serf-core => legacy/serf-core}/src/types/filter/tag_filter.rs (100%) rename {serf-core => legacy/serf-core}/src/types/fuzzy.rs (100%) rename {serf-core => legacy/serf-core}/src/types/join.rs (100%) rename {serf-core => legacy/serf-core}/src/types/key.rs (100%) rename {serf-core => legacy/serf-core}/src/types/leave.rs (100%) rename {serf-core => legacy/serf-core}/src/types/member.rs (100%) rename {serf-core => legacy/serf-core}/src/types/message.rs (100%) rename {serf-core => legacy/serf-core}/src/types/push_pull.rs (100%) rename {serf-core => legacy/serf-core}/src/types/query.rs (100%) rename {serf-core => legacy/serf-core}/src/types/query/response.rs (100%) rename {serf-core => legacy/serf-core}/src/types/quickcheck_impl.rs (100%) rename {serf-core => legacy/serf-core}/src/types/tags.rs (100%) rename {serf-core => legacy/serf-core}/src/types/tests.rs (100%) rename {serf-core => legacy/serf-core}/src/types/user_event.rs (100%) rename {serf-core => legacy/serf-core}/src/types/user_event/message.rs (100%) rename {serf-core => legacy/serf-core}/src/types/user_event/user_events.rs (100%) rename {serf-core => legacy/serf-core}/src/types/version.rs (100%) rename {serf => legacy/serf}/Cargo.toml (100%) create mode 120000 legacy/serf/README.md rename {serf => legacy/serf}/src/lib.rs (100%) rename {serf => legacy/serf}/src/smol.rs (100%) rename {serf => legacy/serf}/src/tokio.rs (100%) rename {serf => legacy/serf}/test/main.rs (100%) rename {serf => legacy/serf}/test/main/net.rs (100%) rename {serf => legacy/serf}/test/main/net/coordinates.rs (100%) rename {serf => legacy/serf}/test/main/net/delegate.rs (100%) rename {serf => legacy/serf}/test/main/net/delegate/local_state.rs (100%) rename {serf => legacy/serf}/test/main/net/delegate/nodemeta.rs (100%) rename {serf => legacy/serf}/test/main/net/delegate/ping_delegate.rs (100%) rename {serf => legacy/serf}/test/main/net/delegate/remote_state.rs (100%) rename {serf => legacy/serf}/test/main/net/event.rs (100%) rename {serf => legacy/serf}/test/main/net/event/default_query.rs (100%) rename {serf => legacy/serf}/test/main/net/event/event_user.rs (100%) rename {serf => legacy/serf}/test/main/net/event/event_user_size_limit.rs (100%) rename {serf => legacy/serf}/test/main/net/event/events_failed.rs (100%) rename {serf => legacy/serf}/test/main/net/event/events_join.rs (100%) rename {serf => legacy/serf}/test/main/net/event/events_leave.rs (100%) rename {serf => legacy/serf}/test/main/net/event/events_leave_avoid_infinite_rebroadcast.rs (100%) rename {serf => legacy/serf}/test/main/net/event/query.rs (100%) rename {serf => legacy/serf}/test/main/net/event/query_deduplicate.rs (100%) rename {serf => legacy/serf}/test/main/net/event/query_filter.rs (100%) rename {serf => legacy/serf}/test/main/net/event/query_old_message.rs (100%) rename {serf => legacy/serf}/test/main/net/event/query_same_clock.rs (100%) rename {serf => legacy/serf}/test/main/net/event/query_size_limit.rs (100%) rename {serf => legacy/serf}/test/main/net/event/query_size_limit_increased.rs (100%) rename {serf => legacy/serf}/test/main/net/event/remove_failed_events_leave.rs (100%) rename {serf => legacy/serf}/test/main/net/event/should_process.rs (100%) rename {serf => legacy/serf}/test/main/net/event/user_event_old_message.rs (100%) rename {serf => legacy/serf}/test/main/net/event/user_event_same_clock.rs (100%) rename {serf => legacy/serf}/test/main/net/get_queue_max.rs (100%) rename {serf => legacy/serf}/test/main/net/join.rs (100%) rename {serf => legacy/serf}/test/main/net/join/intent_buffer_early.rs (100%) rename {serf => legacy/serf}/test/main/net/join/intent_newer.rs (100%) rename {serf => legacy/serf}/test/main/net/join/intent_old_message.rs (100%) rename {serf => legacy/serf}/test/main/net/join/intent_reset_leaving.rs (100%) rename {serf => legacy/serf}/test/main/net/join/join_cancel.rs (100%) rename {serf => legacy/serf}/test/main/net/join/join_ignore_old.rs (100%) rename {serf => legacy/serf}/test/main/net/join/join_leave.rs (100%) rename {serf => legacy/serf}/test/main/net/join/join_leave_join.rs (100%) rename {serf => legacy/serf}/test/main/net/join/leave_ltime.rs (100%) rename {serf => legacy/serf}/test/main/net/join/pending_intent.rs (100%) rename {serf => legacy/serf}/test/main/net/join/pending_intents.rs (100%) rename {serf => legacy/serf}/test/main/net/leave.rs (100%) rename {serf => legacy/serf}/test/main/net/leave/force_leave_failed.rs (100%) rename {serf => legacy/serf}/test/main/net/leave/force_leave_leaving.rs (100%) rename {serf => legacy/serf}/test/main/net/leave/force_leave_left.rs (100%) rename {serf => legacy/serf}/test/main/net/leave/intent_buffer_early.rs (100%) rename {serf => legacy/serf}/test/main/net/leave/intent_newer.rs (100%) rename {serf => legacy/serf}/test/main/net/leave/intent_old_message.rs (100%) rename {serf => legacy/serf}/test/main/net/leave/rejoin_different_role.rs (100%) rename {serf => legacy/serf}/test/main/net/leave/snapshot_recovery.rs (100%) rename {serf => legacy/serf}/test/main/net/local_member.rs (100%) rename {serf => legacy/serf}/test/main/net/name_resolution.rs (100%) rename {serf => legacy/serf}/test/main/net/num_nodes.rs (100%) rename {serf => legacy/serf}/test/main/net/reap.rs (100%) rename {serf => legacy/serf}/test/main/net/reap/handler.rs (100%) rename {serf => legacy/serf}/test/main/net/reap/handler_shutdown.rs (100%) rename {serf => legacy/serf}/test/main/net/reap/reap.rs (100%) rename {serf => legacy/serf}/test/main/net/reconnect.rs (100%) rename {serf => legacy/serf}/test/main/net/reconnect/reconnect.rs (100%) rename {serf => legacy/serf}/test/main/net/reconnect/same_ip.rs (100%) rename {serf => legacy/serf}/test/main/net/reconnect/timeout.rs (100%) rename {serf => legacy/serf}/test/main/net/remove.rs (100%) rename {serf => legacy/serf}/test/main/net/remove/failed_node.rs (100%) rename {serf => legacy/serf}/test/main/net/remove/failed_node_ourself.rs (100%) rename {serf => legacy/serf}/test/main/net/remove/failed_node_prune.rs (100%) rename {serf => legacy/serf}/test/main/net/role.rs (100%) rename {serf => legacy/serf}/test/main/net/set_tags.rs (100%) rename {serf => legacy/serf}/test/main/net/snapshot.rs (100%) rename {serf => legacy/serf}/test/main/net/snapshot/snapshoter.rs (100%) rename {serf => legacy/serf}/test/main/net/snapshot/snapshoter_force_compact.rs (100%) rename {serf => legacy/serf}/test/main/net/snapshot/snapshoter_leave.rs (100%) rename {serf => legacy/serf}/test/main/net/snapshot/snapshoter_leave_rejoin.rs (100%) rename {serf => legacy/serf}/test/main/net/snapshot/snapshoter_recovery.rs (100%) rename {serf => legacy/serf}/test/main/net/state.rs (100%) rename {serf => legacy/serf}/test/main/net/stats.rs (100%) rename {serf => legacy/serf}/test/main/net/update.rs (100%) rename {serf => legacy/serf}/test/main/net/write_keyring_file.rs (100%) create mode 100644 serf-proto/Cargo.toml create mode 100644 serf-proto/src/lib.rs diff --git a/.gitignore b/.gitignore index 4fffb2f8..f5842c58 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ /target /Cargo.lock +/docs/ +/legacy/Cargo.lock diff --git a/Cargo.toml b/Cargo.toml index 6cdb726f..6954d6c7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,10 +1,8 @@ [workspace] members = [ - "serf", - "serf-core", - "examples/toyconsul", - "fuzz", + "serf-proto", ] +exclude = ["legacy"] resolver = "3" [workspace.package] @@ -20,6 +18,7 @@ categories = ["network-programming", "asynchronous", "algorithms"] readme = "README.md" [workspace.dependencies] +bytes = "1" byteorder = "1" derive_more = { version = "2", features = ["full"] } futures = { version = "0.3", default-features = false } @@ -29,7 +28,7 @@ indexmap = "2" thiserror = { version = "2", default-features = false } viewit = "0.1.5" regex = "1" -smol_str = "^0.3.6" +smol_str = "0.3" smallvec = "1" rand = "0.10" diff --git a/legacy/Cargo.toml b/legacy/Cargo.toml new file mode 100644 index 00000000..68e9b09c --- /dev/null +++ b/legacy/Cargo.toml @@ -0,0 +1,45 @@ +[workspace] +members = [ + "serf-core", + "serf", + "examples/toyconsul", + "fuzz", +] +resolver = "3" + +[workspace.package] +version = "0.5.2" +edition = "2024" +rust-version = "1.85.0" +description = "A decentralized solution for service discovery and orchestration that is lightweight, highly available, and fault tolerant." +repository = "https://github.com/al8n/serf" +homepage = "https://github.com/al8n/serf" +license = "MPL-2.0" +keywords = ["gossip", "service-discovery", "cluster", "membership", "distributed"] +categories = ["network-programming", "asynchronous", "algorithms"] +readme = "README.md" + +[workspace.dependencies] +bytes = "1" +byteorder = "1" +derive_more = { version = "2", features = ["full"] } +futures = { version = "0.3", default-features = false } +serde = { version = "1", features = ["derive"] } +humantime-serde = "1" +indexmap = "2" +thiserror = { version = "2", default-features = false } +viewit = "0.1.5" +regex = "1" +smol_str = "0.3" +smallvec = "1" +rand = "0.10" + +arbitrary = { version = "1", default-features = false, features = ["derive"] } +quickcheck = { version = "1", default-features = false } + +memberlist-proto = { version = "0.3", default-features = false } +memberlist-core = { version = "0.8.5", default-features = false } +memberlist = { version = "0.8.5", default-features = false } + +serf-core = { path = "serf-core", version = "0.5", default-features = false } +serf = { path = "serf", version = "0.5", default-features = false } diff --git a/serf-core/README.md b/legacy/README.md similarity index 100% rename from serf-core/README.md rename to legacy/README.md diff --git a/examples/toyconsul/Cargo.toml b/legacy/examples/toyconsul/Cargo.toml similarity index 100% rename from examples/toyconsul/Cargo.toml rename to legacy/examples/toyconsul/Cargo.toml diff --git a/examples/toyconsul/README.md b/legacy/examples/toyconsul/README.md similarity index 100% rename from examples/toyconsul/README.md rename to legacy/examples/toyconsul/README.md diff --git a/examples/toyconsul/src/main.rs b/legacy/examples/toyconsul/src/main.rs similarity index 100% rename from examples/toyconsul/src/main.rs rename to legacy/examples/toyconsul/src/main.rs diff --git a/fuzz/.gitignore b/legacy/fuzz/.gitignore similarity index 100% rename from fuzz/.gitignore rename to legacy/fuzz/.gitignore diff --git a/fuzz/Cargo.toml b/legacy/fuzz/Cargo.toml similarity index 100% rename from fuzz/Cargo.toml rename to legacy/fuzz/Cargo.toml diff --git a/fuzz/fuzz_targets/messages.rs b/legacy/fuzz/fuzz_targets/messages.rs similarity index 100% rename from fuzz/fuzz_targets/messages.rs rename to legacy/fuzz/fuzz_targets/messages.rs diff --git a/serf-core/Cargo.toml b/legacy/serf-core/Cargo.toml similarity index 100% rename from serf-core/Cargo.toml rename to legacy/serf-core/Cargo.toml diff --git a/serf/README.md b/legacy/serf-core/README.md similarity index 100% rename from serf/README.md rename to legacy/serf-core/README.md diff --git a/serf-core/src/broadcast.rs b/legacy/serf-core/src/broadcast.rs similarity index 100% rename from serf-core/src/broadcast.rs rename to legacy/serf-core/src/broadcast.rs diff --git a/serf-core/src/coalesce.rs b/legacy/serf-core/src/coalesce.rs similarity index 100% rename from serf-core/src/coalesce.rs rename to legacy/serf-core/src/coalesce.rs diff --git a/serf-core/src/coalesce/member.rs b/legacy/serf-core/src/coalesce/member.rs similarity index 100% rename from serf-core/src/coalesce/member.rs rename to legacy/serf-core/src/coalesce/member.rs diff --git a/serf-core/src/coalesce/user.rs b/legacy/serf-core/src/coalesce/user.rs similarity index 100% rename from serf-core/src/coalesce/user.rs rename to legacy/serf-core/src/coalesce/user.rs diff --git a/serf-core/src/delegate.rs b/legacy/serf-core/src/delegate.rs similarity index 100% rename from serf-core/src/delegate.rs rename to legacy/serf-core/src/delegate.rs diff --git a/serf-core/src/delegate/composite.rs b/legacy/serf-core/src/delegate/composite.rs similarity index 100% rename from serf-core/src/delegate/composite.rs rename to legacy/serf-core/src/delegate/composite.rs diff --git a/serf-core/src/delegate/merge.rs b/legacy/serf-core/src/delegate/merge.rs similarity index 100% rename from serf-core/src/delegate/merge.rs rename to legacy/serf-core/src/delegate/merge.rs diff --git a/serf-core/src/delegate/reconnect.rs b/legacy/serf-core/src/delegate/reconnect.rs similarity index 100% rename from serf-core/src/delegate/reconnect.rs rename to legacy/serf-core/src/delegate/reconnect.rs diff --git a/serf-core/src/error.rs b/legacy/serf-core/src/error.rs similarity index 100% rename from serf-core/src/error.rs rename to legacy/serf-core/src/error.rs diff --git a/serf-core/src/event.rs b/legacy/serf-core/src/event.rs similarity index 100% rename from serf-core/src/event.rs rename to legacy/serf-core/src/event.rs diff --git a/serf-core/src/event/crate_event.rs b/legacy/serf-core/src/event/crate_event.rs similarity index 100% rename from serf-core/src/event/crate_event.rs rename to legacy/serf-core/src/event/crate_event.rs diff --git a/serf-core/src/key_manager.rs b/legacy/serf-core/src/key_manager.rs similarity index 100% rename from serf-core/src/key_manager.rs rename to legacy/serf-core/src/key_manager.rs diff --git a/serf-core/src/lib.rs b/legacy/serf-core/src/lib.rs similarity index 100% rename from serf-core/src/lib.rs rename to legacy/serf-core/src/lib.rs diff --git a/serf-core/src/options.rs b/legacy/serf-core/src/options.rs similarity index 100% rename from serf-core/src/options.rs rename to legacy/serf-core/src/options.rs diff --git a/serf-core/src/serf.rs b/legacy/serf-core/src/serf.rs similarity index 100% rename from serf-core/src/serf.rs rename to legacy/serf-core/src/serf.rs diff --git a/serf-core/src/serf/api.rs b/legacy/serf-core/src/serf/api.rs similarity index 100% rename from serf-core/src/serf/api.rs rename to legacy/serf-core/src/serf/api.rs diff --git a/serf-core/src/serf/base.rs b/legacy/serf-core/src/serf/base.rs similarity index 100% rename from serf-core/src/serf/base.rs rename to legacy/serf-core/src/serf/base.rs diff --git a/serf-core/src/serf/base/tests.rs b/legacy/serf-core/src/serf/base/tests.rs similarity index 100% rename from serf-core/src/serf/base/tests.rs rename to legacy/serf-core/src/serf/base/tests.rs diff --git a/serf-core/src/serf/base/tests/serf.rs b/legacy/serf-core/src/serf/base/tests/serf.rs similarity index 100% rename from serf-core/src/serf/base/tests/serf.rs rename to legacy/serf-core/src/serf/base/tests/serf.rs diff --git a/serf-core/src/serf/base/tests/serf/delegate.rs b/legacy/serf-core/src/serf/base/tests/serf/delegate.rs similarity index 100% rename from serf-core/src/serf/base/tests/serf/delegate.rs rename to legacy/serf-core/src/serf/base/tests/serf/delegate.rs diff --git a/serf-core/src/serf/base/tests/serf/event.rs b/legacy/serf-core/src/serf/base/tests/serf/event.rs similarity index 100% rename from serf-core/src/serf/base/tests/serf/event.rs rename to legacy/serf-core/src/serf/base/tests/serf/event.rs diff --git a/serf-core/src/serf/base/tests/serf/join.rs b/legacy/serf-core/src/serf/base/tests/serf/join.rs similarity index 100% rename from serf-core/src/serf/base/tests/serf/join.rs rename to legacy/serf-core/src/serf/base/tests/serf/join.rs diff --git a/serf-core/src/serf/base/tests/serf/leave.rs b/legacy/serf-core/src/serf/base/tests/serf/leave.rs similarity index 100% rename from serf-core/src/serf/base/tests/serf/leave.rs rename to legacy/serf-core/src/serf/base/tests/serf/leave.rs diff --git a/serf-core/src/serf/base/tests/serf/reap.rs b/legacy/serf-core/src/serf/base/tests/serf/reap.rs similarity index 100% rename from serf-core/src/serf/base/tests/serf/reap.rs rename to legacy/serf-core/src/serf/base/tests/serf/reap.rs diff --git a/serf-core/src/serf/base/tests/serf/reconnect.rs b/legacy/serf-core/src/serf/base/tests/serf/reconnect.rs similarity index 100% rename from serf-core/src/serf/base/tests/serf/reconnect.rs rename to legacy/serf-core/src/serf/base/tests/serf/reconnect.rs diff --git a/serf-core/src/serf/base/tests/serf/remove.rs b/legacy/serf-core/src/serf/base/tests/serf/remove.rs similarity index 100% rename from serf-core/src/serf/base/tests/serf/remove.rs rename to legacy/serf-core/src/serf/base/tests/serf/remove.rs diff --git a/serf-core/src/serf/base/tests/serf/snapshot.rs b/legacy/serf-core/src/serf/base/tests/serf/snapshot.rs similarity index 100% rename from serf-core/src/serf/base/tests/serf/snapshot.rs rename to legacy/serf-core/src/serf/base/tests/serf/snapshot.rs diff --git a/serf-core/src/serf/delegate.rs b/legacy/serf-core/src/serf/delegate.rs similarity index 100% rename from serf-core/src/serf/delegate.rs rename to legacy/serf-core/src/serf/delegate.rs diff --git a/serf-core/src/serf/internal_query.rs b/legacy/serf-core/src/serf/internal_query.rs similarity index 100% rename from serf-core/src/serf/internal_query.rs rename to legacy/serf-core/src/serf/internal_query.rs diff --git a/serf-core/src/serf/query.rs b/legacy/serf-core/src/serf/query.rs similarity index 100% rename from serf-core/src/serf/query.rs rename to legacy/serf-core/src/serf/query.rs diff --git a/serf-core/src/snapshot.rs b/legacy/serf-core/src/snapshot.rs similarity index 100% rename from serf-core/src/snapshot.rs rename to legacy/serf-core/src/snapshot.rs diff --git a/serf-core/src/types.rs b/legacy/serf-core/src/types.rs similarity index 100% rename from serf-core/src/types.rs rename to legacy/serf-core/src/types.rs diff --git a/serf-core/src/types/arbitrary_impl.rs b/legacy/serf-core/src/types/arbitrary_impl.rs similarity index 100% rename from serf-core/src/types/arbitrary_impl.rs rename to legacy/serf-core/src/types/arbitrary_impl.rs diff --git a/serf-core/src/types/clock.rs b/legacy/serf-core/src/types/clock.rs similarity index 100% rename from serf-core/src/types/clock.rs rename to legacy/serf-core/src/types/clock.rs diff --git a/serf-core/src/types/conflict.rs b/legacy/serf-core/src/types/conflict.rs similarity index 100% rename from serf-core/src/types/conflict.rs rename to legacy/serf-core/src/types/conflict.rs diff --git a/serf-core/src/types/coordinate.rs b/legacy/serf-core/src/types/coordinate.rs similarity index 100% rename from serf-core/src/types/coordinate.rs rename to legacy/serf-core/src/types/coordinate.rs diff --git a/serf-core/src/types/filter.rs b/legacy/serf-core/src/types/filter.rs similarity index 100% rename from serf-core/src/types/filter.rs rename to legacy/serf-core/src/types/filter.rs diff --git a/serf-core/src/types/filter/tag_filter.rs b/legacy/serf-core/src/types/filter/tag_filter.rs similarity index 100% rename from serf-core/src/types/filter/tag_filter.rs rename to legacy/serf-core/src/types/filter/tag_filter.rs diff --git a/serf-core/src/types/fuzzy.rs b/legacy/serf-core/src/types/fuzzy.rs similarity index 100% rename from serf-core/src/types/fuzzy.rs rename to legacy/serf-core/src/types/fuzzy.rs diff --git a/serf-core/src/types/join.rs b/legacy/serf-core/src/types/join.rs similarity index 100% rename from serf-core/src/types/join.rs rename to legacy/serf-core/src/types/join.rs diff --git a/serf-core/src/types/key.rs b/legacy/serf-core/src/types/key.rs similarity index 100% rename from serf-core/src/types/key.rs rename to legacy/serf-core/src/types/key.rs diff --git a/serf-core/src/types/leave.rs b/legacy/serf-core/src/types/leave.rs similarity index 100% rename from serf-core/src/types/leave.rs rename to legacy/serf-core/src/types/leave.rs diff --git a/serf-core/src/types/member.rs b/legacy/serf-core/src/types/member.rs similarity index 100% rename from serf-core/src/types/member.rs rename to legacy/serf-core/src/types/member.rs diff --git a/serf-core/src/types/message.rs b/legacy/serf-core/src/types/message.rs similarity index 100% rename from serf-core/src/types/message.rs rename to legacy/serf-core/src/types/message.rs diff --git a/serf-core/src/types/push_pull.rs b/legacy/serf-core/src/types/push_pull.rs similarity index 100% rename from serf-core/src/types/push_pull.rs rename to legacy/serf-core/src/types/push_pull.rs diff --git a/serf-core/src/types/query.rs b/legacy/serf-core/src/types/query.rs similarity index 100% rename from serf-core/src/types/query.rs rename to legacy/serf-core/src/types/query.rs diff --git a/serf-core/src/types/query/response.rs b/legacy/serf-core/src/types/query/response.rs similarity index 100% rename from serf-core/src/types/query/response.rs rename to legacy/serf-core/src/types/query/response.rs diff --git a/serf-core/src/types/quickcheck_impl.rs b/legacy/serf-core/src/types/quickcheck_impl.rs similarity index 100% rename from serf-core/src/types/quickcheck_impl.rs rename to legacy/serf-core/src/types/quickcheck_impl.rs diff --git a/serf-core/src/types/tags.rs b/legacy/serf-core/src/types/tags.rs similarity index 100% rename from serf-core/src/types/tags.rs rename to legacy/serf-core/src/types/tags.rs diff --git a/serf-core/src/types/tests.rs b/legacy/serf-core/src/types/tests.rs similarity index 100% rename from serf-core/src/types/tests.rs rename to legacy/serf-core/src/types/tests.rs diff --git a/serf-core/src/types/user_event.rs b/legacy/serf-core/src/types/user_event.rs similarity index 100% rename from serf-core/src/types/user_event.rs rename to legacy/serf-core/src/types/user_event.rs diff --git a/serf-core/src/types/user_event/message.rs b/legacy/serf-core/src/types/user_event/message.rs similarity index 100% rename from serf-core/src/types/user_event/message.rs rename to legacy/serf-core/src/types/user_event/message.rs diff --git a/serf-core/src/types/user_event/user_events.rs b/legacy/serf-core/src/types/user_event/user_events.rs similarity index 100% rename from serf-core/src/types/user_event/user_events.rs rename to legacy/serf-core/src/types/user_event/user_events.rs diff --git a/serf-core/src/types/version.rs b/legacy/serf-core/src/types/version.rs similarity index 100% rename from serf-core/src/types/version.rs rename to legacy/serf-core/src/types/version.rs diff --git a/serf/Cargo.toml b/legacy/serf/Cargo.toml similarity index 100% rename from serf/Cargo.toml rename to legacy/serf/Cargo.toml diff --git a/legacy/serf/README.md b/legacy/serf/README.md new file mode 120000 index 00000000..32d46ee8 --- /dev/null +++ b/legacy/serf/README.md @@ -0,0 +1 @@ +../README.md \ No newline at end of file diff --git a/serf/src/lib.rs b/legacy/serf/src/lib.rs similarity index 100% rename from serf/src/lib.rs rename to legacy/serf/src/lib.rs diff --git a/serf/src/smol.rs b/legacy/serf/src/smol.rs similarity index 100% rename from serf/src/smol.rs rename to legacy/serf/src/smol.rs diff --git a/serf/src/tokio.rs b/legacy/serf/src/tokio.rs similarity index 100% rename from serf/src/tokio.rs rename to legacy/serf/src/tokio.rs diff --git a/serf/test/main.rs b/legacy/serf/test/main.rs similarity index 100% rename from serf/test/main.rs rename to legacy/serf/test/main.rs diff --git a/serf/test/main/net.rs b/legacy/serf/test/main/net.rs similarity index 100% rename from serf/test/main/net.rs rename to legacy/serf/test/main/net.rs diff --git a/serf/test/main/net/coordinates.rs b/legacy/serf/test/main/net/coordinates.rs similarity index 100% rename from serf/test/main/net/coordinates.rs rename to legacy/serf/test/main/net/coordinates.rs diff --git a/serf/test/main/net/delegate.rs b/legacy/serf/test/main/net/delegate.rs similarity index 100% rename from serf/test/main/net/delegate.rs rename to legacy/serf/test/main/net/delegate.rs diff --git a/serf/test/main/net/delegate/local_state.rs b/legacy/serf/test/main/net/delegate/local_state.rs similarity index 100% rename from serf/test/main/net/delegate/local_state.rs rename to legacy/serf/test/main/net/delegate/local_state.rs diff --git a/serf/test/main/net/delegate/nodemeta.rs b/legacy/serf/test/main/net/delegate/nodemeta.rs similarity index 100% rename from serf/test/main/net/delegate/nodemeta.rs rename to legacy/serf/test/main/net/delegate/nodemeta.rs diff --git a/serf/test/main/net/delegate/ping_delegate.rs b/legacy/serf/test/main/net/delegate/ping_delegate.rs similarity index 100% rename from serf/test/main/net/delegate/ping_delegate.rs rename to legacy/serf/test/main/net/delegate/ping_delegate.rs diff --git a/serf/test/main/net/delegate/remote_state.rs b/legacy/serf/test/main/net/delegate/remote_state.rs similarity index 100% rename from serf/test/main/net/delegate/remote_state.rs rename to legacy/serf/test/main/net/delegate/remote_state.rs diff --git a/serf/test/main/net/event.rs b/legacy/serf/test/main/net/event.rs similarity index 100% rename from serf/test/main/net/event.rs rename to legacy/serf/test/main/net/event.rs diff --git a/serf/test/main/net/event/default_query.rs b/legacy/serf/test/main/net/event/default_query.rs similarity index 100% rename from serf/test/main/net/event/default_query.rs rename to legacy/serf/test/main/net/event/default_query.rs diff --git a/serf/test/main/net/event/event_user.rs b/legacy/serf/test/main/net/event/event_user.rs similarity index 100% rename from serf/test/main/net/event/event_user.rs rename to legacy/serf/test/main/net/event/event_user.rs diff --git a/serf/test/main/net/event/event_user_size_limit.rs b/legacy/serf/test/main/net/event/event_user_size_limit.rs similarity index 100% rename from serf/test/main/net/event/event_user_size_limit.rs rename to legacy/serf/test/main/net/event/event_user_size_limit.rs diff --git a/serf/test/main/net/event/events_failed.rs b/legacy/serf/test/main/net/event/events_failed.rs similarity index 100% rename from serf/test/main/net/event/events_failed.rs rename to legacy/serf/test/main/net/event/events_failed.rs diff --git a/serf/test/main/net/event/events_join.rs b/legacy/serf/test/main/net/event/events_join.rs similarity index 100% rename from serf/test/main/net/event/events_join.rs rename to legacy/serf/test/main/net/event/events_join.rs diff --git a/serf/test/main/net/event/events_leave.rs b/legacy/serf/test/main/net/event/events_leave.rs similarity index 100% rename from serf/test/main/net/event/events_leave.rs rename to legacy/serf/test/main/net/event/events_leave.rs diff --git a/serf/test/main/net/event/events_leave_avoid_infinite_rebroadcast.rs b/legacy/serf/test/main/net/event/events_leave_avoid_infinite_rebroadcast.rs similarity index 100% rename from serf/test/main/net/event/events_leave_avoid_infinite_rebroadcast.rs rename to legacy/serf/test/main/net/event/events_leave_avoid_infinite_rebroadcast.rs diff --git a/serf/test/main/net/event/query.rs b/legacy/serf/test/main/net/event/query.rs similarity index 100% rename from serf/test/main/net/event/query.rs rename to legacy/serf/test/main/net/event/query.rs diff --git a/serf/test/main/net/event/query_deduplicate.rs b/legacy/serf/test/main/net/event/query_deduplicate.rs similarity index 100% rename from serf/test/main/net/event/query_deduplicate.rs rename to legacy/serf/test/main/net/event/query_deduplicate.rs diff --git a/serf/test/main/net/event/query_filter.rs b/legacy/serf/test/main/net/event/query_filter.rs similarity index 100% rename from serf/test/main/net/event/query_filter.rs rename to legacy/serf/test/main/net/event/query_filter.rs diff --git a/serf/test/main/net/event/query_old_message.rs b/legacy/serf/test/main/net/event/query_old_message.rs similarity index 100% rename from serf/test/main/net/event/query_old_message.rs rename to legacy/serf/test/main/net/event/query_old_message.rs diff --git a/serf/test/main/net/event/query_same_clock.rs b/legacy/serf/test/main/net/event/query_same_clock.rs similarity index 100% rename from serf/test/main/net/event/query_same_clock.rs rename to legacy/serf/test/main/net/event/query_same_clock.rs diff --git a/serf/test/main/net/event/query_size_limit.rs b/legacy/serf/test/main/net/event/query_size_limit.rs similarity index 100% rename from serf/test/main/net/event/query_size_limit.rs rename to legacy/serf/test/main/net/event/query_size_limit.rs diff --git a/serf/test/main/net/event/query_size_limit_increased.rs b/legacy/serf/test/main/net/event/query_size_limit_increased.rs similarity index 100% rename from serf/test/main/net/event/query_size_limit_increased.rs rename to legacy/serf/test/main/net/event/query_size_limit_increased.rs diff --git a/serf/test/main/net/event/remove_failed_events_leave.rs b/legacy/serf/test/main/net/event/remove_failed_events_leave.rs similarity index 100% rename from serf/test/main/net/event/remove_failed_events_leave.rs rename to legacy/serf/test/main/net/event/remove_failed_events_leave.rs diff --git a/serf/test/main/net/event/should_process.rs b/legacy/serf/test/main/net/event/should_process.rs similarity index 100% rename from serf/test/main/net/event/should_process.rs rename to legacy/serf/test/main/net/event/should_process.rs diff --git a/serf/test/main/net/event/user_event_old_message.rs b/legacy/serf/test/main/net/event/user_event_old_message.rs similarity index 100% rename from serf/test/main/net/event/user_event_old_message.rs rename to legacy/serf/test/main/net/event/user_event_old_message.rs diff --git a/serf/test/main/net/event/user_event_same_clock.rs b/legacy/serf/test/main/net/event/user_event_same_clock.rs similarity index 100% rename from serf/test/main/net/event/user_event_same_clock.rs rename to legacy/serf/test/main/net/event/user_event_same_clock.rs diff --git a/serf/test/main/net/get_queue_max.rs b/legacy/serf/test/main/net/get_queue_max.rs similarity index 100% rename from serf/test/main/net/get_queue_max.rs rename to legacy/serf/test/main/net/get_queue_max.rs diff --git a/serf/test/main/net/join.rs b/legacy/serf/test/main/net/join.rs similarity index 100% rename from serf/test/main/net/join.rs rename to legacy/serf/test/main/net/join.rs diff --git a/serf/test/main/net/join/intent_buffer_early.rs b/legacy/serf/test/main/net/join/intent_buffer_early.rs similarity index 100% rename from serf/test/main/net/join/intent_buffer_early.rs rename to legacy/serf/test/main/net/join/intent_buffer_early.rs diff --git a/serf/test/main/net/join/intent_newer.rs b/legacy/serf/test/main/net/join/intent_newer.rs similarity index 100% rename from serf/test/main/net/join/intent_newer.rs rename to legacy/serf/test/main/net/join/intent_newer.rs diff --git a/serf/test/main/net/join/intent_old_message.rs b/legacy/serf/test/main/net/join/intent_old_message.rs similarity index 100% rename from serf/test/main/net/join/intent_old_message.rs rename to legacy/serf/test/main/net/join/intent_old_message.rs diff --git a/serf/test/main/net/join/intent_reset_leaving.rs b/legacy/serf/test/main/net/join/intent_reset_leaving.rs similarity index 100% rename from serf/test/main/net/join/intent_reset_leaving.rs rename to legacy/serf/test/main/net/join/intent_reset_leaving.rs diff --git a/serf/test/main/net/join/join_cancel.rs b/legacy/serf/test/main/net/join/join_cancel.rs similarity index 100% rename from serf/test/main/net/join/join_cancel.rs rename to legacy/serf/test/main/net/join/join_cancel.rs diff --git a/serf/test/main/net/join/join_ignore_old.rs b/legacy/serf/test/main/net/join/join_ignore_old.rs similarity index 100% rename from serf/test/main/net/join/join_ignore_old.rs rename to legacy/serf/test/main/net/join/join_ignore_old.rs diff --git a/serf/test/main/net/join/join_leave.rs b/legacy/serf/test/main/net/join/join_leave.rs similarity index 100% rename from serf/test/main/net/join/join_leave.rs rename to legacy/serf/test/main/net/join/join_leave.rs diff --git a/serf/test/main/net/join/join_leave_join.rs b/legacy/serf/test/main/net/join/join_leave_join.rs similarity index 100% rename from serf/test/main/net/join/join_leave_join.rs rename to legacy/serf/test/main/net/join/join_leave_join.rs diff --git a/serf/test/main/net/join/leave_ltime.rs b/legacy/serf/test/main/net/join/leave_ltime.rs similarity index 100% rename from serf/test/main/net/join/leave_ltime.rs rename to legacy/serf/test/main/net/join/leave_ltime.rs diff --git a/serf/test/main/net/join/pending_intent.rs b/legacy/serf/test/main/net/join/pending_intent.rs similarity index 100% rename from serf/test/main/net/join/pending_intent.rs rename to legacy/serf/test/main/net/join/pending_intent.rs diff --git a/serf/test/main/net/join/pending_intents.rs b/legacy/serf/test/main/net/join/pending_intents.rs similarity index 100% rename from serf/test/main/net/join/pending_intents.rs rename to legacy/serf/test/main/net/join/pending_intents.rs diff --git a/serf/test/main/net/leave.rs b/legacy/serf/test/main/net/leave.rs similarity index 100% rename from serf/test/main/net/leave.rs rename to legacy/serf/test/main/net/leave.rs diff --git a/serf/test/main/net/leave/force_leave_failed.rs b/legacy/serf/test/main/net/leave/force_leave_failed.rs similarity index 100% rename from serf/test/main/net/leave/force_leave_failed.rs rename to legacy/serf/test/main/net/leave/force_leave_failed.rs diff --git a/serf/test/main/net/leave/force_leave_leaving.rs b/legacy/serf/test/main/net/leave/force_leave_leaving.rs similarity index 100% rename from serf/test/main/net/leave/force_leave_leaving.rs rename to legacy/serf/test/main/net/leave/force_leave_leaving.rs diff --git a/serf/test/main/net/leave/force_leave_left.rs b/legacy/serf/test/main/net/leave/force_leave_left.rs similarity index 100% rename from serf/test/main/net/leave/force_leave_left.rs rename to legacy/serf/test/main/net/leave/force_leave_left.rs diff --git a/serf/test/main/net/leave/intent_buffer_early.rs b/legacy/serf/test/main/net/leave/intent_buffer_early.rs similarity index 100% rename from serf/test/main/net/leave/intent_buffer_early.rs rename to legacy/serf/test/main/net/leave/intent_buffer_early.rs diff --git a/serf/test/main/net/leave/intent_newer.rs b/legacy/serf/test/main/net/leave/intent_newer.rs similarity index 100% rename from serf/test/main/net/leave/intent_newer.rs rename to legacy/serf/test/main/net/leave/intent_newer.rs diff --git a/serf/test/main/net/leave/intent_old_message.rs b/legacy/serf/test/main/net/leave/intent_old_message.rs similarity index 100% rename from serf/test/main/net/leave/intent_old_message.rs rename to legacy/serf/test/main/net/leave/intent_old_message.rs diff --git a/serf/test/main/net/leave/rejoin_different_role.rs b/legacy/serf/test/main/net/leave/rejoin_different_role.rs similarity index 100% rename from serf/test/main/net/leave/rejoin_different_role.rs rename to legacy/serf/test/main/net/leave/rejoin_different_role.rs diff --git a/serf/test/main/net/leave/snapshot_recovery.rs b/legacy/serf/test/main/net/leave/snapshot_recovery.rs similarity index 100% rename from serf/test/main/net/leave/snapshot_recovery.rs rename to legacy/serf/test/main/net/leave/snapshot_recovery.rs diff --git a/serf/test/main/net/local_member.rs b/legacy/serf/test/main/net/local_member.rs similarity index 100% rename from serf/test/main/net/local_member.rs rename to legacy/serf/test/main/net/local_member.rs diff --git a/serf/test/main/net/name_resolution.rs b/legacy/serf/test/main/net/name_resolution.rs similarity index 100% rename from serf/test/main/net/name_resolution.rs rename to legacy/serf/test/main/net/name_resolution.rs diff --git a/serf/test/main/net/num_nodes.rs b/legacy/serf/test/main/net/num_nodes.rs similarity index 100% rename from serf/test/main/net/num_nodes.rs rename to legacy/serf/test/main/net/num_nodes.rs diff --git a/serf/test/main/net/reap.rs b/legacy/serf/test/main/net/reap.rs similarity index 100% rename from serf/test/main/net/reap.rs rename to legacy/serf/test/main/net/reap.rs diff --git a/serf/test/main/net/reap/handler.rs b/legacy/serf/test/main/net/reap/handler.rs similarity index 100% rename from serf/test/main/net/reap/handler.rs rename to legacy/serf/test/main/net/reap/handler.rs diff --git a/serf/test/main/net/reap/handler_shutdown.rs b/legacy/serf/test/main/net/reap/handler_shutdown.rs similarity index 100% rename from serf/test/main/net/reap/handler_shutdown.rs rename to legacy/serf/test/main/net/reap/handler_shutdown.rs diff --git a/serf/test/main/net/reap/reap.rs b/legacy/serf/test/main/net/reap/reap.rs similarity index 100% rename from serf/test/main/net/reap/reap.rs rename to legacy/serf/test/main/net/reap/reap.rs diff --git a/serf/test/main/net/reconnect.rs b/legacy/serf/test/main/net/reconnect.rs similarity index 100% rename from serf/test/main/net/reconnect.rs rename to legacy/serf/test/main/net/reconnect.rs diff --git a/serf/test/main/net/reconnect/reconnect.rs b/legacy/serf/test/main/net/reconnect/reconnect.rs similarity index 100% rename from serf/test/main/net/reconnect/reconnect.rs rename to legacy/serf/test/main/net/reconnect/reconnect.rs diff --git a/serf/test/main/net/reconnect/same_ip.rs b/legacy/serf/test/main/net/reconnect/same_ip.rs similarity index 100% rename from serf/test/main/net/reconnect/same_ip.rs rename to legacy/serf/test/main/net/reconnect/same_ip.rs diff --git a/serf/test/main/net/reconnect/timeout.rs b/legacy/serf/test/main/net/reconnect/timeout.rs similarity index 100% rename from serf/test/main/net/reconnect/timeout.rs rename to legacy/serf/test/main/net/reconnect/timeout.rs diff --git a/serf/test/main/net/remove.rs b/legacy/serf/test/main/net/remove.rs similarity index 100% rename from serf/test/main/net/remove.rs rename to legacy/serf/test/main/net/remove.rs diff --git a/serf/test/main/net/remove/failed_node.rs b/legacy/serf/test/main/net/remove/failed_node.rs similarity index 100% rename from serf/test/main/net/remove/failed_node.rs rename to legacy/serf/test/main/net/remove/failed_node.rs diff --git a/serf/test/main/net/remove/failed_node_ourself.rs b/legacy/serf/test/main/net/remove/failed_node_ourself.rs similarity index 100% rename from serf/test/main/net/remove/failed_node_ourself.rs rename to legacy/serf/test/main/net/remove/failed_node_ourself.rs diff --git a/serf/test/main/net/remove/failed_node_prune.rs b/legacy/serf/test/main/net/remove/failed_node_prune.rs similarity index 100% rename from serf/test/main/net/remove/failed_node_prune.rs rename to legacy/serf/test/main/net/remove/failed_node_prune.rs diff --git a/serf/test/main/net/role.rs b/legacy/serf/test/main/net/role.rs similarity index 100% rename from serf/test/main/net/role.rs rename to legacy/serf/test/main/net/role.rs diff --git a/serf/test/main/net/set_tags.rs b/legacy/serf/test/main/net/set_tags.rs similarity index 100% rename from serf/test/main/net/set_tags.rs rename to legacy/serf/test/main/net/set_tags.rs diff --git a/serf/test/main/net/snapshot.rs b/legacy/serf/test/main/net/snapshot.rs similarity index 100% rename from serf/test/main/net/snapshot.rs rename to legacy/serf/test/main/net/snapshot.rs diff --git a/serf/test/main/net/snapshot/snapshoter.rs b/legacy/serf/test/main/net/snapshot/snapshoter.rs similarity index 100% rename from serf/test/main/net/snapshot/snapshoter.rs rename to legacy/serf/test/main/net/snapshot/snapshoter.rs diff --git a/serf/test/main/net/snapshot/snapshoter_force_compact.rs b/legacy/serf/test/main/net/snapshot/snapshoter_force_compact.rs similarity index 100% rename from serf/test/main/net/snapshot/snapshoter_force_compact.rs rename to legacy/serf/test/main/net/snapshot/snapshoter_force_compact.rs diff --git a/serf/test/main/net/snapshot/snapshoter_leave.rs b/legacy/serf/test/main/net/snapshot/snapshoter_leave.rs similarity index 100% rename from serf/test/main/net/snapshot/snapshoter_leave.rs rename to legacy/serf/test/main/net/snapshot/snapshoter_leave.rs diff --git a/serf/test/main/net/snapshot/snapshoter_leave_rejoin.rs b/legacy/serf/test/main/net/snapshot/snapshoter_leave_rejoin.rs similarity index 100% rename from serf/test/main/net/snapshot/snapshoter_leave_rejoin.rs rename to legacy/serf/test/main/net/snapshot/snapshoter_leave_rejoin.rs diff --git a/serf/test/main/net/snapshot/snapshoter_recovery.rs b/legacy/serf/test/main/net/snapshot/snapshoter_recovery.rs similarity index 100% rename from serf/test/main/net/snapshot/snapshoter_recovery.rs rename to legacy/serf/test/main/net/snapshot/snapshoter_recovery.rs diff --git a/serf/test/main/net/state.rs b/legacy/serf/test/main/net/state.rs similarity index 100% rename from serf/test/main/net/state.rs rename to legacy/serf/test/main/net/state.rs diff --git a/serf/test/main/net/stats.rs b/legacy/serf/test/main/net/stats.rs similarity index 100% rename from serf/test/main/net/stats.rs rename to legacy/serf/test/main/net/stats.rs diff --git a/serf/test/main/net/update.rs b/legacy/serf/test/main/net/update.rs similarity index 100% rename from serf/test/main/net/update.rs rename to legacy/serf/test/main/net/update.rs diff --git a/serf/test/main/net/write_keyring_file.rs b/legacy/serf/test/main/net/write_keyring_file.rs similarity index 100% rename from serf/test/main/net/write_keyring_file.rs rename to legacy/serf/test/main/net/write_keyring_file.rs diff --git a/serf-proto/Cargo.toml b/serf-proto/Cargo.toml new file mode 100644 index 00000000..62c76601 --- /dev/null +++ b/serf-proto/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "serf-proto" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +[dependencies] +memberlist-proto = { git = "https://github.com/al8n/memberlist", branch = "main", default-features = false, features = ["std"] } +bytes = { workspace = true } +byteorder.workspace = true +smallvec.workspace = true +indexmap.workspace = true +derive_more.workspace = true +viewit.workspace = true +thiserror.workspace = true +bitflags = "2" + +[dev-dependencies] diff --git a/serf-proto/src/lib.rs b/serf-proto/src/lib.rs new file mode 100644 index 00000000..d26c44d5 --- /dev/null +++ b/serf-proto/src/lib.rs @@ -0,0 +1,5 @@ +//! The serf wire codec — pure, no-I/O message types shared by the serf driver crates. +//! +//! Depends on `memberlist-proto` for the `Data`/`DataRef` codec primitives; defines serf's +//! own message set and framing on top of them. +#![deny(missing_docs)] From 98693b6be79b1cabb0114fd67698937f493bc6c7 Mon Sep 17 00:00:00 2001 From: al8n Date: Tue, 23 Jun 2026 22:19:04 +1200 Subject: [PATCH 002/128] feat(serf-proto): buffa codegen + UserEventMessage round-trip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add buffa-build codegen (proto/serf/v1/messages.proto → OUT_DIR/serf_wire_generated.rs), the typed UserEventMessage + LamportTime, the user_event_{to,from}_pb bridge, and two round-trip tests (happy-path + missing-ltime rejection). Build and test both green. --- serf-proto/Cargo.toml | 5 +++ serf-proto/build.rs | 19 ++++++++++ serf-proto/proto/serf/v1/messages.proto | 16 ++++++++ serf-proto/src/bridge/mod.rs | 50 +++++++++++++++++++++++++ serf-proto/src/lib.rs | 41 ++++++++++++++++++++ serf-proto/src/messages/mod.rs | 20 ++++++++++ serf-proto/src/messages/tests.rs | 39 +++++++++++++++++++ serf-proto/src/typed/mod.rs | 22 +++++++++++ 8 files changed, 212 insertions(+) create mode 100644 serf-proto/build.rs create mode 100644 serf-proto/proto/serf/v1/messages.proto create mode 100644 serf-proto/src/bridge/mod.rs create mode 100644 serf-proto/src/messages/mod.rs create mode 100644 serf-proto/src/messages/tests.rs create mode 100644 serf-proto/src/typed/mod.rs diff --git a/serf-proto/Cargo.toml b/serf-proto/Cargo.toml index 62c76601..f3187ddd 100644 --- a/serf-proto/Cargo.toml +++ b/serf-proto/Cargo.toml @@ -16,5 +16,10 @@ derive_more.workspace = true viewit.workspace = true thiserror.workspace = true bitflags = "2" +buffa = { version = "0.7", default-features = false } +smol_str = { workspace = true } + +[build-dependencies] +buffa-build = "0.7" [dev-dependencies] diff --git a/serf-proto/build.rs b/serf-proto/build.rs new file mode 100644 index 00000000..6f1228f3 --- /dev/null +++ b/serf-proto/build.rs @@ -0,0 +1,19 @@ +//! Build-time codegen for the `messages` module. +//! +//! Uses `buffa-build` to invoke `protoc` against +//! `proto/serf/v1/messages.proto` and produce Rust types via +//! `buffa-codegen`. The output is written to `OUT_DIR` and pulled in by +//! `src/messages/mod.rs` via `include!`. + +fn main() { + println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rerun-if-changed=proto"); + + buffa_build::Config::new() + .files(&["proto/serf/v1/messages.proto"]) + .includes(&["proto"]) + .use_bytes_type() + .include_file("serf_wire_generated.rs") + .compile() + .expect("buffa codegen failed"); +} diff --git a/serf-proto/proto/serf/v1/messages.proto b/serf-proto/proto/serf/v1/messages.proto new file mode 100644 index 00000000..3a35678c --- /dev/null +++ b/serf-proto/proto/serf/v1/messages.proto @@ -0,0 +1,16 @@ +// Wire message schemas for the serf gossip protocol. +// +// Wire format is standard protobuf (buffa) — not byte-compatible with serf's old hand-rolled codec; new↔new clusters only. + +syntax = "proto3"; + +package serf.v1; + +// UserEventMessage is broadcast when a user-generated event is emitted. +// `optional` on ltime because the decoder must reject a missing lamport clock. +message UserEventMessage { + optional uint64 ltime = 1; + bool cc = 2; + string name = 3; + bytes payload = 4; +} diff --git a/serf-proto/src/bridge/mod.rs b/serf-proto/src/bridge/mod.rs new file mode 100644 index 00000000..2556fe1c --- /dev/null +++ b/serf-proto/src/bridge/mod.rs @@ -0,0 +1,50 @@ +//! Conversion between serf typed message shapes and the buffa-generated codec types. +//! +//! The typed side holds rich Rust types (`LamportTime`, `SmolStr`, `Bytes`); +//! the buffa side stores them as primitive protobuf types. These functions are +//! the single boundary where those conversions happen. + +use std::borrow::Cow; + +use crate::{ + LamportTime, + messages::serf::v1 as pb, + typed::UserEventMessage, +}; + +// ─── BridgeError ───────────────────────────────────────────────────────────── + +/// Errors that can occur when converting between typed shapes and buffa types. +#[non_exhaustive] +#[derive(Debug, Clone, thiserror::Error)] +pub enum BridgeError { + /// A required field was absent in the wire message. + #[error("missing required field: {0}")] + MissingField(Cow<'static, str>), +} + +// ─── UserEventMessage ──────────────────────────────────────────────────────── + +/// Convert a typed [`UserEventMessage`] → `pb::UserEventMessage`. +pub fn user_event_to_pb(t: &UserEventMessage) -> pb::UserEventMessage { + pb::UserEventMessage { + ltime: Some(t.ltime.into()), + cc: t.cc, + name: t.name.to_string(), + payload: t.payload.clone(), + ..Default::default() + } +} + +/// Convert `pb::UserEventMessage` → typed [`UserEventMessage`]. +pub fn user_event_from_pb(b: &pb::UserEventMessage) -> Result { + let ltime = b + .ltime + .ok_or(BridgeError::MissingField("UserEventMessage.ltime".into()))?; + Ok(UserEventMessage { + ltime: LamportTime::from(ltime), + cc: b.cc, + name: smol_str::SmolStr::from(b.name.as_str()), + payload: b.payload.clone(), + }) +} diff --git a/serf-proto/src/lib.rs b/serf-proto/src/lib.rs index d26c44d5..aaa9b90f 100644 --- a/serf-proto/src/lib.rs +++ b/serf-proto/src/lib.rs @@ -3,3 +3,44 @@ //! Depends on `memberlist-proto` for the `Data`/`DataRef` codec primitives; defines serf's //! own message set and framing on top of them. #![deny(missing_docs)] + +pub use bridge::{BridgeError, user_event_from_pb, user_event_to_pb}; +pub use typed::UserEventMessage; + +/// A lamport logical clock value — a monotonically increasing counter used to +/// order serf events. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +#[repr(transparent)] +pub struct LamportTime(pub(crate) u64); + +impl core::fmt::Display for LamportTime { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "{}", self.0) + } +} + +impl From for LamportTime { + fn from(t: u64) -> Self { + Self(t) + } +} + +impl From for u64 { + fn from(t: LamportTime) -> Self { + t.0 + } +} + +impl LamportTime { + /// Zero lamport time. + pub const ZERO: Self = LamportTime(0); + + /// Creates a new `LamportTime` from a `u64`. + pub const fn new(t: u64) -> Self { + Self(t) + } +} + +pub mod bridge; +pub mod messages; +pub mod typed; diff --git a/serf-proto/src/messages/mod.rs b/serf-proto/src/messages/mod.rs new file mode 100644 index 00000000..0df1620a --- /dev/null +++ b/serf-proto/src/messages/mod.rs @@ -0,0 +1,20 @@ +//! Buffa-generated owned + view types for serf wire messages. +//! +//! The types in this module are generated at build time from +//! `proto/serf/v1/messages.proto` via `buffa-build` + `buffa-codegen`. +//! Do not edit this file; edit the `.proto` schema instead. +//! +//! # Generated types +//! +//! - [`serf::v1::UserEventMessage`] — user-generated event broadcast +//! +//! All `bytes` fields use [`bytes::Bytes`] (configured via +//! `buffa_build::Config::use_bytes_type()`). + +#![allow(missing_docs)] +#![allow(clippy::all)] + +include!(concat!(env!("OUT_DIR"), "/serf_wire_generated.rs")); + +#[cfg(test)] +mod tests; diff --git a/serf-proto/src/messages/tests.rs b/serf-proto/src/messages/tests.rs new file mode 100644 index 00000000..32884ff7 --- /dev/null +++ b/serf-proto/src/messages/tests.rs @@ -0,0 +1,39 @@ +use buffa::Message as _; + +use super::serf::v1::UserEventMessage as PbUserEventMessage; +use crate::{LamportTime, UserEventMessage, user_event_from_pb, user_event_to_pb}; + +#[test] +fn user_event_message_roundtrip_pb() { + let typed = UserEventMessage { + ltime: LamportTime::new(42), + cc: true, + name: smol_str::SmolStr::from("deploy"), + payload: bytes::Bytes::from_static(b"hello-serf"), + }; + + // typed → pb → bytes → pb → typed + let pb = user_event_to_pb(&typed); + let encoded = pb.encode_to_vec(); + let decoded_pb = PbUserEventMessage::decode_from_slice(encoded.as_slice()) + .expect("decode_from_slice failed"); + let roundtripped = user_event_from_pb(&decoded_pb).expect("user_event_from_pb failed"); + + assert_eq!(roundtripped, typed); +} + +#[test] +fn user_event_message_ltime_required() { + // A pb message with no ltime must be rejected by user_event_from_pb. + let pb = PbUserEventMessage { + ltime: None, + cc: false, + name: String::from("x"), + payload: bytes::Bytes::new(), + ..Default::default() + }; + assert!( + user_event_from_pb(&pb).is_err(), + "expected BridgeError::MissingField for absent ltime" + ); +} diff --git a/serf-proto/src/typed/mod.rs b/serf-proto/src/typed/mod.rs new file mode 100644 index 00000000..1ed5f679 --- /dev/null +++ b/serf-proto/src/typed/mod.rs @@ -0,0 +1,22 @@ +//! Typed serf message shapes — the canonical in-memory representations. +//! +//! These types are what the serf state machine works with directly. The +//! `bridge` module converts them to/from the buffa-generated codec types. + +use bytes::Bytes; +use smol_str::SmolStr; + +use crate::LamportTime; + +/// A user-generated event broadcast through the serf cluster. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct UserEventMessage { + /// The lamport clock value at the time the event was emitted. + pub ltime: LamportTime, + /// "Can Coalesce" — whether the event may be merged with later identical events. + pub cc: bool, + /// The event name. + pub name: SmolStr, + /// The event payload. + pub payload: Bytes, +} From 2ac06c1becf6ac4733bda41aeef88053732c6127 Mon Sep 17 00:00:00 2001 From: al8n Date: Tue, 23 Jun 2026 22:37:32 +1200 Subject: [PATCH 003/128] feat(serf-proto): MessageType enum + plain-frame encode/decode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `serf-proto::framing`: - `MessageType` enum with 10 variants (Leave/Join/PushPull/UserEvent/ Query/QueryResponse/ConflictResponse/Relay/KeyRequest/KeyResponse) + `Unknown(u8)` for forward compatibility; tag bytes 1–10 match the legacy serf-core constants. - `encode_message(ty, &impl buffa::Message) -> Result, FrameError>`: writes `[tag][LEB128 body_len][buffa body]` into a single allocation. - `decode_message(&Bytes) -> Result<(MessageType, Bytes, usize), FrameError>`: zero-copy body slice via `Bytes::slice`, returns bytes consumed. - `FrameError` covers Empty / Incomplete / VarintOverflow / FrameTooLarge / Decode. All four items re-exported from the crate root. 6 framing tests + 2 pre-existing messages tests — 8 total, all green. --- serf-proto/src/framing/mod.rs | 260 ++++++++++++++++++++++++++++++++ serf-proto/src/framing/tests.rs | 178 ++++++++++++++++++++++ serf-proto/src/lib.rs | 2 + 3 files changed, 440 insertions(+) create mode 100644 serf-proto/src/framing/mod.rs create mode 100644 serf-proto/src/framing/tests.rs diff --git a/serf-proto/src/framing/mod.rs b/serf-proto/src/framing/mod.rs new file mode 100644 index 00000000..d33d6fa8 --- /dev/null +++ b/serf-proto/src/framing/mod.rs @@ -0,0 +1,260 @@ +//! Plain-frame encoder/decoder: `[TAG_BYTE][VARINT_LEN][BUFFA_BODY]`. +//! +//! # Wire format +//! +//! A serf message on the wire is three contiguous regions: +//! +//! ```text +//! [ 1 byte : MessageType tag ][ LEB128 u32 : body length ][ body bytes ] +//! ``` +//! +//! The tag byte is the raw numeric discriminant of [`MessageType`] (e.g. +//! `UserEvent` = `4`). The body is the buffa-encoded protobuf bytes for the +//! concrete message type. This framing is serf-specific and is NOT +//! byte-compatible with the legacy `serf-core` hand-rolled codec (which used +//! the memberlist-proto `merge(WireType::LengthDelimited, TAG)` scheme); +//! serf-proto forms new-wire-only clusters. + +use bytes::Bytes; + +// ── MessageType ────────────────────────────────────────────────────────────── + +/// Tag constants — the raw numeric discriminants used as the envelope byte. +const LEAVE_TAG: u8 = 1; +const JOIN_TAG: u8 = 2; +const PUSH_PULL_TAG: u8 = 3; +const USER_EVENT_TAG: u8 = 4; +const QUERY_TAG: u8 = 5; +const QUERY_RESPONSE_TAG: u8 = 6; +const CONFLICT_RESPONSE_TAG: u8 = 7; +const RELAY_TAG: u8 = 8; +const KEY_REQUEST_TAG: u8 = 9; +const KEY_RESPONSE_TAG: u8 = 10; + +/// One-byte discriminant that opens every serf message frame. +/// +/// Numeric values are identical to the legacy `serf-core` tag constants so +/// that future mixed-version migration tooling can map them trivially. +/// `Unknown(u8)` provides forward compatibility for tag values not yet +/// recognised by this build. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +#[non_exhaustive] +pub enum MessageType { + /// Leave — node announcing it is leaving the cluster. + Leave, + /// Join — node joining the cluster. + Join, + /// PushPull — full state-sync exchange. + PushPull, + /// UserEvent — application-level event broadcast. + UserEvent, + /// Query — serf RPC query fanout. + Query, + /// QueryResponse — response to a Query. + QueryResponse, + /// ConflictResponse — tie-breaker for conflicting node names. + ConflictResponse, + /// Relay — message relayed through an intermediary node. + Relay, + /// KeyRequest — encryption key management request. + KeyRequest, + /// KeyResponse — encryption key management response. + KeyResponse, + /// A tag not recognised by this build — preserved for forward compatibility. + Unknown(u8), +} + +impl From for MessageType { + fn from(b: u8) -> Self { + match b { + LEAVE_TAG => Self::Leave, + JOIN_TAG => Self::Join, + PUSH_PULL_TAG => Self::PushPull, + USER_EVENT_TAG => Self::UserEvent, + QUERY_TAG => Self::Query, + QUERY_RESPONSE_TAG => Self::QueryResponse, + CONFLICT_RESPONSE_TAG => Self::ConflictResponse, + RELAY_TAG => Self::Relay, + KEY_REQUEST_TAG => Self::KeyRequest, + KEY_RESPONSE_TAG => Self::KeyResponse, + val => Self::Unknown(val), + } + } +} + +impl From for u8 { + fn from(ty: MessageType) -> Self { + match ty { + MessageType::Leave => LEAVE_TAG, + MessageType::Join => JOIN_TAG, + MessageType::PushPull => PUSH_PULL_TAG, + MessageType::UserEvent => USER_EVENT_TAG, + MessageType::Query => QUERY_TAG, + MessageType::QueryResponse => QUERY_RESPONSE_TAG, + MessageType::ConflictResponse => CONFLICT_RESPONSE_TAG, + MessageType::Relay => RELAY_TAG, + MessageType::KeyRequest => KEY_REQUEST_TAG, + MessageType::KeyResponse => KEY_RESPONSE_TAG, + MessageType::Unknown(val) => val, + } + } +} + +// ── Errors ─────────────────────────────────────────────────────────────────── + +/// The `(available, required)` byte-count pair carried by +/// [`FrameError::Incomplete`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("incomplete frame: {available} bytes available, {required} required")] +pub struct IncompleteFrame { + available: usize, + required: usize, +} + +impl IncompleteFrame { + /// Construct an incomplete-frame payload. + #[inline(always)] + pub const fn new(available: usize, required: usize) -> Self { + Self { available, required } + } + + /// Bytes available in the buffer. + #[inline(always)] + pub const fn available(&self) -> usize { + self.available + } + + /// Bytes required to complete the frame. + #[inline(always)] + pub const fn required(&self) -> usize { + self.required + } +} + +/// Errors returned by [`encode_message`] and [`decode_message`]. +#[non_exhaustive] +#[derive(Debug, Clone, thiserror::Error)] +pub enum FrameError { + /// The input buffer is empty; no frame to decode. + #[error("frame buffer is empty")] + Empty, + /// The buffer holds a partial frame — more bytes are required. + #[error(transparent)] + Incomplete(IncompleteFrame), + /// The varint length field overflows a `u32`. + #[error("varint length overflows u32")] + VarintOverflow, + /// The buffa `encoded_len` / actual-write lengths disagree, which would + /// desynchronize the receiver's length prefix. Carries the actual number of + /// bytes written by `buffa::Message::encode`. + #[error("frame encode length mismatch: encoded_len predicted {0} bytes but encode wrote a different count")] + FrameTooLarge(usize), + /// The buffa decoder rejected the body bytes. + #[error("buffa decode error: body bytes could not be decoded")] + Decode, +} + +// ── Varint helpers ─────────────────────────────────────────────────────────── + +/// Append a LEB128-encoded `u32` to `out`. +fn encode_varint_u32(mut value: u32, out: &mut Vec) { + while value >= 0x80 { + out.push(((value & 0x7f) as u8) | 0x80); + value >>= 7; + } + out.push(value as u8); +} + +/// Decode a LEB128 `u32` from the front of `buf`. +/// +/// Returns `(value, bytes_consumed)`. +fn decode_varint_u32(buf: &[u8]) -> Result<(u32, usize), FrameError> { + let mut value: u32 = 0; + let mut shift: u32 = 0; + for (i, &byte) in buf.iter().enumerate().take(5) { + // The 5th byte of a u32 LEB128 can only have 4 significant bits (bits + // 28–31); a byte larger than 0x0f would set bits 32+ and overflow. + if i == 4 && byte > 0x0f { + return Err(FrameError::VarintOverflow); + } + value |= u32::from(byte & 0x7f) << shift; + if byte & 0x80 == 0 { + return Ok((value, i + 1)); + } + shift += 7; + } + // Fell through without a terminating byte — truncated length prefix. + Err(FrameError::Incomplete(IncompleteFrame::new(buf.len(), buf.len() + 1))) +} + +// ── Public API ─────────────────────────────────────────────────────────────── + +/// Encode `msg` into a plain serf frame: `[tag][varint body_len][buffa body]`. +/// +/// Returns a freshly allocated `Vec` containing the complete frame. +/// Fails only when the buffa body length exceeds `u32::MAX` (which buffa's +/// `u32` return type for `encoded_len` already guarantees cannot happen). +pub fn encode_message(ty: MessageType, msg: &M) -> Result, FrameError> +where + M: buffa::Message, +{ + // buffa::Message::encoded_len() returns u32 — already bounded. + let body_len_u32: u32 = msg.encoded_len(); + let body_len: usize = body_len_u32 as usize; + + // Pre-size: 1 (tag) + up to 5 (varint) + body. + let mut out = Vec::with_capacity(1 + 5 + body_len); + out.push(u8::from(ty)); + encode_varint_u32(body_len_u32, &mut out); + let body_start = out.len(); + msg.encode(&mut out); + + // Guard against an `encoded_len()` / actual-write disagreement that would + // desynchronise the receiver's length prefix. + let written = out.len() - body_start; + if written != body_len { + return Err(FrameError::FrameTooLarge(written)); + } + + Ok(out) +} + +/// Decode the leading plain frame from `buf`. +/// +/// Returns `(MessageType, body_bytes, bytes_consumed)` on success. +/// `body_bytes` is a zero-copy sub-slice of the input [`Bytes`]; the caller +/// is responsible for decoding it to the appropriate concrete type with +/// `buffa::Message::decode` / `decode_from_slice`. +/// +/// `bytes_consumed` is the total number of bytes read (tag + varint + body), +/// allowing a streaming caller to advance its read cursor. +pub fn decode_message(frame: &Bytes) -> Result<(MessageType, Bytes, usize), FrameError> { + let buf = frame.as_ref(); + if buf.is_empty() { + return Err(FrameError::Empty); + } + + let ty = MessageType::from(buf[0]); + + let (body_len, varint_bytes) = match decode_varint_u32(&buf[1..]) { + Ok(v) => v, + Err(FrameError::Incomplete(_)) => { + return Err(FrameError::Incomplete(IncompleteFrame::new(buf.len(), buf.len() + 1))); + } + Err(e) => return Err(e), + }; + + let header_len = 1 + varint_bytes; + let frame_end = header_len + body_len as usize; + + if buf.len() < frame_end { + return Err(FrameError::Incomplete(IncompleteFrame::new(buf.len(), frame_end))); + } + + // Zero-copy slice of the body out of the `Bytes` allocation. + let body = frame.slice(header_len..frame_end); + Ok((ty, body, frame_end)) +} + +#[cfg(test)] +mod tests; diff --git a/serf-proto/src/framing/tests.rs b/serf-proto/src/framing/tests.rs new file mode 100644 index 00000000..77054c9d --- /dev/null +++ b/serf-proto/src/framing/tests.rs @@ -0,0 +1,178 @@ +use buffa::Message as _; +use bytes::Bytes; + +use super::{FrameError, MessageType, decode_message, encode_message}; +use crate::messages::serf::v1::UserEventMessage as PbUserEventMessage; +use crate::{LamportTime, UserEventMessage, user_event_from_pb, user_event_to_pb}; + +// ── MessageType round-trips ─────────────────────────────────────────────────── + +#[test] +fn message_type_tag_round_trip() { + let cases: &[(MessageType, u8)] = &[ + (MessageType::Leave, 1), + (MessageType::Join, 2), + (MessageType::PushPull, 3), + (MessageType::UserEvent, 4), + (MessageType::Query, 5), + (MessageType::QueryResponse, 6), + (MessageType::ConflictResponse, 7), + (MessageType::Relay, 8), + (MessageType::KeyRequest, 9), + (MessageType::KeyResponse, 10), + ]; + + for &(ref ty, expected_byte) in cases { + let byte = u8::from(*ty); + assert_eq!( + byte, expected_byte, + "{ty:?} should have tag byte {expected_byte}" + ); + assert_eq!( + MessageType::from(byte), + *ty, + "u8 {byte} should round-trip to {ty:?}" + ); + } +} + +#[test] +fn message_type_unknown_preserved() { + let unknown_byte: u8 = 42; + let ty = MessageType::from(unknown_byte); + assert_eq!(ty, MessageType::Unknown(42)); + assert_eq!(u8::from(ty), unknown_byte); +} + +// ── encode_message / decode_message ────────────────────────────────────────── + +#[test] +fn user_event_frame_round_trip() { + // Build a typed UserEventMessage and convert to the pb shape. + let typed = UserEventMessage { + ltime: LamportTime::new(7), + cc: true, + name: smol_str::SmolStr::from("deploy"), + payload: Bytes::from_static(b"hello-serf"), + }; + let pb = user_event_to_pb(&typed); + + // Encode into a serf frame. + let frame_vec = + encode_message(MessageType::UserEvent, &pb).expect("encode_message should succeed"); + assert!( + !frame_vec.is_empty(), + "encoded frame must not be empty" + ); + + // The leading byte must be the UserEvent tag. + assert_eq!( + frame_vec[0], + u8::from(MessageType::UserEvent), + "first byte must be the UserEvent tag (4)" + ); + + // decode_message recovers the tag and zero-copy body slice. + let frame = Bytes::from(frame_vec); + let (recovered_ty, body, consumed) = + decode_message(&frame).expect("decode_message should succeed"); + + assert_eq!(recovered_ty, MessageType::UserEvent); + assert_eq!( + consumed, + frame.len(), + "consumed must equal the full frame length" + ); + + // The body bytes decode back to the original pb message. + let decoded_pb = + PbUserEventMessage::decode_from_slice(body.as_ref()).expect("decode_from_slice should succeed"); + let roundtripped = user_event_from_pb(&decoded_pb).expect("user_event_from_pb should succeed"); + + assert_eq!(roundtripped, typed); +} + +#[test] +fn decode_message_empty_errors() { + let empty = Bytes::new(); + assert!(matches!(decode_message(&empty), Err(super::FrameError::Empty))); +} + +#[test] +fn decode_message_truncated_errors() { + // Only the tag byte — no varint length yet. + let truncated = Bytes::from_static(&[4u8]); + assert!(matches!( + decode_message(&truncated), + Err(super::FrameError::Incomplete(_)) + )); +} + +#[test] +fn encode_message_encodes_empty_body() { + // An empty buffa message (all fields at default) should produce a valid + // frame containing just the tag + a single zero-varint (body_len=0). + let pb = PbUserEventMessage::default(); + let frame_vec = + encode_message(MessageType::UserEvent, &pb).expect("encode_message should succeed"); + let frame = Bytes::from(frame_vec); + let (ty, body, consumed) = decode_message(&frame).expect("decode_message should succeed"); + assert_eq!(ty, MessageType::UserEvent); + assert_eq!(body.len(), 0); + assert_eq!(consumed, frame.len()); +} + +// ── Security-relevant decoder edge cases ───────────────────────────────────── + +#[test] +fn decode_varint_overflow_is_error_not_panic() { + // A 5-byte LEB128 whose 5th byte has bits beyond position 28 set would + // overflow u32. The decoder must reject it with VarintOverflow, not panic. + // Byte sequence: 4 continuation bytes (all 0x80) + one final byte > 0x0f. + let buf: &[u8] = &[ + u8::from(MessageType::UserEvent), // tag + 0x80, 0x80, 0x80, 0x80, 0x10, // 5-byte LEB128 with 5th byte = 0x10 > 0x0f + ]; + let frame = Bytes::copy_from_slice(buf); + assert!( + matches!(decode_message(&frame), Err(FrameError::VarintOverflow)), + "a 5th LEB128 byte > 0x0f must yield VarintOverflow" + ); +} + +#[test] +fn decode_large_declared_body_len_is_incomplete_not_alloc() { + // A well-formed header declaring body_len = 1_000_000 in a 5-byte buffer + // must yield Incomplete, not a panic or a huge allocation attempt. + // Encode body_len = 1_000_000 as LEB128: 0xC0 0x84 0x3D (3 bytes). + let body_len: u32 = 1_000_000; + let mut header = vec![u8::from(MessageType::UserEvent)]; + let mut v = body_len; + while v >= 0x80 { + header.push(((v & 0x7f) as u8) | 0x80); + v >>= 7; + } + header.push(v as u8); + // The buffer is just the header — no body bytes follow. + let frame = Bytes::from(header); + assert!( + matches!(decode_message(&frame), Err(FrameError::Incomplete(_))), + "a well-formed header with body_len=1_000_000 but no body bytes must yield Incomplete" + ); +} + +#[test] +fn unknown_tag_round_trips_encode_decode() { + // An Unknown(200) tag must survive encode + decode: the tag byte is + // preserved and the body (empty default message) is recovered intact. + let pb = PbUserEventMessage::default(); + let frame_vec = + encode_message(MessageType::Unknown(200), &pb).expect("encode_message with Unknown tag should succeed"); + let frame = Bytes::from(frame_vec); + assert_eq!(frame[0], 200, "first byte must be the Unknown tag value 200"); + let (recovered_ty, body, consumed) = + decode_message(&frame).expect("decode_message with Unknown tag should succeed"); + assert_eq!(recovered_ty, MessageType::Unknown(200)); + assert_eq!(body.len(), 0); + assert_eq!(consumed, frame.len()); +} diff --git a/serf-proto/src/lib.rs b/serf-proto/src/lib.rs index aaa9b90f..1c6e1dc1 100644 --- a/serf-proto/src/lib.rs +++ b/serf-proto/src/lib.rs @@ -5,6 +5,7 @@ #![deny(missing_docs)] pub use bridge::{BridgeError, user_event_from_pb, user_event_to_pb}; +pub use framing::{FrameError, MessageType, decode_message, encode_message}; pub use typed::UserEventMessage; /// A lamport logical clock value — a monotonically increasing counter used to @@ -42,5 +43,6 @@ impl LamportTime { } pub mod bridge; +pub mod framing; pub mod messages; pub mod typed; From f0c5fc89a1c1b4b73e8a23b935f2e576c34d0cac Mon Sep 17 00:00:00 2001 From: al8n Date: Tue, 23 Jun 2026 22:56:55 +1200 Subject: [PATCH 004/128] feat(serf-proto): QueryFlag, Coordinate, Tags, Filter typed types + proto + bridge Adds the four foundation field types: - QueryFlag: bitflags! u32 (ACK | NO_BROADCAST); no standalone proto, u32 wire - Coordinate: typed + pb::Coordinate (repeated double portion, error/adjustment/height) - Tags: typed + pb::Tags (map entries via buffa HashMap) - Filter: typed enum + pb::Filter (oneof NodeIdList|TagFilter, boxed by buffa) Bridges: coordinate_{to,from}_pb, tags_{to,from}_pb, filter_{to,from}_pb. Tests: 14 new round-trip tests (22 total), all green. --- serf-proto/proto/serf/v1/messages.proto | 38 +++++ serf-proto/src/bridge/mod.rs | 88 +++++++++++- serf-proto/src/lib.rs | 14 +- serf-proto/src/messages/tests.rs | 178 +++++++++++++++++++++++- serf-proto/src/typed/mod.rs | 123 ++++++++++++++++ 5 files changed, 434 insertions(+), 7 deletions(-) diff --git a/serf-proto/proto/serf/v1/messages.proto b/serf-proto/proto/serf/v1/messages.proto index 3a35678c..34df8db3 100644 --- a/serf-proto/proto/serf/v1/messages.proto +++ b/serf-proto/proto/serf/v1/messages.proto @@ -14,3 +14,41 @@ message UserEventMessage { string name = 3; bytes payload = 4; } + +// Coordinate is the Vivaldi network-coordinate wire point. +// All values are in units of seconds. +// `portion` is the Euclidean part (variable-length vector of f64). +message Coordinate { + repeated double portion = 1; + double error = 2; + double adjustment = 3; + double height = 4; +} + +// Tags encodes node metadata as a string→string map gossiped via node meta. +message Tags { + map entries = 1; +} + +// TagFilter matches nodes whose tag value satisfies an optional regex. +message TagFilter { + string tag = 1; + // expr is omitted when no regex constraint is needed (match-any). + optional string expr = 2; +} + +// Filter is a single query-scoping predicate. +// Exactly one of the oneof alternatives is set per Filter message. +message Filter { + oneof kind { + // node_ids is a list of node-id strings; only those nodes respond. + NodeIdList node_ids = 1; + // tag is a tag-name / optional-regex pair. + TagFilter tag = 2; + } +} + +// NodeIdList is the payload for a node-id filter. +message NodeIdList { + repeated string ids = 1; +} diff --git a/serf-proto/src/bridge/mod.rs b/serf-proto/src/bridge/mod.rs index 2556fe1c..5073410c 100644 --- a/serf-proto/src/bridge/mod.rs +++ b/serf-proto/src/bridge/mod.rs @@ -6,10 +6,12 @@ use std::borrow::Cow; +use smol_str::SmolStr; + use crate::{ LamportTime, messages::serf::v1 as pb, - typed::UserEventMessage, + typed::{Coordinate, Filter, TagFilter, Tags, UserEventMessage}, }; // ─── BridgeError ───────────────────────────────────────────────────────────── @@ -21,6 +23,9 @@ pub enum BridgeError { /// A required field was absent in the wire message. #[error("missing required field: {0}")] MissingField(Cow<'static, str>), + /// A `oneof` field held no recognised variant. + #[error("unknown or missing oneof variant in {0}")] + UnknownVariant(Cow<'static, str>), } // ─── UserEventMessage ──────────────────────────────────────────────────────── @@ -44,7 +49,86 @@ pub fn user_event_from_pb(b: &pb::UserEventMessage) -> Result pb::Coordinate { + pb::Coordinate { + portion: t.vec.clone(), + error: t.error, + adjustment: t.adjustment, + height: t.height, + ..Default::default() + } +} + +/// Convert `pb::Coordinate` → typed [`Coordinate`]. +pub fn coordinate_from_pb(b: &pb::Coordinate) -> Coordinate { + Coordinate { + vec: b.portion.clone(), + error: b.error, + adjustment: b.adjustment, + height: b.height, + } +} + +// ─── Tags ───────────────────────────────────────────────────────────────────── + +/// Convert typed [`Tags`] → `pb::Tags`. +pub fn tags_to_pb(t: &Tags) -> pb::Tags { + pb::Tags { + entries: t.0.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect(), + ..Default::default() + } +} + +/// Convert `pb::Tags` → typed [`Tags`]. +pub fn tags_from_pb(b: &pb::Tags) -> Tags { + Tags( + b.entries + .iter() + .map(|(k, v)| (SmolStr::from(k.as_str()), SmolStr::from(v.as_str()))) + .collect(), + ) +} + +// ─── Filter ────────────────────────────────────────────────────────────────── + +/// Convert a typed [`Filter`] → `pb::Filter`. +pub fn filter_to_pb(t: &Filter) -> pb::Filter { + let kind = match t { + Filter::Id(ids) => pb::filter::Kind::NodeIds(Box::new(pb::NodeIdList { + ids: ids.iter().map(|s| s.to_string()).collect(), + ..Default::default() + })), + Filter::Tag(tf) => pb::filter::Kind::Tag(Box::new(pb::TagFilter { + tag: tf.tag.to_string(), + expr: tf.expr.as_deref().map(str::to_owned), + ..Default::default() + })), + }; + pb::Filter { + kind: Some(kind), + ..Default::default() + } +} + +/// Convert `pb::Filter` → typed [`Filter`]. +pub fn filter_from_pb(b: &pb::Filter) -> Result { + match b.kind.as_ref() { + Some(pb::filter::Kind::NodeIds(list)) => { + let ids = list.ids.iter().map(|s| SmolStr::from(s.as_str())).collect(); + Ok(Filter::Id(ids)) + } + Some(pb::filter::Kind::Tag(tf)) => Ok(Filter::Tag(TagFilter { + tag: SmolStr::from(tf.tag.as_str()), + expr: tf.expr.as_deref().map(SmolStr::from), + })), + None => Err(BridgeError::UnknownVariant("Filter.kind".into())), + } +} diff --git a/serf-proto/src/lib.rs b/serf-proto/src/lib.rs index 1c6e1dc1..ac143e29 100644 --- a/serf-proto/src/lib.rs +++ b/serf-proto/src/lib.rs @@ -4,9 +4,19 @@ //! own message set and framing on top of them. #![deny(missing_docs)] -pub use bridge::{BridgeError, user_event_from_pb, user_event_to_pb}; +pub use bridge::{ + BridgeError, + coordinate_from_pb, + coordinate_to_pb, + filter_from_pb, + filter_to_pb, + tags_from_pb, + tags_to_pb, + user_event_from_pb, + user_event_to_pb, +}; pub use framing::{FrameError, MessageType, decode_message, encode_message}; -pub use typed::UserEventMessage; +pub use typed::{Coordinate, Filter, QueryFlag, TagFilter, Tags, UserEventMessage}; /// A lamport logical clock value — a monotonically increasing counter used to /// order serf events. diff --git a/serf-proto/src/messages/tests.rs b/serf-proto/src/messages/tests.rs index 32884ff7..1a9cfee6 100644 --- a/serf-proto/src/messages/tests.rs +++ b/serf-proto/src/messages/tests.rs @@ -1,14 +1,38 @@ use buffa::Message as _; +use smol_str::SmolStr; -use super::serf::v1::UserEventMessage as PbUserEventMessage; -use crate::{LamportTime, UserEventMessage, user_event_from_pb, user_event_to_pb}; +use super::serf::v1::{ + Coordinate as PbCoordinate, + Filter as PbFilter, + Tags as PbTags, + UserEventMessage as PbUserEventMessage, +}; +use crate::{ + Coordinate, + Filter, + LamportTime, + QueryFlag, + TagFilter, + Tags, + UserEventMessage, + coordinate_from_pb, + coordinate_to_pb, + filter_from_pb, + filter_to_pb, + tags_from_pb, + tags_to_pb, + user_event_from_pb, + user_event_to_pb, +}; + +// ── UserEventMessage ───────────────────────────────────────────────────────── #[test] fn user_event_message_roundtrip_pb() { let typed = UserEventMessage { ltime: LamportTime::new(42), cc: true, - name: smol_str::SmolStr::from("deploy"), + name: SmolStr::from("deploy"), payload: bytes::Bytes::from_static(b"hello-serf"), }; @@ -37,3 +61,151 @@ fn user_event_message_ltime_required() { "expected BridgeError::MissingField for absent ltime" ); } + +// ── QueryFlag ──────────────────────────────────────────────────────────────── + +#[test] +fn query_flag_bits_roundtrip() { + // ACK and NO_BROADCAST round-trip through u32. + let flags = QueryFlag::ACK | QueryFlag::NO_BROADCAST; + let bits: u32 = flags.bits(); + let recovered = QueryFlag::from_bits_truncate(bits); + assert_eq!(recovered, flags); +} + +#[test] +fn query_flag_individual_bits() { + assert_eq!(QueryFlag::ACK.bits(), 1u32); + assert_eq!(QueryFlag::NO_BROADCAST.bits(), 2u32); +} + +#[test] +fn query_flag_empty_roundtrip() { + let flags = QueryFlag::empty(); + let recovered = QueryFlag::from_bits_truncate(flags.bits()); + assert_eq!(recovered, flags); +} + +// ── Coordinate ─────────────────────────────────────────────────────────────── + +#[test] +fn coordinate_roundtrip_pb() { + let typed = Coordinate { + vec: vec![1.0, -2.5, 3.14], + error: 0.25, + adjustment: -0.001, + height: 0.000_010, + }; + + let pb = coordinate_to_pb(&typed); + let encoded = pb.encode_to_vec(); + let decoded_pb = + PbCoordinate::decode_from_slice(encoded.as_slice()).expect("decode_from_slice failed"); + let roundtripped = coordinate_from_pb(&decoded_pb); + + assert_eq!(roundtripped.vec, typed.vec); + assert_eq!(roundtripped.error, typed.error); + assert_eq!(roundtripped.adjustment, typed.adjustment); + assert_eq!(roundtripped.height, typed.height); +} + +#[test] +fn coordinate_empty_vec_roundtrip() { + let typed = Coordinate { + vec: Vec::new(), + error: 1.5, + adjustment: 0.0, + height: 0.00001, + }; + let pb = coordinate_to_pb(&typed); + let encoded = pb.encode_to_vec(); + let decoded_pb = + PbCoordinate::decode_from_slice(encoded.as_slice()).expect("decode_from_slice failed"); + let roundtripped = coordinate_from_pb(&decoded_pb); + assert_eq!(roundtripped.vec, typed.vec); + assert_eq!(roundtripped.error, typed.error); +} + +// ── Tags ───────────────────────────────────────────────────────────────────── + +#[test] +fn tags_empty_roundtrip() { + let typed = Tags::new(); + let pb = tags_to_pb(&typed); + let encoded = pb.encode_to_vec(); + let decoded_pb = PbTags::decode_from_slice(encoded.as_slice()).expect("decode_from_slice failed"); + let roundtripped = tags_from_pb(&decoded_pb); + assert_eq!(roundtripped, typed); +} + +#[test] +fn tags_multi_entry_roundtrip() { + let typed: Tags = [("role", "web"), ("env", "prod"), ("dc", "us-east")].into_iter().collect(); + + let pb = tags_to_pb(&typed); + let encoded = pb.encode_to_vec(); + let decoded_pb = PbTags::decode_from_slice(encoded.as_slice()).expect("decode_from_slice failed"); + let roundtripped = tags_from_pb(&decoded_pb); + + assert_eq!(roundtripped.len(), typed.len()); + for (k, v) in &typed.0 { + assert_eq!(roundtripped.0.get(k), Some(v)); + } +} + +// ── Filter ─────────────────────────────────────────────────────────────────── + +#[test] +fn filter_node_ids_roundtrip() { + let typed = Filter::Id(vec![SmolStr::from("node-1"), SmolStr::from("node-2")]); + + let pb = filter_to_pb(&typed); + let encoded = pb.encode_to_vec(); + let decoded_pb = PbFilter::decode_from_slice(encoded.as_slice()).expect("decode_from_slice"); + let roundtripped = filter_from_pb(&decoded_pb).expect("filter_from_pb"); + + assert_eq!(roundtripped, typed); +} + +#[test] +fn filter_tag_with_expr_roundtrip() { + let typed = Filter::Tag(TagFilter { + tag: SmolStr::from("role"), + expr: Some(SmolStr::from("^web.*")), + }); + + let pb = filter_to_pb(&typed); + let encoded = pb.encode_to_vec(); + let decoded_pb = PbFilter::decode_from_slice(encoded.as_slice()).expect("decode_from_slice"); + let roundtripped = filter_from_pb(&decoded_pb).expect("filter_from_pb"); + + assert_eq!(roundtripped, typed); +} + +#[test] +fn filter_tag_without_expr_roundtrip() { + let typed = Filter::Tag(TagFilter { + tag: SmolStr::from("dc"), + expr: None, + }); + + let pb = filter_to_pb(&typed); + let encoded = pb.encode_to_vec(); + let decoded_pb = PbFilter::decode_from_slice(encoded.as_slice()).expect("decode_from_slice"); + let roundtripped = filter_from_pb(&decoded_pb).expect("filter_from_pb"); + + assert_eq!(roundtripped, typed); +} + +#[test] +fn filter_missing_kind_is_error() { + // A Filter pb message with no kind set must be rejected. + let pb = PbFilter { + kind: None, + ..Default::default() + }; + assert!( + filter_from_pb(&pb).is_err(), + "expected BridgeError::UnknownVariant for missing kind" + ); +} diff --git a/serf-proto/src/typed/mod.rs b/serf-proto/src/typed/mod.rs index 1ed5f679..897d053d 100644 --- a/serf-proto/src/typed/mod.rs +++ b/serf-proto/src/typed/mod.rs @@ -3,6 +3,8 @@ //! These types are what the serf state machine works with directly. The //! `bridge` module converts them to/from the buffa-generated codec types. +use std::collections::HashMap; + use bytes::Bytes; use smol_str::SmolStr; @@ -20,3 +22,124 @@ pub struct UserEventMessage { /// The event payload. pub payload: Bytes, } + +// ── QueryFlag ──────────────────────────────────────────────────────────────── + +bitflags::bitflags! { + /// Control flags for a serf query message. + /// + /// Rides the wire as a `uint32`; the bit positions are identical to the + /// legacy `serf-core` constants so tooling that inspects raw integers stays + /// compatible. + #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Default)] + pub struct QueryFlag: u32 { + /// Ack — force the receiver to send an acknowledgement back. + const ACK = 1 << 0; + /// NoBroadcast — suppress re-broadcast of the query; useful for targeted + /// unicast queries to individual members. + const NO_BROADCAST = 1 << 1; + } +} + +// ── Coordinate ─────────────────────────────────────────────────────────────── + +/// A Vivaldi network-coordinate wire point. +/// +/// Holds the serialisable fields of the Vivaldi coordinate; the Vivaldi +/// client engine (`CoordinateClient`) is not part of the wire protocol. +/// All values are in units of seconds. +#[derive(Debug, Clone, PartialEq)] +pub struct Coordinate { + /// Euclidean portion of the coordinate (variable-length f64 vector). + pub vec: Vec, + /// Confidence in the coordinate estimate (dimensionless). + pub error: f64, + /// Distance offset derived from peer observations (seconds). + pub adjustment: f64, + /// Non-Euclidean height term modelling access-link latency (seconds). + pub height: f64, +} + +impl Default for Coordinate { + fn default() -> Self { + Self { + vec: Vec::new(), + error: 0.0, + adjustment: 0.0, + height: 0.0, + } + } +} + +// ── Tags ───────────────────────────────────────────────────────────────────── + +/// Node metadata: a string→string map gossiped via node meta. +/// +/// Thin newtype over [`HashMap`] so the rest of the crate +/// can name the concept without spelling out the full map type. +/// Proto3 `map` wire encoding does not guarantee key order, so encoded bytes +/// are not canonical for a given set of tags. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct Tags(pub HashMap); + +impl Tags { + /// Creates an empty `Tags` map. + pub fn new() -> Self { + Self(HashMap::new()) + } + + /// Creates a `Tags` map with the given initial capacity. + pub fn with_capacity(cap: usize) -> Self { + Self(HashMap::with_capacity(cap)) + } + + /// Returns the number of tag entries. + pub fn len(&self) -> usize { + self.0.len() + } + + /// Returns `true` if no tags are set. + pub fn is_empty(&self) -> bool { + self.0.is_empty() + } +} + +impl FromIterator<(K, V)> for Tags +where + K: Into, + V: Into, +{ + fn from_iter(iter: T) -> Self + where + T: IntoIterator, + { + Self(iter.into_iter().map(|(k, v)| (k.into(), v.into())).collect()) + } +} + +// ── Filter ─────────────────────────────────────────────────────────────────── + +/// A tag-name / optional-regex pair for matching nodes by their tags. +/// +/// When `expr` is `None` the filter matches any node that has the named tag, +/// regardless of its value. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TagFilter { + /// The tag key to match on. + pub tag: SmolStr, + /// Optional regex that the tag value must satisfy. `None` = match any value. + pub expr: Option, +} + +/// A single query-scoping predicate. +/// +/// Exactly one variant is active per `Filter`. The `Id` variant restricts +/// the query to the listed node ids; the `Tag` variant restricts it to nodes +/// whose tag satisfies the [`TagFilter`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Filter { + /// Restrict responses to the listed node ids. + Id(Vec), + /// Restrict responses to nodes whose tag value satisfies the filter. + Tag(TagFilter), +} From 9c81d69a7b0b378d4bd4da9873a991bba28b43d3 Mon Sep 17 00:00:00 2001 From: al8n Date: Tue, 23 Jun 2026 23:12:38 +1200 Subject: [PATCH 005/128] feat(proto): membership messages JoinMessage, LeaveMessage, ConflictResponseMessage Add proto definitions, typed generics, and bridge conversions for the three membership messages. Node-id (I) and Node are embedded as opaque `bytes` encoded via memberlist_proto::Data / DataRef, mirroring the bridge pattern from memberlist-proto. ltime is proto3 `optional uint64` (required on decode). Round-trip tests use I=SmolStr, A=SocketAddr. --- serf-proto/proto/serf/v1/messages.proto | 23 ++++ serf-proto/src/bridge/mod.rs | 156 +++++++++++++++++++++++- serf-proto/src/lib.rs | 18 ++- serf-proto/src/messages/tests.rs | 126 +++++++++++++++++++ serf-proto/src/typed/mod.rs | 63 ++++++++++ 5 files changed, 384 insertions(+), 2 deletions(-) diff --git a/serf-proto/proto/serf/v1/messages.proto b/serf-proto/proto/serf/v1/messages.proto index 34df8db3..3bd22c16 100644 --- a/serf-proto/proto/serf/v1/messages.proto +++ b/serf-proto/proto/serf/v1/messages.proto @@ -52,3 +52,26 @@ message Filter { message NodeIdList { repeated string ids = 1; } + +// JoinMessage is broadcast after a node joins to associate it with a lamport clock. +// `optional` on ltime because the decoder must reject a missing lamport clock. +// `id` carries the node-id encoded via memberlist_proto::Data (opaque bytes). +message JoinMessage { + optional uint64 ltime = 1; + bytes id = 2; +} + +// LeaveMessage is broadcast to signal the intent to leave. +// `optional` on ltime because the decoder must reject a missing lamport clock. +// `id` carries the node-id encoded via memberlist_proto::Data (opaque bytes). +message LeaveMessage { + optional uint64 ltime = 1; + bool prune = 2; + bytes id = 3; +} + +// ConflictResponseMessage carries the winning member in a node-name conflict. +// `member` carries the memberlist_proto::Node encoded via Data (opaque bytes). +message ConflictResponseMessage { + bytes member = 1; +} diff --git a/serf-proto/src/bridge/mod.rs b/serf-proto/src/bridge/mod.rs index 5073410c..a9673717 100644 --- a/serf-proto/src/bridge/mod.rs +++ b/serf-proto/src/bridge/mod.rs @@ -6,12 +6,23 @@ use std::borrow::Cow; +use bytes::Bytes; +use memberlist_proto::{Data, DataRef, data::DecodeError, data::EncodeError}; use smol_str::SmolStr; use crate::{ LamportTime, messages::serf::v1 as pb, - typed::{Coordinate, Filter, TagFilter, Tags, UserEventMessage}, + typed::{ + Coordinate, + ConflictResponseMessage, + Filter, + JoinMessage, + LeaveMessage, + TagFilter, + Tags, + UserEventMessage, + }, }; // ─── BridgeError ───────────────────────────────────────────────────────────── @@ -26,6 +37,12 @@ pub enum BridgeError { /// A `oneof` field held no recognised variant. #[error("unknown or missing oneof variant in {0}")] UnknownVariant(Cow<'static, str>), + /// An encode error occurred while serialising a `memberlist_proto::Data` field. + #[error("encode error: {0}")] + Encode(#[from] EncodeError), + /// A decode error occurred while deserialising a `memberlist_proto::Data` field. + #[error("decode error: {0}")] + Decode(#[from] DecodeError), } // ─── UserEventMessage ──────────────────────────────────────────────────────── @@ -132,3 +149,140 @@ pub fn filter_from_pb(b: &pb::Filter) -> Result { None => Err(BridgeError::UnknownVariant("Filter.kind".into())), } } + +// ─── Data ↔ Bytes helpers ──────────────────────────────────────────────────── + +/// Encode a `memberlist_proto::Data` value to a raw `Bytes` buffer (no length prefix). +/// +/// Allocates a buffer sized by `encoded_len`, writes the encoding via +/// `encode`, and wraps it in `Bytes`. This mirrors the pattern in +/// `memberlist_proto::bridge` for serialising opaque `I`/`A` fields. +fn data_to_bytes(val: &T) -> Result +where + T: Data, +{ + let mut buf = vec![0u8; val.encoded_len()]; + val.encode(&mut buf)?; + Ok(Bytes::from(buf)) +} + +/// Decode a `memberlist_proto::Data` value from raw bytes (no length prefix). +/// +/// Rejects trailing data: the whole slice must be consumed so a malformed +/// wire field is caught at the wire→machine boundary. +fn data_from_bytes(buf: &Bytes) -> Result +where + T: Data, +{ + let (bytes_read, val) = as DataRef<'_, T>>::decode(buf.as_ref())?; + if bytes_read != buf.len() { + return Err(BridgeError::Decode(DecodeError::custom(format!( + "trailing data in encoded field: decoder consumed {bytes_read} of {} bytes", + buf.len() + )))); + } + Ok(T::from_ref(val)?) +} + +// ─── JoinMessage ───────────────────────────────────────────────────────────── + +/// Convert a typed [`JoinMessage`] → `pb::JoinMessage`. +/// +/// The node-id `I` is serialised to opaque `bytes` via `memberlist_proto::Data`. +pub fn join_to_pb(t: &JoinMessage) -> Result +where + I: Data, +{ + Ok(pb::JoinMessage { + ltime: Some(t.ltime.into()), + id: data_to_bytes(&t.id)?, + ..Default::default() + }) +} + +/// Convert `pb::JoinMessage` → typed [`JoinMessage`]. +/// +/// Rejects a missing `ltime` (required field). The `id` bytes are decoded via +/// `memberlist_proto::DataRef`. +pub fn join_from_pb(b: &pb::JoinMessage) -> Result, BridgeError> +where + I: Data, +{ + let ltime = b + .ltime + .ok_or(BridgeError::MissingField("JoinMessage.ltime".into()))?; + let id: I = data_from_bytes(&b.id)?; + Ok(JoinMessage { + ltime: LamportTime::from(ltime), + id, + }) +} + +// ─── LeaveMessage ──────────────────────────────────────────────────────────── + +/// Convert a typed [`LeaveMessage`] → `pb::LeaveMessage`. +/// +/// The node-id `I` is serialised to opaque `bytes` via `memberlist_proto::Data`. +pub fn leave_to_pb(t: &LeaveMessage) -> Result +where + I: Data, +{ + Ok(pb::LeaveMessage { + ltime: Some(t.ltime.into()), + prune: t.prune, + id: data_to_bytes(&t.id)?, + ..Default::default() + }) +} + +/// Convert `pb::LeaveMessage` → typed [`LeaveMessage`]. +/// +/// Rejects a missing `ltime` (required field). The `id` bytes are decoded via +/// `memberlist_proto::DataRef`. +pub fn leave_from_pb(b: &pb::LeaveMessage) -> Result, BridgeError> +where + I: Data, +{ + let ltime = b + .ltime + .ok_or(BridgeError::MissingField("LeaveMessage.ltime".into()))?; + let id: I = data_from_bytes(&b.id)?; + Ok(LeaveMessage { + ltime: LamportTime::from(ltime), + id, + prune: b.prune, + }) +} + +// ─── ConflictResponseMessage ───────────────────────────────────────────────── + +/// Convert a typed [`ConflictResponseMessage`] → `pb::ConflictResponseMessage`. +/// +/// The `Node` member is serialised to opaque `bytes` via +/// `memberlist_proto::Data`. +pub fn conflict_response_to_pb( + t: &ConflictResponseMessage, +) -> Result +where + I: Data, + A: Data, +{ + Ok(pb::ConflictResponseMessage { + member: data_to_bytes(&t.member)?, + ..Default::default() + }) +} + +/// Convert `pb::ConflictResponseMessage` → typed [`ConflictResponseMessage`]. +/// +/// The `member` bytes are decoded as a `Node` via `memberlist_proto::DataRef`. +pub fn conflict_response_from_pb( + b: &pb::ConflictResponseMessage, +) -> Result, BridgeError> +where + I: Data, + A: Data, +{ + let member: memberlist_proto::Node = data_from_bytes(&b.member)?; + Ok(ConflictResponseMessage { member }) +} diff --git a/serf-proto/src/lib.rs b/serf-proto/src/lib.rs index ac143e29..dda2e7e8 100644 --- a/serf-proto/src/lib.rs +++ b/serf-proto/src/lib.rs @@ -6,17 +6,33 @@ pub use bridge::{ BridgeError, + conflict_response_from_pb, + conflict_response_to_pb, coordinate_from_pb, coordinate_to_pb, filter_from_pb, filter_to_pb, + join_from_pb, + join_to_pb, + leave_from_pb, + leave_to_pb, tags_from_pb, tags_to_pb, user_event_from_pb, user_event_to_pb, }; pub use framing::{FrameError, MessageType, decode_message, encode_message}; -pub use typed::{Coordinate, Filter, QueryFlag, TagFilter, Tags, UserEventMessage}; +pub use typed::{ + Coordinate, + ConflictResponseMessage, + Filter, + JoinMessage, + LeaveMessage, + QueryFlag, + TagFilter, + Tags, + UserEventMessage, +}; /// A lamport logical clock value — a monotonically increasing counter used to /// order serf events. diff --git a/serf-proto/src/messages/tests.rs b/serf-proto/src/messages/tests.rs index 1a9cfee6..a015bac8 100644 --- a/serf-proto/src/messages/tests.rs +++ b/serf-proto/src/messages/tests.rs @@ -1,24 +1,39 @@ +use std::net::SocketAddr; + use buffa::Message as _; +use memberlist_proto::Node; use smol_str::SmolStr; use super::serf::v1::{ + ConflictResponseMessage as PbConflictResponseMessage, Coordinate as PbCoordinate, Filter as PbFilter, + JoinMessage as PbJoinMessage, + LeaveMessage as PbLeaveMessage, Tags as PbTags, UserEventMessage as PbUserEventMessage, }; use crate::{ + ConflictResponseMessage, Coordinate, Filter, + JoinMessage, LamportTime, + LeaveMessage, QueryFlag, TagFilter, Tags, UserEventMessage, + conflict_response_from_pb, + conflict_response_to_pb, coordinate_from_pb, coordinate_to_pb, filter_from_pb, filter_to_pb, + join_from_pb, + join_to_pb, + leave_from_pb, + leave_to_pb, tags_from_pb, tags_to_pb, user_event_from_pb, @@ -209,3 +224,114 @@ fn filter_missing_kind_is_error() { "expected BridgeError::UnknownVariant for missing kind" ); } + +// ── JoinMessage ────────────────────────────────────────────────────────────── + +type I = SmolStr; +type A = SocketAddr; + +fn sample_addr() -> A { + "127.0.0.1:7946".parse().unwrap() +} + +#[test] +fn join_message_roundtrip_pb() { + let typed: JoinMessage = JoinMessage { + ltime: LamportTime::new(99), + id: SmolStr::from("node-join"), + }; + + let pb = join_to_pb(&typed).expect("join_to_pb failed"); + let encoded = pb.encode_to_vec(); + let decoded_pb = + PbJoinMessage::decode_from_slice(encoded.as_slice()).expect("decode_from_slice failed"); + let roundtripped: JoinMessage = join_from_pb(&decoded_pb).expect("join_from_pb failed"); + + assert_eq!(roundtripped.ltime, typed.ltime); + assert_eq!(roundtripped.id, typed.id); +} + +#[test] +fn join_message_ltime_required() { + let pb = PbJoinMessage { + ltime: None, + id: bytes::Bytes::new(), + ..Default::default() + }; + assert!( + join_from_pb::(&pb).is_err(), + "expected BridgeError::MissingField for absent ltime" + ); +} + +// ── LeaveMessage ───────────────────────────────────────────────────────────── + +#[test] +fn leave_message_roundtrip_pb_no_prune() { + let typed: LeaveMessage = LeaveMessage { + ltime: LamportTime::new(7), + id: SmolStr::from("node-leave"), + prune: false, + }; + + let pb = leave_to_pb(&typed).expect("leave_to_pb failed"); + let encoded = pb.encode_to_vec(); + let decoded_pb = + PbLeaveMessage::decode_from_slice(encoded.as_slice()).expect("decode_from_slice failed"); + let roundtripped: LeaveMessage = leave_from_pb(&decoded_pb).expect("leave_from_pb failed"); + + assert_eq!(roundtripped.ltime, typed.ltime); + assert_eq!(roundtripped.id, typed.id); + assert_eq!(roundtripped.prune, false); +} + +#[test] +fn leave_message_roundtrip_pb_with_prune() { + let typed: LeaveMessage = LeaveMessage { + ltime: LamportTime::new(42), + id: SmolStr::from("node-prune"), + prune: true, + }; + + let pb = leave_to_pb(&typed).expect("leave_to_pb failed"); + let encoded = pb.encode_to_vec(); + let decoded_pb = + PbLeaveMessage::decode_from_slice(encoded.as_slice()).expect("decode_from_slice failed"); + let roundtripped: LeaveMessage = leave_from_pb(&decoded_pb).expect("leave_from_pb failed"); + + assert_eq!(roundtripped.ltime, typed.ltime); + assert_eq!(roundtripped.id, typed.id); + assert_eq!(roundtripped.prune, true); +} + +#[test] +fn leave_message_ltime_required() { + let pb = PbLeaveMessage { + ltime: None, + prune: false, + id: bytes::Bytes::new(), + ..Default::default() + }; + assert!( + leave_from_pb::(&pb).is_err(), + "expected BridgeError::MissingField for absent ltime" + ); +} + +// ── ConflictResponseMessage ─────────────────────────────────────────────────── + +#[test] +fn conflict_response_roundtrip_pb() { + let node: Node = Node::new(SmolStr::from("node-conflict"), sample_addr()); + let typed: ConflictResponseMessage = ConflictResponseMessage::new(node); + + let pb = conflict_response_to_pb(&typed).expect("conflict_response_to_pb failed"); + let encoded = pb.encode_to_vec(); + let decoded_pb = PbConflictResponseMessage::decode_from_slice(encoded.as_slice()) + .expect("decode_from_slice failed"); + let roundtripped: ConflictResponseMessage = + conflict_response_from_pb(&decoded_pb).expect("conflict_response_from_pb failed"); + + assert_eq!(roundtripped.member.id_ref(), typed.member.id_ref()); + assert_eq!(roundtripped.member.addr_ref(), typed.member.addr_ref()); +} diff --git a/serf-proto/src/typed/mod.rs b/serf-proto/src/typed/mod.rs index 897d053d..c13a001f 100644 --- a/serf-proto/src/typed/mod.rs +++ b/serf-proto/src/typed/mod.rs @@ -9,6 +9,7 @@ use bytes::Bytes; use smol_str::SmolStr; use crate::LamportTime; +use memberlist_proto::Node; /// A user-generated event broadcast through the serf cluster. #[derive(Debug, Default, Clone, PartialEq, Eq)] @@ -143,3 +144,65 @@ pub enum Filter { /// Restrict responses to nodes whose tag value satisfies the filter. Tag(TagFilter), } + +// ── Membership messages ─────────────────────────────────────────────────────── + +/// Broadcast after a node joins the cluster to associate it with a lamport clock. +/// +/// Generic over `I`: the node-id type, which must implement +/// `memberlist_proto::Data` so it can be encoded as opaque proto `bytes` in +/// the bridge layer. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct JoinMessage { + /// The lamport clock value at the time the node joined. + pub ltime: crate::LamportTime, + /// The joining node's identifier. + pub id: I, +} + +impl JoinMessage { + /// Construct a new `JoinMessage`. + pub fn new(ltime: crate::LamportTime, id: I) -> Self { + Self { ltime, id } + } +} + +/// Broadcast to signal the intent to leave the cluster. +/// +/// Generic over `I`: the node-id type, which must implement +/// `memberlist_proto::Data` so it can be encoded as opaque proto `bytes` in +/// the bridge layer. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LeaveMessage { + /// The lamport clock value at the time the leave was emitted. + pub ltime: crate::LamportTime, + /// The leaving node's identifier. + pub id: I, + /// Whether the leave is a prune (permanent removal) rather than a graceful leave. + pub prune: bool, +} + +impl LeaveMessage { + /// Construct a new `LeaveMessage`. + pub fn new(ltime: crate::LamportTime, id: I, prune: bool) -> Self { + Self { ltime, id, prune } + } +} + +/// Carries the winning node in a node-name conflict tie-breaker. +/// +/// Generic over `I` and `A`: the node-id and address types, which must +/// implement `memberlist_proto::Data` so the embedded `Node` can be +/// encoded as opaque proto `bytes` in the bridge layer. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ConflictResponseMessage { + /// The winning node in the conflict resolution. + pub member: Node, +} + +impl ConflictResponseMessage { + /// Construct a new `ConflictResponseMessage`. + pub fn new(member: Node) -> Self { + Self { member } + } +} From bc55ae8abea3f5dab4052467c7b67c3ea5a292e1 Mon Sep 17 00:00:00 2001 From: al8n Date: Tue, 23 Jun 2026 23:25:03 +1200 Subject: [PATCH 006/128] feat(serf-proto): Filter retrofit + QueryMessage + QueryResponseMessage - Make Filter generic over I: Id(Vec) with where-clause Data bounds; default I = SmolStr preserves backward compatibility. - NodeIdList proto field changed from repeated string to repeated bytes; each I encodes via memberlist_proto::Data (mirrors the Node bytes pattern). - Add QueryMessage and QueryResponseMessage typed structs with ack()/no_broadcast() helpers; Duration stored as uint64 nanos on the wire. - Add pb::QueryMessage and pb::QueryResponseMessage to messages.proto with field numbers matching the legacy serf-core tag constants. - Add filter_to_pb/filter_from_pb bridge generics (now return Result); add query_to_pb/query_from_pb + query_response_to_pb/query_response_from_pb. - Export all new types and bridge functions from lib.rs. - Update existing Filter tests for new Result-returning bridge API; add 8 new query tests covering roundtrip, filters, and required-field rejection. - All 36 tests pass (cargo test -p serf-proto, exit 0). --- serf-proto/proto/serf/v1/messages.proto | 29 ++- serf-proto/src/bridge/mod.rs | 188 +++++++++++++++- serf-proto/src/lib.rs | 6 + serf-proto/src/messages/tests.rs | 288 +++++++++++++++++++++++- serf-proto/src/typed/mod.rs | 77 ++++++- 5 files changed, 563 insertions(+), 25 deletions(-) diff --git a/serf-proto/proto/serf/v1/messages.proto b/serf-proto/proto/serf/v1/messages.proto index 3bd22c16..3c059000 100644 --- a/serf-proto/proto/serf/v1/messages.proto +++ b/serf-proto/proto/serf/v1/messages.proto @@ -41,7 +41,7 @@ message TagFilter { // Exactly one of the oneof alternatives is set per Filter message. message Filter { oneof kind { - // node_ids is a list of node-id strings; only those nodes respond. + // node_ids is a list of opaque byte-encoded node ids; only those nodes respond. NodeIdList node_ids = 1; // tag is a tag-name / optional-regex pair. TagFilter tag = 2; @@ -49,8 +49,33 @@ message Filter { } // NodeIdList is the payload for a node-id filter. +// Each `id` entry is a node-id encoded via memberlist_proto::Data (opaque bytes). message NodeIdList { - repeated string ids = 1; + repeated bytes ids = 1; +} + +// QueryMessage is a query broadcast to the cluster, optionally scoped by filters. +// Field numbers match the legacy serf-core tag constants. +message QueryMessage { + optional uint64 ltime = 1; + optional uint32 id = 2; + bytes from = 3; + repeated Filter filters = 4; + optional uint32 flags = 5; + optional uint32 relay_factor = 6; + optional uint64 timeout_nanos = 7; + string name = 8; + bytes payload = 9; +} + +// QueryResponseMessage is sent back to the query originator. +// Field numbers match the legacy serf-core tag constants. +message QueryResponseMessage { + optional uint64 ltime = 1; + optional uint32 id = 2; + bytes from = 3; + optional uint32 flags = 4; + bytes payload = 5; } // JoinMessage is broadcast after a node joins to associate it with a lamport clock. diff --git a/serf-proto/src/bridge/mod.rs b/serf-proto/src/bridge/mod.rs index a9673717..d84c5b28 100644 --- a/serf-proto/src/bridge/mod.rs +++ b/serf-proto/src/bridge/mod.rs @@ -19,6 +19,9 @@ use crate::{ Filter, JoinMessage, LeaveMessage, + QueryFlag, + QueryMessage, + QueryResponseMessage, TagFilter, Tags, UserEventMessage, @@ -34,6 +37,9 @@ pub enum BridgeError { /// A required field was absent in the wire message. #[error("missing required field: {0}")] MissingField(Cow<'static, str>), + /// A field value was present but outside the accepted range or domain. + #[error("invalid field value: {0}")] + InvalidValue(Cow<'static, str>), /// A `oneof` field held no recognised variant. #[error("unknown or missing oneof variant in {0}")] UnknownVariant(Cow<'static, str>), @@ -116,30 +122,51 @@ pub fn tags_from_pb(b: &pb::Tags) -> Tags { // ─── Filter ────────────────────────────────────────────────────────────────── -/// Convert a typed [`Filter`] → `pb::Filter`. -pub fn filter_to_pb(t: &Filter) -> pb::Filter { +/// Convert a typed [`Filter`] → `pb::Filter`. +/// +/// The node-id type `I` is encoded as opaque `bytes` via `memberlist_proto::Data` +/// for the `Id` variant; `Tag` variants encode as before. +pub fn filter_to_pb(t: &Filter) -> Result +where + I: Data, +{ let kind = match t { - Filter::Id(ids) => pb::filter::Kind::NodeIds(Box::new(pb::NodeIdList { - ids: ids.iter().map(|s| s.to_string()).collect(), - ..Default::default() - })), + Filter::Id(ids) => { + let mut encoded_ids = Vec::with_capacity(ids.len()); + for id in ids { + encoded_ids.push(data_to_bytes(id)?); + } + pb::filter::Kind::NodeIds(Box::new(pb::NodeIdList { + ids: encoded_ids, + ..Default::default() + })) + } Filter::Tag(tf) => pb::filter::Kind::Tag(Box::new(pb::TagFilter { tag: tf.tag.to_string(), expr: tf.expr.as_deref().map(str::to_owned), ..Default::default() })), }; - pb::Filter { + Ok(pb::Filter { kind: Some(kind), ..Default::default() - } + }) } -/// Convert `pb::Filter` → typed [`Filter`]. -pub fn filter_from_pb(b: &pb::Filter) -> Result { +/// Convert `pb::Filter` → typed [`Filter`]. +/// +/// The `Id` variant decodes each `bytes` entry as `I` via `memberlist_proto::DataRef`. +pub fn filter_from_pb(b: &pb::Filter) -> Result, BridgeError> +where + I: Data, +{ match b.kind.as_ref() { Some(pb::filter::Kind::NodeIds(list)) => { - let ids = list.ids.iter().map(|s| SmolStr::from(s.as_str())).collect(); + let ids = list + .ids + .iter() + .map(|buf| data_from_bytes::(buf)) + .collect::, BridgeError>>()?; Ok(Filter::Id(ids)) } Some(pb::filter::Kind::Tag(tf)) => Ok(Filter::Tag(TagFilter { @@ -286,3 +313,142 @@ where let member: memberlist_proto::Node = data_from_bytes(&b.member)?; Ok(ConflictResponseMessage { member }) } + +// ─── QueryMessage ───────────────────────────────────────────────────────────── + +/// Convert a typed [`QueryMessage`] → `pb::QueryMessage`. +/// +/// - `from: Node` is serialised to opaque `bytes` via `memberlist_proto::Data`. +/// - Each `Filter` in `filters` is encoded via [`filter_to_pb`]. +/// - `timeout` is stored as nanoseconds in a `uint64`. +/// - `flags` is stored as the raw `u32` bit-pattern. +pub fn query_to_pb(t: &QueryMessage) -> Result +where + I: Data, + A: Data, +{ + let filters = t + .filters + .iter() + .map(|f| filter_to_pb::(f)) + .collect::, BridgeError>>()?; + + Ok(pb::QueryMessage { + ltime: Some(t.ltime.into()), + id: Some(t.id), + from: data_to_bytes(&t.from)?, + filters, + flags: Some(t.flags.bits()), + relay_factor: Some(t.relay_factor as u32), + timeout_nanos: Some(t.timeout.as_nanos() as u64), + name: t.name.to_string(), + payload: t.payload.clone(), + ..Default::default() + }) +} + +/// Convert `pb::QueryMessage` → typed [`QueryMessage`]. +/// +/// Rejects missing `ltime`, `id`, `flags`, `relay_factor`, and `timeout_nanos` +/// (all required by the legacy protocol). Rejects `relay_factor` values that +/// exceed `u8::MAX`. Decodes `from` as `Node` and each `Filter` via [`filter_from_pb`]. +pub fn query_from_pb(b: &pb::QueryMessage) -> Result, BridgeError> +where + I: Data, + A: Data, +{ + let ltime = b + .ltime + .ok_or(BridgeError::MissingField("QueryMessage.ltime".into()))?; + let id = b + .id + .ok_or(BridgeError::MissingField("QueryMessage.id".into()))?; + let from: memberlist_proto::Node = data_from_bytes(&b.from)?; + let filters = b + .filters + .iter() + .map(|f| filter_from_pb::(f)) + .collect::>, BridgeError>>()?; + let flags = QueryFlag::from_bits_truncate( + b.flags + .ok_or(BridgeError::MissingField("QueryMessage.flags".into()))?, + ); + let relay_factor = u8::try_from( + b.relay_factor + .ok_or(BridgeError::MissingField("QueryMessage.relay_factor".into()))?, + ) + .map_err(|_| BridgeError::InvalidValue("QueryMessage.relay_factor exceeds u8::MAX".into()))?; + // Safe: query timeouts are measured in seconds to minutes, well within u64::MAX nanoseconds. + let timeout = std::time::Duration::from_nanos( + b.timeout_nanos + .ok_or(BridgeError::MissingField("QueryMessage.timeout_nanos".into()))?, + ); + + Ok(QueryMessage { + ltime: LamportTime::from(ltime), + id, + from, + filters, + flags, + relay_factor, + timeout, + name: SmolStr::from(b.name.as_str()), + payload: b.payload.clone(), + }) +} + +// ─── QueryResponseMessage ───────────────────────────────────────────────────── + +/// Convert a typed [`QueryResponseMessage`] → `pb::QueryResponseMessage`. +/// +/// - `from: Node` is serialised to opaque `bytes` via `memberlist_proto::Data`. +/// - `flags` is stored as the raw `u32` bit-pattern. +pub fn query_response_to_pb( + t: &QueryResponseMessage, +) -> Result +where + I: Data, + A: Data, +{ + Ok(pb::QueryResponseMessage { + ltime: Some(t.ltime.into()), + id: Some(t.id), + from: data_to_bytes(&t.from)?, + flags: Some(t.flags.bits()), + payload: t.payload.clone(), + ..Default::default() + }) +} + +/// Convert `pb::QueryResponseMessage` → typed [`QueryResponseMessage`]. +/// +/// Rejects missing `ltime`, `id`, and `flags` (all required by the legacy protocol). +/// Decodes `from` as `Node` via `memberlist_proto::DataRef`. Flags are +/// decoded with `from_bits_retain` to preserve any future extension bits. +pub fn query_response_from_pb( + b: &pb::QueryResponseMessage, +) -> Result, BridgeError> +where + I: Data, + A: Data, +{ + let ltime = b + .ltime + .ok_or(BridgeError::MissingField("QueryResponseMessage.ltime".into()))?; + let id = b + .id + .ok_or(BridgeError::MissingField("QueryResponseMessage.id".into()))?; + let from: memberlist_proto::Node = data_from_bytes(&b.from)?; + let flags = QueryFlag::from_bits_retain( + b.flags + .ok_or(BridgeError::MissingField("QueryResponseMessage.flags".into()))?, + ); + + Ok(QueryResponseMessage { + ltime: LamportTime::from(ltime), + id, + from, + flags, + payload: b.payload.clone(), + }) +} diff --git a/serf-proto/src/lib.rs b/serf-proto/src/lib.rs index dda2e7e8..3d91f7f2 100644 --- a/serf-proto/src/lib.rs +++ b/serf-proto/src/lib.rs @@ -16,6 +16,10 @@ pub use bridge::{ join_to_pb, leave_from_pb, leave_to_pb, + query_from_pb, + query_response_from_pb, + query_response_to_pb, + query_to_pb, tags_from_pb, tags_to_pb, user_event_from_pb, @@ -29,6 +33,8 @@ pub use typed::{ JoinMessage, LeaveMessage, QueryFlag, + QueryMessage, + QueryResponseMessage, TagFilter, Tags, UserEventMessage, diff --git a/serf-proto/src/messages/tests.rs b/serf-proto/src/messages/tests.rs index a015bac8..3c349b61 100644 --- a/serf-proto/src/messages/tests.rs +++ b/serf-proto/src/messages/tests.rs @@ -10,6 +10,8 @@ use super::serf::v1::{ Filter as PbFilter, JoinMessage as PbJoinMessage, LeaveMessage as PbLeaveMessage, + QueryMessage as PbQueryMessage, + QueryResponseMessage as PbQueryResponseMessage, Tags as PbTags, UserEventMessage as PbUserEventMessage, }; @@ -21,6 +23,8 @@ use crate::{ LamportTime, LeaveMessage, QueryFlag, + QueryMessage, + QueryResponseMessage, TagFilter, Tags, UserEventMessage, @@ -34,6 +38,10 @@ use crate::{ join_to_pb, leave_from_pb, leave_to_pb, + query_from_pb, + query_response_from_pb, + query_response_to_pb, + query_to_pb, tags_from_pb, tags_to_pb, user_event_from_pb, @@ -172,42 +180,44 @@ fn tags_multi_entry_roundtrip() { #[test] fn filter_node_ids_roundtrip() { - let typed = Filter::Id(vec![SmolStr::from("node-1"), SmolStr::from("node-2")]); + // I = SmolStr (the default generic parameter). + let typed: Filter = + Filter::Id(vec![SmolStr::from("node-1"), SmolStr::from("node-2")]); - let pb = filter_to_pb(&typed); + let pb = filter_to_pb(&typed).expect("filter_to_pb"); let encoded = pb.encode_to_vec(); let decoded_pb = PbFilter::decode_from_slice(encoded.as_slice()).expect("decode_from_slice"); - let roundtripped = filter_from_pb(&decoded_pb).expect("filter_from_pb"); + let roundtripped: Filter = filter_from_pb(&decoded_pb).expect("filter_from_pb"); assert_eq!(roundtripped, typed); } #[test] fn filter_tag_with_expr_roundtrip() { - let typed = Filter::Tag(TagFilter { + let typed: Filter = Filter::Tag(TagFilter { tag: SmolStr::from("role"), expr: Some(SmolStr::from("^web.*")), }); - let pb = filter_to_pb(&typed); + let pb = filter_to_pb(&typed).expect("filter_to_pb"); let encoded = pb.encode_to_vec(); let decoded_pb = PbFilter::decode_from_slice(encoded.as_slice()).expect("decode_from_slice"); - let roundtripped = filter_from_pb(&decoded_pb).expect("filter_from_pb"); + let roundtripped: Filter = filter_from_pb(&decoded_pb).expect("filter_from_pb"); assert_eq!(roundtripped, typed); } #[test] fn filter_tag_without_expr_roundtrip() { - let typed = Filter::Tag(TagFilter { + let typed: Filter = Filter::Tag(TagFilter { tag: SmolStr::from("dc"), expr: None, }); - let pb = filter_to_pb(&typed); + let pb = filter_to_pb(&typed).expect("filter_to_pb"); let encoded = pb.encode_to_vec(); let decoded_pb = PbFilter::decode_from_slice(encoded.as_slice()).expect("decode_from_slice"); - let roundtripped = filter_from_pb(&decoded_pb).expect("filter_from_pb"); + let roundtripped: Filter = filter_from_pb(&decoded_pb).expect("filter_from_pb"); assert_eq!(roundtripped, typed); } @@ -220,7 +230,7 @@ fn filter_missing_kind_is_error() { ..Default::default() }; assert!( - filter_from_pb(&pb).is_err(), + filter_from_pb::(&pb).is_err(), "expected BridgeError::UnknownVariant for missing kind" ); } @@ -335,3 +345,261 @@ fn conflict_response_roundtrip_pb() { assert_eq!(roundtripped.member.id_ref(), typed.member.id_ref()); assert_eq!(roundtripped.member.addr_ref(), typed.member.addr_ref()); } + +// ── QueryMessage ────────────────────────────────────────────────────────────── + +#[test] +fn query_message_roundtrip_pb_no_filters() { + let typed: QueryMessage = QueryMessage { + ltime: LamportTime::new(5), + id: 1234, + from: Node::new(SmolStr::from("node-q"), sample_addr()), + filters: vec![], + flags: QueryFlag::ACK, + relay_factor: 3, + timeout: std::time::Duration::from_millis(500), + name: smol_str::SmolStr::from("my-query"), + payload: bytes::Bytes::from_static(b"query-payload"), + }; + + let pb = query_to_pb(&typed).expect("query_to_pb failed"); + let encoded = pb.encode_to_vec(); + let decoded_pb = + PbQueryMessage::decode_from_slice(encoded.as_slice()).expect("decode_from_slice failed"); + let roundtripped: QueryMessage = query_from_pb(&decoded_pb).expect("query_from_pb failed"); + + assert_eq!(roundtripped.ltime, typed.ltime); + assert_eq!(roundtripped.id, typed.id); + assert_eq!(roundtripped.from.id_ref(), typed.from.id_ref()); + assert_eq!(roundtripped.from.addr_ref(), typed.from.addr_ref()); + assert_eq!(roundtripped.filters.len(), 0); + assert_eq!(roundtripped.flags, typed.flags); + assert_eq!(roundtripped.relay_factor, typed.relay_factor); + assert_eq!(roundtripped.timeout, typed.timeout); + assert_eq!(roundtripped.name, typed.name); + assert_eq!(roundtripped.payload, typed.payload); +} + +#[test] +fn query_message_roundtrip_pb_with_filters() { + let typed: QueryMessage = QueryMessage { + ltime: LamportTime::new(10), + id: 9999, + from: Node::new(SmolStr::from("node-q2"), sample_addr()), + filters: vec![ + Filter::Id(vec![SmolStr::from("target-1"), SmolStr::from("target-2")]), + Filter::Tag(TagFilter { + tag: SmolStr::from("role"), + expr: Some(SmolStr::from("^db.*")), + }), + ], + flags: QueryFlag::ACK | QueryFlag::NO_BROADCAST, + relay_factor: 0, + timeout: std::time::Duration::from_secs(2), + name: smol_str::SmolStr::from("filtered-query"), + payload: bytes::Bytes::new(), + }; + + let pb = query_to_pb(&typed).expect("query_to_pb failed"); + let encoded = pb.encode_to_vec(); + let decoded_pb = + PbQueryMessage::decode_from_slice(encoded.as_slice()).expect("decode_from_slice failed"); + let roundtripped: QueryMessage = query_from_pb(&decoded_pb).expect("query_from_pb failed"); + + assert_eq!(roundtripped.ltime, typed.ltime); + assert_eq!(roundtripped.id, typed.id); + assert_eq!(roundtripped.flags, typed.flags); + assert_eq!(roundtripped.timeout, typed.timeout); + assert_eq!(roundtripped.name, typed.name); + assert_eq!(roundtripped.filters.len(), 2); + + // Check Id filter + match &roundtripped.filters[0] { + Filter::Id(ids) => { + assert_eq!(ids.len(), 2); + assert_eq!(ids[0], SmolStr::from("target-1")); + assert_eq!(ids[1], SmolStr::from("target-2")); + } + other => panic!("expected Filter::Id, got {:?}", other), + } + + // Check Tag filter + match &roundtripped.filters[1] { + Filter::Tag(tf) => { + assert_eq!(tf.tag, SmolStr::from("role")); + assert_eq!(tf.expr, Some(SmolStr::from("^db.*"))); + } + other => panic!("expected Filter::Tag, got {:?}", other), + } +} + +#[test] +fn query_message_ltime_required() { + let pb = PbQueryMessage { + ltime: None, + id: Some(1), + from: bytes::Bytes::new(), + ..Default::default() + }; + assert!( + query_from_pb::(&pb).is_err(), + "expected BridgeError::MissingField for absent ltime" + ); +} + +#[test] +fn query_message_id_required() { + let pb = PbQueryMessage { + ltime: Some(1), + id: None, + from: bytes::Bytes::new(), + ..Default::default() + }; + assert!( + query_from_pb::(&pb).is_err(), + "expected BridgeError::MissingField for absent id" + ); +} + +// ── QueryResponseMessage ────────────────────────────────────────────────────── + +#[test] +fn query_response_message_roundtrip_pb_ack() { + let typed: QueryResponseMessage = QueryResponseMessage { + ltime: LamportTime::new(3), + id: 42, + from: Node::new(SmolStr::from("node-resp"), sample_addr()), + flags: QueryFlag::ACK, + payload: bytes::Bytes::new(), + }; + + let pb = query_response_to_pb(&typed).expect("query_response_to_pb failed"); + let encoded = pb.encode_to_vec(); + let decoded_pb = + PbQueryResponseMessage::decode_from_slice(encoded.as_slice()).expect("decode_from_slice failed"); + let roundtripped: QueryResponseMessage = + query_response_from_pb(&decoded_pb).expect("query_response_from_pb failed"); + + assert_eq!(roundtripped.ltime, typed.ltime); + assert_eq!(roundtripped.id, typed.id); + assert_eq!(roundtripped.from.id_ref(), typed.from.id_ref()); + assert_eq!(roundtripped.from.addr_ref(), typed.from.addr_ref()); + assert_eq!(roundtripped.flags, typed.flags); + assert!(roundtripped.ack()); + assert_eq!(roundtripped.payload, typed.payload); +} + +#[test] +fn query_response_message_roundtrip_pb_with_payload() { + let typed: QueryResponseMessage = QueryResponseMessage { + ltime: LamportTime::new(7), + id: 100, + from: Node::new(SmolStr::from("node-resp2"), sample_addr()), + flags: QueryFlag::empty(), + payload: bytes::Bytes::from_static(b"response-data"), + }; + + let pb = query_response_to_pb(&typed).expect("query_response_to_pb failed"); + let encoded = pb.encode_to_vec(); + let decoded_pb = + PbQueryResponseMessage::decode_from_slice(encoded.as_slice()).expect("decode_from_slice failed"); + let roundtripped: QueryResponseMessage = + query_response_from_pb(&decoded_pb).expect("query_response_from_pb failed"); + + assert_eq!(roundtripped.ltime, typed.ltime); + assert_eq!(roundtripped.id, typed.id); + assert!(!roundtripped.ack()); + assert_eq!(roundtripped.payload, bytes::Bytes::from_static(b"response-data")); +} + +#[test] +fn query_response_message_ltime_required() { + let pb = PbQueryResponseMessage { + ltime: None, + id: Some(1), + from: bytes::Bytes::new(), + ..Default::default() + }; + assert!( + query_response_from_pb::(&pb).is_err(), + "expected BridgeError::MissingField for absent ltime" + ); +} + +#[test] +fn query_response_message_id_required() { + let pb = PbQueryResponseMessage { + ltime: Some(1), + id: None, + from: bytes::Bytes::new(), + ..Default::default() + }; + assert!( + query_response_from_pb::(&pb).is_err(), + "expected BridgeError::MissingField for absent id" + ); +} + +#[test] +fn query_response_message_flags_required() { + // A QueryResponseMessage pb with flags absent must be rejected. + let pb = PbQueryResponseMessage { + ltime: Some(1), + id: Some(42), + from: bytes::Bytes::new(), + flags: None, + ..Default::default() + }; + assert!( + query_response_from_pb::(&pb).is_err(), + "expected BridgeError::MissingField for absent flags" + ); +} + +#[test] +fn query_message_required_fields() { + // flags absent — must be rejected. + let pb_no_flags = PbQueryMessage { + ltime: Some(1), + id: Some(1), + from: bytes::Bytes::new(), + flags: None, + relay_factor: Some(0), + timeout_nanos: Some(1_000_000_000), + ..Default::default() + }; + assert!( + query_from_pb::(&pb_no_flags).is_err(), + "expected error for absent flags" + ); + + // relay_factor absent — must be rejected. + let pb_no_relay = PbQueryMessage { + ltime: Some(1), + id: Some(1), + from: bytes::Bytes::new(), + flags: Some(0), + relay_factor: None, + timeout_nanos: Some(1_000_000_000), + ..Default::default() + }; + assert!( + query_from_pb::(&pb_no_relay).is_err(), + "expected error for absent relay_factor" + ); + + // timeout_nanos absent — must be rejected. + let pb_no_timeout = PbQueryMessage { + ltime: Some(1), + id: Some(1), + from: bytes::Bytes::new(), + flags: Some(0), + relay_factor: Some(0), + timeout_nanos: None, + ..Default::default() + }; + assert!( + query_from_pb::(&pb_no_timeout).is_err(), + "expected error for absent timeout_nanos" + ); +} diff --git a/serf-proto/src/typed/mod.rs b/serf-proto/src/typed/mod.rs index c13a001f..57b00dc1 100644 --- a/serf-proto/src/typed/mod.rs +++ b/serf-proto/src/typed/mod.rs @@ -137,14 +137,87 @@ pub struct TagFilter { /// Exactly one variant is active per `Filter`. The `Id` variant restricts /// the query to the listed node ids; the `Tag` variant restricts it to nodes /// whose tag satisfies the [`TagFilter`]. +/// +/// Generic over `I`: the node-id type. When `I = SmolStr` this mirrors the +/// legacy hand-rolled codec; when `I` is a custom type it encodes each id via +/// `memberlist_proto::Data` (opaque `bytes` on the wire). #[derive(Debug, Clone, PartialEq, Eq)] -pub enum Filter { +pub enum Filter { /// Restrict responses to the listed node ids. - Id(Vec), + Id(Vec), /// Restrict responses to nodes whose tag value satisfies the filter. Tag(TagFilter), } +// ── QueryMessage ────────────────────────────────────────────────────────────── + +/// A query broadcast through the cluster, optionally scoped by filters. +/// +/// Generic over `I` (node-id) and `A` (node-address); both must implement +/// `memberlist_proto::Data` so the embedded `Node` and any `Filter` +/// node-ids can be encoded as opaque `bytes` on the wire. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct QueryMessage { + /// The lamport clock value when the query was issued. + pub ltime: LamportTime, + /// Randomly generated query identifier used to correlate responses. + pub id: u32, + /// The node that originated the query. + pub from: memberlist_proto::Node, + /// Optional list of node-id / tag predicates that scope which nodes respond. + pub filters: Vec>, + /// Control flags (ACK, NO_BROADCAST, …). + pub flags: QueryFlag, + /// Number of relayed duplicate responses requested. + pub relay_factor: u8, + /// Maximum time allowed between delivery and response. + pub timeout: std::time::Duration, + /// Query name. + pub name: SmolStr, + /// Query payload. + pub payload: Bytes, +} + +impl QueryMessage { + /// Returns `true` if the ACK flag is set. + pub fn ack(&self) -> bool { + self.flags.contains(QueryFlag::ACK) + } + + /// Returns `true` if the NO_BROADCAST flag is set. + pub fn no_broadcast(&self) -> bool { + self.flags.contains(QueryFlag::NO_BROADCAST) + } +} + +// ── QueryResponseMessage ────────────────────────────────────────────────────── + +/// A response to a [`QueryMessage`], sent back to the originator. +/// +/// Generic over `I` (node-id) and `A` (node-address); both must implement +/// `memberlist_proto::Data` so the embedded `Node` can be encoded as +/// opaque `bytes` on the wire. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct QueryResponseMessage { + /// The lamport clock value when the response was emitted. + pub ltime: LamportTime, + /// Identifier of the query being responded to. + pub id: u32, + /// The node sending this response. + pub from: memberlist_proto::Node, + /// Control flags (e.g. ACK to acknowledge the query). + pub flags: QueryFlag, + /// Optional response payload. + pub payload: Bytes, +} + +impl QueryResponseMessage { + /// Returns `true` if the ACK flag is set (this message is an acknowledgement). + pub fn ack(&self) -> bool { + self.flags.contains(QueryFlag::ACK) + } +} + // ── Membership messages ─────────────────────────────────────────────────────── /// Broadcast after a node joins the cluster to associate it with a lamport clock. From c85c2be8fd1893b7f2848d47a08baeda0088ca03 Mon Sep 17 00:00:00 2001 From: al8n Date: Tue, 23 Jun 2026 23:45:15 +1200 Subject: [PATCH 007/128] feat(serf-proto): PushPullMessage, UserEvents, KeyRequest/Response, RelayMessage PushPullMessage: 3 lamport clocks (ltime/event_ltime/query_ltime) required-on-decode; status_ltimes as repeated NodeStatusTime (map forbidden in proto3); left_members as repeated bytes; events as repeated UserEvents. UserEvent + UserEvents batch added as separate typed + proto + bridge types. KeyRequestMessage: optional bytes key (None = list-keys). KeyResponseMessage: result + message + repeated bytes keys + optional primary_key. KeyResponse (machine aggregation) intentionally NOT ported per spec. RelayMessage: destination Node via Data bytes; payload carried verbatim. 54 tests pass (24 new). --- serf-proto/Cargo.toml | 12 + serf-proto/proto/serf/v1/messages.proto | 63 ++++ serf-proto/src/bridge/mod.rs | 327 +++++++++++++++++++ serf-proto/src/framing/mod.rs | 19 +- serf-proto/src/framing/tests.rs | 28 ++ serf-proto/src/lib.rs | 18 ++ serf-proto/src/messages/tests.rs | 402 ++++++++++++++++++++++++ serf-proto/src/typed/mod.rs | 174 ++++++++++ 8 files changed, 1042 insertions(+), 1 deletion(-) diff --git a/serf-proto/Cargo.toml b/serf-proto/Cargo.toml index f3187ddd..164c1a2b 100644 --- a/serf-proto/Cargo.toml +++ b/serf-proto/Cargo.toml @@ -6,6 +6,18 @@ license.workspace = true repository.workspace = true rust-version.workspace = true +[features] +# Encryption key management messages (KeyRequestMessage / KeyResponseMessage). +# Pulls in the memberlist-proto encryption backend (at least one AEAD cipher +# must be selected; both are forwarded here so the caller can pick). +# Activating this feature makes the key message types available and ensures +# key material is held in memberlist_proto::SecretKey (typed, cipher-tagged) +# rather than raw bytes. +aes-gcm = ["memberlist-proto/aes-gcm"] +chacha20-poly1305 = ["memberlist-proto/chacha20-poly1305"] +# Umbrella that activates both backends (convenient for tests / CI). +encryption = ["aes-gcm", "chacha20-poly1305"] + [dependencies] memberlist-proto = { git = "https://github.com/al8n/memberlist", branch = "main", default-features = false, features = ["std"] } bytes = { workspace = true } diff --git a/serf-proto/proto/serf/v1/messages.proto b/serf-proto/proto/serf/v1/messages.proto index 3c059000..2ec97794 100644 --- a/serf-proto/proto/serf/v1/messages.proto +++ b/serf-proto/proto/serf/v1/messages.proto @@ -100,3 +100,66 @@ message LeaveMessage { message ConflictResponseMessage { bytes member = 1; } + +// NodeStatusTime is one entry in the PushPullMessage.status_ltimes repeated field. +// proto3 map is not allowed (bytes keys are forbidden), so the +// status-times map is transmitted as a repeated message instead. +// `id` carries the node-id encoded via memberlist_proto::Data (opaque bytes). +message NodeStatusTime { + bytes id = 1; + uint64 ltime = 2; +} + +// UserEvent is a single named event with an optional payload. +// Mirrors legacy serf-core UserEvent. +message UserEvent { + bool cc = 1; + string name = 2; + bytes payload = 3; +} + +// UserEvents is a batch of user events associated with a single lamport clock. +// Mirrors legacy serf-core UserEvents. +// `optional` on ltime because the decoder must reject a missing lamport clock. +message UserEvents { + optional uint64 ltime = 1; + repeated UserEvent events = 2; +} + +// PushPullMessage carries the full cluster state during a state-sync exchange. +// `optional` on the three lamport clocks because the decoder must reject missing values. +// `status_ltimes` is a repeated NodeStatusTime rather than map because +// proto3 map keys cannot be bytes. +message PushPullMessage { + optional uint64 ltime = 1; + repeated NodeStatusTime status_ltimes = 2; + repeated bytes left_members = 3; + optional uint64 event_ltime = 4; + repeated UserEvents events = 5; + optional uint64 query_ltime = 6; +} + +// KeyRequestMessage is broadcast to all nodes as part of a key query operation. +// `key` is absent for list-keys requests; present for install/use/remove operations. +message KeyRequestMessage { + optional bytes key = 1; +} + +// KeyResponseMessage carries the result of a key operation from one node. +// `keys` carries encoded SecretKey values (each as opaque bytes). +// `primary_key` is absent when the primary key is not being reported. +message KeyResponseMessage { + bool result = 1; + string message = 2; + repeated bytes keys = 3; + optional bytes primary_key = 4; +} + +// RelayMessage wraps a serf message for forwarding to a destination node. +// `destination` carries the Node encoded via memberlist_proto::Data (opaque bytes). +// `payload` carries the inner serf framed message verbatim (tag + varint-len + buffa body); +// the relay target re-decodes it with decode_message. +message RelayMessage { + bytes destination = 1; + bytes payload = 2; +} diff --git a/serf-proto/src/bridge/mod.rs b/serf-proto/src/bridge/mod.rs index d84c5b28..a73cb32c 100644 --- a/serf-proto/src/bridge/mod.rs +++ b/serf-proto/src/bridge/mod.rs @@ -8,6 +8,8 @@ use std::borrow::Cow; use bytes::Bytes; use memberlist_proto::{Data, DataRef, data::DecodeError, data::EncodeError}; +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +use memberlist_proto::SecretKey; use smol_str::SmolStr; use crate::{ @@ -19,14 +21,20 @@ use crate::{ Filter, JoinMessage, LeaveMessage, + PushPullMessage, QueryFlag, QueryMessage, QueryResponseMessage, + RelayMessage, TagFilter, Tags, + UserEvent, UserEventMessage, + UserEvents, }, }; +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +use crate::typed::{KeyRequestMessage, KeyResponseMessage}; // ─── BridgeError ───────────────────────────────────────────────────────────── @@ -452,3 +460,322 @@ where payload: b.payload.clone(), }) } + +// ─── UserEvent ──────────────────────────────────────────────────────────────── + +/// Convert a typed [`UserEvent`] → `pb::UserEvent`. +pub fn user_event_single_to_pb(t: &UserEvent) -> pb::UserEvent { + pb::UserEvent { + cc: t.cc, + name: t.name.to_string(), + payload: t.payload.clone(), + ..Default::default() + } +} + +/// Convert `pb::UserEvent` → typed [`UserEvent`]. +pub fn user_event_single_from_pb(b: &pb::UserEvent) -> UserEvent { + UserEvent { + cc: b.cc, + name: SmolStr::from(b.name.as_str()), + payload: b.payload.clone(), + } +} + +// ─── UserEvents ─────────────────────────────────────────────────────────────── + +/// Convert a typed [`UserEvents`] → `pb::UserEvents`. +pub fn user_events_to_pb(t: &UserEvents) -> pb::UserEvents { + pb::UserEvents { + ltime: Some(t.ltime.into()), + events: t.events.iter().map(user_event_single_to_pb).collect(), + ..Default::default() + } +} + +/// Convert `pb::UserEvents` → typed [`UserEvents`]. +/// +/// Rejects a missing `ltime` and an empty `events` list — a batch with zero +/// events carries no information and would silently consume buffer history +/// entries. The legacy `serf-core` invariant is `OneOrMore` (at least one event +/// per batch); this decoder enforces the same constraint. +pub fn user_events_from_pb(b: &pb::UserEvents) -> Result { + let ltime = b + .ltime + .ok_or(BridgeError::MissingField("UserEvents.ltime".into()))?; + if b.events.is_empty() { + return Err(BridgeError::MissingField("UserEvents.events (must be non-empty)".into())); + } + Ok(UserEvents { + ltime: LamportTime::from(ltime), + events: b.events.iter().map(user_event_single_from_pb).collect(), + }) +} + +// ─── PushPullMessage ───────────────────────────────────────────────────────── + +/// Convert a typed [`PushPullMessage`] → `pb::PushPullMessage`. +/// +/// - `status_ltimes`: each `(I, LamportTime)` pair is encoded as a +/// `pb::NodeStatusTime` with the node-id in the `id` bytes field. +/// - `left_members`: each `I` is encoded to opaque `bytes` via +/// `memberlist_proto::Data`. +/// - `events`: each [`UserEvents`] batch is encoded via [`user_events_to_pb`]. +pub fn push_pull_to_pb(t: &PushPullMessage) -> Result +where + I: Data, +{ + let status_ltimes = t + .status_ltimes + .iter() + .map(|(id, ltime)| { + data_to_bytes(id).map(|id_bytes| pb::NodeStatusTime { + id: id_bytes, + ltime: (*ltime).into(), + ..Default::default() + }) + }) + .collect::, BridgeError>>()?; + + let left_members = t + .left_members + .iter() + .map(|id| data_to_bytes(id)) + .collect::, BridgeError>>()?; + + let events = t.events.iter().map(user_events_to_pb).collect(); + + Ok(pb::PushPullMessage { + ltime: Some(t.ltime.into()), + status_ltimes, + left_members, + event_ltime: Some(t.event_ltime.into()), + events, + query_ltime: Some(t.query_ltime.into()), + ..Default::default() + }) +} + +/// Convert `pb::PushPullMessage` → typed [`PushPullMessage`]. +/// +/// Rejects missing `ltime`, `event_ltime`, and `query_ltime` (all required by +/// the legacy protocol). Decodes each `NodeStatusTime.id` and each +/// `left_members` entry as `I` via `memberlist_proto::DataRef`. +pub fn push_pull_from_pb(b: &pb::PushPullMessage) -> Result, BridgeError> +where + I: Data, +{ + let ltime = b + .ltime + .ok_or(BridgeError::MissingField("PushPullMessage.ltime".into()))?; + let event_ltime = b + .event_ltime + .ok_or(BridgeError::MissingField("PushPullMessage.event_ltime".into()))?; + let query_ltime = b + .query_ltime + .ok_or(BridgeError::MissingField("PushPullMessage.query_ltime".into()))?; + + let status_ltimes = b + .status_ltimes + .iter() + .map(|nst| { + let id: I = data_from_bytes(&nst.id)?; + Ok((id, LamportTime::from(nst.ltime))) + }) + .collect::, BridgeError>>()?; + + let left_members = b + .left_members + .iter() + .map(|buf| data_from_bytes::(buf)) + .collect::, BridgeError>>()?; + + let events = b + .events + .iter() + .map(user_events_from_pb) + .collect::, BridgeError>>()?; + + Ok(PushPullMessage { + ltime: LamportTime::from(ltime), + status_ltimes, + left_members, + event_ltime: LamportTime::from(event_ltime), + events, + query_ltime: LamportTime::from(query_ltime), + }) +} + +// ─── SecretKey ↔ Bytes helpers ─────────────────────────────────────────────── + +/// Encode a [`SecretKey`] as `[algorithm_tag][raw_key_bytes]`. +/// +/// The leading algorithm tag byte makes the wire encoding self-describing so +/// decoding is unambiguous even when two ciphers share the same key length +/// (e.g. AES-256 and ChaCha20-Poly1305 are both 32 bytes). +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +fn secret_key_to_bytes(key: &SecretKey) -> Bytes { + let raw = key.as_bytes(); + let mut buf = Vec::with_capacity(1 + raw.len()); + buf.push(key.algorithm().tag()); + buf.extend_from_slice(raw); + Bytes::from(buf) +} + +/// Decode a [`SecretKey`] from `[algorithm_tag][raw_key_bytes]` wire bytes. +/// +/// Returns [`BridgeError::InvalidValue`] when the tag is unknown to this build +/// or the byte count does not match the algorithm's expected key length. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +fn secret_key_from_bytes(buf: &Bytes) -> Result { + if buf.is_empty() { + return Err(BridgeError::InvalidValue( + "key bytes must carry at least the algorithm tag".into(), + )); + } + let tag = buf[0]; + let raw = &buf[1..]; + match tag { + #[cfg(feature = "aes-gcm")] + 1 => { + // AES-GCM: length determines the AES variant. + match raw.len() { + 16 => { + let k: [u8; 16] = raw.try_into().unwrap(); + Ok(SecretKey::Aes128(k)) + } + 24 => { + let k: [u8; 24] = raw.try_into().unwrap(); + Ok(SecretKey::Aes192(k)) + } + 32 => { + let k: [u8; 32] = raw.try_into().unwrap(); + Ok(SecretKey::Aes256(k)) + } + n => Err(BridgeError::InvalidValue( + format!("AES-GCM key must be 16, 24, or 32 bytes; got {n}").into(), + )), + } + } + #[cfg(feature = "chacha20-poly1305")] + 2 => { + // ChaCha20-Poly1305: always 32 bytes. + if raw.len() != 32 { + return Err(BridgeError::InvalidValue( + format!("ChaCha20-Poly1305 key must be 32 bytes; got {}", raw.len()).into(), + )); + } + let k: [u8; 32] = raw.try_into().unwrap(); + Ok(SecretKey::ChaCha20Poly1305(k)) + } + other => Err(BridgeError::InvalidValue( + format!("unknown or unsupported algorithm tag {other}").into(), + )), + } +} + +// ─── KeyRequestMessage ──────────────────────────────────────────────────────── + +/// Convert a typed [`KeyRequestMessage`] → `pb::KeyRequestMessage`. +/// +/// The key (if present) is encoded as `[algorithm_tag][raw_key_bytes]` so the +/// wire encoding is self-describing. Requires the `aes-gcm` or +/// `chacha20-poly1305` feature. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))))] +pub fn key_request_to_pb(t: &KeyRequestMessage) -> pb::KeyRequestMessage { + pb::KeyRequestMessage { + key: t.key.as_ref().map(secret_key_to_bytes), + ..Default::default() + } +} + +/// Convert `pb::KeyRequestMessage` → typed [`KeyRequestMessage`]. +/// +/// The key bytes (if present) are decoded as `[algorithm_tag][raw_key_bytes]`. +/// Returns [`BridgeError::InvalidValue`] if the bytes are present but malformed. +/// Requires the `aes-gcm` or `chacha20-poly1305` feature. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))))] +pub fn key_request_from_pb(b: &pb::KeyRequestMessage) -> Result { + let key = b.key.as_ref().map(secret_key_from_bytes).transpose()?; + Ok(KeyRequestMessage { key }) +} + +// ─── KeyResponseMessage ─────────────────────────────────────────────────────── + +/// Convert a typed [`KeyResponseMessage`] → `pb::KeyResponseMessage`. +/// +/// Each key is encoded as `[algorithm_tag][raw_key_bytes]`. Requires the +/// `aes-gcm` or `chacha20-poly1305` feature. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))))] +pub fn key_response_to_pb(t: &KeyResponseMessage) -> pb::KeyResponseMessage { + pb::KeyResponseMessage { + result: t.result, + message: t.message.to_string(), + keys: t.keys.iter().map(secret_key_to_bytes).collect(), + primary_key: t.primary_key.as_ref().map(secret_key_to_bytes), + ..Default::default() + } +} + +/// Convert `pb::KeyResponseMessage` → typed [`KeyResponseMessage`]. +/// +/// Each key bytes entry is decoded as `[algorithm_tag][raw_key_bytes]`. +/// Returns [`BridgeError::InvalidValue`] if any entry is malformed. Requires +/// the `aes-gcm` or `chacha20-poly1305` feature. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))))] +pub fn key_response_from_pb(b: &pb::KeyResponseMessage) -> Result { + let keys = b + .keys + .iter() + .map(secret_key_from_bytes) + .collect::, BridgeError>>()?; + let primary_key = b + .primary_key + .as_ref() + .map(secret_key_from_bytes) + .transpose()?; + Ok(KeyResponseMessage { + result: b.result, + message: SmolStr::from(b.message.as_str()), + keys, + primary_key, + }) +} + +// ─── RelayMessage ───────────────────────────────────────────────────────────── + +/// Convert a typed [`RelayMessage`] → `pb::RelayMessage`. +/// +/// The `destination: Node` is serialised to opaque `bytes` via +/// `memberlist_proto::Data`. The `payload` bytes are copied verbatim. +pub fn relay_to_pb(t: &RelayMessage) -> Result +where + I: Data, + A: Data, +{ + Ok(pb::RelayMessage { + destination: data_to_bytes(&t.destination)?, + payload: t.payload.clone(), + ..Default::default() + }) +} + +/// Convert `pb::RelayMessage` → typed [`RelayMessage`]. +/// +/// Decodes `destination` as `Node` via `memberlist_proto::DataRef`. +/// The `payload` bytes are preserved verbatim without parsing. +pub fn relay_from_pb(b: &pb::RelayMessage) -> Result, BridgeError> +where + I: Data, + A: Data, +{ + let destination: memberlist_proto::Node = data_from_bytes(&b.destination)?; + Ok(RelayMessage { + destination, + payload: b.payload.clone(), + }) +} diff --git a/serf-proto/src/framing/mod.rs b/serf-proto/src/framing/mod.rs index d33d6fa8..6c4554ed 100644 --- a/serf-proto/src/framing/mod.rs +++ b/serf-proto/src/framing/mod.rs @@ -28,7 +28,9 @@ const QUERY_TAG: u8 = 5; const QUERY_RESPONSE_TAG: u8 = 6; const CONFLICT_RESPONSE_TAG: u8 = 7; const RELAY_TAG: u8 = 8; +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] const KEY_REQUEST_TAG: u8 = 9; +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] const KEY_RESPONSE_TAG: u8 = 10; /// One-byte discriminant that opens every serf message frame. @@ -36,7 +38,8 @@ const KEY_RESPONSE_TAG: u8 = 10; /// Numeric values are identical to the legacy `serf-core` tag constants so /// that future mixed-version migration tooling can map them trivially. /// `Unknown(u8)` provides forward compatibility for tag values not yet -/// recognised by this build. +/// recognised by this build, including key-management messages when this build +/// was compiled without an encryption backend. #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] #[non_exhaustive] pub enum MessageType { @@ -57,8 +60,18 @@ pub enum MessageType { /// Relay — message relayed through an intermediary node. Relay, /// KeyRequest — encryption key management request. + /// + /// Requires the `aes-gcm` or `chacha20-poly1305` feature; without an + /// encryption backend the tag byte decodes as [`MessageType::Unknown`]. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))))] KeyRequest, /// KeyResponse — encryption key management response. + /// + /// Requires the `aes-gcm` or `chacha20-poly1305` feature; without an + /// encryption backend the tag byte decodes as [`MessageType::Unknown`]. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))))] KeyResponse, /// A tag not recognised by this build — preserved for forward compatibility. Unknown(u8), @@ -75,7 +88,9 @@ impl From for MessageType { QUERY_RESPONSE_TAG => Self::QueryResponse, CONFLICT_RESPONSE_TAG => Self::ConflictResponse, RELAY_TAG => Self::Relay, + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] KEY_REQUEST_TAG => Self::KeyRequest, + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] KEY_RESPONSE_TAG => Self::KeyResponse, val => Self::Unknown(val), } @@ -93,7 +108,9 @@ impl From for u8 { MessageType::QueryResponse => QUERY_RESPONSE_TAG, MessageType::ConflictResponse => CONFLICT_RESPONSE_TAG, MessageType::Relay => RELAY_TAG, + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] MessageType::KeyRequest => KEY_REQUEST_TAG, + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] MessageType::KeyResponse => KEY_RESPONSE_TAG, MessageType::Unknown(val) => val, } diff --git a/serf-proto/src/framing/tests.rs b/serf-proto/src/framing/tests.rs index 77054c9d..7bab146b 100644 --- a/serf-proto/src/framing/tests.rs +++ b/serf-proto/src/framing/tests.rs @@ -18,6 +18,26 @@ fn message_type_tag_round_trip() { (MessageType::QueryResponse, 6), (MessageType::ConflictResponse, 7), (MessageType::Relay, 8), + ]; + + for &(ref ty, expected_byte) in cases { + let byte = u8::from(*ty); + assert_eq!( + byte, expected_byte, + "{ty:?} should have tag byte {expected_byte}" + ); + assert_eq!( + MessageType::from(byte), + *ty, + "u8 {byte} should round-trip to {ty:?}" + ); + } +} + +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn message_type_key_tag_round_trip() { + let cases: &[(MessageType, u8)] = &[ (MessageType::KeyRequest, 9), (MessageType::KeyResponse, 10), ]; @@ -36,6 +56,14 @@ fn message_type_tag_round_trip() { } } +/// Without an encryption feature, key tag bytes must decode as Unknown. +#[cfg(not(any(feature = "aes-gcm", feature = "chacha20-poly1305")))] +#[test] +fn message_type_key_tags_are_unknown_without_encryption() { + assert_eq!(MessageType::from(9u8), MessageType::Unknown(9)); + assert_eq!(MessageType::from(10u8), MessageType::Unknown(10)); +} + #[test] fn message_type_unknown_preserved() { let unknown_byte: u8 = 42; diff --git a/serf-proto/src/lib.rs b/serf-proto/src/lib.rs index 3d91f7f2..646cc5b7 100644 --- a/serf-proto/src/lib.rs +++ b/serf-proto/src/lib.rs @@ -16,15 +16,26 @@ pub use bridge::{ join_to_pb, leave_from_pb, leave_to_pb, + push_pull_from_pb, + push_pull_to_pb, query_from_pb, query_response_from_pb, query_response_to_pb, query_to_pb, + relay_from_pb, + relay_to_pb, tags_from_pb, tags_to_pb, user_event_from_pb, user_event_to_pb, + user_event_single_from_pb, + user_event_single_to_pb, + user_events_from_pb, + user_events_to_pb, }; +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))))] +pub use bridge::{key_request_from_pb, key_request_to_pb, key_response_from_pb, key_response_to_pb}; pub use framing::{FrameError, MessageType, decode_message, encode_message}; pub use typed::{ Coordinate, @@ -32,13 +43,20 @@ pub use typed::{ Filter, JoinMessage, LeaveMessage, + PushPullMessage, QueryFlag, QueryMessage, QueryResponseMessage, + RelayMessage, TagFilter, Tags, + UserEvent, UserEventMessage, + UserEvents, }; +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))))] +pub use typed::{KeyRequestMessage, KeyResponseMessage}; /// A lamport logical clock value — a monotonically increasing counter used to /// order serf events. diff --git a/serf-proto/src/messages/tests.rs b/serf-proto/src/messages/tests.rs index 3c349b61..ea9cd381 100644 --- a/serf-proto/src/messages/tests.rs +++ b/serf-proto/src/messages/tests.rs @@ -10,10 +10,19 @@ use super::serf::v1::{ Filter as PbFilter, JoinMessage as PbJoinMessage, LeaveMessage as PbLeaveMessage, + PushPullMessage as PbPushPullMessage, QueryMessage as PbQueryMessage, QueryResponseMessage as PbQueryResponseMessage, + RelayMessage as PbRelayMessage, Tags as PbTags, + UserEvent as PbUserEvent, UserEventMessage as PbUserEventMessage, + UserEvents as PbUserEvents, +}; +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +use super::serf::v1::{ + KeyRequestMessage as PbKeyRequestMessage, + KeyResponseMessage as PbKeyResponseMessage, }; use crate::{ ConflictResponseMessage, @@ -22,12 +31,16 @@ use crate::{ JoinMessage, LamportTime, LeaveMessage, + PushPullMessage, QueryFlag, QueryMessage, QueryResponseMessage, + RelayMessage, TagFilter, Tags, + UserEvent, UserEventMessage, + UserEvents, conflict_response_from_pb, conflict_response_to_pb, coordinate_from_pb, @@ -38,15 +51,34 @@ use crate::{ join_to_pb, leave_from_pb, leave_to_pb, + push_pull_from_pb, + push_pull_to_pb, query_from_pb, query_response_from_pb, query_response_to_pb, query_to_pb, + relay_from_pb, + relay_to_pb, tags_from_pb, tags_to_pb, user_event_from_pb, + user_event_single_from_pb, + user_event_single_to_pb, user_event_to_pb, + user_events_from_pb, + user_events_to_pb, +}; +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +use crate::{ + KeyRequestMessage, + KeyResponseMessage, + key_request_from_pb, + key_request_to_pb, + key_response_from_pb, + key_response_to_pb, }; +#[cfg(feature = "aes-gcm")] +use memberlist_proto::SecretKey; // ── UserEventMessage ───────────────────────────────────────────────────────── @@ -603,3 +635,373 @@ fn query_message_required_fields() { "expected error for absent timeout_nanos" ); } + +// ── UserEvent (single) ──────────────────────────────────────────────────────── + +#[test] +fn user_event_single_roundtrip_pb() { + let typed = UserEvent { + cc: true, + name: SmolStr::from("deploy"), + payload: bytes::Bytes::from_static(b"ev-payload"), + }; + + let pb = user_event_single_to_pb(&typed); + let encoded = pb.encode_to_vec(); + let decoded_pb = + PbUserEvent::decode_from_slice(encoded.as_slice()).expect("decode_from_slice failed"); + let roundtripped = user_event_single_from_pb(&decoded_pb); + + assert_eq!(roundtripped.cc, typed.cc); + assert_eq!(roundtripped.name, typed.name); + assert_eq!(roundtripped.payload, typed.payload); +} + +#[test] +fn user_event_single_empty_payload_roundtrip() { + let typed = UserEvent { + cc: false, + name: SmolStr::from("ping"), + payload: bytes::Bytes::new(), + }; + + let pb = user_event_single_to_pb(&typed); + let encoded = pb.encode_to_vec(); + let decoded_pb = + PbUserEvent::decode_from_slice(encoded.as_slice()).expect("decode_from_slice failed"); + let roundtripped = user_event_single_from_pb(&decoded_pb); + + assert_eq!(roundtripped.cc, typed.cc); + assert_eq!(roundtripped.name, typed.name); + assert!(roundtripped.payload.is_empty()); +} + +// ── UserEvents (batch) ──────────────────────────────────────────────────────── + +#[test] +fn user_events_roundtrip_pb() { + let typed = UserEvents { + ltime: LamportTime::new(7), + events: vec![ + UserEvent { + cc: true, + name: SmolStr::from("deploy"), + payload: bytes::Bytes::from_static(b"v1"), + }, + UserEvent { + cc: false, + name: SmolStr::from("alert"), + payload: bytes::Bytes::from_static(b"critical"), + }, + ], + }; + + let pb = user_events_to_pb(&typed); + let encoded = pb.encode_to_vec(); + let decoded_pb = + PbUserEvents::decode_from_slice(encoded.as_slice()).expect("decode_from_slice failed"); + let roundtripped = user_events_from_pb(&decoded_pb).expect("user_events_from_pb failed"); + + assert_eq!(roundtripped.ltime, typed.ltime); + assert_eq!(roundtripped.events.len(), 2); + assert_eq!(roundtripped.events[0].name, SmolStr::from("deploy")); + assert_eq!(roundtripped.events[1].name, SmolStr::from("alert")); +} + +#[test] +fn user_events_ltime_required() { + let pb = PbUserEvents { + ltime: None, + events: vec![], + ..Default::default() + }; + assert!( + user_events_from_pb(&pb).is_err(), + "expected BridgeError::MissingField for absent ltime" + ); +} + +#[test] +fn user_events_empty_events_rejected() { + // A UserEvents batch with an empty events list must be rejected — the legacy + // invariant is OneOrMore (at least one event per batch). An empty batch + // carries no information and would silently consume a buffer history slot. + let pb = PbUserEvents { + ltime: Some(1), + events: vec![], + ..Default::default() + }; + assert!( + user_events_from_pb(&pb).is_err(), + "expected BridgeError::MissingField for empty events list" + ); +} + +// ── PushPullMessage ─────────────────────────────────────────────────────────── + +#[test] +fn push_pull_message_roundtrip_pb_full() { + let typed: PushPullMessage = PushPullMessage { + ltime: LamportTime::new(100), + status_ltimes: vec![ + (SmolStr::from("node-a"), LamportTime::new(10)), + (SmolStr::from("node-b"), LamportTime::new(20)), + ], + left_members: vec![SmolStr::from("node-gone")], + event_ltime: LamportTime::new(50), + events: vec![UserEvents { + ltime: LamportTime::new(49), + events: vec![UserEvent { + cc: false, + name: SmolStr::from("deploy"), + payload: bytes::Bytes::from_static(b"v2"), + }], + }], + query_ltime: LamportTime::new(75), + }; + + let pb = push_pull_to_pb(&typed).expect("push_pull_to_pb failed"); + let encoded = pb.encode_to_vec(); + let decoded_pb = + PbPushPullMessage::decode_from_slice(encoded.as_slice()).expect("decode_from_slice failed"); + let roundtripped: PushPullMessage = + push_pull_from_pb(&decoded_pb).expect("push_pull_from_pb failed"); + + assert_eq!(roundtripped.ltime, typed.ltime); + assert_eq!(roundtripped.event_ltime, typed.event_ltime); + assert_eq!(roundtripped.query_ltime, typed.query_ltime); + + assert_eq!(roundtripped.status_ltimes.len(), 2); + assert_eq!(roundtripped.status_ltimes[0].0, SmolStr::from("node-a")); + assert_eq!(roundtripped.status_ltimes[0].1, LamportTime::new(10)); + assert_eq!(roundtripped.status_ltimes[1].0, SmolStr::from("node-b")); + assert_eq!(roundtripped.status_ltimes[1].1, LamportTime::new(20)); + + assert_eq!(roundtripped.left_members.len(), 1); + assert_eq!(roundtripped.left_members[0], SmolStr::from("node-gone")); + + assert_eq!(roundtripped.events.len(), 1); + assert_eq!(roundtripped.events[0].ltime, LamportTime::new(49)); + assert_eq!(roundtripped.events[0].events.len(), 1); + assert_eq!(roundtripped.events[0].events[0].name, SmolStr::from("deploy")); +} + +#[test] +fn push_pull_message_roundtrip_pb_empty() { + let typed: PushPullMessage = PushPullMessage { + ltime: LamportTime::new(1), + status_ltimes: vec![], + left_members: vec![], + event_ltime: LamportTime::new(2), + events: vec![], + query_ltime: LamportTime::new(3), + }; + + let pb = push_pull_to_pb(&typed).expect("push_pull_to_pb failed"); + let encoded = pb.encode_to_vec(); + let decoded_pb = + PbPushPullMessage::decode_from_slice(encoded.as_slice()).expect("decode_from_slice failed"); + let roundtripped: PushPullMessage = + push_pull_from_pb(&decoded_pb).expect("push_pull_from_pb failed"); + + assert_eq!(roundtripped.ltime, typed.ltime); + assert_eq!(roundtripped.event_ltime, typed.event_ltime); + assert_eq!(roundtripped.query_ltime, typed.query_ltime); + assert!(roundtripped.status_ltimes.is_empty()); + assert!(roundtripped.left_members.is_empty()); + assert!(roundtripped.events.is_empty()); +} + +#[test] +fn push_pull_message_ltime_required() { + let pb = PbPushPullMessage { + ltime: None, + event_ltime: Some(1), + query_ltime: Some(1), + ..Default::default() + }; + assert!( + push_pull_from_pb::(&pb).is_err(), + "expected BridgeError::MissingField for absent ltime" + ); +} + +#[test] +fn push_pull_message_event_ltime_required() { + let pb = PbPushPullMessage { + ltime: Some(1), + event_ltime: None, + query_ltime: Some(1), + ..Default::default() + }; + assert!( + push_pull_from_pb::(&pb).is_err(), + "expected BridgeError::MissingField for absent event_ltime" + ); +} + +#[test] +fn push_pull_message_query_ltime_required() { + let pb = PbPushPullMessage { + ltime: Some(1), + event_ltime: Some(1), + query_ltime: None, + ..Default::default() + }; + assert!( + push_pull_from_pb::(&pb).is_err(), + "expected BridgeError::MissingField for absent query_ltime" + ); +} + +// ── KeyRequestMessage ───────────────────────────────────────────────────────── + +#[cfg(feature = "aes-gcm")] +#[test] +fn key_request_roundtrip_pb_with_key() { + let key = SecretKey::Aes256([0xABu8; 32]); + let typed = KeyRequestMessage { key: Some(key) }; + + let pb = key_request_to_pb(&typed); + let encoded = pb.encode_to_vec(); + let decoded_pb = + PbKeyRequestMessage::decode_from_slice(encoded.as_slice()).expect("decode_from_slice failed"); + let roundtripped = key_request_from_pb(&decoded_pb).expect("key_request_from_pb failed"); + + assert_eq!(roundtripped.key, typed.key); +} + +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn key_request_roundtrip_pb_no_key() { + let typed = KeyRequestMessage { key: None }; + + let pb = key_request_to_pb(&typed); + let encoded = pb.encode_to_vec(); + let decoded_pb = + PbKeyRequestMessage::decode_from_slice(encoded.as_slice()).expect("decode_from_slice failed"); + let roundtripped = key_request_from_pb(&decoded_pb).expect("key_request_from_pb failed"); + + assert!(roundtripped.key.is_none()); +} + +// ── KeyResponseMessage ──────────────────────────────────────────────────────── + +#[cfg(feature = "aes-gcm")] +#[test] +fn key_response_roundtrip_pb_success_with_keys() { + let k1 = SecretKey::Aes128([0x11u8; 16]); + let k2 = SecretKey::Aes256([0x22u8; 32]); + let primary = SecretKey::Aes128([0x11u8; 16]); + let typed = KeyResponseMessage { + result: true, + message: SmolStr::from("ok"), + keys: vec![k1, k2], + primary_key: Some(primary), + }; + + let pb = key_response_to_pb(&typed); + let encoded = pb.encode_to_vec(); + let decoded_pb = + PbKeyResponseMessage::decode_from_slice(encoded.as_slice()).expect("decode_from_slice failed"); + let roundtripped = key_response_from_pb(&decoded_pb).expect("key_response_from_pb failed"); + + assert!(roundtripped.result); + assert_eq!(roundtripped.message, SmolStr::from("ok")); + assert_eq!(roundtripped.keys.len(), 2); + assert_eq!(roundtripped.keys[0], typed.keys[0]); + assert_eq!(roundtripped.keys[1], typed.keys[1]); + assert_eq!(roundtripped.primary_key, typed.primary_key); +} + +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn key_response_roundtrip_pb_failure_no_keys() { + let typed = KeyResponseMessage { + result: false, + message: SmolStr::from("permission denied"), + keys: vec![], + primary_key: None, + }; + + let pb = key_response_to_pb(&typed); + let encoded = pb.encode_to_vec(); + let decoded_pb = + PbKeyResponseMessage::decode_from_slice(encoded.as_slice()).expect("decode_from_slice failed"); + let roundtripped = key_response_from_pb(&decoded_pb).expect("key_response_from_pb failed"); + + assert!(!roundtripped.result); + assert_eq!(roundtripped.message, SmolStr::from("permission denied")); + assert!(roundtripped.keys.is_empty()); + assert!(roundtripped.primary_key.is_none()); +} + +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn key_response_malformed_bytes_rejected() { + // A key entry with no algorithm tag byte must be rejected. + let pb = PbKeyResponseMessage { + result: true, + message: String::from("ok"), + keys: vec![bytes::Bytes::new()], // empty — missing algorithm tag + primary_key: None, + ..Default::default() + }; + assert!( + key_response_from_pb(&pb).is_err(), + "expected BridgeError::InvalidValue for empty key bytes" + ); +} + +// ── RelayMessage ────────────────────────────────────────────────────────────── + +#[test] +fn relay_message_roundtrip_pb() { + let destination: Node = Node::new(SmolStr::from("relay-target"), sample_addr()); + // The payload carries a raw framed serf message; here we use arbitrary bytes + // to verify the passthrough without interpreting the content. + let payload = bytes::Bytes::from_static(b"\x04\x05hello"); + let typed: RelayMessage = RelayMessage::new(destination.clone(), payload.clone()); + + let pb = relay_to_pb(&typed).expect("relay_to_pb failed"); + let encoded = pb.encode_to_vec(); + let decoded_pb = + PbRelayMessage::decode_from_slice(encoded.as_slice()).expect("decode_from_slice failed"); + let roundtripped: RelayMessage = + relay_from_pb(&decoded_pb).expect("relay_from_pb failed"); + + assert_eq!(roundtripped.destination.id_ref(), typed.destination.id_ref()); + assert_eq!(roundtripped.destination.addr_ref(), typed.destination.addr_ref()); + assert_eq!(roundtripped.payload, payload); +} + +#[test] +fn relay_message_empty_payload_roundtrip() { + let destination: Node = Node::new(SmolStr::from("target"), sample_addr()); + let typed: RelayMessage = RelayMessage::new(destination, bytes::Bytes::new()); + + let pb = relay_to_pb(&typed).expect("relay_to_pb failed"); + let encoded = pb.encode_to_vec(); + let decoded_pb = + PbRelayMessage::decode_from_slice(encoded.as_slice()).expect("decode_from_slice failed"); + let roundtripped: RelayMessage = + relay_from_pb(&decoded_pb).expect("relay_from_pb failed"); + + assert!(roundtripped.payload.is_empty()); +} + +#[test] +fn relay_message_empty_destination_rejected() { + // A RelayMessage pb with `destination = b""` must be rejected by + // relay_from_pb: an empty byte slice is not a valid encoded Node, + // so data_from_bytes returns a Data decode error. + let pb = PbRelayMessage { + destination: bytes::Bytes::new(), + payload: bytes::Bytes::from_static(b"some-payload"), + ..Default::default() + }; + assert!( + relay_from_pb::(&pb).is_err(), + "expected a BridgeError for empty destination bytes" + ); +} diff --git a/serf-proto/src/typed/mod.rs b/serf-proto/src/typed/mod.rs index 57b00dc1..83e8c932 100644 --- a/serf-proto/src/typed/mod.rs +++ b/serf-proto/src/typed/mod.rs @@ -10,6 +10,8 @@ use smol_str::SmolStr; use crate::LamportTime; use memberlist_proto::Node; +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +use memberlist_proto::SecretKey; /// A user-generated event broadcast through the serf cluster. #[derive(Debug, Default, Clone, PartialEq, Eq)] @@ -220,6 +222,178 @@ impl QueryResponseMessage { // ── Membership messages ─────────────────────────────────────────────────────── +// ── UserEvent ───────────────────────────────────────────────────────────────── + +/// A single named user event with an optional payload. +/// +/// Mirrors the legacy `serf-core` `UserEvent` struct. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UserEvent { + /// Whether the event may be coalesced with later identical events. + pub cc: bool, + /// The event name. + pub name: SmolStr, + /// The event payload. + pub payload: Bytes, +} + +// ── UserEvents ──────────────────────────────────────────────────────────────── + +/// A batch of user events associated with a single lamport clock value. +/// +/// Serf buffers received events in `UserEvents` entries to prevent re-delivery. +/// Mirrors the legacy `serf-core` `UserEvents` struct. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct UserEvents { + /// The lamport clock value for this event batch. + pub ltime: LamportTime, + /// The events in this batch. + pub events: Vec, +} + +// ── PushPullMessage ─────────────────────────────────────────────────────────── + +/// Full cluster state exchanged during a state-sync (push-pull) operation. +/// +/// This is the largest serf message but is sent infrequently — only during +/// the anti-entropy state exchange between two nodes. +/// +/// Generic over `I`: the node-id type, which must implement +/// `memberlist_proto::Data` so node-ids can be encoded as opaque proto `bytes` +/// in the bridge layer. +/// +/// The `status_ltimes` map is transmitted as a `repeated NodeStatusTime` rather +/// than a proto3 `map` because proto3 forbids `bytes` map keys. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PushPullMessage { + /// The lamport clock value of the sending node at the time of the exchange. + pub ltime: LamportTime, + /// Maps each known node-id to its last-seen status lamport time. + pub status_ltimes: Vec<(I, LamportTime)>, + /// Wire list of node-ids that have left the cluster. + /// + /// The codec does NOT dedup this list; the consuming machine is responsible + /// for treating it as a set (duplicate entries are idempotent leave events). + pub left_members: Vec, + /// The lamport clock value for the event subsystem. + pub event_ltime: LamportTime, + /// Buffered user-event batches. + pub events: Vec, + /// The lamport clock value for the query subsystem. + pub query_ltime: LamportTime, +} + +impl PushPullMessage { + /// Construct a new `PushPullMessage`. + pub fn new( + ltime: LamportTime, + status_ltimes: Vec<(I, LamportTime)>, + left_members: Vec, + event_ltime: LamportTime, + events: Vec, + query_ltime: LamportTime, + ) -> Self { + Self { + ltime, + status_ltimes, + left_members, + event_ltime, + events, + query_ltime, + } + } +} + +// ── KeyRequestMessage ───────────────────────────────────────────────────────── + +/// Encryption key management request, broadcast to all nodes. +/// +/// `key` is absent for list-keys requests and present for install/use/remove +/// operations. +/// +/// Requires the `aes-gcm` or `chacha20-poly1305` feature. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))))] +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct KeyRequestMessage { + /// The encryption key, or `None` for a list-keys request. + pub key: Option, +} + +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +impl KeyRequestMessage { + /// Construct a new `KeyRequestMessage`. + pub fn new(key: Option) -> Self { + Self { key } + } +} + +// ── KeyResponseMessage ──────────────────────────────────────────────────────── + +/// Result of a key operation from a single node. +/// +/// The aggregation type (`KeyResponse`) is machine-side and is NOT part of +/// the wire codec; only this per-node response message is encoded on the wire. +/// +/// Requires the `aes-gcm` or `chacha20-poly1305` feature. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))))] +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct KeyResponseMessage { + /// `true` if the operation succeeded on this node. + pub result: bool, + /// Human-readable result or error description. + pub message: SmolStr, + /// Installed keys (used by list-keys responses). + pub keys: Vec, + /// The current primary key, if reporting it. + pub primary_key: Option, +} + +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +impl KeyResponseMessage { + /// Construct a default (failure, no keys) `KeyResponseMessage`. + pub fn new() -> Self { + Self { + result: false, + message: SmolStr::default(), + keys: Vec::new(), + primary_key: None, + } + } +} + +// ── RelayMessage ────────────────────────────────────────────────────────────── + +/// A serf message forwarded through an intermediary node. +/// +/// The `payload` carries the inner serf framed message verbatim +/// (`[tag][varint_len][buffa_body]`). The relay target re-decodes it with +/// `decode_message` without this layer parsing the content. +/// +/// Generic over `I` (node-id) and `A` (node-address). The `memberlist_proto::Data` +/// bound is enforced at the bridge layer (`relay_to_pb` / `relay_from_pb`) where +/// the destination is encoded/decoded as opaque `bytes`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RelayMessage { + /// The node to forward the message to. + pub destination: Node, + /// The inner serf framed message, carried unparsed. + pub payload: Bytes, +} + +impl RelayMessage { + /// Construct a new `RelayMessage`. + pub fn new(destination: Node, payload: Bytes) -> Self { + Self { + destination, + payload, + } + } +} + +// ── Membership messages ─────────────────────────────────────────────────────── + /// Broadcast after a node joins the cluster to associate it with a lamport clock. /// /// Generic over `I`: the node-id type, which must implement From d33cfc7a1017e6c6aa96043023980a3ceeac14f1 Mon Sep 17 00:00:00 2001 From: al8n Date: Wed, 24 Jun 2026 00:10:53 +1200 Subject: [PATCH 008/128] feat(serf-proto): AnyMessage enum + minor sweeps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add `src/any/mod.rs`: `AnyMessage` enum (one variant per MessageType) with `decode(buf: &Bytes) -> Result` that dispatches on the framing tag byte → buffa decode → bridge → typed variant. `DecodeError` wraps `FrameError`, `BridgeError`, `Buffa { tag }`, and `UnknownTag(u8)`. KeyRequest/KeyResponse variants and DecodeError::UnknownTag arms are gated on `any(aes-gcm, chacha20-poly1305)` with matching doc(cfg). - Add `src/any/tests.rs`: end-to-end `encode_message → AnyMessage::decode` round- trip for every message type (11 plain + 2 encryption-only). Unknown tag and empty buffer error cases. No-encryption guard for key tags → UnknownTag. - Re-export `AnyMessage` and `DecodeError` from `lib.rs`; add `pub mod any`. - Sweep minors: - `data_to_bytes`: delegate to `Data::encode_to_bytes()` (eliminates local reimpl). - `data_from_bytes`: widen signature from `&Bytes` to `&[u8]`; all call sites coerce via Deref unchanged. - `KeyResponseMessage`: add `#[derive(Default)]` (fixes clippy::new_without_default + clippy::derivable_impls; `new()` delegates to `Self::default()`). - Full gate: all 6 commands (build/test/clippy × plain/encryption) RC=0. 63 tests plain, 69 tests encryption. --- serf-proto/Cargo.toml | 10 +- serf-proto/proto/serf/v1/messages.proto | 45 +- serf-proto/src/any/mod.rs | 304 ++++++++++++ serf-proto/src/any/tests.rs | 614 ++++++++++++++++++++++++ serf-proto/src/bridge/mod.rs | 280 ++++++----- serf-proto/src/framing/mod.rs | 50 +- serf-proto/src/framing/tests.rs | 37 +- serf-proto/src/lib.rs | 66 +-- serf-proto/src/messages/tests.rs | 205 ++++---- serf-proto/src/typed/mod.rs | 32 +- 10 files changed, 1311 insertions(+), 332 deletions(-) create mode 100644 serf-proto/src/any/mod.rs create mode 100644 serf-proto/src/any/tests.rs diff --git a/serf-proto/Cargo.toml b/serf-proto/Cargo.toml index 164c1a2b..44347f1c 100644 --- a/serf-proto/Cargo.toml +++ b/serf-proto/Cargo.toml @@ -13,23 +13,19 @@ rust-version.workspace = true # Activating this feature makes the key message types available and ensures # key material is held in memberlist_proto::SecretKey (typed, cipher-tagged) # rather than raw bytes. -aes-gcm = ["memberlist-proto/aes-gcm"] -chacha20-poly1305 = ["memberlist-proto/chacha20-poly1305"] +aes-gcm = ["memberlist-proto/aes-gcm", "dep:zeroize"] +chacha20-poly1305 = ["memberlist-proto/chacha20-poly1305", "dep:zeroize"] # Umbrella that activates both backends (convenient for tests / CI). encryption = ["aes-gcm", "chacha20-poly1305"] [dependencies] memberlist-proto = { git = "https://github.com/al8n/memberlist", branch = "main", default-features = false, features = ["std"] } bytes = { workspace = true } -byteorder.workspace = true -smallvec.workspace = true -indexmap.workspace = true -derive_more.workspace = true -viewit.workspace = true thiserror.workspace = true bitflags = "2" buffa = { version = "0.7", default-features = false } smol_str = { workspace = true } +zeroize = { version = "1", optional = true } [build-dependencies] buffa-build = "0.7" diff --git a/serf-proto/proto/serf/v1/messages.proto b/serf-proto/proto/serf/v1/messages.proto index 2ec97794..e3f8b66e 100644 --- a/serf-proto/proto/serf/v1/messages.proto +++ b/serf-proto/proto/serf/v1/messages.proto @@ -56,10 +56,13 @@ message NodeIdList { // QueryMessage is a query broadcast to the cluster, optionally scoped by filters. // Field numbers match the legacy serf-core tag constants. +// `optional` on the legacy-required fields (ltime, id, from, flags, relay_factor, +// timeout_nanos) so the decoder can reject any being absent — proto3 plain +// bytes/scalars have no presence bit. `from` carries the Node via Data. message QueryMessage { optional uint64 ltime = 1; optional uint32 id = 2; - bytes from = 3; + optional bytes from = 3; repeated Filter filters = 4; optional uint32 flags = 5; optional uint32 relay_factor = 6; @@ -70,52 +73,62 @@ message QueryMessage { // QueryResponseMessage is sent back to the query originator. // Field numbers match the legacy serf-core tag constants. +// `optional` on the legacy-required fields (ltime, id, from, flags) so the +// decoder can reject any being absent — proto3 plain bytes/scalars have no +// presence bit. `from` carries the Node via Data. message QueryResponseMessage { optional uint64 ltime = 1; optional uint32 id = 2; - bytes from = 3; + optional bytes from = 3; optional uint32 flags = 4; bytes payload = 5; } // JoinMessage is broadcast after a node joins to associate it with a lamport clock. -// `optional` on ltime because the decoder must reject a missing lamport clock. +// `optional` on ltime and id because the decoder must reject either being absent +// (proto3 plain bytes/scalars have no presence bit). // `id` carries the node-id encoded via memberlist_proto::Data (opaque bytes). message JoinMessage { optional uint64 ltime = 1; - bytes id = 2; + optional bytes id = 2; } // LeaveMessage is broadcast to signal the intent to leave. -// `optional` on ltime because the decoder must reject a missing lamport clock. +// `optional` on ltime and id because the decoder must reject either being absent +// (proto3 plain bytes/scalars have no presence bit). // `id` carries the node-id encoded via memberlist_proto::Data (opaque bytes). message LeaveMessage { optional uint64 ltime = 1; bool prune = 2; - bytes id = 3; + optional bytes id = 3; } // ConflictResponseMessage carries the winning member in a node-name conflict. +// `optional` on member so the decoder can reject its absence (proto3 plain bytes +// has no presence bit). // `member` carries the memberlist_proto::Node encoded via Data (opaque bytes). message ConflictResponseMessage { - bytes member = 1; + optional bytes member = 1; } // NodeStatusTime is one entry in the PushPullMessage.status_ltimes repeated field. // proto3 map is not allowed (bytes keys are forbidden), so the // status-times map is transmitted as a repeated message instead. // `id` carries the node-id encoded via memberlist_proto::Data (opaque bytes). +// `optional` on id and ltime because the decoder must reject either being absent +// (proto3 plain bytes/scalars have no presence bit). message NodeStatusTime { - bytes id = 1; - uint64 ltime = 2; + optional bytes id = 1; + optional uint64 ltime = 2; } // UserEvent is a single named event with an optional payload. -// Mirrors legacy serf-core UserEvent. +// Mirrors legacy serf-core UserEvent (name + payload only). +// The "can coalesce" flag lives on the broadcast UserEventMessage wrapper, +// not on individual buffered entries. message UserEvent { - bool cc = 1; - string name = 2; - bytes payload = 3; + string name = 1; + bytes payload = 2; } // UserEvents is a batch of user events associated with a single lamport clock. @@ -156,10 +169,12 @@ message KeyResponseMessage { } // RelayMessage wraps a serf message for forwarding to a destination node. +// `optional` on destination so the decoder can reject its absence (proto3 plain +// bytes has no presence bit). // `destination` carries the Node encoded via memberlist_proto::Data (opaque bytes). // `payload` carries the inner serf framed message verbatim (tag + varint-len + buffa body); // the relay target re-decodes it with decode_message. message RelayMessage { - bytes destination = 1; - bytes payload = 2; + optional bytes destination = 1; + bytes payload = 2; } diff --git a/serf-proto/src/any/mod.rs b/serf-proto/src/any/mod.rs new file mode 100644 index 00000000..cb5b6ded --- /dev/null +++ b/serf-proto/src/any/mod.rs @@ -0,0 +1,304 @@ +//! The [`AnyMessage`] enum — an owned, tag-dispatched serf message. +//! +//! [`AnyMessage`] is the owned counterpart to the legacy `MessageRef<'a>`. +//! It wraps one typed message variant per [`MessageType`] and provides a +//! single encode ([`AnyMessage::encode`]) and decode ([`AnyMessage::decode`]) +//! entry-point that accepts a framed `Bytes` buffer, splits the tag byte and +//! body with the crate-internal frame decoder, then delegates to the +//! appropriate bridge function to produce the typed variant. +//! +//! Unknown tag bytes and empty / truncated buffers propagate as [`DecodeError`]. + +use bytes::Bytes; +use memberlist_proto::Data; + +use crate::{ + BridgeError, ConflictResponseMessage, FrameError, JoinMessage, LeaveMessage, MessageType, + PushPullMessage, QueryMessage, QueryResponseMessage, RelayMessage, UserEventMessage, + bridge::{ + conflict_response_from_pb, conflict_response_to_pb, join_from_pb, join_to_pb, leave_from_pb, + leave_to_pb, push_pull_from_pb, push_pull_to_pb, query_from_pb, query_response_from_pb, + query_response_to_pb, query_to_pb, relay_from_pb, relay_to_pb, user_event_from_pb, + user_event_to_pb, + }, + framing::{decode_message, encode_message}, + messages::serf::v1 as pb, +}; +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +use crate::{ + KeyRequestMessage, KeyResponseMessage, + bridge::{key_request_from_pb, key_request_to_pb, key_response_from_pb, key_response_to_pb}, +}; + +// ─── DecodeError ────────────────────────────────────────────────────────────── + +/// Error returned by [`AnyMessage::decode`]. +/// +/// Wraps both framing errors (truncated / empty / varint-overflow) and +/// bridge errors (missing required fields, invalid field values). +#[non_exhaustive] +#[derive(Debug, Clone, thiserror::Error)] +pub enum DecodeError { + /// The frame is malformed or incomplete. + #[error("frame error: {0}")] + Frame(#[from] FrameError), + /// A field could not be converted from the wire shape. + #[error("bridge error: {0}")] + Bridge(#[from] BridgeError), + /// The buffa body bytes could not be decoded to the expected pb type. + /// Carries the [`MessageType`] tag byte whose body could not be decoded. + #[error("buffa decode failed for message type {0}")] + Buffa(u8), + /// The tag byte is not recognised by this build (e.g. key messages without + /// an encryption feature, or a tag from a future protocol version). + #[error("unknown or unsupported message tag: {0}")] + UnknownTag(u8), +} + +// ─── EncodeError ────────────────────────────────────────────────────────────── + +/// Error returned by [`AnyMessage::encode`]. +/// +/// Wraps bridge conversion errors (e.g. timeout overflow) and framing errors +/// (e.g. body length mismatch). +#[non_exhaustive] +#[derive(Debug, Clone, thiserror::Error)] +pub enum EncodeError { + /// A field value could not be converted to the wire shape. + #[error("bridge error: {0}")] + Bridge(#[from] BridgeError), + /// The framing encoder rejected the encoded body. + #[error("frame error: {0}")] + Frame(#[from] FrameError), +} + +// ─── AnyMessage ─────────────────────────────────────────────────────────────── + +/// An owned serf message, one variant per [`MessageType`]. +/// +/// Dispatch key: the leading tag byte of the serf plain frame determines the +/// variant. Generic over `I` (node-id type) and `A` (node-address type); both +/// must implement [`Data`] so the bridge layer can encode/decode them from the +/// opaque `bytes` fields in the wire representation. +/// +/// # Key-management variants +/// +/// `KeyRequest` and `KeyResponse` are available only when the `aes-gcm` or +/// `chacha20-poly1305` feature is enabled. Frames carrying those tag bytes +/// (9 / 10) decode as [`DecodeError::UnknownTag`] in builds without an +/// encryption backend. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub enum AnyMessage { + /// Leave — node announcing it is leaving the cluster. + Leave(LeaveMessage), + /// Join — node joining the cluster. + Join(JoinMessage), + /// PushPull — full state-sync exchange. + PushPull(PushPullMessage), + /// UserEvent — application-level event broadcast. + UserEvent(UserEventMessage), + /// Query — serf RPC query fanout. + Query(QueryMessage), + /// QueryResponse — response to a Query. + QueryResponse(QueryResponseMessage), + /// ConflictResponse — tie-breaker for conflicting node names. + ConflictResponse(ConflictResponseMessage), + /// Relay — message relayed through an intermediary node. + Relay(RelayMessage), + /// KeyRequest — encryption key management request. + /// + /// Requires the `aes-gcm` or `chacha20-poly1305` feature. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + KeyRequest(KeyRequestMessage), + /// KeyResponse — encryption key management response. + /// + /// Requires the `aes-gcm` or `chacha20-poly1305` feature. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + KeyResponse(KeyResponseMessage), +} + +impl AnyMessage +where + I: Data, + A: Data, +{ + /// Returns the [`MessageType`] tag for this message variant. + pub fn message_type(&self) -> MessageType { + match self { + Self::Leave(_) => MessageType::Leave, + Self::Join(_) => MessageType::Join, + Self::PushPull(_) => MessageType::PushPull, + Self::UserEvent(_) => MessageType::UserEvent, + Self::Query(_) => MessageType::Query, + Self::QueryResponse(_) => MessageType::QueryResponse, + Self::ConflictResponse(_) => MessageType::ConflictResponse, + Self::Relay(_) => MessageType::Relay, + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + Self::KeyRequest(_) => MessageType::KeyRequest, + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + Self::KeyResponse(_) => MessageType::KeyResponse, + } + } + + /// Encode this message into a framed `Bytes` buffer. + /// + /// Converts the typed message to its buffa protobuf representation via the + /// bridge layer, then wraps it in the standard serf plain frame: + /// `[tag][varint body_len][buffa body]`. + /// + /// This is the single public encoding entry-point — raw key bytes never + /// leave the `SecretKey` type; the bridge converts them internally. + /// + /// # Errors + /// + /// - [`EncodeError::Bridge`] — a field value could not be mapped to the wire + /// shape (e.g. a query timeout exceeding `u64::MAX` nanoseconds). + /// - [`EncodeError::Frame`] — the buffa body length disagrees between + /// `encoded_len` and the actual write (should not happen in practice). + pub fn encode(&self) -> Result + where + I: Data, + A: Data, + { + let frame_vec = match self { + Self::Leave(m) => { + let pb = leave_to_pb::(m)?; + encode_message(MessageType::Leave, &pb)? + } + Self::Join(m) => { + let pb = join_to_pb::(m)?; + encode_message(MessageType::Join, &pb)? + } + Self::PushPull(m) => { + let pb = push_pull_to_pb::(m)?; + encode_message(MessageType::PushPull, &pb)? + } + Self::UserEvent(m) => { + let pb = user_event_to_pb(m); + encode_message(MessageType::UserEvent, &pb)? + } + Self::Query(m) => { + let pb = query_to_pb::(m)?; + encode_message(MessageType::Query, &pb)? + } + Self::QueryResponse(m) => { + let pb = query_response_to_pb::(m)?; + encode_message(MessageType::QueryResponse, &pb)? + } + Self::ConflictResponse(m) => { + let pb = conflict_response_to_pb::(m)?; + encode_message(MessageType::ConflictResponse, &pb)? + } + Self::Relay(m) => { + let pb = relay_to_pb::(m)?; + encode_message(MessageType::Relay, &pb)? + } + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + Self::KeyRequest(m) => { + let pb = key_request_to_pb(m); + encode_message(MessageType::KeyRequest, &pb)? + } + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + Self::KeyResponse(m) => { + let pb = key_response_to_pb(m); + encode_message(MessageType::KeyResponse, &pb)? + } + }; + Ok(Bytes::from(frame_vec)) + } + + /// Decode the leading serf frame in `buf` into an [`AnyMessage`]. + /// + /// Splits the tag byte and body with the crate-internal frame decoder, then + /// dispatches on the tag to the appropriate buffa decoder and bridge function. + /// + /// # Errors + /// + /// - [`DecodeError::Frame`] — the buffer is empty, truncated, or has a + /// bad varint length prefix. + /// - [`DecodeError::Buffa`] — the buffa decode of the body bytes failed for + /// the identified message type. + /// - [`DecodeError::Bridge`] — a required field was absent or a field value + /// was out of range. + /// - [`DecodeError::UnknownTag`] — the tag byte is not recognised by this + /// build. + pub fn decode(buf: &Bytes) -> Result + where + I: Data, + A: Data, + { + use buffa::Message as _; + + let (ty, body, _consumed) = decode_message(buf)?; + + match ty { + MessageType::Leave => { + let pb = pb::LeaveMessage::decode_from_slice(body.as_ref()) + .map_err(|_| DecodeError::Buffa(u8::from(MessageType::Leave)))?; + Ok(Self::Leave(leave_from_pb::(&pb)?)) + } + MessageType::Join => { + let pb = pb::JoinMessage::decode_from_slice(body.as_ref()) + .map_err(|_| DecodeError::Buffa(u8::from(MessageType::Join)))?; + Ok(Self::Join(join_from_pb::(&pb)?)) + } + MessageType::PushPull => { + let pb = pb::PushPullMessage::decode_from_slice(body.as_ref()) + .map_err(|_| DecodeError::Buffa(u8::from(MessageType::PushPull)))?; + Ok(Self::PushPull(push_pull_from_pb::(&pb)?)) + } + MessageType::UserEvent => { + let pb = pb::UserEventMessage::decode_from_slice(body.as_ref()) + .map_err(|_| DecodeError::Buffa(u8::from(MessageType::UserEvent)))?; + Ok(Self::UserEvent(user_event_from_pb(&pb)?)) + } + MessageType::Query => { + let pb = pb::QueryMessage::decode_from_slice(body.as_ref()) + .map_err(|_| DecodeError::Buffa(u8::from(MessageType::Query)))?; + Ok(Self::Query(query_from_pb::(&pb)?)) + } + MessageType::QueryResponse => { + let pb = pb::QueryResponseMessage::decode_from_slice(body.as_ref()) + .map_err(|_| DecodeError::Buffa(u8::from(MessageType::QueryResponse)))?; + Ok(Self::QueryResponse(query_response_from_pb::(&pb)?)) + } + MessageType::ConflictResponse => { + let pb = pb::ConflictResponseMessage::decode_from_slice(body.as_ref()) + .map_err(|_| DecodeError::Buffa(u8::from(MessageType::ConflictResponse)))?; + Ok(Self::ConflictResponse(conflict_response_from_pb::( + &pb, + )?)) + } + MessageType::Relay => { + let pb = pb::RelayMessage::decode_from_slice(body.as_ref()) + .map_err(|_| DecodeError::Buffa(u8::from(MessageType::Relay)))?; + Ok(Self::Relay(relay_from_pb::(&pb)?)) + } + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + MessageType::KeyRequest => { + let pb = pb::KeyRequestMessage::decode_from_slice(body.as_ref()) + .map_err(|_| DecodeError::Buffa(u8::from(MessageType::KeyRequest)))?; + Ok(Self::KeyRequest(key_request_from_pb(&pb)?)) + } + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + MessageType::KeyResponse => { + let pb = pb::KeyResponseMessage::decode_from_slice(body.as_ref()) + .map_err(|_| DecodeError::Buffa(u8::from(MessageType::KeyResponse)))?; + Ok(Self::KeyResponse(key_response_from_pb(&pb)?)) + } + MessageType::Unknown(tag) => Err(DecodeError::UnknownTag(tag)), + } + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-proto/src/any/tests.rs b/serf-proto/src/any/tests.rs new file mode 100644 index 00000000..8bda09cf --- /dev/null +++ b/serf-proto/src/any/tests.rs @@ -0,0 +1,614 @@ +//! Round-trip tests for AnyMessage encode and decode. + +use std::net::SocketAddr; + +use bytes::Bytes; +use memberlist_proto::Node; +use smol_str::SmolStr; + +use super::{AnyMessage, DecodeError}; +#[cfg(feature = "aes-gcm")] +use crate::bridge::{key_request_to_pb, key_response_to_pb}; +use crate::{ + BridgeError, ConflictResponseMessage, JoinMessage, LamportTime, LeaveMessage, MessageType, + PushPullMessage, QueryFlag, QueryMessage, QueryResponseMessage, RelayMessage, UserEvent, + UserEventMessage, UserEvents, + bridge::{ + conflict_response_to_pb, join_to_pb, leave_to_pb, push_pull_to_pb, query_response_to_pb, + query_to_pb, relay_to_pb, user_event_to_pb, + }, + framing::encode_message, + messages::serf::v1 as pb, +}; +#[cfg(feature = "aes-gcm")] +use crate::{KeyRequestMessage, KeyResponseMessage}; +#[cfg(feature = "aes-gcm")] +use memberlist_proto::SecretKey; + +// Convenience aliases used throughout. +type I = SmolStr; +type A = SocketAddr; + +fn sample_addr() -> A { + "127.0.0.1:7946".parse().unwrap() +} + +// ─── UserEvent ──────────────────────────────────────────────────────────────── + +#[test] +fn any_message_user_event_round_trip() { + let typed = UserEventMessage { + ltime: LamportTime::new(1), + cc: true, + name: SmolStr::from("deploy"), + payload: Bytes::from_static(b"hello"), + }; + let pb = user_event_to_pb(&typed); + let frame = encode_message(MessageType::UserEvent, &pb).expect("encode_message"); + let frame = Bytes::from(frame); + + let msg: AnyMessage = AnyMessage::decode(&frame).expect("AnyMessage::decode"); + match msg { + AnyMessage::UserEvent(m) => { + assert_eq!(m.ltime, typed.ltime); + assert_eq!(m.cc, typed.cc); + assert_eq!(m.name, typed.name); + assert_eq!(m.payload, typed.payload); + } + other => panic!("expected UserEvent, got {:?}", other.message_type()), + } +} + +// ─── JoinMessage ───────────────────────────────────────────────────────────── + +#[test] +fn any_message_join_round_trip() { + let typed: JoinMessage = JoinMessage::new(LamportTime::new(2), SmolStr::from("node-join")); + let pb = join_to_pb(&typed).expect("join_to_pb"); + let frame = encode_message(MessageType::Join, &pb).expect("encode_message"); + let frame = Bytes::from(frame); + + let msg: AnyMessage = AnyMessage::decode(&frame).expect("AnyMessage::decode"); + match msg { + AnyMessage::Join(m) => { + assert_eq!(m.ltime, typed.ltime); + assert_eq!(m.id, typed.id); + } + other => panic!("expected Join, got {:?}", other.message_type()), + } +} + +// ─── LeaveMessage ───────────────────────────────────────────────────────────── + +#[test] +fn any_message_leave_round_trip() { + let typed: LeaveMessage = + LeaveMessage::new(LamportTime::new(3), SmolStr::from("node-leave"), true); + let pb = leave_to_pb(&typed).expect("leave_to_pb"); + let frame = encode_message(MessageType::Leave, &pb).expect("encode_message"); + let frame = Bytes::from(frame); + + let msg: AnyMessage = AnyMessage::decode(&frame).expect("AnyMessage::decode"); + match msg { + AnyMessage::Leave(m) => { + assert_eq!(m.ltime, typed.ltime); + assert_eq!(m.id, typed.id); + assert_eq!(m.prune, typed.prune); + } + other => panic!("expected Leave, got {:?}", other.message_type()), + } +} + +// ─── PushPullMessage ────────────────────────────────────────────────────────── + +#[test] +fn any_message_push_pull_round_trip() { + let typed: PushPullMessage = PushPullMessage { + ltime: LamportTime::new(10), + status_ltimes: vec![(SmolStr::from("node-a"), LamportTime::new(5))], + left_members: vec![SmolStr::from("node-gone")], + event_ltime: LamportTime::new(8), + events: vec![UserEvents { + ltime: LamportTime::new(7), + events: vec![UserEvent { + name: SmolStr::from("ev"), + payload: Bytes::from_static(b"p"), + }], + }], + query_ltime: LamportTime::new(9), + }; + let pb = push_pull_to_pb(&typed).expect("push_pull_to_pb"); + let frame = encode_message(MessageType::PushPull, &pb).expect("encode_message"); + let frame = Bytes::from(frame); + + let msg: AnyMessage = AnyMessage::decode(&frame).expect("AnyMessage::decode"); + match msg { + AnyMessage::PushPull(m) => { + assert_eq!(m.ltime, typed.ltime); + assert_eq!(m.event_ltime, typed.event_ltime); + assert_eq!(m.query_ltime, typed.query_ltime); + assert_eq!(m.status_ltimes.len(), 1); + assert_eq!(m.left_members.len(), 1); + assert_eq!(m.events.len(), 1); + } + other => panic!("expected PushPull, got {:?}", other.message_type()), + } +} + +// ─── QueryMessage ───────────────────────────────────────────────────────────── + +#[test] +fn any_message_query_round_trip() { + let typed: QueryMessage = QueryMessage { + ltime: LamportTime::new(20), + id: 42, + from: Node::new(SmolStr::from("node-q"), sample_addr()), + filters: vec![], + flags: QueryFlag::ACK, + relay_factor: 1, + timeout: std::time::Duration::from_secs(1), + name: SmolStr::from("my-query"), + payload: Bytes::from_static(b"qp"), + }; + let pb = query_to_pb(&typed).expect("query_to_pb"); + let frame = encode_message(MessageType::Query, &pb).expect("encode_message"); + let frame = Bytes::from(frame); + + let msg: AnyMessage = AnyMessage::decode(&frame).expect("AnyMessage::decode"); + match msg { + AnyMessage::Query(m) => { + assert_eq!(m.ltime, typed.ltime); + assert_eq!(m.id, typed.id); + assert_eq!(m.name, typed.name); + assert_eq!(m.flags, typed.flags); + } + other => panic!("expected Query, got {:?}", other.message_type()), + } +} + +// ─── QueryResponseMessage ───────────────────────────────────────────────────── + +#[test] +fn any_message_query_response_round_trip() { + let typed: QueryResponseMessage = QueryResponseMessage { + ltime: LamportTime::new(21), + id: 42, + from: Node::new(SmolStr::from("node-resp"), sample_addr()), + flags: QueryFlag::ACK, + payload: Bytes::from_static(b"rp"), + }; + let pb = query_response_to_pb(&typed).expect("query_response_to_pb"); + let frame = encode_message(MessageType::QueryResponse, &pb).expect("encode_message"); + let frame = Bytes::from(frame); + + let msg: AnyMessage = AnyMessage::decode(&frame).expect("AnyMessage::decode"); + match msg { + AnyMessage::QueryResponse(m) => { + assert_eq!(m.ltime, typed.ltime); + assert_eq!(m.id, typed.id); + assert!(m.ack()); + } + other => panic!("expected QueryResponse, got {:?}", other.message_type()), + } +} + +// ─── ConflictResponseMessage ────────────────────────────────────────────────── + +#[test] +fn any_message_conflict_response_round_trip() { + let typed: ConflictResponseMessage = + ConflictResponseMessage::new(Node::new(SmolStr::from("winner"), sample_addr())); + let pb = conflict_response_to_pb(&typed).expect("conflict_response_to_pb"); + let frame = encode_message(MessageType::ConflictResponse, &pb).expect("encode_message"); + let frame = Bytes::from(frame); + + let msg: AnyMessage = AnyMessage::decode(&frame).expect("AnyMessage::decode"); + match msg { + AnyMessage::ConflictResponse(m) => { + assert_eq!(m.member.id_ref(), typed.member.id_ref()); + assert_eq!(m.member.addr_ref(), typed.member.addr_ref()); + } + other => panic!("expected ConflictResponse, got {:?}", other.message_type()), + } +} + +// ─── RelayMessage ───────────────────────────────────────────────────────────── + +#[test] +fn any_message_relay_round_trip() { + let typed: RelayMessage = RelayMessage::new( + Node::new(SmolStr::from("relay-target"), sample_addr()), + Bytes::from_static(b"\x04\x03xyz"), + ); + let pb = relay_to_pb(&typed).expect("relay_to_pb"); + let frame = encode_message(MessageType::Relay, &pb).expect("encode_message"); + let frame = Bytes::from(frame); + + let msg: AnyMessage = AnyMessage::decode(&frame).expect("AnyMessage::decode"); + match msg { + AnyMessage::Relay(m) => { + assert_eq!(m.destination.id_ref(), typed.destination.id_ref()); + assert_eq!(m.destination.addr_ref(), typed.destination.addr_ref()); + assert_eq!(m.payload, typed.payload); + } + other => panic!("expected Relay, got {:?}", other.message_type()), + } +} + +// ─── KeyRequestMessage ──────────────────────────────────────────────────────── + +#[cfg(feature = "aes-gcm")] +#[test] +fn any_message_key_request_round_trip() { + let typed = KeyRequestMessage::new(Some(SecretKey::Aes256([0xABu8; 32]))); + let pb = key_request_to_pb(&typed); + let frame = encode_message(MessageType::KeyRequest, &pb).expect("encode_message"); + let frame = Bytes::from(frame); + + let msg: AnyMessage = AnyMessage::decode(&frame).expect("AnyMessage::decode"); + match msg { + AnyMessage::KeyRequest(m) => { + assert_eq!(m.key, typed.key); + } + other => panic!("expected KeyRequest, got {:?}", other.message_type()), + } +} + +// ─── KeyResponseMessage ─────────────────────────────────────────────────────── + +#[cfg(feature = "aes-gcm")] +#[test] +fn any_message_key_response_round_trip() { + let typed = KeyResponseMessage { + result: true, + message: SmolStr::from("ok"), + keys: vec![SecretKey::Aes128([0x11u8; 16])], + primary_key: Some(SecretKey::Aes128([0x11u8; 16])), + }; + let pb = key_response_to_pb(&typed); + let frame = encode_message(MessageType::KeyResponse, &pb).expect("encode_message"); + let frame = Bytes::from(frame); + + let msg: AnyMessage = AnyMessage::decode(&frame).expect("AnyMessage::decode"); + match msg { + AnyMessage::KeyResponse(m) => { + assert_eq!(m.result, typed.result); + assert_eq!(m.message, typed.message); + assert_eq!(m.keys.len(), 1); + } + other => panic!("expected KeyResponse, got {:?}", other.message_type()), + } +} + +// ─── AnyMessage::encode round-trips (encode → decode → same variant+fields) ─── + +#[test] +fn any_message_encode_user_event_round_trip() { + let msg: AnyMessage = AnyMessage::UserEvent(UserEventMessage { + ltime: LamportTime::new(7), + cc: false, + name: SmolStr::from("ev"), + payload: Bytes::from_static(b"data"), + }); + let frame = msg.encode().expect("encode"); + let decoded: AnyMessage = AnyMessage::decode(&frame).expect("decode"); + match decoded { + AnyMessage::UserEvent(m) => { + assert_eq!(m.ltime, LamportTime::new(7)); + assert_eq!(m.name, "ev"); + } + other => panic!("expected UserEvent, got {:?}", other.message_type()), + } +} + +#[test] +fn any_message_encode_join_round_trip() { + let msg: AnyMessage = + AnyMessage::Join(JoinMessage::new(LamportTime::new(2), SmolStr::from("n"))); + let frame = msg.encode().expect("encode"); + let decoded: AnyMessage = AnyMessage::decode(&frame).expect("decode"); + match decoded { + AnyMessage::Join(m) => assert_eq!(m.ltime, LamportTime::new(2)), + other => panic!("expected Join, got {:?}", other.message_type()), + } +} + +#[test] +fn any_message_encode_leave_round_trip() { + let msg: AnyMessage = AnyMessage::Leave(LeaveMessage::new( + LamportTime::new(3), + SmolStr::from("n"), + true, + )); + let frame = msg.encode().expect("encode"); + let decoded: AnyMessage = AnyMessage::decode(&frame).expect("decode"); + match decoded { + AnyMessage::Leave(m) => { + assert_eq!(m.ltime, LamportTime::new(3)); + assert!(m.prune); + } + other => panic!("expected Leave, got {:?}", other.message_type()), + } +} + +#[test] +fn any_message_encode_push_pull_round_trip() { + let msg: AnyMessage = AnyMessage::PushPull(PushPullMessage { + ltime: LamportTime::new(10), + status_ltimes: vec![], + left_members: vec![], + event_ltime: LamportTime::new(8), + events: vec![], + query_ltime: LamportTime::new(9), + }); + let frame = msg.encode().expect("encode"); + let decoded: AnyMessage = AnyMessage::decode(&frame).expect("decode"); + match decoded { + AnyMessage::PushPull(m) => { + assert_eq!(m.ltime, LamportTime::new(10)); + assert_eq!(m.event_ltime, LamportTime::new(8)); + assert_eq!(m.query_ltime, LamportTime::new(9)); + } + other => panic!("expected PushPull, got {:?}", other.message_type()), + } +} + +#[test] +fn any_message_encode_query_round_trip() { + let msg: AnyMessage = AnyMessage::Query(QueryMessage { + ltime: LamportTime::new(20), + id: 1, + from: Node::new(SmolStr::from("q"), sample_addr()), + filters: vec![], + flags: QueryFlag::ACK, + relay_factor: 0, + timeout: std::time::Duration::from_secs(1), + name: SmolStr::from("qname"), + payload: Bytes::new(), + }); + let frame = msg.encode().expect("encode"); + let decoded: AnyMessage = AnyMessage::decode(&frame).expect("decode"); + match decoded { + AnyMessage::Query(m) => { + assert_eq!(m.ltime, LamportTime::new(20)); + assert_eq!(m.name, "qname"); + } + other => panic!("expected Query, got {:?}", other.message_type()), + } +} + +#[test] +fn any_message_encode_query_response_round_trip() { + let msg: AnyMessage = AnyMessage::QueryResponse(QueryResponseMessage { + ltime: LamportTime::new(21), + id: 1, + from: Node::new(SmolStr::from("r"), sample_addr()), + flags: QueryFlag::ACK, + payload: Bytes::new(), + }); + let frame = msg.encode().expect("encode"); + let decoded: AnyMessage = AnyMessage::decode(&frame).expect("decode"); + match decoded { + AnyMessage::QueryResponse(m) => assert_eq!(m.ltime, LamportTime::new(21)), + other => panic!("expected QueryResponse, got {:?}", other.message_type()), + } +} + +#[test] +fn any_message_encode_conflict_response_round_trip() { + let msg: AnyMessage = AnyMessage::ConflictResponse(ConflictResponseMessage::new( + Node::new(SmolStr::from("winner"), sample_addr()), + )); + let frame = msg.encode().expect("encode"); + let decoded: AnyMessage = AnyMessage::decode(&frame).expect("decode"); + match decoded { + AnyMessage::ConflictResponse(m) => assert_eq!(m.member.id_ref(), "winner"), + other => panic!("expected ConflictResponse, got {:?}", other.message_type()), + } +} + +#[test] +fn any_message_encode_relay_round_trip() { + let msg: AnyMessage = AnyMessage::Relay(RelayMessage::new( + Node::new(SmolStr::from("dest"), sample_addr()), + Bytes::from_static(b"\x04\x01x"), + )); + let frame = msg.encode().expect("encode"); + let decoded: AnyMessage = AnyMessage::decode(&frame).expect("decode"); + match decoded { + AnyMessage::Relay(m) => assert_eq!(m.destination.id_ref(), "dest"), + other => panic!("expected Relay, got {:?}", other.message_type()), + } +} + +#[cfg(feature = "aes-gcm")] +#[test] +fn any_message_encode_key_request_round_trip() { + let msg: AnyMessage = AnyMessage::KeyRequest(KeyRequestMessage::new(Some( + SecretKey::Aes256([0xCCu8; 32]), + ))); + let frame = msg.encode().expect("encode"); + let decoded: AnyMessage = AnyMessage::decode(&frame).expect("decode"); + match decoded { + AnyMessage::KeyRequest(m) => { + assert!(m.key.is_some()); + } + other => panic!("expected KeyRequest, got {:?}", other.message_type()), + } +} + +#[cfg(feature = "aes-gcm")] +#[test] +fn any_message_encode_key_response_round_trip() { + let msg: AnyMessage = AnyMessage::KeyResponse(KeyResponseMessage { + result: true, + message: SmolStr::from("ok"), + keys: vec![SecretKey::Aes128([0xAAu8; 16])], + primary_key: None, + }); + let frame = msg.encode().expect("encode"); + let decoded: AnyMessage = AnyMessage::decode(&frame).expect("decode"); + match decoded { + AnyMessage::KeyResponse(m) => { + assert!(m.result); + assert_eq!(m.keys.len(), 1); + } + other => panic!("expected KeyResponse, got {:?}", other.message_type()), + } +} + +// ─── Error cases ────────────────────────────────────────────────────────────── + +#[test] +fn any_message_empty_buf_is_error() { + let empty = Bytes::new(); + assert!( + matches!( + AnyMessage::::decode(&empty), + Err(DecodeError::Frame(_)) + ), + "empty buffer must yield DecodeError::Frame" + ); +} + +#[test] +fn any_message_unknown_tag_is_error() { + // A well-formed frame with tag = 200 (unrecognised). + use crate::messages::serf::v1::UserEventMessage as PbUserEventMessage; + let pb = PbUserEventMessage::default(); + let frame = Bytes::from(encode_message(MessageType::Unknown(200), &pb).expect("encode_message")); + assert!( + matches!( + AnyMessage::::decode(&frame), + Err(DecodeError::UnknownTag(200)) + ), + "unknown tag must yield DecodeError::UnknownTag(200)" + ); +} + +/// Without an encryption feature, key tag bytes must decode as UnknownTag. +#[cfg(not(any(feature = "aes-gcm", feature = "chacha20-poly1305")))] +#[test] +fn any_message_key_tags_unknown_without_encryption() { + use crate::messages::serf::v1::UserEventMessage as PbUserEventMessage; + let pb = PbUserEventMessage::default(); + + let frame9 = Bytes::from(encode_message(MessageType::Unknown(9), &pb).expect("encode_message")); + assert!(matches!( + AnyMessage::::decode(&frame9), + Err(DecodeError::UnknownTag(9)) + )); + + let frame10 = Bytes::from(encode_message(MessageType::Unknown(10), &pb).expect("encode_message")); + assert!(matches!( + AnyMessage::::decode(&frame10), + Err(DecodeError::UnknownTag(10)) + )); +} + +// ─── Absent required opaque-bytes fields must be rejected ────────────────────── +// +// proto3 plain `bytes` carries no presence bit, so an absent legacy-required +// opaque field would otherwise silently decode as empty bytes. Each field is +// declared `optional` in the schema and rejected as `BridgeError::MissingField` +// before the bytes are handed to the `Data` decoder. + +/// A `JoinMessage` whose `id` bytes are absent must be rejected. +#[test] +fn any_message_join_absent_id_is_rejected() { + let body = pb::JoinMessage { + ltime: Some(1), + id: None, + ..Default::default() + }; + let frame = Bytes::from(encode_message(MessageType::Join, &body).expect("encode_message")); + assert!( + matches!( + AnyMessage::::decode(&frame), + Err(DecodeError::Bridge(BridgeError::MissingField(_))) + ), + "absent JoinMessage.id must yield DecodeError::Bridge(MissingField)" + ); +} + +/// A `QueryMessage` whose `from` bytes are absent must be rejected. +#[test] +fn any_message_query_absent_from_is_rejected() { + let body = pb::QueryMessage { + ltime: Some(1), + id: Some(1), + from: None, + flags: Some(0), + relay_factor: Some(0), + timeout_nanos: Some(0), + ..Default::default() + }; + let frame = Bytes::from(encode_message(MessageType::Query, &body).expect("encode_message")); + assert!( + matches!( + AnyMessage::::decode(&frame), + Err(DecodeError::Bridge(BridgeError::MissingField(_))) + ), + "absent QueryMessage.from must yield DecodeError::Bridge(MissingField)" + ); +} + +/// A `ConflictResponseMessage` whose `member` bytes are absent must be rejected. +#[test] +fn any_message_conflict_response_absent_member_is_rejected() { + let body = pb::ConflictResponseMessage { + member: None, + ..Default::default() + }; + let frame = + Bytes::from(encode_message(MessageType::ConflictResponse, &body).expect("encode_message")); + assert!( + matches!( + AnyMessage::::decode(&frame), + Err(DecodeError::Bridge(BridgeError::MissingField(_))) + ), + "absent ConflictResponseMessage.member must yield DecodeError::Bridge(MissingField)" + ); +} + +/// A `RelayMessage` whose `destination` bytes are absent must be rejected. +#[test] +fn any_message_relay_absent_destination_is_rejected() { + let body = pb::RelayMessage { + destination: None, + payload: Bytes::from_static(b"\x04\x01x"), + ..Default::default() + }; + let frame = Bytes::from(encode_message(MessageType::Relay, &body).expect("encode_message")); + assert!( + matches!( + AnyMessage::::decode(&frame), + Err(DecodeError::Bridge(BridgeError::MissingField(_))) + ), + "absent RelayMessage.destination must yield DecodeError::Bridge(MissingField)" + ); +} + +/// A `PushPullMessage` whose `NodeStatusTime.id` bytes are absent must be rejected. +#[test] +fn any_message_push_pull_absent_status_id_is_rejected() { + let body = pb::PushPullMessage { + ltime: Some(1), + status_ltimes: vec![pb::NodeStatusTime { + id: None, + ltime: Some(5), + ..Default::default() + }], + event_ltime: Some(2), + query_ltime: Some(3), + ..Default::default() + }; + let frame = Bytes::from(encode_message(MessageType::PushPull, &body).expect("encode_message")); + assert!( + matches!( + AnyMessage::::decode(&frame), + Err(DecodeError::Bridge(BridgeError::MissingField(_))) + ), + "absent NodeStatusTime.id must yield DecodeError::Bridge(MissingField)" + ); +} diff --git a/serf-proto/src/bridge/mod.rs b/serf-proto/src/bridge/mod.rs index a73cb32c..764c649d 100644 --- a/serf-proto/src/bridge/mod.rs +++ b/serf-proto/src/bridge/mod.rs @@ -7,34 +7,27 @@ use std::borrow::Cow; use bytes::Bytes; -use memberlist_proto::{Data, DataRef, data::DecodeError, data::EncodeError}; #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] use memberlist_proto::SecretKey; +use memberlist_proto::{ + Data, DataRef, + data::{DecodeError, EncodeError}, +}; use smol_str::SmolStr; +#[cfg(test)] +use crate::typed::{Coordinate, Tags}; +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +use crate::typed::{KeyRequestMessage, KeyResponseMessage}; use crate::{ LamportTime, messages::serf::v1 as pb, typed::{ - Coordinate, - ConflictResponseMessage, - Filter, - JoinMessage, - LeaveMessage, - PushPullMessage, - QueryFlag, - QueryMessage, - QueryResponseMessage, - RelayMessage, - TagFilter, - Tags, - UserEvent, - UserEventMessage, + ConflictResponseMessage, Filter, JoinMessage, LeaveMessage, PushPullMessage, QueryFlag, + QueryMessage, QueryResponseMessage, RelayMessage, TagFilter, UserEvent, UserEventMessage, UserEvents, }, }; -#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] -use crate::typed::{KeyRequestMessage, KeyResponseMessage}; // ─── BridgeError ───────────────────────────────────────────────────────────── @@ -62,7 +55,7 @@ pub enum BridgeError { // ─── UserEventMessage ──────────────────────────────────────────────────────── /// Convert a typed [`UserEventMessage`] → `pb::UserEventMessage`. -pub fn user_event_to_pb(t: &UserEventMessage) -> pb::UserEventMessage { +pub(crate) fn user_event_to_pb(t: &UserEventMessage) -> pb::UserEventMessage { pb::UserEventMessage { ltime: Some(t.ltime.into()), cc: t.cc, @@ -73,7 +66,9 @@ pub fn user_event_to_pb(t: &UserEventMessage) -> pb::UserEventMessage { } /// Convert `pb::UserEventMessage` → typed [`UserEventMessage`]. -pub fn user_event_from_pb(b: &pb::UserEventMessage) -> Result { +pub(crate) fn user_event_from_pb( + b: &pb::UserEventMessage, +) -> Result { let ltime = b .ltime .ok_or(BridgeError::MissingField("UserEventMessage.ltime".into()))?; @@ -88,7 +83,8 @@ pub fn user_event_from_pb(b: &pb::UserEventMessage) -> Result pb::Coordinate { +#[cfg(test)] +pub(crate) fn coordinate_to_pb(t: &Coordinate) -> pb::Coordinate { pb::Coordinate { portion: t.vec.clone(), error: t.error, @@ -99,7 +95,8 @@ pub fn coordinate_to_pb(t: &Coordinate) -> pb::Coordinate { } /// Convert `pb::Coordinate` → typed [`Coordinate`]. -pub fn coordinate_from_pb(b: &pb::Coordinate) -> Coordinate { +#[cfg(test)] +pub(crate) fn coordinate_from_pb(b: &pb::Coordinate) -> Coordinate { Coordinate { vec: b.portion.clone(), error: b.error, @@ -111,15 +108,21 @@ pub fn coordinate_from_pb(b: &pb::Coordinate) -> Coordinate { // ─── Tags ───────────────────────────────────────────────────────────────────── /// Convert typed [`Tags`] → `pb::Tags`. -pub fn tags_to_pb(t: &Tags) -> pb::Tags { +#[cfg(test)] +pub(crate) fn tags_to_pb(t: &Tags) -> pb::Tags { pb::Tags { - entries: t.0.iter().map(|(k, v)| (k.to_string(), v.to_string())).collect(), + entries: t + .0 + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), ..Default::default() } } /// Convert `pb::Tags` → typed [`Tags`]. -pub fn tags_from_pb(b: &pb::Tags) -> Tags { +#[cfg(test)] +pub(crate) fn tags_from_pb(b: &pb::Tags) -> Tags { Tags( b.entries .iter() @@ -134,7 +137,7 @@ pub fn tags_from_pb(b: &pb::Tags) -> Tags { /// /// The node-id type `I` is encoded as opaque `bytes` via `memberlist_proto::Data` /// for the `Id` variant; `Tag` variants encode as before. -pub fn filter_to_pb(t: &Filter) -> Result +pub(crate) fn filter_to_pb(t: &Filter) -> Result where I: Data, { @@ -164,7 +167,7 @@ where /// Convert `pb::Filter` → typed [`Filter`]. /// /// The `Id` variant decodes each `bytes` entry as `I` via `memberlist_proto::DataRef`. -pub fn filter_from_pb(b: &pb::Filter) -> Result, BridgeError> +pub(crate) fn filter_from_pb(b: &pb::Filter) -> Result, BridgeError> where I: Data, { @@ -189,27 +192,28 @@ where /// Encode a `memberlist_proto::Data` value to a raw `Bytes` buffer (no length prefix). /// -/// Allocates a buffer sized by `encoded_len`, writes the encoding via -/// `encode`, and wraps it in `Bytes`. This mirrors the pattern in +/// Delegates to [`Data::encode_to_bytes`], which allocates a correctly-sized +/// buffer and returns it as `Bytes`. This mirrors the pattern in /// `memberlist_proto::bridge` for serialising opaque `I`/`A` fields. fn data_to_bytes(val: &T) -> Result where T: Data, { - let mut buf = vec![0u8; val.encoded_len()]; - val.encode(&mut buf)?; - Ok(Bytes::from(buf)) + Ok(val.encode_to_bytes()?) } /// Decode a `memberlist_proto::Data` value from raw bytes (no length prefix). /// +/// Accepts any `&[u8]` slice — the caller may pass a `Bytes` ref via +/// `buf.as_ref()` or a plain slice directly. +/// /// Rejects trailing data: the whole slice must be consumed so a malformed /// wire field is caught at the wire→machine boundary. -fn data_from_bytes(buf: &Bytes) -> Result +fn data_from_bytes(buf: &[u8]) -> Result where T: Data, { - let (bytes_read, val) = as DataRef<'_, T>>::decode(buf.as_ref())?; + let (bytes_read, val) = as DataRef<'_, T>>::decode(buf)?; if bytes_read != buf.len() { return Err(BridgeError::Decode(DecodeError::custom(format!( "trailing data in encoded field: decoder consumed {bytes_read} of {} bytes", @@ -224,29 +228,33 @@ where /// Convert a typed [`JoinMessage`] → `pb::JoinMessage`. /// /// The node-id `I` is serialised to opaque `bytes` via `memberlist_proto::Data`. -pub fn join_to_pb(t: &JoinMessage) -> Result +pub(crate) fn join_to_pb(t: &JoinMessage) -> Result where I: Data, { Ok(pb::JoinMessage { ltime: Some(t.ltime.into()), - id: data_to_bytes(&t.id)?, + id: Some(data_to_bytes(&t.id)?), ..Default::default() }) } /// Convert `pb::JoinMessage` → typed [`JoinMessage`]. /// -/// Rejects a missing `ltime` (required field). The `id` bytes are decoded via -/// `memberlist_proto::DataRef`. -pub fn join_from_pb(b: &pb::JoinMessage) -> Result, BridgeError> +/// Rejects a missing `ltime` and a missing `id` (both required fields). The +/// `id` bytes are decoded via `memberlist_proto::DataRef`. +pub(crate) fn join_from_pb(b: &pb::JoinMessage) -> Result, BridgeError> where I: Data, { let ltime = b .ltime .ok_or(BridgeError::MissingField("JoinMessage.ltime".into()))?; - let id: I = data_from_bytes(&b.id)?; + let id_bytes = b + .id + .as_ref() + .ok_or(BridgeError::MissingField("JoinMessage.id".into()))?; + let id: I = data_from_bytes(id_bytes)?; Ok(JoinMessage { ltime: LamportTime::from(ltime), id, @@ -258,30 +266,34 @@ where /// Convert a typed [`LeaveMessage`] → `pb::LeaveMessage`. /// /// The node-id `I` is serialised to opaque `bytes` via `memberlist_proto::Data`. -pub fn leave_to_pb(t: &LeaveMessage) -> Result +pub(crate) fn leave_to_pb(t: &LeaveMessage) -> Result where I: Data, { Ok(pb::LeaveMessage { ltime: Some(t.ltime.into()), prune: t.prune, - id: data_to_bytes(&t.id)?, + id: Some(data_to_bytes(&t.id)?), ..Default::default() }) } /// Convert `pb::LeaveMessage` → typed [`LeaveMessage`]. /// -/// Rejects a missing `ltime` (required field). The `id` bytes are decoded via -/// `memberlist_proto::DataRef`. -pub fn leave_from_pb(b: &pb::LeaveMessage) -> Result, BridgeError> +/// Rejects a missing `ltime` and a missing `id` (both required fields). The +/// `id` bytes are decoded via `memberlist_proto::DataRef`. +pub(crate) fn leave_from_pb(b: &pb::LeaveMessage) -> Result, BridgeError> where I: Data, { let ltime = b .ltime .ok_or(BridgeError::MissingField("LeaveMessage.ltime".into()))?; - let id: I = data_from_bytes(&b.id)?; + let id_bytes = b + .id + .as_ref() + .ok_or(BridgeError::MissingField("LeaveMessage.id".into()))?; + let id: I = data_from_bytes(id_bytes)?; Ok(LeaveMessage { ltime: LamportTime::from(ltime), id, @@ -295,7 +307,7 @@ where /// /// The `Node` member is serialised to opaque `bytes` via /// `memberlist_proto::Data`. -pub fn conflict_response_to_pb( +pub(crate) fn conflict_response_to_pb( t: &ConflictResponseMessage, ) -> Result where @@ -303,22 +315,26 @@ where A: Data, { Ok(pb::ConflictResponseMessage { - member: data_to_bytes(&t.member)?, + member: Some(data_to_bytes(&t.member)?), ..Default::default() }) } /// Convert `pb::ConflictResponseMessage` → typed [`ConflictResponseMessage`]. /// -/// The `member` bytes are decoded as a `Node` via `memberlist_proto::DataRef`. -pub fn conflict_response_from_pb( +/// Rejects a missing `member` (required field). The `member` bytes are decoded +/// as a `Node` via `memberlist_proto::DataRef`. +pub(crate) fn conflict_response_from_pb( b: &pb::ConflictResponseMessage, ) -> Result, BridgeError> where I: Data, A: Data, { - let member: memberlist_proto::Node = data_from_bytes(&b.member)?; + let member_bytes = b.member.as_ref().ok_or(BridgeError::MissingField( + "ConflictResponseMessage.member".into(), + ))?; + let member: memberlist_proto::Node = data_from_bytes(member_bytes)?; Ok(ConflictResponseMessage { member }) } @@ -330,7 +346,7 @@ where /// - Each `Filter` in `filters` is encoded via [`filter_to_pb`]. /// - `timeout` is stored as nanoseconds in a `uint64`. /// - `flags` is stored as the raw `u32` bit-pattern. -pub fn query_to_pb(t: &QueryMessage) -> Result +pub(crate) fn query_to_pb(t: &QueryMessage) -> Result where I: Data, A: Data, @@ -341,14 +357,18 @@ where .map(|f| filter_to_pb::(f)) .collect::, BridgeError>>()?; + let timeout_nanos = u64::try_from(t.timeout.as_nanos()).map_err(|_| { + BridgeError::InvalidValue("QueryMessage.timeout exceeds u64::MAX nanoseconds".into()) + })?; + Ok(pb::QueryMessage { ltime: Some(t.ltime.into()), id: Some(t.id), - from: data_to_bytes(&t.from)?, + from: Some(data_to_bytes(&t.from)?), filters, flags: Some(t.flags.bits()), relay_factor: Some(t.relay_factor as u32), - timeout_nanos: Some(t.timeout.as_nanos() as u64), + timeout_nanos: Some(timeout_nanos), name: t.name.to_string(), payload: t.payload.clone(), ..Default::default() @@ -357,10 +377,11 @@ where /// Convert `pb::QueryMessage` → typed [`QueryMessage`]. /// -/// Rejects missing `ltime`, `id`, `flags`, `relay_factor`, and `timeout_nanos` -/// (all required by the legacy protocol). Rejects `relay_factor` values that -/// exceed `u8::MAX`. Decodes `from` as `Node` and each `Filter` via [`filter_from_pb`]. -pub fn query_from_pb(b: &pb::QueryMessage) -> Result, BridgeError> +/// Rejects missing `ltime`, `id`, `from`, `flags`, `relay_factor`, and +/// `timeout_nanos` (all required by the legacy protocol). Rejects `relay_factor` +/// values that exceed `u8::MAX`. Decodes `from` as `Node` and each `Filter` +/// via [`filter_from_pb`]. +pub(crate) fn query_from_pb(b: &pb::QueryMessage) -> Result, BridgeError> where I: Data, A: Data, @@ -371,7 +392,11 @@ where let id = b .id .ok_or(BridgeError::MissingField("QueryMessage.id".into()))?; - let from: memberlist_proto::Node = data_from_bytes(&b.from)?; + let from_bytes = b + .from + .as_ref() + .ok_or(BridgeError::MissingField("QueryMessage.from".into()))?; + let from: memberlist_proto::Node = data_from_bytes(from_bytes)?; let filters = b .filters .iter() @@ -381,16 +406,14 @@ where b.flags .ok_or(BridgeError::MissingField("QueryMessage.flags".into()))?, ); - let relay_factor = u8::try_from( - b.relay_factor - .ok_or(BridgeError::MissingField("QueryMessage.relay_factor".into()))?, - ) + let relay_factor = u8::try_from(b.relay_factor.ok_or(BridgeError::MissingField( + "QueryMessage.relay_factor".into(), + ))?) .map_err(|_| BridgeError::InvalidValue("QueryMessage.relay_factor exceeds u8::MAX".into()))?; // Safe: query timeouts are measured in seconds to minutes, well within u64::MAX nanoseconds. - let timeout = std::time::Duration::from_nanos( - b.timeout_nanos - .ok_or(BridgeError::MissingField("QueryMessage.timeout_nanos".into()))?, - ); + let timeout = std::time::Duration::from_nanos(b.timeout_nanos.ok_or( + BridgeError::MissingField("QueryMessage.timeout_nanos".into()), + )?); Ok(QueryMessage { ltime: LamportTime::from(ltime), @@ -411,7 +434,7 @@ where /// /// - `from: Node` is serialised to opaque `bytes` via `memberlist_proto::Data`. /// - `flags` is stored as the raw `u32` bit-pattern. -pub fn query_response_to_pb( +pub(crate) fn query_response_to_pb( t: &QueryResponseMessage, ) -> Result where @@ -421,7 +444,7 @@ where Ok(pb::QueryResponseMessage { ltime: Some(t.ltime.into()), id: Some(t.id), - from: data_to_bytes(&t.from)?, + from: Some(data_to_bytes(&t.from)?), flags: Some(t.flags.bits()), payload: t.payload.clone(), ..Default::default() @@ -430,27 +453,29 @@ where /// Convert `pb::QueryResponseMessage` → typed [`QueryResponseMessage`]. /// -/// Rejects missing `ltime`, `id`, and `flags` (all required by the legacy protocol). -/// Decodes `from` as `Node` via `memberlist_proto::DataRef`. Flags are -/// decoded with `from_bits_retain` to preserve any future extension bits. -pub fn query_response_from_pb( +/// Rejects missing `ltime`, `id`, `from`, and `flags` (all required by the +/// legacy protocol). Decodes `from` as `Node` via `memberlist_proto::DataRef`. +/// Flags are decoded with `from_bits_retain` to preserve any future extension bits. +pub(crate) fn query_response_from_pb( b: &pb::QueryResponseMessage, ) -> Result, BridgeError> where I: Data, A: Data, { - let ltime = b - .ltime - .ok_or(BridgeError::MissingField("QueryResponseMessage.ltime".into()))?; + let ltime = b.ltime.ok_or(BridgeError::MissingField( + "QueryResponseMessage.ltime".into(), + ))?; let id = b .id .ok_or(BridgeError::MissingField("QueryResponseMessage.id".into()))?; - let from: memberlist_proto::Node = data_from_bytes(&b.from)?; - let flags = QueryFlag::from_bits_retain( - b.flags - .ok_or(BridgeError::MissingField("QueryResponseMessage.flags".into()))?, - ); + let from_bytes = b.from.as_ref().ok_or(BridgeError::MissingField( + "QueryResponseMessage.from".into(), + ))?; + let from: memberlist_proto::Node = data_from_bytes(from_bytes)?; + let flags = QueryFlag::from_bits_retain(b.flags.ok_or(BridgeError::MissingField( + "QueryResponseMessage.flags".into(), + ))?); Ok(QueryResponseMessage { ltime: LamportTime::from(ltime), @@ -464,9 +489,8 @@ where // ─── UserEvent ──────────────────────────────────────────────────────────────── /// Convert a typed [`UserEvent`] → `pb::UserEvent`. -pub fn user_event_single_to_pb(t: &UserEvent) -> pb::UserEvent { +pub(crate) fn user_event_single_to_pb(t: &UserEvent) -> pb::UserEvent { pb::UserEvent { - cc: t.cc, name: t.name.to_string(), payload: t.payload.clone(), ..Default::default() @@ -474,9 +498,8 @@ pub fn user_event_single_to_pb(t: &UserEvent) -> pb::UserEvent { } /// Convert `pb::UserEvent` → typed [`UserEvent`]. -pub fn user_event_single_from_pb(b: &pb::UserEvent) -> UserEvent { +pub(crate) fn user_event_single_from_pb(b: &pb::UserEvent) -> UserEvent { UserEvent { - cc: b.cc, name: SmolStr::from(b.name.as_str()), payload: b.payload.clone(), } @@ -485,7 +508,7 @@ pub fn user_event_single_from_pb(b: &pb::UserEvent) -> UserEvent { // ─── UserEvents ─────────────────────────────────────────────────────────────── /// Convert a typed [`UserEvents`] → `pb::UserEvents`. -pub fn user_events_to_pb(t: &UserEvents) -> pb::UserEvents { +pub(crate) fn user_events_to_pb(t: &UserEvents) -> pb::UserEvents { pb::UserEvents { ltime: Some(t.ltime.into()), events: t.events.iter().map(user_event_single_to_pb).collect(), @@ -499,12 +522,14 @@ pub fn user_events_to_pb(t: &UserEvents) -> pb::UserEvents { /// events carries no information and would silently consume buffer history /// entries. The legacy `serf-core` invariant is `OneOrMore` (at least one event /// per batch); this decoder enforces the same constraint. -pub fn user_events_from_pb(b: &pb::UserEvents) -> Result { +pub(crate) fn user_events_from_pb(b: &pb::UserEvents) -> Result { let ltime = b .ltime .ok_or(BridgeError::MissingField("UserEvents.ltime".into()))?; if b.events.is_empty() { - return Err(BridgeError::MissingField("UserEvents.events (must be non-empty)".into())); + return Err(BridgeError::MissingField( + "UserEvents.events (must be non-empty)".into(), + )); } Ok(UserEvents { ltime: LamportTime::from(ltime), @@ -521,7 +546,7 @@ pub fn user_events_from_pb(b: &pb::UserEvents) -> Result(t: &PushPullMessage) -> Result +pub(crate) fn push_pull_to_pb(t: &PushPullMessage) -> Result where I: Data, { @@ -530,8 +555,8 @@ where .iter() .map(|(id, ltime)| { data_to_bytes(id).map(|id_bytes| pb::NodeStatusTime { - id: id_bytes, - ltime: (*ltime).into(), + id: Some(id_bytes), + ltime: Some((*ltime).into()), ..Default::default() }) }) @@ -558,29 +583,39 @@ where /// Convert `pb::PushPullMessage` → typed [`PushPullMessage`]. /// -/// Rejects missing `ltime`, `event_ltime`, and `query_ltime` (all required by -/// the legacy protocol). Decodes each `NodeStatusTime.id` and each -/// `left_members` entry as `I` via `memberlist_proto::DataRef`. -pub fn push_pull_from_pb(b: &pb::PushPullMessage) -> Result, BridgeError> +/// Rejects missing `ltime`, `event_ltime`, `query_ltime`, and each +/// `NodeStatusTime.id` / `NodeStatusTime.ltime` (all required by the legacy +/// protocol). Decodes each `NodeStatusTime.id` and each `left_members` entry as +/// `I` via `memberlist_proto::DataRef`. +pub(crate) fn push_pull_from_pb( + b: &pb::PushPullMessage, +) -> Result, BridgeError> where I: Data, { let ltime = b .ltime .ok_or(BridgeError::MissingField("PushPullMessage.ltime".into()))?; - let event_ltime = b - .event_ltime - .ok_or(BridgeError::MissingField("PushPullMessage.event_ltime".into()))?; - let query_ltime = b - .query_ltime - .ok_or(BridgeError::MissingField("PushPullMessage.query_ltime".into()))?; + let event_ltime = b.event_ltime.ok_or(BridgeError::MissingField( + "PushPullMessage.event_ltime".into(), + ))?; + let query_ltime = b.query_ltime.ok_or(BridgeError::MissingField( + "PushPullMessage.query_ltime".into(), + ))?; let status_ltimes = b .status_ltimes .iter() .map(|nst| { - let id: I = data_from_bytes(&nst.id)?; - Ok((id, LamportTime::from(nst.ltime))) + let id_bytes = nst + .id + .as_ref() + .ok_or(BridgeError::MissingField("NodeStatusTime.id".into()))?; + let id: I = data_from_bytes(id_bytes)?; + let ltime = nst + .ltime + .ok_or(BridgeError::MissingField("NodeStatusTime.ltime".into()))?; + Ok((id, LamportTime::from(ltime))) }) .collect::, BridgeError>>()?; @@ -613,13 +648,19 @@ where /// The leading algorithm tag byte makes the wire encoding self-describing so /// decoding is unambiguous even when two ciphers share the same key length /// (e.g. AES-256 and ChaCha20-Poly1305 are both 32 bytes). +/// +/// The transient plaintext buffer is zeroed before it is freed so the raw key +/// material does not linger in heap memory after this call returns. #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] fn secret_key_to_bytes(key: &SecretKey) -> Bytes { + use zeroize::Zeroize as _; let raw = key.as_bytes(); let mut buf = Vec::with_capacity(1 + raw.len()); buf.push(key.algorithm().tag()); buf.extend_from_slice(raw); - Bytes::from(buf) + let out = Bytes::copy_from_slice(&buf); + buf.zeroize(); + out } /// Decode a [`SecretKey`] from `[algorithm_tag][raw_key_bytes]` wire bytes. @@ -682,8 +723,7 @@ fn secret_key_from_bytes(buf: &Bytes) -> Result { /// wire encoding is self-describing. Requires the `aes-gcm` or /// `chacha20-poly1305` feature. #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] -#[cfg_attr(docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))))] -pub fn key_request_to_pb(t: &KeyRequestMessage) -> pb::KeyRequestMessage { +pub(crate) fn key_request_to_pb(t: &KeyRequestMessage) -> pb::KeyRequestMessage { pb::KeyRequestMessage { key: t.key.as_ref().map(secret_key_to_bytes), ..Default::default() @@ -696,8 +736,9 @@ pub fn key_request_to_pb(t: &KeyRequestMessage) -> pb::KeyRequestMessage { /// Returns [`BridgeError::InvalidValue`] if the bytes are present but malformed. /// Requires the `aes-gcm` or `chacha20-poly1305` feature. #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] -#[cfg_attr(docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))))] -pub fn key_request_from_pb(b: &pb::KeyRequestMessage) -> Result { +pub(crate) fn key_request_from_pb( + b: &pb::KeyRequestMessage, +) -> Result { let key = b.key.as_ref().map(secret_key_from_bytes).transpose()?; Ok(KeyRequestMessage { key }) } @@ -709,8 +750,7 @@ pub fn key_request_from_pb(b: &pb::KeyRequestMessage) -> Result pb::KeyResponseMessage { +pub(crate) fn key_response_to_pb(t: &KeyResponseMessage) -> pb::KeyResponseMessage { pb::KeyResponseMessage { result: t.result, message: t.message.to_string(), @@ -726,8 +766,9 @@ pub fn key_response_to_pb(t: &KeyResponseMessage) -> pb::KeyResponseMessage { /// Returns [`BridgeError::InvalidValue`] if any entry is malformed. Requires /// the `aes-gcm` or `chacha20-poly1305` feature. #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] -#[cfg_attr(docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))))] -pub fn key_response_from_pb(b: &pb::KeyResponseMessage) -> Result { +pub(crate) fn key_response_from_pb( + b: &pb::KeyResponseMessage, +) -> Result { let keys = b .keys .iter() @@ -752,13 +793,13 @@ pub fn key_response_from_pb(b: &pb::KeyResponseMessage) -> Result` is serialised to opaque `bytes` via /// `memberlist_proto::Data`. The `payload` bytes are copied verbatim. -pub fn relay_to_pb(t: &RelayMessage) -> Result +pub(crate) fn relay_to_pb(t: &RelayMessage) -> Result where I: Data, A: Data, { Ok(pb::RelayMessage { - destination: data_to_bytes(&t.destination)?, + destination: Some(data_to_bytes(&t.destination)?), payload: t.payload.clone(), ..Default::default() }) @@ -766,14 +807,19 @@ where /// Convert `pb::RelayMessage` → typed [`RelayMessage`]. /// -/// Decodes `destination` as `Node` via `memberlist_proto::DataRef`. -/// The `payload` bytes are preserved verbatim without parsing. -pub fn relay_from_pb(b: &pb::RelayMessage) -> Result, BridgeError> +/// Rejects a missing `destination` (required field). Decodes `destination` as +/// `Node` via `memberlist_proto::DataRef`. The `payload` bytes are +/// preserved verbatim without parsing. +pub(crate) fn relay_from_pb(b: &pb::RelayMessage) -> Result, BridgeError> where I: Data, A: Data, { - let destination: memberlist_proto::Node = data_from_bytes(&b.destination)?; + let destination_bytes = b + .destination + .as_ref() + .ok_or(BridgeError::MissingField("RelayMessage.destination".into()))?; + let destination: memberlist_proto::Node = data_from_bytes(destination_bytes)?; Ok(RelayMessage { destination, payload: b.payload.clone(), diff --git a/serf-proto/src/framing/mod.rs b/serf-proto/src/framing/mod.rs index 6c4554ed..a05e2b72 100644 --- a/serf-proto/src/framing/mod.rs +++ b/serf-proto/src/framing/mod.rs @@ -64,14 +64,20 @@ pub enum MessageType { /// Requires the `aes-gcm` or `chacha20-poly1305` feature; without an /// encryption backend the tag byte decodes as [`MessageType::Unknown`]. #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] - #[cfg_attr(docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))))] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] KeyRequest, /// KeyResponse — encryption key management response. /// /// Requires the `aes-gcm` or `chacha20-poly1305` feature; without an /// encryption backend the tag byte decodes as [`MessageType::Unknown`]. #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] - #[cfg_attr(docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))))] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] KeyResponse, /// A tag not recognised by this build — preserved for forward compatibility. Unknown(u8), @@ -119,8 +125,8 @@ impl From for u8 { // ── Errors ─────────────────────────────────────────────────────────────────── -/// The `(available, required)` byte-count pair carried by -/// [`FrameError::Incomplete`]. +/// The `(available, required)` byte-count pair carried by the +/// `FrameError::Incomplete` variant. #[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] #[error("incomplete frame: {available} bytes available, {required} required")] pub struct IncompleteFrame { @@ -132,7 +138,10 @@ impl IncompleteFrame { /// Construct an incomplete-frame payload. #[inline(always)] pub const fn new(available: usize, required: usize) -> Self { - Self { available, required } + Self { + available, + required, + } } /// Bytes available in the buffer. @@ -148,7 +157,9 @@ impl IncompleteFrame { } } -/// Errors returned by [`encode_message`] and [`decode_message`]. +/// Errors returned by the serf plain-frame encoder / decoder underlying +/// [`AnyMessage::encode`](crate::AnyMessage::encode) and +/// [`AnyMessage::decode`](crate::AnyMessage::decode). #[non_exhaustive] #[derive(Debug, Clone, thiserror::Error)] pub enum FrameError { @@ -164,7 +175,9 @@ pub enum FrameError { /// The buffa `encoded_len` / actual-write lengths disagree, which would /// desynchronize the receiver's length prefix. Carries the actual number of /// bytes written by `buffa::Message::encode`. - #[error("frame encode length mismatch: encoded_len predicted {0} bytes but encode wrote a different count")] + #[error( + "frame encode length mismatch: encoded_len predicted {0} bytes but encode wrote a different count" + )] FrameTooLarge(usize), /// The buffa decoder rejected the body bytes. #[error("buffa decode error: body bytes could not be decoded")] @@ -201,7 +214,10 @@ fn decode_varint_u32(buf: &[u8]) -> Result<(u32, usize), FrameError> { shift += 7; } // Fell through without a terminating byte — truncated length prefix. - Err(FrameError::Incomplete(IncompleteFrame::new(buf.len(), buf.len() + 1))) + Err(FrameError::Incomplete(IncompleteFrame::new( + buf.len(), + buf.len() + 1, + ))) } // ── Public API ─────────────────────────────────────────────────────────────── @@ -211,7 +227,7 @@ fn decode_varint_u32(buf: &[u8]) -> Result<(u32, usize), FrameError> { /// Returns a freshly allocated `Vec` containing the complete frame. /// Fails only when the buffa body length exceeds `u32::MAX` (which buffa's /// `u32` return type for `encoded_len` already guarantees cannot happen). -pub fn encode_message(ty: MessageType, msg: &M) -> Result, FrameError> +pub(crate) fn encode_message(ty: MessageType, msg: &M) -> Result, FrameError> where M: buffa::Message, { @@ -245,7 +261,7 @@ where /// /// `bytes_consumed` is the total number of bytes read (tag + varint + body), /// allowing a streaming caller to advance its read cursor. -pub fn decode_message(frame: &Bytes) -> Result<(MessageType, Bytes, usize), FrameError> { +pub(crate) fn decode_message(frame: &Bytes) -> Result<(MessageType, Bytes, usize), FrameError> { let buf = frame.as_ref(); if buf.is_empty() { return Err(FrameError::Empty); @@ -256,16 +272,24 @@ pub fn decode_message(frame: &Bytes) -> Result<(MessageType, Bytes, usize), Fram let (body_len, varint_bytes) = match decode_varint_u32(&buf[1..]) { Ok(v) => v, Err(FrameError::Incomplete(_)) => { - return Err(FrameError::Incomplete(IncompleteFrame::new(buf.len(), buf.len() + 1))); + return Err(FrameError::Incomplete(IncompleteFrame::new( + buf.len(), + buf.len() + 1, + ))); } Err(e) => return Err(e), }; let header_len = 1 + varint_bytes; - let frame_end = header_len + body_len as usize; + let frame_end = header_len + .checked_add(body_len as usize) + .ok_or(FrameError::VarintOverflow)?; if buf.len() < frame_end { - return Err(FrameError::Incomplete(IncompleteFrame::new(buf.len(), frame_end))); + return Err(FrameError::Incomplete(IncompleteFrame::new( + buf.len(), + frame_end, + ))); } // Zero-copy slice of the body out of the `Bytes` allocation. diff --git a/serf-proto/src/framing/tests.rs b/serf-proto/src/framing/tests.rs index 7bab146b..42f68513 100644 --- a/serf-proto/src/framing/tests.rs +++ b/serf-proto/src/framing/tests.rs @@ -2,8 +2,11 @@ use buffa::Message as _; use bytes::Bytes; use super::{FrameError, MessageType, decode_message, encode_message}; -use crate::messages::serf::v1::UserEventMessage as PbUserEventMessage; -use crate::{LamportTime, UserEventMessage, user_event_from_pb, user_event_to_pb}; +use crate::{ + LamportTime, UserEventMessage, + bridge::{user_event_from_pb, user_event_to_pb}, + messages::serf::v1::UserEventMessage as PbUserEventMessage, +}; // ── MessageType round-trips ─────────────────────────────────────────────────── @@ -37,10 +40,7 @@ fn message_type_tag_round_trip() { #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] #[test] fn message_type_key_tag_round_trip() { - let cases: &[(MessageType, u8)] = &[ - (MessageType::KeyRequest, 9), - (MessageType::KeyResponse, 10), - ]; + let cases: &[(MessageType, u8)] = &[(MessageType::KeyRequest, 9), (MessageType::KeyResponse, 10)]; for &(ref ty, expected_byte) in cases { let byte = u8::from(*ty); @@ -88,10 +88,7 @@ fn user_event_frame_round_trip() { // Encode into a serf frame. let frame_vec = encode_message(MessageType::UserEvent, &pb).expect("encode_message should succeed"); - assert!( - !frame_vec.is_empty(), - "encoded frame must not be empty" - ); + assert!(!frame_vec.is_empty(), "encoded frame must not be empty"); // The leading byte must be the UserEvent tag. assert_eq!( @@ -123,7 +120,10 @@ fn user_event_frame_round_trip() { #[test] fn decode_message_empty_errors() { let empty = Bytes::new(); - assert!(matches!(decode_message(&empty), Err(super::FrameError::Empty))); + assert!(matches!( + decode_message(&empty), + Err(super::FrameError::Empty) + )); } #[test] @@ -159,7 +159,11 @@ fn decode_varint_overflow_is_error_not_panic() { // Byte sequence: 4 continuation bytes (all 0x80) + one final byte > 0x0f. let buf: &[u8] = &[ u8::from(MessageType::UserEvent), // tag - 0x80, 0x80, 0x80, 0x80, 0x10, // 5-byte LEB128 with 5th byte = 0x10 > 0x0f + 0x80, + 0x80, + 0x80, + 0x80, + 0x10, // 5-byte LEB128 with 5th byte = 0x10 > 0x0f ]; let frame = Bytes::copy_from_slice(buf); assert!( @@ -194,10 +198,13 @@ fn unknown_tag_round_trips_encode_decode() { // An Unknown(200) tag must survive encode + decode: the tag byte is // preserved and the body (empty default message) is recovered intact. let pb = PbUserEventMessage::default(); - let frame_vec = - encode_message(MessageType::Unknown(200), &pb).expect("encode_message with Unknown tag should succeed"); + let frame_vec = encode_message(MessageType::Unknown(200), &pb) + .expect("encode_message with Unknown tag should succeed"); let frame = Bytes::from(frame_vec); - assert_eq!(frame[0], 200, "first byte must be the Unknown tag value 200"); + assert_eq!( + frame[0], 200, + "first byte must be the Unknown tag value 200" + ); let (recovered_ty, body, consumed) = decode_message(&frame).expect("decode_message with Unknown tag should succeed"); assert_eq!(recovered_ty, MessageType::Unknown(200)); diff --git a/serf-proto/src/lib.rs b/serf-proto/src/lib.rs index 646cc5b7..f2fdd302 100644 --- a/serf-proto/src/lib.rs +++ b/serf-proto/src/lib.rs @@ -4,58 +4,19 @@ //! own message set and framing on top of them. #![deny(missing_docs)] -pub use bridge::{ - BridgeError, - conflict_response_from_pb, - conflict_response_to_pb, - coordinate_from_pb, - coordinate_to_pb, - filter_from_pb, - filter_to_pb, - join_from_pb, - join_to_pb, - leave_from_pb, - leave_to_pb, - push_pull_from_pb, - push_pull_to_pb, - query_from_pb, - query_response_from_pb, - query_response_to_pb, - query_to_pb, - relay_from_pb, - relay_to_pb, - tags_from_pb, - tags_to_pb, - user_event_from_pb, - user_event_to_pb, - user_event_single_from_pb, - user_event_single_to_pb, - user_events_from_pb, - user_events_to_pb, -}; -#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] -#[cfg_attr(docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))))] -pub use bridge::{key_request_from_pb, key_request_to_pb, key_response_from_pb, key_response_to_pb}; -pub use framing::{FrameError, MessageType, decode_message, encode_message}; +pub use any::{AnyMessage, DecodeError, EncodeError}; +pub use bridge::BridgeError; +pub use framing::{FrameError, IncompleteFrame, MessageType}; pub use typed::{ - Coordinate, - ConflictResponseMessage, - Filter, - JoinMessage, - LeaveMessage, - PushPullMessage, - QueryFlag, - QueryMessage, - QueryResponseMessage, - RelayMessage, - TagFilter, - Tags, - UserEvent, - UserEventMessage, - UserEvents, + ConflictResponseMessage, Coordinate, Filter, JoinMessage, LeaveMessage, PushPullMessage, + QueryFlag, QueryMessage, QueryResponseMessage, RelayMessage, TagFilter, Tags, UserEvent, + UserEventMessage, UserEvents, }; #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] -#[cfg_attr(docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))))] +#[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) +)] pub use typed::{KeyRequestMessage, KeyResponseMessage}; /// A lamport logical clock value — a monotonically increasing counter used to @@ -92,7 +53,8 @@ impl LamportTime { } } -pub mod bridge; -pub mod framing; -pub mod messages; +pub mod any; +pub(crate) mod bridge; +pub(crate) mod framing; +pub(crate) mod messages; pub mod typed; diff --git a/serf-proto/src/messages/tests.rs b/serf-proto/src/messages/tests.rs index ea9cd381..b7c8df51 100644 --- a/serf-proto/src/messages/tests.rs +++ b/serf-proto/src/messages/tests.rs @@ -5,78 +5,35 @@ use memberlist_proto::Node; use smol_str::SmolStr; use super::serf::v1::{ - ConflictResponseMessage as PbConflictResponseMessage, - Coordinate as PbCoordinate, - Filter as PbFilter, - JoinMessage as PbJoinMessage, - LeaveMessage as PbLeaveMessage, - PushPullMessage as PbPushPullMessage, - QueryMessage as PbQueryMessage, - QueryResponseMessage as PbQueryResponseMessage, - RelayMessage as PbRelayMessage, - Tags as PbTags, - UserEvent as PbUserEvent, - UserEventMessage as PbUserEventMessage, - UserEvents as PbUserEvents, + ConflictResponseMessage as PbConflictResponseMessage, Coordinate as PbCoordinate, + Filter as PbFilter, JoinMessage as PbJoinMessage, LeaveMessage as PbLeaveMessage, + PushPullMessage as PbPushPullMessage, QueryMessage as PbQueryMessage, + QueryResponseMessage as PbQueryResponseMessage, RelayMessage as PbRelayMessage, Tags as PbTags, + UserEvent as PbUserEvent, UserEventMessage as PbUserEventMessage, UserEvents as PbUserEvents, }; #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] use super::serf::v1::{ - KeyRequestMessage as PbKeyRequestMessage, - KeyResponseMessage as PbKeyResponseMessage, -}; -use crate::{ - ConflictResponseMessage, - Coordinate, - Filter, - JoinMessage, - LamportTime, - LeaveMessage, - PushPullMessage, - QueryFlag, - QueryMessage, - QueryResponseMessage, - RelayMessage, - TagFilter, - Tags, - UserEvent, - UserEventMessage, - UserEvents, - conflict_response_from_pb, - conflict_response_to_pb, - coordinate_from_pb, - coordinate_to_pb, - filter_from_pb, - filter_to_pb, - join_from_pb, - join_to_pb, - leave_from_pb, - leave_to_pb, - push_pull_from_pb, - push_pull_to_pb, - query_from_pb, - query_response_from_pb, - query_response_to_pb, - query_to_pb, - relay_from_pb, - relay_to_pb, - tags_from_pb, - tags_to_pb, - user_event_from_pb, - user_event_single_from_pb, - user_event_single_to_pb, - user_event_to_pb, - user_events_from_pb, - user_events_to_pb, + KeyRequestMessage as PbKeyRequestMessage, KeyResponseMessage as PbKeyResponseMessage, }; #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +use crate::bridge::{ + key_request_from_pb, key_request_to_pb, key_response_from_pb, key_response_to_pb, +}; use crate::{ - KeyRequestMessage, - KeyResponseMessage, - key_request_from_pb, - key_request_to_pb, - key_response_from_pb, - key_response_to_pb, + ConflictResponseMessage, Coordinate, Filter, JoinMessage, LamportTime, LeaveMessage, + PushPullMessage, QueryFlag, QueryMessage, QueryResponseMessage, RelayMessage, TagFilter, Tags, + UserEvent, UserEventMessage, UserEvents, + bridge::{ + conflict_response_from_pb, conflict_response_to_pb, coordinate_from_pb, coordinate_to_pb, + filter_from_pb, filter_to_pb, join_from_pb, join_to_pb, leave_from_pb, leave_to_pb, + push_pull_from_pb, push_pull_to_pb, query_from_pb, query_response_from_pb, + query_response_to_pb, query_to_pb, relay_from_pb, relay_to_pb, tags_from_pb, tags_to_pb, + user_event_from_pb, user_event_single_from_pb, user_event_single_to_pb, user_event_to_pb, + user_events_from_pb, user_events_to_pb, + }, }; +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +use crate::{KeyRequestMessage, KeyResponseMessage}; #[cfg(feature = "aes-gcm")] use memberlist_proto::SecretKey; @@ -94,8 +51,8 @@ fn user_event_message_roundtrip_pb() { // typed → pb → bytes → pb → typed let pb = user_event_to_pb(&typed); let encoded = pb.encode_to_vec(); - let decoded_pb = PbUserEventMessage::decode_from_slice(encoded.as_slice()) - .expect("decode_from_slice failed"); + let decoded_pb = + PbUserEventMessage::decode_from_slice(encoded.as_slice()).expect("decode_from_slice failed"); let roundtripped = user_event_from_pb(&decoded_pb).expect("user_event_from_pb failed"); assert_eq!(roundtripped, typed); @@ -195,7 +152,9 @@ fn tags_empty_roundtrip() { #[test] fn tags_multi_entry_roundtrip() { - let typed: Tags = [("role", "web"), ("env", "prod"), ("dc", "us-east")].into_iter().collect(); + let typed: Tags = [("role", "web"), ("env", "prod"), ("dc", "us-east")] + .into_iter() + .collect(); let pb = tags_to_pb(&typed); let encoded = pb.encode_to_vec(); @@ -213,8 +172,7 @@ fn tags_multi_entry_roundtrip() { #[test] fn filter_node_ids_roundtrip() { // I = SmolStr (the default generic parameter). - let typed: Filter = - Filter::Id(vec![SmolStr::from("node-1"), SmolStr::from("node-2")]); + let typed: Filter = Filter::Id(vec![SmolStr::from("node-1"), SmolStr::from("node-2")]); let pb = filter_to_pb(&typed).expect("filter_to_pb"); let encoded = pb.encode_to_vec(); @@ -297,7 +255,7 @@ fn join_message_roundtrip_pb() { fn join_message_ltime_required() { let pb = PbJoinMessage { ltime: None, - id: bytes::Bytes::new(), + id: Some(bytes::Bytes::new()), ..Default::default() }; assert!( @@ -351,7 +309,7 @@ fn leave_message_ltime_required() { let pb = PbLeaveMessage { ltime: None, prune: false, - id: bytes::Bytes::new(), + id: Some(bytes::Bytes::new()), ..Default::default() }; assert!( @@ -470,7 +428,7 @@ fn query_message_ltime_required() { let pb = PbQueryMessage { ltime: None, id: Some(1), - from: bytes::Bytes::new(), + from: Some(bytes::Bytes::new()), ..Default::default() }; assert!( @@ -484,7 +442,7 @@ fn query_message_id_required() { let pb = PbQueryMessage { ltime: Some(1), id: None, - from: bytes::Bytes::new(), + from: Some(bytes::Bytes::new()), ..Default::default() }; assert!( @@ -493,6 +451,29 @@ fn query_message_id_required() { ); } +#[test] +fn query_to_pb_timeout_overflow_is_error() { + // A duration whose nanosecond count exceeds u64::MAX (requires u128) must + // produce BridgeError::InvalidValue, not a silent truncation. + // u64::MAX nanos ≈ 584 years; add one second to guarantee overflow. + let huge_timeout = std::time::Duration::from_nanos(u64::MAX) + std::time::Duration::from_secs(1); + let typed: QueryMessage = QueryMessage { + ltime: LamportTime::new(1), + id: 1, + from: Node::new(SmolStr::from("n"), sample_addr()), + filters: vec![], + flags: QueryFlag::empty(), + relay_factor: 0, + timeout: huge_timeout, + name: SmolStr::from("q"), + payload: bytes::Bytes::new(), + }; + assert!( + query_to_pb(&typed).is_err(), + "timeout exceeding u64::MAX nanoseconds must yield BridgeError::InvalidValue" + ); +} + // ── QueryResponseMessage ────────────────────────────────────────────────────── #[test] @@ -507,8 +488,8 @@ fn query_response_message_roundtrip_pb_ack() { let pb = query_response_to_pb(&typed).expect("query_response_to_pb failed"); let encoded = pb.encode_to_vec(); - let decoded_pb = - PbQueryResponseMessage::decode_from_slice(encoded.as_slice()).expect("decode_from_slice failed"); + let decoded_pb = PbQueryResponseMessage::decode_from_slice(encoded.as_slice()) + .expect("decode_from_slice failed"); let roundtripped: QueryResponseMessage = query_response_from_pb(&decoded_pb).expect("query_response_from_pb failed"); @@ -533,15 +514,18 @@ fn query_response_message_roundtrip_pb_with_payload() { let pb = query_response_to_pb(&typed).expect("query_response_to_pb failed"); let encoded = pb.encode_to_vec(); - let decoded_pb = - PbQueryResponseMessage::decode_from_slice(encoded.as_slice()).expect("decode_from_slice failed"); + let decoded_pb = PbQueryResponseMessage::decode_from_slice(encoded.as_slice()) + .expect("decode_from_slice failed"); let roundtripped: QueryResponseMessage = query_response_from_pb(&decoded_pb).expect("query_response_from_pb failed"); assert_eq!(roundtripped.ltime, typed.ltime); assert_eq!(roundtripped.id, typed.id); assert!(!roundtripped.ack()); - assert_eq!(roundtripped.payload, bytes::Bytes::from_static(b"response-data")); + assert_eq!( + roundtripped.payload, + bytes::Bytes::from_static(b"response-data") + ); } #[test] @@ -549,7 +533,7 @@ fn query_response_message_ltime_required() { let pb = PbQueryResponseMessage { ltime: None, id: Some(1), - from: bytes::Bytes::new(), + from: Some(bytes::Bytes::new()), ..Default::default() }; assert!( @@ -563,7 +547,7 @@ fn query_response_message_id_required() { let pb = PbQueryResponseMessage { ltime: Some(1), id: None, - from: bytes::Bytes::new(), + from: Some(bytes::Bytes::new()), ..Default::default() }; assert!( @@ -578,7 +562,7 @@ fn query_response_message_flags_required() { let pb = PbQueryResponseMessage { ltime: Some(1), id: Some(42), - from: bytes::Bytes::new(), + from: Some(bytes::Bytes::new()), flags: None, ..Default::default() }; @@ -594,7 +578,7 @@ fn query_message_required_fields() { let pb_no_flags = PbQueryMessage { ltime: Some(1), id: Some(1), - from: bytes::Bytes::new(), + from: Some(bytes::Bytes::new()), flags: None, relay_factor: Some(0), timeout_nanos: Some(1_000_000_000), @@ -609,7 +593,7 @@ fn query_message_required_fields() { let pb_no_relay = PbQueryMessage { ltime: Some(1), id: Some(1), - from: bytes::Bytes::new(), + from: Some(bytes::Bytes::new()), flags: Some(0), relay_factor: None, timeout_nanos: Some(1_000_000_000), @@ -624,7 +608,7 @@ fn query_message_required_fields() { let pb_no_timeout = PbQueryMessage { ltime: Some(1), id: Some(1), - from: bytes::Bytes::new(), + from: Some(bytes::Bytes::new()), flags: Some(0), relay_factor: Some(0), timeout_nanos: None, @@ -641,7 +625,6 @@ fn query_message_required_fields() { #[test] fn user_event_single_roundtrip_pb() { let typed = UserEvent { - cc: true, name: SmolStr::from("deploy"), payload: bytes::Bytes::from_static(b"ev-payload"), }; @@ -652,7 +635,6 @@ fn user_event_single_roundtrip_pb() { PbUserEvent::decode_from_slice(encoded.as_slice()).expect("decode_from_slice failed"); let roundtripped = user_event_single_from_pb(&decoded_pb); - assert_eq!(roundtripped.cc, typed.cc); assert_eq!(roundtripped.name, typed.name); assert_eq!(roundtripped.payload, typed.payload); } @@ -660,7 +642,6 @@ fn user_event_single_roundtrip_pb() { #[test] fn user_event_single_empty_payload_roundtrip() { let typed = UserEvent { - cc: false, name: SmolStr::from("ping"), payload: bytes::Bytes::new(), }; @@ -671,7 +652,6 @@ fn user_event_single_empty_payload_roundtrip() { PbUserEvent::decode_from_slice(encoded.as_slice()).expect("decode_from_slice failed"); let roundtripped = user_event_single_from_pb(&decoded_pb); - assert_eq!(roundtripped.cc, typed.cc); assert_eq!(roundtripped.name, typed.name); assert!(roundtripped.payload.is_empty()); } @@ -684,12 +664,10 @@ fn user_events_roundtrip_pb() { ltime: LamportTime::new(7), events: vec![ UserEvent { - cc: true, name: SmolStr::from("deploy"), payload: bytes::Bytes::from_static(b"v1"), }, UserEvent { - cc: false, name: SmolStr::from("alert"), payload: bytes::Bytes::from_static(b"critical"), }, @@ -752,7 +730,6 @@ fn push_pull_message_roundtrip_pb_full() { events: vec![UserEvents { ltime: LamportTime::new(49), events: vec![UserEvent { - cc: false, name: SmolStr::from("deploy"), payload: bytes::Bytes::from_static(b"v2"), }], @@ -783,7 +760,10 @@ fn push_pull_message_roundtrip_pb_full() { assert_eq!(roundtripped.events.len(), 1); assert_eq!(roundtripped.events[0].ltime, LamportTime::new(49)); assert_eq!(roundtripped.events[0].events.len(), 1); - assert_eq!(roundtripped.events[0].events[0].name, SmolStr::from("deploy")); + assert_eq!( + roundtripped.events[0].events[0].name, + SmolStr::from("deploy") + ); } #[test] @@ -854,6 +834,27 @@ fn push_pull_message_query_ltime_required() { ); } +#[test] +fn push_pull_node_status_time_ltime_required() { + // A status_ltimes entry with ltime absent must cause push_pull_from_pb to reject the message. + use super::serf::v1::NodeStatusTime; + let pb = PbPushPullMessage { + ltime: Some(1), + event_ltime: Some(1), + query_ltime: Some(1), + status_ltimes: vec![NodeStatusTime { + id: Some(bytes::Bytes::new()), + ltime: None, // absent ltime — must be rejected + ..Default::default() + }], + ..Default::default() + }; + assert!( + push_pull_from_pb::(&pb).is_err(), + "expected BridgeError::MissingField for absent NodeStatusTime.ltime" + ); +} + // ── KeyRequestMessage ───────────────────────────────────────────────────────── #[cfg(feature = "aes-gcm")] @@ -967,11 +968,16 @@ fn relay_message_roundtrip_pb() { let encoded = pb.encode_to_vec(); let decoded_pb = PbRelayMessage::decode_from_slice(encoded.as_slice()).expect("decode_from_slice failed"); - let roundtripped: RelayMessage = - relay_from_pb(&decoded_pb).expect("relay_from_pb failed"); + let roundtripped: RelayMessage = relay_from_pb(&decoded_pb).expect("relay_from_pb failed"); - assert_eq!(roundtripped.destination.id_ref(), typed.destination.id_ref()); - assert_eq!(roundtripped.destination.addr_ref(), typed.destination.addr_ref()); + assert_eq!( + roundtripped.destination.id_ref(), + typed.destination.id_ref() + ); + assert_eq!( + roundtripped.destination.addr_ref(), + typed.destination.addr_ref() + ); assert_eq!(roundtripped.payload, payload); } @@ -984,8 +990,7 @@ fn relay_message_empty_payload_roundtrip() { let encoded = pb.encode_to_vec(); let decoded_pb = PbRelayMessage::decode_from_slice(encoded.as_slice()).expect("decode_from_slice failed"); - let roundtripped: RelayMessage = - relay_from_pb(&decoded_pb).expect("relay_from_pb failed"); + let roundtripped: RelayMessage = relay_from_pb(&decoded_pb).expect("relay_from_pb failed"); assert!(roundtripped.payload.is_empty()); } @@ -996,7 +1001,7 @@ fn relay_message_empty_destination_rejected() { // relay_from_pb: an empty byte slice is not a valid encoded Node, // so data_from_bytes returns a Data decode error. let pb = PbRelayMessage { - destination: bytes::Bytes::new(), + destination: Some(bytes::Bytes::new()), payload: bytes::Bytes::from_static(b"some-payload"), ..Default::default() }; diff --git a/serf-proto/src/typed/mod.rs b/serf-proto/src/typed/mod.rs index 83e8c932..9e7cd5b7 100644 --- a/serf-proto/src/typed/mod.rs +++ b/serf-proto/src/typed/mod.rs @@ -116,7 +116,12 @@ where where T: IntoIterator, { - Self(iter.into_iter().map(|(k, v)| (k.into(), v.into())).collect()) + Self( + iter + .into_iter() + .map(|(k, v)| (k.into(), v.into())) + .collect(), + ) } } @@ -226,11 +231,11 @@ impl QueryResponseMessage { /// A single named user event with an optional payload. /// -/// Mirrors the legacy `serf-core` `UserEvent` struct. +/// Mirrors the legacy `serf-core` `UserEvent` struct (name + payload only). +/// The "can coalesce" flag lives on the broadcast wrapper [`UserEventMessage`], +/// not on individual buffered entries. #[derive(Debug, Clone, PartialEq, Eq)] pub struct UserEvent { - /// Whether the event may be coalesced with later identical events. - pub cc: bool, /// The event name. pub name: SmolStr, /// The event payload. @@ -313,7 +318,10 @@ impl PushPullMessage { /// /// Requires the `aes-gcm` or `chacha20-poly1305` feature. #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] -#[cfg_attr(docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))))] +#[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) +)] #[derive(Debug, Clone, PartialEq, Eq)] pub struct KeyRequestMessage { /// The encryption key, or `None` for a list-keys request. @@ -337,8 +345,11 @@ impl KeyRequestMessage { /// /// Requires the `aes-gcm` or `chacha20-poly1305` feature. #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] -#[cfg_attr(docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))))] -#[derive(Debug, Clone, PartialEq, Eq)] +#[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) +)] +#[derive(Debug, Default, Clone, PartialEq, Eq)] pub struct KeyResponseMessage { /// `true` if the operation succeeded on this node. pub result: bool, @@ -354,12 +365,7 @@ pub struct KeyResponseMessage { impl KeyResponseMessage { /// Construct a default (failure, no keys) `KeyResponseMessage`. pub fn new() -> Self { - Self { - result: false, - message: SmolStr::default(), - keys: Vec::new(), - primary_key: None, - } + Self::default() } } From 8c79ff730f9f251743717063031029c16241f38c Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 26 Jun 2026 12:56:55 +1200 Subject: [PATCH 009/128] feat(serf-proto): serf Endpoint super-machine over memberlist-proto Pure Sans-I/O serf state machine over memberlist_proto::Endpoint: membership FSM, three Lamport clocks, user events, queries/responses/relay, conflict and key-management interceptors, push-pull anti-entropy, Vivaldi coordinates, and snapshot replay. --- serf-proto/Cargo.toml | 16 + serf-proto/src/any/mod.rs | 45 +- serf-proto/src/any/tests.rs | 13 + serf-proto/src/bridge/mod.rs | 11 +- serf-proto/src/coordinate_client/mod.rs | 602 ++ serf-proto/src/coordinate_client/tests.rs | 586 ++ serf-proto/src/endpoint/mod.rs | 5021 +++++++++++++ serf-proto/src/endpoint/serf_parity_tests.rs | 520 ++ serf-proto/src/endpoint/tests.rs | 6621 ++++++++++++++++++ serf-proto/src/event/mod.rs | 499 ++ serf-proto/src/event/tests.rs | 33 + serf-proto/src/framing/mod.rs | 37 + serf-proto/src/lib.rs | 55 +- serf-proto/src/members/mod.rs | 404 ++ serf-proto/src/members/tests.rs | 178 + serf-proto/src/options/mod.rs | 438 ++ serf-proto/src/options/tests.rs | 117 + serf-proto/src/snapshot/mod.rs | 580 ++ serf-proto/src/snapshot/tests.rs | 487 ++ serf-proto/src/typed/mod.rs | 80 +- 20 files changed, 16273 insertions(+), 70 deletions(-) create mode 100644 serf-proto/src/coordinate_client/mod.rs create mode 100644 serf-proto/src/coordinate_client/tests.rs create mode 100644 serf-proto/src/endpoint/mod.rs create mode 100644 serf-proto/src/endpoint/serf_parity_tests.rs create mode 100644 serf-proto/src/endpoint/tests.rs create mode 100644 serf-proto/src/event/mod.rs create mode 100644 serf-proto/src/event/tests.rs create mode 100644 serf-proto/src/members/mod.rs create mode 100644 serf-proto/src/members/tests.rs create mode 100644 serf-proto/src/options/mod.rs create mode 100644 serf-proto/src/options/tests.rs create mode 100644 serf-proto/src/snapshot/mod.rs create mode 100644 serf-proto/src/snapshot/tests.rs diff --git a/serf-proto/Cargo.toml b/serf-proto/Cargo.toml index 44347f1c..e39a1e30 100644 --- a/serf-proto/Cargo.toml +++ b/serf-proto/Cargo.toml @@ -7,6 +7,7 @@ repository.workspace = true rust-version.workspace = true [features] +default = ["tag-regex"] # Encryption key management messages (KeyRequestMessage / KeyResponseMessage). # Pulls in the memberlist-proto encryption backend (at least one AEAD cipher # must be selected; both are forwarded here so the caller can pick). @@ -17,15 +18,30 @@ aes-gcm = ["memberlist-proto/aes-gcm", "dep:zeroize"] chacha20-poly1305 = ["memberlist-proto/chacha20-poly1305", "dep:zeroize"] # Umbrella that activates both backends (convenient for tests / CI). encryption = ["aes-gcm", "chacha20-poly1305"] +# Vivaldi network coordinate estimation. Enabled by default under std. +# Disable for embedded / no_std targets or when the coordinate overhead is +# not desired. +coordinates = [] +# Regex-backed tag-filter matching (mirrors Go serf `regexp.MatchString`). +# When enabled (default), a `Filter::Tag` expression is compiled as a regex +# and matched with partial (anywhere-in-value) semantics. When disabled +# (`--no-default-features`), tag matching falls back to exact string equality. +# This crate currently requires std regardless of this flag; disabling +# `tag-regex` is the forward-compatible preparation for a future no_std port, +# not current no_std support. +tag-regex = ["dep:regex"] [dependencies] memberlist-proto = { git = "https://github.com/al8n/memberlist", branch = "main", default-features = false, features = ["std"] } bytes = { workspace = true } +derive_more = { workspace = true } +rand = { workspace = true } thiserror.workspace = true bitflags = "2" buffa = { version = "0.7", default-features = false } smol_str = { workspace = true } zeroize = { version = "1", optional = true } +regex = { version = "1", optional = true } [build-dependencies] buffa-build = "0.7" diff --git a/serf-proto/src/any/mod.rs b/serf-proto/src/any/mod.rs index cb5b6ded..003a4cf4 100644 --- a/serf-proto/src/any/mod.rs +++ b/serf-proto/src/any/mod.rs @@ -132,6 +132,7 @@ where A: Data, { /// Returns the [`MessageType`] tag for this message variant. + #[cfg(test)] pub fn message_type(&self) -> MessageType { match self { Self::Leave(_) => MessageType::Leave, @@ -216,10 +217,33 @@ where Ok(Bytes::from(frame_vec)) } + /// Decode the leading serf frame in `buf` into an [`AnyMessage`], also + /// returning the number of bytes consumed from `buf`. + /// + /// Same as [`AnyMessage::decode`] but surfaces the consumed byte count so + /// callers can verify that the frame is the only content in the buffer + /// (exact-consumption check: `consumed == buf.len()` must hold on the gossip + /// ingress path to reject packets with trailing junk). + /// + /// # Errors + /// + /// See [`AnyMessage::decode`]. + pub(crate) fn decode_with_consumed(buf: &Bytes) -> Result<(Self, usize), DecodeError> + where + I: Data, + A: Data, + { + let (ty, body, consumed) = decode_message(buf)?; + let msg = Self::decode_body(ty, body)?; + Ok((msg, consumed)) + } + /// Decode the leading serf frame in `buf` into an [`AnyMessage`]. /// - /// Splits the tag byte and body with the crate-internal frame decoder, then - /// dispatches on the tag to the appropriate buffa decoder and bridge function. + /// All production decode sites use [`AnyMessage::decode_with_consumed`] for + /// exact-consumption enforcement; this convenience wrapper is retained for + /// round-trip tests in `any/tests.rs` and the test adapter in + /// `endpoint/mod.rs`. /// /// # Errors /// @@ -231,14 +255,27 @@ where /// was out of range. /// - [`DecodeError::UnknownTag`] — the tag byte is not recognised by this /// build. + #[cfg(test)] pub fn decode(buf: &Bytes) -> Result where I: Data, A: Data, { - use buffa::Message as _; - let (ty, body, _consumed) = decode_message(buf)?; + Self::decode_body(ty, body) + } + + /// Dispatch-decode a typed body given the tag and the raw body bytes. + /// + /// Shared implementation used by both [`AnyMessage::decode`] and + /// [`AnyMessage::decode_with_consumed`] to avoid duplicating the match arm + /// logic. + fn decode_body(ty: MessageType, body: Bytes) -> Result + where + I: Data, + A: Data, + { + use buffa::Message as _; match ty { MessageType::Leave => { diff --git a/serf-proto/src/any/tests.rs b/serf-proto/src/any/tests.rs index 8bda09cf..af8fef6e 100644 --- a/serf-proto/src/any/tests.rs +++ b/serf-proto/src/any/tests.rs @@ -457,6 +457,19 @@ fn any_message_encode_key_response_round_trip() { } } +// ─── Visibility seal test ───────────────────────────────────────────────────── + +#[test] +fn sealed_types_are_crate_internal() { + // In-crate: the sealed types still EXIST after the seal (resolve here), + // while no longer being `pub` to dependents. + fn _assert_crate_visible( + _: core::marker::PhantomData>, + ) { + } + fn _assert_public(_: core::marker::PhantomData) {} +} + // ─── Error cases ────────────────────────────────────────────────────────────── #[test] diff --git a/serf-proto/src/bridge/mod.rs b/serf-proto/src/bridge/mod.rs index 764c649d..e6f3d0aa 100644 --- a/serf-proto/src/bridge/mod.rs +++ b/serf-proto/src/bridge/mod.rs @@ -15,8 +15,8 @@ use memberlist_proto::{ }; use smol_str::SmolStr; -#[cfg(test)] -use crate::typed::{Coordinate, Tags}; +#[cfg(any(feature = "coordinates", test))] +use crate::typed::Coordinate; #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] use crate::typed::{KeyRequestMessage, KeyResponseMessage}; use crate::{ @@ -24,7 +24,7 @@ use crate::{ messages::serf::v1 as pb, typed::{ ConflictResponseMessage, Filter, JoinMessage, LeaveMessage, PushPullMessage, QueryFlag, - QueryMessage, QueryResponseMessage, RelayMessage, TagFilter, UserEvent, UserEventMessage, + QueryMessage, QueryResponseMessage, RelayMessage, TagFilter, Tags, UserEvent, UserEventMessage, UserEvents, }, }; @@ -83,7 +83,7 @@ pub(crate) fn user_event_from_pb( // ─── Coordinate ────────────────────────────────────────────────────────────── /// Convert a typed [`Coordinate`] → `pb::Coordinate`. -#[cfg(test)] +#[cfg(any(feature = "coordinates", test))] pub(crate) fn coordinate_to_pb(t: &Coordinate) -> pb::Coordinate { pb::Coordinate { portion: t.vec.clone(), @@ -95,7 +95,7 @@ pub(crate) fn coordinate_to_pb(t: &Coordinate) -> pb::Coordinate { } /// Convert `pb::Coordinate` → typed [`Coordinate`]. -#[cfg(test)] +#[cfg(any(feature = "coordinates", test))] pub(crate) fn coordinate_from_pb(b: &pb::Coordinate) -> Coordinate { Coordinate { vec: b.portion.clone(), @@ -121,7 +121,6 @@ pub(crate) fn tags_to_pb(t: &Tags) -> pb::Tags { } /// Convert `pb::Tags` → typed [`Tags`]. -#[cfg(test)] pub(crate) fn tags_from_pb(b: &pb::Tags) -> Tags { Tags( b.entries diff --git a/serf-proto/src/coordinate_client/mod.rs b/serf-proto/src/coordinate_client/mod.rs new file mode 100644 index 00000000..7120f6f4 --- /dev/null +++ b/serf-proto/src/coordinate_client/mod.rs @@ -0,0 +1,602 @@ +//! Pure Vivaldi-based network coordinate engine. +//! +//! Ported faithfully from `legacy/serf-core/src/types/coordinate.rs`. +//! +//! This module is a *pure* numeric engine — no I/O, no async, no wall-clock +//! reads, and no global RNG draws. Any randomness (the jitter applied when two +//! coordinates occupy the same point) is drawn from an injected [`Rng`]. +//! +//! References used throughout: +//! - \[1\] Dabek, Frank, et al. "Vivaldi: A decentralized network coordinate +//! system." ACM SIGCOMM 2004. +//! - \[2\] Ledlie, Jonathan, Paul Gardner, and Margo I. Seltzer. "Network +//! Coordinates in the Wild." NSDI 2007. +//! - \[3\] Lee, Sanghwan, et al. "On suitability of Euclidean embedding for +//! host-based network coordinate systems." IEEE/ACM Transactions on +//! Networking, 2010. + +use std::{collections::HashMap, time::Duration}; + +use memberlist_proto::Rng; +use rand::RngExt; + +use crate::Coordinate; + +/// Used to decide if two coordinates are on top of each other. +const ZERO_THRESHOLD: f64 = 1.0e-6; + +/// Convert float seconds to nanoseconds. +const SECONDS_TO_NANOSECONDS: f64 = 1.0e9; + +// ── CoordinateOptions ──────────────────────────────────────────────────────── + +/// Tuning parameters for the Vivaldi-based coordinate mapping algorithm. +/// +/// The default values are suitable for basic algorithm testing but are not +/// tuned for any particular cluster topology. +/// +/// See `CoordinateClient::new` for the concrete defaults. +#[derive(Debug, Clone, PartialEq)] +#[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] +pub struct CoordinateOptions { + /// Dimensionality of the coordinate space. + /// + /// Per [2], 8 dimensions plus a non-Euclidean height gives a good + /// accuracy/complexity balance. + dimensionality: usize, + /// Upper bound on the error value. + /// + /// Serves as the initial error estimate when a node has not yet made any + /// observations, and caps runaway error growth. + vivaldi_error_max: f64, + /// Controls the maximum impact an observation has on error confidence. + /// + /// See [1] for derivation. + vivaldi_ce: f64, + /// Controls the maximum impact an observation has on the coordinate itself. + /// + /// See [1] for derivation. + vivaldi_cc: f64, + /// Number of RTT samples retained per observation window for the adjustment + /// factor described in [3]. Setting this to zero disables the feature. + adjustment_window_size: usize, + /// Minimum value for the height parameter. + /// + /// Always positive; introduces a small error so the value should be + /// relatively small compared to typical coordinates. + height_min: f64, + /// Maximum RTT samples retained per peer for computing the median + /// latency-filter value. + /// + /// The intent is to smooth over measurement blips. See [2]. + latency_filter_size: usize, + /// Gravity coefficient. + /// + /// Determines how strongly coordinates drift back toward the origin to + /// combat unbounded drift. See [2]. + gravity_rho: f64, +} + +impl Default for CoordinateOptions { + fn default() -> Self { + Self::new() + } +} + +impl CoordinateOptions { + /// Returns default `CoordinateOptions` tuned for basic algorithm testing. + pub fn new() -> Self { + Self { + dimensionality: 8, + vivaldi_error_max: 1.5, + vivaldi_ce: 0.25, + vivaldi_cc: 0.25, + adjustment_window_size: 20, + height_min: 10.0e-6, + latency_filter_size: 3, + gravity_rho: 150.0, + } + } + + /// Returns the dimensionality of the coordinate space. + pub const fn dimensionality(&self) -> usize { + self.dimensionality + } + + /// Sets the dimensionality of the coordinate space. + /// + /// A value of 0 is clamped to 1: the `unit_vector_at` coincident-point + /// retry loop iterates over the component vector; a zero-length vector + /// yields no components to randomize, so `jmag` stays zero and the loop + /// spins forever — a CPU hang. 1 dimension is the minimum coordinate system. + /// Mirrors `with_latency_filter_size`'s `.max(1)` guard. + pub fn with_dimensionality(mut self, v: usize) -> Self { + self.dimensionality = v.max(1); + self + } + + /// Returns the maximum error value. + pub const fn vivaldi_error_max(&self) -> f64 { + self.vivaldi_error_max + } + + /// Sets the maximum error value. + pub fn with_vivaldi_error_max(mut self, v: f64) -> Self { + self.vivaldi_error_max = v; + self + } + + /// Returns the `vivaldi_ce` confidence tuning factor. + pub const fn vivaldi_ce(&self) -> f64 { + self.vivaldi_ce + } + + /// Sets the `vivaldi_ce` confidence tuning factor. + pub fn with_vivaldi_ce(mut self, v: f64) -> Self { + self.vivaldi_ce = v; + self + } + + /// Returns the `vivaldi_cc` position tuning factor. + pub const fn vivaldi_cc(&self) -> f64 { + self.vivaldi_cc + } + + /// Sets the `vivaldi_cc` position tuning factor. + pub fn with_vivaldi_cc(mut self, v: f64) -> Self { + self.vivaldi_cc = v; + self + } + + /// Returns the adjustment window size. + pub const fn adjustment_window_size(&self) -> usize { + self.adjustment_window_size + } + + /// Sets the adjustment window size. + pub fn with_adjustment_window_size(mut self, v: usize) -> Self { + self.adjustment_window_size = v; + self + } + + /// Returns the minimum height value. + pub const fn height_min(&self) -> f64 { + self.height_min + } + + /// Sets the minimum height value. + pub fn with_height_min(mut self, v: f64) -> Self { + self.height_min = v; + self + } + + /// Returns the latency filter window size (samples per peer). + pub const fn latency_filter_size(&self) -> usize { + self.latency_filter_size + } + + /// Sets the latency filter window size. + /// + /// A value of 0 is clamped to 1: `latency_filter` computes a median on the + /// sliding window and indexes `tmp[tmp.len() / 2]`; a zero-length window + /// would push a sample, evict it immediately (len > 0 → remove), then index + /// an empty vec — a panic. Mirrors `EventBuffer::new`'s `.max(1)` guard. + pub fn with_latency_filter_size(mut self, v: usize) -> Self { + self.latency_filter_size = v.max(1); + self + } + + /// Returns the gravity coefficient. + pub const fn gravity_rho(&self) -> f64 { + self.gravity_rho + } + + /// Sets the gravity coefficient. + pub fn with_gravity_rho(mut self, v: f64) -> Self { + self.gravity_rho = v; + self + } +} + +// ── CoordinateError ────────────────────────────────────────────────────────── + +/// Errors returned by [`CoordinateClient`] operations. +#[derive(Debug, thiserror::Error, PartialEq, Eq)] +#[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] +pub enum CoordinateError { + /// The two coordinates have incompatible dimensionalities. + #[error("coordinate dimensions are not compatible")] + DimensionalityMismatch, + /// A coordinate contains non-finite (NaN or infinite) values. + #[error("coordinate contains invalid (non-finite) values")] + InvalidCoordinate, + /// The supplied RTT is outside the valid range [0, 10s]. + #[error("round-trip time {0:?} is not in the valid range (must be ≤ 10 seconds)")] + InvalidRtt(Duration), +} + +// ── CoordinateClientStats ──────────────────────────────────────────────────── + +/// Counters recorded by [`CoordinateClient`] during updates. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +#[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] +pub struct CoordinateClientStats { + /// Number of times the coordinate was reset to the origin because an + /// update produced an invalid (non-finite) result. + resets: usize, +} + +impl CoordinateClientStats { + /// Returns the number of coordinate resets. + pub const fn resets(&self) -> usize { + self.resets + } +} + +// ── Coordinate helpers ─────────────────────────────────────────────────────── +// +// All maths operates on `Vec` to match the serf-proto `Coordinate::vec` +// field (the oracle uses `SmallVec<[f64; 8]>` which is equivalent at our dim). + +/// Add `rhs` into `lhs` in place (lhs += rhs, element-wise). +#[inline] +fn vec_add_in_place(lhs: &mut [f64], rhs: &[f64]) { + for (x, y) in lhs.iter_mut().zip(rhs) { + *x += y; + } +} + +/// Element-wise difference: returns lhs - rhs as an owned Vec. +#[inline] +fn vec_diff(lhs: &[f64], rhs: &[f64]) -> Vec { + lhs.iter().zip(rhs).map(|(x, y)| x - y).collect() +} + +/// Scale `vec` by `factor` in place, returning the slice. +#[inline] +fn vec_scale_in_place(vec: &mut [f64], factor: f64) -> &mut [f64] { + for x in vec.iter_mut() { + *x *= factor; + } + vec +} + +/// L2 magnitude of an iterator of `f64` values. +#[inline] +fn vec_magnitude(iter: impl Iterator) -> f64 { + iter.fold(0.0_f64, |acc, x| acc + x * x).sqrt() +} + +/// Returns a unit vector pointing *from* `b` *toward* `a`, plus the distance +/// between `a` and `b`. +/// +/// When the two points coincide (distance below [`ZERO_THRESHOLD`]), a random +/// unit direction is drawn from `rng` so that the update step does not stall. +/// The distance is reported as `0.0` in that case. +/// +/// FIX (oracle used `rand::rng()` / global thread-local): jitter now draws +/// from the caller's injected `rng`, keeping the engine fully deterministic +/// under test. +fn unit_vector_at(a: &[f64], b: &[f64], rng: &mut R) -> (Vec, f64) +where + R: Rng, +{ + let mut delta = vec_diff(a, b); + let mag = vec_magnitude(delta.iter().copied()); + + if mag > ZERO_THRESHOLD { + vec_scale_in_place(&mut delta, mag.recip()); + return (delta, mag); + } + + // Coincident points: draw a random direction so the update does not stall. + loop { + for x in delta.iter_mut() { + // Draw a uniform float in [0, 1) via the injected rng. + *x = rng.random_range::(0.0..1.0) - 0.5; + } + let jmag = vec_magnitude(delta.iter().copied()); + if jmag > ZERO_THRESHOLD { + vec_scale_in_place(&mut delta, jmag.recip()); + return (delta, 0.0); + } + // Extraordinarily unlikely to loop more than once; retry rather than + // silently emitting a zero vector. + } +} + +// ── Coordinate methods (extension helpers) ─────────────────────────────────── + +/// Raw distance between two coordinates (no adjustment, sum of Euclidean + +/// height terms). +fn raw_distance(a: &Coordinate, b: &Coordinate) -> f64 { + vec_magnitude(a.vec.iter().zip(&b.vec).map(|(x, y)| x - y)) + a.height + b.height +} + +/// Distance including both nodes' adjustment terms, floored to the raw value +/// if the sum goes negative. +fn adjusted_distance_secs(a: &Coordinate, b: &Coordinate) -> f64 { + let raw = raw_distance(a, b); + let adj = raw + a.adjustment + b.adjustment; + if adj > 0.0 { adj } else { raw } +} + +/// Apply force `f` (from `other`'s direction) to `coord` in place. +fn apply_force_in_place( + coord: &mut Coordinate, + height_min: f64, + force: f64, + other: &Coordinate, + rng: &mut R, +) where + R: Rng, +{ + let (mut unit, mag) = unit_vector_at(&coord.vec, &other.vec, rng); + vec_scale_in_place(&mut unit, force); + vec_add_in_place(&mut coord.vec, &unit); + + if mag > ZERO_THRESHOLD { + coord.height = (coord.height + other.height) * force / mag + coord.height; + coord.height = coord.height.max(height_min); + } +} + +// ── CoordinateClient ───────────────────────────────────────────────────────── + +/// Manages the estimated Vivaldi network coordinate for a local node. +/// +/// Adjusts the coordinate as the node observes round-trip times and remote +/// coordinates from peers. This is a *pure, single-threaded* engine: +/// +/// - No I/O, no async, no threads. +/// - No wall-clock reads — the RTT is always passed in by the caller. +/// - No global RNG — random jitter (for coincident-point handling) is drawn +/// from the `rng` parameter of [`update`]. +/// - `stats.resets` is a plain `usize` (no atomics — single-threaded machine). +#[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] +pub struct CoordinateClient { + /// Current coordinate estimate for this node. + coord: Coordinate, + /// Coordinate at the origin (all zeros) used for gravity updates. + origin: Coordinate, + /// Algorithm tuning parameters. + opts: CoordinateOptions, + /// Circular index into `adjustment_samples`. + adjustment_index: usize, + /// Rolling window of `(rtt - raw_distance)` samples used to compute the + /// adjustment factor described in [3]. + adjustment_samples: Vec, + /// Per-peer sliding windows of raw RTT samples for the latency filter. + latency_filter_samples: HashMap>, + /// Lifetime counters. + stats: CoordinateClientStats, +} + +impl CoordinateClient +where + I: Eq + core::hash::Hash + Clone, +{ + /// Creates a new client with the given tuning options. + pub fn new(opts: CoordinateOptions) -> Self { + let mut coord_vec = Vec::with_capacity(opts.dimensionality); + coord_vec.resize(opts.dimensionality, 0.0_f64); + + let coord = Coordinate { + vec: coord_vec.clone(), + error: opts.vivaldi_error_max, + adjustment: 0.0, + height: opts.height_min, + }; + let origin = Coordinate { + vec: coord_vec, + error: opts.vivaldi_error_max, + adjustment: 0.0, + height: opts.height_min, + }; + + let adj_size = if opts.adjustment_window_size > 0 { + opts.adjustment_window_size + } else { + 1 // avoid a zero-length vec; update_adjustment is a no-op when size==0 + }; + let adjustment_samples = vec![0.0_f64; adj_size]; + + Self { + coord, + origin, + opts, + adjustment_index: 0, + adjustment_samples, + latency_filter_samples: HashMap::new(), + stats: CoordinateClientStats::default(), + } + } + + /// Returns a copy of the current coordinate estimate. + pub fn get_coordinate(&self) -> Coordinate { + self.coord.clone() + } + + /// Forces the coordinate to a known state, for testing or snapshot restore. + /// + /// Returns an error if `coord` is incompatible with the current + /// dimensionality or contains non-finite values. + pub fn set_coordinate(&mut self, coord: Coordinate) -> Result<(), CoordinateError> { + Self::check_coordinate(&self.coord, &coord)?; + self.coord = coord; + Ok(()) + } + + /// Returns the estimated RTT from this node to `other`. + pub fn distance_to(&self, other: &Coordinate) -> Duration { + let secs = adjusted_distance_secs(&self.coord, other); + Duration::from_nanos((secs * SECONDS_TO_NANOSECONDS) as u64) + } + + /// Returns a copy of the lifetime counters. + pub fn stats(&self) -> CoordinateClientStats { + self.stats + } + + /// Removes any latency-filter history for `node` (call on member reap so + /// stale samples do not affect a rejoining peer). + pub fn forget_node(&mut self, node: &I) { + self.latency_filter_samples.remove(node); + } + + /// Observes a measured round-trip time (`rtt`) to a peer and updates the + /// local coordinate estimate. + /// + /// `rng` supplies the random jitter needed if this node and `other` happen + /// to occupy the same point in coordinate space (exceedingly rare in + /// practice). Passing the same seeded RNG produces deterministic results. + /// + /// Returns the updated coordinate on success. + /// + /// # Errors + /// + /// - [`CoordinateError::DimensionalityMismatch`] — `other` has a different + /// dimensionality than this client. + /// - [`CoordinateError::InvalidCoordinate`] — `other` contains a non-finite + /// value. + /// - [`CoordinateError::InvalidRtt`] — `rtt` exceeds 10 seconds (a hard cap + /// that guards against stale or wildly incorrect measurements). + pub fn update( + &mut self, + node: &I, + other: &Coordinate, + rtt: Duration, + rng: &mut R, + ) -> Result + where + R: Rng, + { + Self::check_coordinate(&self.coord, other)?; + + const MAX_RTT: Duration = Duration::from_secs(10); + if rtt > MAX_RTT { + return Err(CoordinateError::InvalidRtt(rtt)); + } + + // Zero RTTs are valid (coarse-grained monotonic clocks can produce them); + // the algorithm handles them gracefully via the ZERO_THRESHOLD floor. + + let rtt_seconds = self.latency_filter(node, rtt.as_secs_f64()); + self.update_vivaldi(other, rtt_seconds, rng); + self.update_adjustment(other, rtt_seconds); + self.update_gravity(rng); + + if !self.is_valid() { + // The update produced a degenerate coordinate; reset to the origin and + // count the event so callers can observe it. + self.stats.resets += 1; + let mut vec = Vec::with_capacity(self.opts.dimensionality); + vec.resize(self.opts.dimensionality, 0.0_f64); + self.coord = Coordinate { + vec, + error: self.opts.vivaldi_error_max, + adjustment: 0.0, + height: self.opts.height_min, + }; + } + + Ok(self.coord.clone()) + } + + // ── private helpers ──────────────────────────────────────────────────────── + + /// Returns `true` if the current coordinate contains only finite values. + fn is_valid(&self) -> bool { + self.coord.vec.iter().all(|f| f.is_finite()) + && self.coord.error.is_finite() + && self.coord.adjustment.is_finite() + && self.coord.height.is_finite() + } + + /// Returns an error if `coord` is dimensionally incompatible with `reference` + /// or contains non-finite values. + fn check_coordinate(reference: &Coordinate, coord: &Coordinate) -> Result<(), CoordinateError> { + if reference.vec.len() != coord.vec.len() { + return Err(CoordinateError::DimensionalityMismatch); + } + let valid = coord.vec.iter().all(|f| f.is_finite()) + && coord.error.is_finite() + && coord.adjustment.is_finite() + && coord.height.is_finite(); + if !valid { + return Err(CoordinateError::InvalidCoordinate); + } + Ok(()) + } + + /// Returns the median of the per-peer RTT sample window, inserting `rtt` + /// first. Older samples are evicted when the window is full (sliding + /// window of size `latency_filter_size`). + fn latency_filter(&mut self, node: &I, rtt_seconds: f64) -> f64 { + let size = self.opts.latency_filter_size; + let samples = self + .latency_filter_samples + .entry(node.clone()) + .or_insert_with(|| Vec::with_capacity(size)); + + samples.push(rtt_seconds); + if samples.len() > size { + samples.remove(0); + } + + let mut tmp = samples.clone(); + tmp.sort_unstable_by(|a, b| a.partial_cmp(b).unwrap_or(core::cmp::Ordering::Equal)); + tmp[tmp.len() / 2] + } + + /// Applies the Vivaldi spring force to move the local coordinate toward + /// `other`, weighted by confidence. See [1]. + fn update_vivaldi(&mut self, other: &Coordinate, rtt_seconds: f64, rng: &mut R) + where + R: Rng, + { + let dist = adjusted_distance_secs(&self.coord, other); + let rtt_seconds = rtt_seconds.max(ZERO_THRESHOLD); + + let wrongness = ((dist - rtt_seconds) / rtt_seconds).abs(); + + let total_error = (self.coord.error + other.error).max(ZERO_THRESHOLD); + let weight = self.coord.error / total_error; + + self.coord.error = ((self.opts.vivaldi_ce * weight * wrongness) + + self.coord.error * (1.0 - self.opts.vivaldi_ce * weight)) + .min(self.opts.vivaldi_error_max); + + let force = self.opts.vivaldi_cc * weight * (rtt_seconds - dist); + apply_force_in_place(&mut self.coord, self.opts.height_min, force, other, rng); + } + + /// Updates the rolling adjustment factor window ([3]). No-op when + /// `adjustment_window_size == 0`. + fn update_adjustment(&mut self, other: &Coordinate, rtt_seconds: f64) { + if self.opts.adjustment_window_size == 0 { + return; + } + let dist = raw_distance(&self.coord, other); + self.adjustment_samples[self.adjustment_index] = rtt_seconds - dist; + self.adjustment_index = (self.adjustment_index + 1) % self.opts.adjustment_window_size; + + self.coord.adjustment = + self.adjustment_samples.iter().sum::() / (2.0 * self.opts.adjustment_window_size as f64); + } + + /// Applies a small gravity force pulling the coordinate back toward the + /// origin to combat long-term drift ([2]). + fn update_gravity(&mut self, rng: &mut R) + where + R: Rng, + { + let dist = adjusted_distance_secs(&self.origin, &self.coord); + let force = -f64::powi(dist / self.opts.gravity_rho, 2); + let origin = self.origin.clone(); + apply_force_in_place(&mut self.coord, self.opts.height_min, force, &origin, rng); + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-proto/src/coordinate_client/tests.rs b/serf-proto/src/coordinate_client/tests.rs new file mode 100644 index 00000000..b3d025ce --- /dev/null +++ b/serf-proto/src/coordinate_client/tests.rs @@ -0,0 +1,586 @@ +//! Unit tests for the pure Vivaldi `CoordinateClient` engine. + +use std::time::Duration; + +use memberlist_proto::SmallRng; +use rand::SeedableRng; + +use crate::Coordinate; + +use super::{ + CoordinateClient, CoordinateClientStats, CoordinateError, CoordinateOptions, ZERO_THRESHOLD, +}; + +// ── Helpers ─────────────────────────────────────────────────────────────────── + +/// Returns a deterministic seeded RNG for use in tests. +fn test_rng() -> SmallRng { + SmallRng::seed_from_u64(0xdead_beef_cafe_babe) +} + +/// Returns a `CoordinateOptions` with the given dimensionality and zero +/// height_min (makes distance assertions easier). +fn opts_dim(d: usize) -> CoordinateOptions { + CoordinateOptions::new() + .with_dimensionality(d) + .with_height_min(0.0) +} + +/// Build a zero coordinate with `dim` dimensions. +fn zero_coord(dim: usize) -> Coordinate { + Coordinate { + vec: vec![0.0; dim], + error: CoordinateOptions::new().vivaldi_error_max(), + adjustment: 0.0, + height: 0.0, + } +} + +/// Assert two floats are within `ZERO_THRESHOLD` of each other. +fn assert_float_eq(a: f64, b: f64) { + assert!( + (a - b).abs() <= ZERO_THRESHOLD, + "float mismatch: {a:.9} != {b:.9}" + ); +} + +// ── Construction ────────────────────────────────────────────────────────────── + +/// A freshly constructed client begins at the origin with default error and +/// zero height (when `height_min` is 0). +#[test] +fn new_client_starts_at_origin() { + let opts = opts_dim(3); + let c = CoordinateClient::::new(opts.clone()); + let coord = c.get_coordinate(); + assert_eq!(coord.vec, vec![0.0; 3]); + assert_float_eq(coord.error, opts.vivaldi_error_max()); + assert_float_eq(coord.adjustment, 0.0); +} + +/// `default()` and `new()` produce identical defaults. +#[test] +fn options_default_matches_new() { + let a = CoordinateOptions::default(); + let b = CoordinateOptions::new(); + assert_eq!(a, b); +} + +// ── Update convergence ──────────────────────────────────────────────────────── + +/// After one update, the coordinate moves to reduce the error between the +/// predicted and observed RTT. Here the peer is above the origin and the +/// observed RTT is twice the peer's displacement — the client should scoot +/// *away* from the peer (downward in dim 2) to increase its predicted distance. +#[test] +fn coordinate_update_moves_toward_peer() { + let opts = opts_dim(3); + let mut c = CoordinateClient::::new(opts.clone()); + let before = c.get_coordinate(); + + let mut peer = zero_coord(3); + peer.vec[2] = 0.001; // peer is 0.001 s away in dim 2 + + let rtt = Duration::from_nanos((2.0 * peer.vec[2] * 1.0e9) as u64); + let mut rng = test_rng(); + let updated = c.update(&2u32, &peer, rtt, &mut rng).unwrap(); + + // The client should have moved (coordinate changed from origin). + assert_ne!(updated.vec, before.vec); + // Specifically, it should move away from the peer (dim 2 goes negative). + assert!( + updated.vec[2] < 0.0, + "client should move away from peer in dim 2" + ); +} + +/// Repeated updates between two symmetrically placed nodes converge: the +/// coordinate difference should approach the true RTT (within some tolerance). +#[test] +fn update_converges_over_multiple_observations() { + let opts = CoordinateOptions::new() + .with_dimensionality(3) + .with_height_min(0.0); + let mut a = CoordinateClient::::new(opts.clone()); + let mut b = CoordinateClient::::new(opts); + + let true_rtt = Duration::from_millis(20); + let mut rng = test_rng(); + + for _ in 0..100 { + let coord_b = b.get_coordinate(); + a.update(&1u32, &coord_b, true_rtt, &mut rng).unwrap(); + let coord_a = a.get_coordinate(); + b.update(&0u32, &coord_a, true_rtt, &mut rng).unwrap(); + } + + let predicted = a.distance_to(&b.get_coordinate()); + let diff_ms = (predicted.as_secs_f64() - true_rtt.as_secs_f64()).abs() * 1000.0; + // After 100 rounds the prediction should be within 5 ms of the true RTT. + assert!( + diff_ms < 5.0, + "convergence failed: predicted {predicted:?}, true {true_rtt:?}, diff {diff_ms:.3} ms" + ); +} + +// ── distance_to ─────────────────────────────────────────────────────────────── + +/// `distance_to` is symmetric: distance from C1 to C2 equals distance from C2 to C1. +/// +/// Both `Coordinate::distance_to` and `CoordinateClient::distance_to` are tested since +/// the `Coordinate` wire type carries adjustments that feed into the symmetric formula. +#[test] +fn distance_is_symmetric() { + let opts = opts_dim(3); + + // Two clients both starting at the origin; give them different coordinates manually. + let mut c1 = Coordinate { + vec: vec![1.0, 2.0, 3.0], + error: 1.5, + adjustment: 0.0, + height: 0.0, + }; + let mut c2 = Coordinate { + vec: vec![4.0, -1.0, 0.0], + error: 1.5, + adjustment: 0.0, + height: 0.0, + }; + + // Test at the Coordinate level (both directions). + let mut a = CoordinateClient::::new(opts.clone()); + let mut b = CoordinateClient::::new(opts); + + a.set_coordinate(c1.clone()).unwrap(); + b.set_coordinate(c2.clone()).unwrap(); + + let d_ab = a.distance_to(&c2); + let d_ba = b.distance_to(&c1); + assert_eq!( + d_ab, d_ba, + "distance must be symmetric: {d_ab:?} != {d_ba:?}" + ); + + // Also confirm the formula is symmetric when adjustments are non-zero. + c1.adjustment = 0.001; + c2.adjustment = 0.002; + a.set_coordinate(c1.clone()).unwrap(); + b.set_coordinate(c2.clone()).unwrap(); + let d_adj_ab = a.distance_to(&c2); + let d_adj_ba = b.distance_to(&c1); + assert_eq!( + d_adj_ab, d_adj_ba, + "distance must be symmetric with adjustments" + ); +} + +/// With zero height, a coordinate exactly `x` seconds away in one dimension +/// should report a distance of `x` seconds. +#[test] +fn distance_to_exact_value_when_height_zero() { + let opts = opts_dim(3).with_height_min(0.0); + let a = CoordinateClient::::new(opts.clone()); + + let mut other = zero_coord(3); + other.height = 0.0; + other.adjustment = 0.0; + other.vec[2] = 12.345; + + let expected = Duration::from_nanos((12.345 * 1.0e9) as u64); + let got = a.distance_to(&other); + assert_eq!(got, expected); +} + +// ── Height term ─────────────────────────────────────────────────────────────── + +/// When `height_min > 0` the height term introduces a positive floor on the +/// distance estimate even for co-located coordinates. +#[test] +fn height_term_adds_floor_to_distance() { + // Use CoordinateOptions::new() so that height_min = 10e-6 (non-zero). + let opts = CoordinateOptions::new().with_dimensionality(3); + let a = CoordinateClient::::new(opts.clone()); + let other = Coordinate { + vec: vec![0.0; 3], + error: opts.vivaldi_error_max(), + adjustment: 0.0, + height: opts.height_min(), + }; + // Both nodes have the same coordinate vector: distance = 2 * height_min. + let dist = a.distance_to(&other).as_secs_f64(); + assert!( + dist > 0.0, + "height term should produce a positive floor, got {dist}" + ); + assert_float_eq(dist, 2.0 * opts.height_min()); +} + +// ── Adjustment term ─────────────────────────────────────────────────────────── + +/// After several updates where the observed RTT differs from the predicted +/// distance, the adjustment field becomes non-zero. +#[test] +fn adjustment_becomes_nonzero_after_updates() { + let opts = CoordinateOptions::new() + .with_dimensionality(3) + .with_height_min(0.0); + let mut c = CoordinateClient::::new(opts); + + // Peer at the origin — the client starts at the origin too, so the + // Euclidean distance is 0 while we observe a non-zero RTT. The + // adjustment window will accumulate `rtt - raw_distance = 0.1 - ~0`, + // driving the adjustment to a positive value. + let peer = zero_coord(3); + + let mut rng = test_rng(); + for _ in 0..50 { + c.update(&42u32, &peer, Duration::from_millis(100), &mut rng) + .unwrap(); + } + + let coord = c.get_coordinate(); + // The adjustment must have drifted away from its initial zero. + assert_ne!( + coord.adjustment, 0.0, + "adjustment should be non-zero after updates with mismatched RTT" + ); +} + +/// When `adjustment_window_size == 0` the adjustment field stays at zero. +#[test] +fn adjustment_stays_zero_when_window_disabled() { + let opts = CoordinateOptions::new() + .with_dimensionality(3) + .with_height_min(0.0) + .with_adjustment_window_size(0); + let mut c = CoordinateClient::::new(opts); + + let mut peer = zero_coord(3); + peer.vec[2] = 0.05; + + let mut rng = test_rng(); + for _ in 0..10 { + c.update(&1u32, &peer, Duration::from_millis(50), &mut rng) + .unwrap(); + } + + assert_float_eq(c.get_coordinate().adjustment, 0.0); +} + +// ── Gravity term ────────────────────────────────────────────────────────────── + +/// The gravity term keeps the coordinate from drifting to infinity. After +/// many updates to a peer far from the origin the coordinate should remain +/// finite and not grow unboundedly. +#[test] +fn gravity_prevents_unbounded_drift() { + let opts = CoordinateOptions::new() + .with_dimensionality(3) + .with_height_min(0.0); + let mut c = CoordinateClient::::new(opts); + + // Peer sits far from the origin. + let peer = Coordinate { + vec: vec![1000.0, 0.0, 0.0], + error: 1.5, + adjustment: 0.0, + height: 0.0, + }; + let mut rng = test_rng(); + for _ in 0..200 { + // Provide an RTT consistent with the distance so Vivaldi pushes outward. + let _ = c.update(&1u32, &peer, Duration::from_secs(1), &mut rng); + } + + let coord = c.get_coordinate(); + assert!( + coord.vec.iter().all(|f| f.is_finite()), + "coordinate must stay finite" + ); + assert!( + coord.vec[0].abs() < 2000.0, + "gravity should prevent unbounded drift" + ); +} + +// ── Latency filter ──────────────────────────────────────────────────────────── + +/// The latency filter returns the median of the sliding window of RTT samples +/// and ages out old samples correctly. +#[test] +fn latency_filter_returns_median() { + let opts = CoordinateOptions::new().with_latency_filter_size(3); + let mut c = CoordinateClient::::new(opts); + + // First sample: median of [0.201] = 0.201. + assert_float_eq(c.latency_filter(&1u32, 0.201), 0.201); + // Second: median of [0.201, 0.200] = 0.201 (middle of sorted pair). + assert_float_eq(c.latency_filter(&1u32, 0.200), 0.201); + // Third: median of [0.201, 0.200, 0.207] sorted = [0.200, 0.201, 0.207] -> 0.201. + assert_float_eq(c.latency_filter(&1u32, 0.207), 0.201); + + // A glitch pushed in: window slides to [0.200, 0.207, 1.9] -> median 0.207. + assert_float_eq(c.latency_filter(&1u32, 1.9), 0.207); + // Next: [0.207, 1.9, 0.203] -> sorted [0.203, 0.207, 1.9] -> 0.207. + assert_float_eq(c.latency_filter(&1u32, 0.203), 0.207); +} + +/// Different peers have independent sample windows. +#[test] +fn latency_filter_independent_per_peer() { + let opts = CoordinateOptions::new().with_latency_filter_size(3); + let mut c = CoordinateClient::::new(opts); + + c.latency_filter(&1u32, 0.201); + c.latency_filter(&1u32, 0.200); + // Peer 2 has never been seen; first sample is the median. + assert_float_eq(c.latency_filter(&2u32, 0.310), 0.310); +} + +/// `forget_node` clears per-peer history so the next sample starts fresh. +#[test] +fn forget_node_clears_peer_history() { + let opts = CoordinateOptions::new().with_latency_filter_size(3); + let mut c = CoordinateClient::::new(opts); + + c.latency_filter(&1u32, 0.888); + c.latency_filter(&1u32, 0.888); + c.forget_node(&1u32); + // After forgetting, a new sample should be the only one in the window. + assert_float_eq(c.latency_filter(&1u32, 0.123), 0.123); +} + +// ── Error and stats bookkeeping ─────────────────────────────────────────────── + +/// `stats()` starts at zero and records resets when an update produces an +/// invalid result (NaN poisoned internally, then a valid update triggers the +/// reset path). +#[test] +fn stats_records_resets() { + let opts = CoordinateOptions::new().with_dimensionality(3); + let mut c = CoordinateClient::::new(opts.clone()); + + assert_eq!(c.stats().resets(), 0); + + // Poison the coordinate directly (simulates internal NaN production). + c.coord.vec[0] = f64::NAN; + + let peer = zero_coord(3); + let mut rng = test_rng(); + // The update should detect the invalid result, reset, and increment the counter. + let result = c + .update(&1u32, &peer, Duration::from_millis(250), &mut rng) + .unwrap(); + assert!( + result.vec.iter().all(|f| f.is_finite()), + "reset should produce valid coordinate" + ); + assert_eq!(c.stats().resets(), 1); +} + +/// `CoordinateClientStats::resets()` accessor works correctly. +#[test] +fn stats_struct_default_is_zero() { + let s = CoordinateClientStats::default(); + assert_eq!(s.resets(), 0); +} + +// ── NaN / invalid input defense ─────────────────────────────────────────────── + +/// Feeding a NaN coordinate from a peer is rejected before any mutation. +#[test] +fn nan_peer_coordinate_is_rejected() { + let opts = CoordinateOptions::new().with_dimensionality(3); + let mut c = CoordinateClient::::new(opts.clone()); + + let mut bad = zero_coord(3); + bad.vec[0] = f64::NAN; + + let mut rng = test_rng(); + let err = c + .update(&1u32, &bad, Duration::from_millis(250), &mut rng) + .unwrap_err(); + assert_eq!(err, CoordinateError::InvalidCoordinate); + // Client coordinate must be unchanged. + assert!(c.get_coordinate().vec.iter().all(|f| f.is_finite())); +} + +/// RTTs above 10 seconds are rejected. +#[test] +fn rtt_above_10s_is_rejected() { + let opts = CoordinateOptions::new().with_dimensionality(3); + let mut c = CoordinateClient::::new(opts); + let peer = zero_coord(3); + let mut rng = test_rng(); + + let err = c + .update(&1u32, &peer, Duration::from_secs(11), &mut rng) + .unwrap_err(); + assert!(matches!(err, CoordinateError::InvalidRtt(_))); +} + +/// A coordinate with mismatched dimensionality is rejected. +#[test] +fn dimensionality_mismatch_is_rejected() { + let opts = CoordinateOptions::new().with_dimensionality(3); + let mut c = CoordinateClient::::new(opts); + // Peer has 4 dimensions. + let alien = Coordinate { + vec: vec![0.0; 4], + error: 1.5, + adjustment: 0.0, + height: 0.0, + }; + let mut rng = test_rng(); + + let err = c + .update(&1u32, &alien, Duration::from_millis(10), &mut rng) + .unwrap_err(); + assert_eq!(err, CoordinateError::DimensionalityMismatch); +} + +/// `set_coordinate` rejects a coordinate with incompatible dimensions. +#[test] +fn set_coordinate_rejects_wrong_dimensions() { + let opts = CoordinateOptions::new().with_dimensionality(3); + let mut c = CoordinateClient::::new(opts); + let alien = Coordinate { + vec: vec![0.0; 6], + error: 1.5, + adjustment: 0.0, + height: 0.0, + }; + + let err = c.set_coordinate(alien).unwrap_err(); + assert_eq!(err, CoordinateError::DimensionalityMismatch); +} + +/// `set_coordinate` accepts a valid coordinate and updates the estimate. +#[test] +fn set_coordinate_updates_estimate() { + let opts = CoordinateOptions::new().with_dimensionality(3); + let mut c = CoordinateClient::::new(opts); + let new_coord = Coordinate { + vec: vec![1.0, 2.0, 3.0], + error: 0.5, + adjustment: 0.0, + height: 0.0, + }; + c.set_coordinate(new_coord.clone()).unwrap(); + assert_eq!(c.get_coordinate().vec, new_coord.vec); +} + +// ── Zero-RTT handling ───────────────────────────────────────────────────────── + +// ── FIX: zero latency_filter_size does not panic ────────────────────────────── + +/// `CoordinateOptions::with_latency_filter_size(0)` must be clamped to 1 so +/// that the first `update` call does not panic. +/// +/// Without the clamp: `latency_filter` pushes a sample (len = 1 > 0), then +/// removes index 0 (len = 0), then indexes `tmp[tmp.len() / 2]` = `tmp[0]` +/// on an empty vec — a bounds-check panic. +/// +/// Regression for `CoordinateOptions::with_latency_filter_size(0)`. +#[test] +fn latency_filter_size_zero_is_clamped_and_does_not_panic() { + let opts = CoordinateOptions::new() + .with_dimensionality(3) + .with_latency_filter_size(0); + // The clamp must have taken effect. + assert_eq!( + opts.latency_filter_size(), + 1, + "latency_filter_size(0) must be clamped to 1" + ); + + let mut c = CoordinateClient::::new(opts); + let peer = Coordinate { + vec: vec![0.0; 3], + error: CoordinateOptions::new().vivaldi_error_max(), + adjustment: 0.0, + height: 0.0, + }; + let mut rng = test_rng(); + // Must not panic. + let result = c.update(&1u32, &peer, Duration::from_millis(10), &mut rng); + assert!( + result.is_ok(), + "update with latency_filter_size clamped to 1 must not panic or error" + ); +} + +/// Zero-RTT observations are valid (can occur with coarse-grained monotonic +/// clocks) and do not panic or produce NaN. +#[test] +fn zero_rtt_is_handled_gracefully() { + let opts = CoordinateOptions::new().with_dimensionality(3); + let mut c = CoordinateClient::::new(opts); + let peer = zero_coord(3); + let mut rng = test_rng(); + + let result = c.update(&1u32, &peer, Duration::ZERO, &mut rng).unwrap(); + assert!(result.vec.iter().all(|f| f.is_finite())); +} + +// ── Coincident-point jitter ─────────────────────────────────────────────────── + +/// When the client and peer share the same position, `update` must not panic +/// (the `unit_vector_at` fallback applies random jitter from the injected rng). +#[test] +fn coincident_coordinates_do_not_panic() { + let opts = opts_dim(3); + let mut c = CoordinateClient::::new(opts.clone()); + + // Peer at same position as origin (all zeros). + let peer = Coordinate { + vec: vec![0.0; 3], + error: opts.vivaldi_error_max(), + adjustment: 0.0, + height: 0.0, + }; + let mut rng = test_rng(); + let result = c.update(&1u32, &peer, Duration::from_millis(10), &mut rng); + assert!( + result.is_ok(), + "coincident-point update should not panic or error" + ); +} + +// ── FIX 2: zero dimensionality must not cause an infinite spin ──────────────── + +/// `CoordinateOptions::with_dimensionality(0)` must be clamped to 1. +/// +/// Without the clamp: `CoordinateClient::new` builds a zero-length coordinate +/// vector; on `update`, `unit_vector_at` enters the coincident-point retry +/// loop. The loop iterates over the empty component vector (yielding nothing +/// to randomize), so `jmag` stays 0.0 forever — an infinite CPU spin. +/// +/// With the clamp the update completes in bounded time. +#[cfg(feature = "coordinates")] +#[test] +fn dimensionality_zero_is_clamped_to_one_and_update_does_not_spin() { + let opts = CoordinateOptions::new().with_dimensionality(0); + assert_eq!( + opts.dimensionality(), + 1, + "with_dimensionality(0) must be clamped to 1" + ); + + let mut c = CoordinateClient::::new(opts); + + // Peer also at dimensionality 1 (clamped), all-zero vector. + let peer = Coordinate { + vec: vec![0.0; 1], + error: CoordinateOptions::new().vivaldi_error_max(), + adjustment: 0.0, + height: 0.0, + }; + let mut rng = test_rng(); + // This call must return (not spin) within the test timeout. + let result = c.update(&1u32, &peer, Duration::from_millis(10), &mut rng); + assert!( + result.is_ok(), + "update with dimensionality clamped to 1 must not spin or error: {result:?}" + ); +} diff --git a/serf-proto/src/endpoint/mod.rs b/serf-proto/src/endpoint/mod.rs new file mode 100644 index 00000000..dc42fea6 --- /dev/null +++ b/serf-proto/src/endpoint/mod.rs @@ -0,0 +1,5021 @@ +//! The serf `Endpoint` — a pure Sans-I/O super-machine. +//! +//! Owns a [`memberlist_proto::Endpoint`] by value and drives it, intercepting +//! its gossip to implement serf's membership FSM, three Lamport clocks, user +//! events, queries/responses/relays, push-pull anti-entropy, and network +//! coordinates. +//! +//! The driver moves opaque `Bytes` in (`handle_packet`) and +//! `Transmit`/`Bytes` out (`poll_transmit`) and ticks `poll_timeout(now)`; +//! it carries zero serf logic. All wall-clock reads are threaded in as a +//! `now: Instant` parameter — no clock reads occur inside this module. +//! +//! # Threat model +//! +//! The machine sits **inside** the keyring trust boundary: cluster members are +//! mutually trusted through a shared AEAD keyring (joining requires possession +//! of the shared key). +//! +//! **In scope:** well-formed reordered, duplicate, or delayed gossip from +//! honest peers; malformed or truncated bytes (the machine never panics on bad +//! input — it drops); bounded memory under honest-but-pathological message +//! volume (floods of distinct event/query/intent ids from honest peers). +//! +//! **Out of scope:** Byzantine or compromised key-holding members (forged +//! responder identities in query acks, crafted Lamport values from a peer +//! that holds the keyring secret). This is consistent with Go serf, which +//! performs no such defenses. +//! +//! The [`LTIME_MAX`] watermark is an **integrity floor**: any Lamport time at +//! or above it is rejected at ingress as out-of-range, so the local clocks +//! can never be driven to `u64::MAX` via ordinary `+1` advancement. +//! `saturating_add` is the no-UB backstop for the rare cases where a stored +//! clock is already at `u64::MAX`. A value at or above `LTIME_MAX` cannot +//! arise organically — a clock starting at 0 and advancing by at most 1 per +//! event needs 2^63 events to reach `LTIME_MAX`, which is unreachable in any +//! finite cluster lifetime. A clock driven near or above `LTIME_MAX` by a +//! corrupt snapshot or crafted peer value parks there: its local emissions +//! are `>= LTIME_MAX` and are rejected by peers, a degraded-but-safe state +//! (no panic, hang, or wrap). Full functional recovery from such a state is +//! out of scope, consistent with upstream Go serf. The internal +//! conflict/key-tally membership gate (checking that a response comes from a +//! known cluster member before counting it) protects the self-shutdown control +//! decision; forged responder ids in application-query acks are out of threat +//! model and left to the driver. + +//! **Directed transmit output** (ACKs, relay sends, directed query responses) +//! is bounded per operation: at most one ACK plus at most `relay_factor` +//! (≤ 255) relay sends per received query. The cross-operation transmit +//! backlog — the inner `Endpoint`'s `pending_transmits` queue — is drained by +//! the driver via `poll_transmit` on every tick; that is the driver's +//! Sans-I/O contract. Per-source rate-limiting of a flooding peer is likewise +//! driver-side responsibility. + +use std::collections::VecDeque; + +use bytes::Bytes; +use memberlist_proto::{ + CheapClone, Data, EndpointEvent, Id, Instant, Node, PushPullKind, Rng, SeedableRng, SmallRng, + StreamCommand, Transmit, parse_message, typed::NodeState, +}; +use rand::RngExt; +use smol_str::SmolStr; + +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +use crate::KeyRequestMessage; +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +use crate::event::{KeyRequest as KeyRequestEvent, KeyRequestOperation, KeyResponseArgs}; +use crate::{ + AnyMessage, ConflictResponseMessage, EncodeError, LamportTime, MessageType, + bridge::{tags_from_pb, user_event_to_pb}, + event::{ + DialPassthrough, Event, MemberEvent, MemberEventKind, QueryAck, QueryEvent, + QueryResponse as QueryResponseEvent, + }, + framing::{encode_message, peek_frame_header}, + members::{IntentKind, Member, MemberState, MemberStatus, Members, SerfState}, + options::Options, + typed::{ + Filter, JoinMessage, LeaveMessage, PushPullMessage, QueryFlag, QueryMessage, + QueryResponseMessage, RelayMessage, Tags, UserEvent, UserEventMessage, UserEvents, + }, +}; + +// ── EventBuffer ────────────────────────────────────────────────────────────── + +/// Ring-buffer dedup store for received user events. +/// +/// Mirrors Go serf `types.go` `eventCore` (buffer + min_time). +/// The ring is indexed by `ltime % buffer.len()`. Each slot holds the +/// batch of `UserEvent` values that arrived at that lamport time, allowing +/// the machine to detect and discard exact duplicates (same name + payload) +/// at the same ltime without re-sending them. +struct EventBuffer { + /// Slots indexed by `ltime % len`. `None` = empty / never used. + buffer: Vec>, + /// Events with `ltime < min_time` are unconditionally dropped. + /// + /// Set to `old_event_clock + 1` when replaying a snapshot (H8/G5 recovery), + /// or to the current event-clock's value bumped by 1 after a push-pull that + /// carries an `eventJoinIgnore` flag (H8/G4). `0` means no floor is active. + min_time: u64, +} + +impl EventBuffer { + /// Allocate a ring of `size` empty slots with `min_time = 0`. + /// + /// A `size` of 0 is clamped to 1 — the ring modulus must never be zero. + /// Mirrors `QueryBuffer::new`'s `.max(1)` guard. + fn new(size: usize) -> Self { + Self { + buffer: vec![None; size.max(1)], + min_time: 0, + } + } + + /// Record and deduplicate one user event. + /// + /// Returns `true` if this is the **first sight** of `(ltime, ev)` (and + /// therefore the event should be rebroadcast and emitted locally). + /// + /// Returns `false` (drop) when: + /// - `ltime < self.min_time` (below the recovery floor). + /// - `ltime` is older than the entire ring relative to `cur_time` + /// (`cur_time > buffer_len && ltime < cur_time - buffer_len`). + /// - The exact `(name, payload)` pair already exists in the slot for this ltime. + /// + /// `cur_time` is the event clock value **after** it has been witnessed. + fn witness_event(&mut self, cur_time: u64, ltime: u64, ev: UserEvent) -> bool { + // Below min_time floor: drop. + if ltime < self.min_time { + return false; + } + + let bltime = self.buffer.len() as u64; + // Too old relative to the ring (mirrors Go serf `handle_user_event` check): + // if cur_time > bltime && ltime < cur_time - bltime { drop } + if cur_time > bltime && ltime < cur_time - bltime { + return false; + } + + let idx = (ltime % bltime) as usize; + + if let Some(slot) = &mut self.buffer[idx] { + if slot.ltime.0 == ltime { + // Same ltime: dedup against existing events. + for prev in slot.events.iter() { + if *prev == ev { + return false; // exact duplicate + } + } + // Per-slot cap: once a slot is saturated, further events at this ltime are + // treated as already-seen. This bounds memory under a flood of distinct + // (name, payload) pairs at the same ltime. + if slot.events.len() >= MAX_EVENTS_PER_LTIME { + return false; + } + slot.events.push(ev); + } else { + // Stale entry from a different ltime wrapped onto this index: replace. + *slot = UserEvents { + ltime: LamportTime(ltime), + events: vec![ev], + }; + } + } else { + self.buffer[idx] = Some(UserEvents { + ltime: LamportTime(ltime), + events: vec![ev], + }); + } + true + } +} + +// ── QueryBuffer ─────────────────────────────────────────────────────────────── + +/// Maximum number of distinct `(name, payload)` user events recorded per ring slot (per Lamport time). +/// +/// A single ltime can accumulate many coalesced or distinct events from different senders. +/// Without a cap a flooder can send unlimited distinct `(name, payload)` pairs at the same +/// accepted ltime and grow the `Vec` in the slot without bound — a memory DoS. +/// Once a slot is saturated every subsequent event at that ltime is treated as already-seen +/// (return `false`) so it is not rebroadcast or emitted locally. +/// +/// Mirrors the same DoS-bound rationale as `MAX_QUERY_IDS_PER_LTIME`. Normal clusters never +/// approach this limit. +const MAX_EVENTS_PER_LTIME: usize = 256; + +/// Maximum number of distinct query ids recorded per ring slot (per Lamport time). +/// +/// A ring slot accumulates all `id` values seen at one `ltime`. Without a cap, +/// a flooder can send unlimited unique ids at the same accepted `ltime` (local +/// queries do not advance `query_clock`, so a single ltime can stay current for +/// many rounds) and grow the `Vec` without bound — a memory DoS. Once a +/// slot reaches this limit every subsequent id at that ltime is treated as +/// already-seen (return `false`) so it is not rebroadcast or delivered locally. +/// +/// This is a port-specific per-slot DoS hardening. Go/oracle lack a per-slot +/// id cap; their `QueryBufferSize` (default 512) is the ring *length* (number +/// of ltime slots), not a per-slot limit. The ring length is configured +/// separately via `Options::with_query_buffer_size`. Normal clusters never +/// issue more than a handful of queries per Lamport tick, so this cap is only +/// reachable under adversarial conditions. +const MAX_QUERY_IDS_PER_LTIME: usize = 2048; + +/// Maximum cardinality of `received_queries`. +/// +/// A flood of distinct `(ltime, id)` queries with huge timeouts would grow +/// `received_queries` without bound: the deadline-pruning in `handle_timeout` +/// only fires periodically, and a fast flooder can insert faster than the +/// pruner evicts. When the map is already at this cap, the incoming query is +/// DROPPED before any state mutation — no clock witness, no dedup write, no +/// ACK, no event emission, no rebroadcast. This preserves every already-surfaced +/// token (which may still be waiting on `respond` / `respond_key`) and keeps +/// memory bounded. Normal clusters never approach this limit; it is reachable +/// only under adversarial flood. +const MAX_RECEIVED_QUERIES: usize = 2048; + +/// Maximum inbound query timeout accepted from a peer. +/// +/// A peer-supplied `msg.timeout` is clamped to this value before computing +/// the response deadline, preventing a flooder from pinning `received_queries` +/// entries open for an arbitrarily long time. The bound mirrors the originator's +/// own default-timeout heuristic ceiling (several minutes at cluster scale). +const MAX_QUERY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(600); + +/// Distinguishes the two call sites of `handle_query`. +/// +/// The inbound overflow cap (`MAX_RECEIVED_QUERIES`) applies only to +/// `Inbound` queries so that peer flooding cannot silently suppress a locally- +/// originated query from self-processing on the initiating node. +#[derive(Clone, Copy, PartialEq, Eq)] +enum QueryOrigin { + /// The query arrived from a peer via `handle_user_packet`. Subject to the + /// inbound overflow cap. + Inbound, + /// The query was originated locally via `query()` or `internal_query()`. + /// The cap is bypassed; the initiating node MUST always self-process. + Local, +} + +/// Per-ltime slot stored in the query ring buffer. +/// +/// Holds all `id` values (random u32 per query) seen at a given Lamport time. +/// Multiple queries may share the same `ltime` if they were issued concurrently; +/// they are distinguished by their random `id`. +#[derive(Clone)] +struct Queries { + ltime: LamportTime, + /// All query ids seen at this ltime. + query_ids: Vec, +} + +/// Ring-buffer dedup store for received queries. +/// +/// Mirrors Go serf `types.go` `queryCore` / `handleQuery` dedup logic. +/// The ring is indexed by `ltime % buffer.len()`. Each slot holds a `Queries` +/// value recording all `id`s seen at that Lamport time. +/// +/// Queries older than the entire ring (`cur_time > ring_len && ltime < cur_time +/// - ring_len`) are also rejected as "too old" to prevent stale re-delivery. +struct QueryBuffer { + /// Slots indexed by `ltime % len`. `None` = never used. + buffer: Vec>, + /// Queries with `ltime < min_time` are unconditionally dropped. + /// + /// Raised to `old_query_clock + 1` when replaying a snapshot (G5 + /// clock-recovery analog for queries). `0` means no floor is active. + min_time: u64, +} + +impl QueryBuffer { + /// Allocate a ring of `size` empty slots with `min_time = 0`. + fn new(size: usize) -> Self { + Self { + buffer: vec![None; size.max(1)], + min_time: 0, + } + } + + /// Record and deduplicate one query by `(ltime, id)`. + /// + /// Returns `true` if this is the **first sight** of this `(ltime, id)` pair + /// (and therefore the query should be rebroadcast and dispatched locally). + /// + /// Returns `false` (drop) when: + /// - `ltime < self.min_time` (below the recovery floor). + /// - `ltime` is older than the entire ring relative to `cur_time` + /// (`cur_time > ring_len && ltime < cur_time - ring_len`). + /// - The same `(ltime, id)` pair already exists in the slot for this ltime. + /// + /// `cur_time` is the query clock value **after** it has been witnessed. + fn witness_query(&mut self, cur_time: u64, ltime: u64, id: u32) -> bool { + // Below min_time floor: drop. + if ltime < self.min_time { + return false; + } + + let bltime = self.buffer.len() as u64; + // Too old relative to the ring (mirrors Go serf `handleQuery` check). + if cur_time > bltime && ltime < cur_time - bltime { + return false; + } + + let idx = (ltime % bltime) as usize; + + match self.buffer[idx].as_mut() { + // Slot holds this ltime: dedup by id, then record this id. + Some(seen) if seen.ltime.0 == ltime => { + for &prev in &seen.query_ids { + if prev == id { + return false; // exact (ltime, id) duplicate + } + } + // Per-slot cap: once a slot is saturated, further ids at this ltime are + // treated as already-seen. This bounds memory use under a flood of + // unique ids at the same ltime (the slot stays owned by the current + // ltime but new ids are silently rejected — not rebroadcast, not + // delivered locally). Normal clusters never approach this limit. + if seen.query_ids.len() >= MAX_QUERY_IDS_PER_LTIME { + return false; + } + seen.query_ids.push(id); + // The slot already existed for this ltime, so this id is a *new* query + // at the same ltime (two concurrent queries with same ltime, different + // random ids). Return true so it is processed. + true + } + // Empty slot, or stale entry from a different ltime that wrapped onto this + // ring index: start a fresh record. Replacing a stale entry is correct — + // keeping it would cause the new ltime's dedup to miss re-arrivals. + _ => { + self.buffer[idx] = Some(Queries { + ltime: LamportTime(ltime), + query_ids: vec![id], + }); + true + } + } + } +} + +// ── QueryId ─────────────────────────────────────────────────────────────────── + +/// Opaque identifier for an issued query, keyed by `(ltime, id)`. +/// +/// `ltime` is the query-clock value stamped at issue time (READ, not +/// incremented — G8 / H8). `id` is a random `u32` drawn from `self.rng` +/// so that multiple queries issued at the same Lamport time are distinguishable +/// by the dedup ring. Together `(ltime, id)` is the canonical composite key +/// used by the `QueryBuffer` and `PendingQuery` registry. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct QueryId { + /// The query-clock value at query-issue time (read, not incremented). + pub ltime: LamportTime, + /// A random u32 drawn from `self.rng` at query-issue time. + pub id: u32, +} + +// ── QueryParams ─────────────────────────────────────────────────────────────── + +/// Parameters for issuing a query via [`Endpoint::query`]. +/// +/// `Default` provides the "no filters, no relay, no ack, zero timeout" +/// baseline; callers supply a non-zero `timeout` (or the machine computes one +/// via `default_query_timeout` if zero is given). +#[derive(Debug, Clone, Default)] +pub struct QueryParams { + /// Optional node-id / tag-filter list. Empty means "broadcast to all". + pub filters: Vec>, + /// Number of relay hops to request from responders. + pub relay_factor: u8, + /// If `true`, the machine sets the ACK flag and waits for per-hop acks. + pub request_ack: bool, + /// Maximum time allowed for responses. + /// + /// A value of `Duration::ZERO` causes the machine to substitute + /// `gossip_interval * query_timeout_mult * log10(n_members + 1)`. + pub timeout: std::time::Duration, +} + +// ── PendingQuery ────────────────────────────────────────────────────────────── + +/// Purpose of a pending query, used by the internal-query interceptor. +/// +/// `App` queries surface responses as `Event::QueryResponse`; `Conflict` and +/// `Key` queries (encryption-gated) are folded internally into `Event::KeyResponse`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum QueryPurpose { + /// An application-issued query; responses surface as `Event::QueryResponse`. + App, + /// An internal conflict-resolution query; responses tally into a vote. + Conflict, + /// An internal key-management query; responses aggregate into a `KeyResponse`. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + Key, +} + +// ── KeyResponseTally ────────────────────────────────────────────────────────── + +/// Accumulator for folding per-node `KeyResponseMessage`s into a `KeyResponse`. +/// +/// Created inside `PendingQuery.key_tally` when `kind == Key`. +/// Folded per response in `handle_key_response_fold`; materialized into a +/// `KeyResponse` and emitted as `Event::KeyResponse` in `close_key_query`. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +pub(crate) struct KeyResponseTally { + /// Number of nodes that responded with `result = false`. + pub(crate) num_err: usize, + /// Key → count of nodes reporting that key. + pub(crate) keys: std::collections::HashMap, + /// Primary key → count of nodes reporting it as primary. + pub(crate) primary_keys: std::collections::HashMap, + /// Per-node error message for nodes that reported result=false. + pub(crate) messages: std::collections::HashMap, +} + +/// Bookkeeping for an in-flight query this node originated. +/// +/// Created in `query()` and kept alive until the `deadline` elapses (in +/// `handle_timeout`). On expiry, `App` queries are silently closed; internal +/// query closes trigger tallying / emitting the result event. +pub(crate) struct PendingQuery { + /// Why this query was issued — controls how responses are folded. + pub(crate) kind: QueryPurpose, + /// Wall-clock deadline after which no more responses are useful. + /// + /// Armed at `now + timeout` in `query()`. + pub(crate) deadline: Instant, + /// Tracks which responders have already replied (dedup by id). + /// + /// The value is `()` — membership is the only information we need. + pub(crate) responses: std::collections::HashMap, + /// Tracks which peers have already acknowledged (dedup by id). + /// + /// Kept separate from `responses` because a single peer may send both an ack + /// (on receipt) and a later response; deduping them in the same set would + /// drop the response after the ack. Non-empty only for `request_ack` queries. + pub(crate) acks: std::collections::HashMap, + /// The composite `(ltime, id)` key of the query. + pub(crate) query_id: QueryId, + /// Whether the originating `query()` requested acks from responders. + /// + /// Only queries issued with `params.request_ack = true` set the ACK flag in + /// the wire message. Acks received for queries that did NOT request acks are + /// silently dropped here (after the ingress gate in `handle_query_response`). + pub(crate) request_ack: bool, + /// Conflict-resolution vote tally: count of responses where the peer's + /// reported address matched the local advertise address. + /// + /// Non-zero only for `kind == Conflict`; always 0 for `App` and `Key`. + pub(crate) conflict_matching: usize, + /// Number of nodes the query was targeted at, captured at issue time. + /// + /// For `Key` queries this is `members.states.len()` at the moment + /// `internal_query()` fires (mirrors Go serf `key_manager.go` + /// `streamKeyResponse` which initialises `resp.num_nodes` from + /// `this.num_members()`). Set to 0 for `App` and `Conflict` queries where + /// `num_nodes` is not surfaced. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + pub(crate) num_nodes: usize, + /// Key aggregation state; `Some` only when `kind == Key`. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + pub(crate) key_tally: Option>, +} + +// ── ReceivedQuery ───────────────────────────────────────────────────────────── + +/// Bookkeeping for a query this node received (surfaced as [`Event::Query`]). +/// +/// Kept until `respond()` fires (entry removed on success) or the deadline +/// passes (pruned by `handle_timeout`). The once-only constraint (G7 guard 2) +/// is enforced by removing the entry on the first successful `respond()`; a +/// second call then falls through the `.ok_or(Error::AlreadyResponded)` guard. +pub(crate) struct ReceivedQuery { + /// Address of the originating node (the `respond()` directed-send target). + pub(crate) from: A, + /// Deadline after which a response is no longer useful (G7 guard 3). + pub(crate) deadline: Instant, +} + +// ── endpoint errors ─────────────────────────────────────────────────────────── + +/// Errors returned by `Endpoint` operations. +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// A `leave()` or `force_leave()` was called from an invalid lifecycle state. + /// + /// `leave()` is idempotent when already `Left` (returns `Ok`), but rejected + /// when already `Leaving` or `Shutdown`. + #[error("leave called from invalid state: {0}")] + BadLeaveState(SerfState), + /// `join()` was called while the local endpoint is not `Alive`. + /// + /// serf only announces its own join intent from the `Alive` state; a + /// `Leaving`, `Left`, or `Shutdown` endpoint rejects the call. Mirrors the + /// `bad_join_status` guard in serf-core `api.rs` `join()`. + #[error("join called from invalid state: {0}")] + BadJoinState(SerfState), + /// The inner memberlist `leave()` returned an error. + #[error("inner leave error: {0}")] + InnerLeave(#[from] memberlist_proto::Error), + /// A user event or its name+payload exceeds the configured + /// `max_user_event_size`. + /// + /// Carries `(actual_size, limit)`. + #[error("user event too large: {0} bytes exceeds limit of {1}")] + UserEventTooLarge(usize, usize), + /// The wire codec failed to encode a user event for broadcast. + #[error("user event encode error: {0}")] + UserEventEncode(#[from] EncodeError), + /// A query payload exceeds `query_size_limit`. + /// + /// Carries `(actual_encoded_size, limit)`. + #[error("query too large: {0} bytes exceeds limit of {1}")] + QueryTooLarge(usize, usize), + /// The response payload exceeds `query_response_size_limit` (G7 guard 1). + /// + /// Carries `(actual_size, limit)`. + #[error("query response too large: {0} bytes exceeds limit of {1}")] + RespondTooLarge(usize, usize), + /// `respond()` was called a second time for the same query token (G7 guard 2). + /// + /// Each received query may be responded to at most once. The span is zeroed + /// on the first successful `respond()`; subsequent calls return this error. + /// Mirrors Go serf `event.go` `query_already_responsed`. + #[error("query already responded")] + AlreadyResponded, + /// `respond()` was called after the query's deadline elapsed (G7 guard 3). + /// + /// Mirrors Go serf `event.go` `query_timeout`. + #[error("query response deadline exceeded")] + RespondAfterDeadline, + /// The wire codec failed to encode a query response for sending. + #[error("query response encode error: {0}")] + RespondEncode(EncodeError), + /// The directed send for a query response failed. + /// + /// The `responded` flag is not set; the caller may retry `respond()`. + #[error("query response send error: {0}")] + RespondSend(memberlist_proto::Error), + /// A `Filter::Tag` in the query carries a regex pattern that fails to + /// compile. No side effects occur: no `PendingQuery` is registered, no + /// broadcast is queued, no clock is advanced. + /// + /// Only reachable when the `tag-regex` feature is enabled; without it, + /// tag-filter matching uses exact string equality, which has no compile step + /// and cannot produce an invalid pattern. + #[cfg(feature = "tag-regex")] + #[cfg_attr(docsrs, doc(cfg(feature = "tag-regex")))] + #[error("query filter contains an invalid tag regex")] + InvalidQueryFilter, +} + +// ── clock witness ───────────────────────────────────────────────────────────── + +/// The ingress reject watermark for Lamport clock values. +/// +/// Any Lamport time at or above this value is rejected on every ingress path +/// as out-of-range and untrustworthy. The upper half `[LTIME_MAX, u64::MAX]` +/// can only be reached after 2^63 events — unreachable in any finite cluster +/// lifetime — so any wire value in that range signals corruption or a crafted +/// packet and is silently dropped before any state mutation. +/// +/// Mirrors Go serf's naked-wrapping clock: no storage clamp is applied, but +/// out-of-range ingress values are unconditionally rejected. +pub(crate) const LTIME_MAX: u64 = 1u64 << 63; + +/// Return `true` if `t` is a valid Lamport time that can be safely witnessed. +/// +/// Accepts times in `[0, LTIME_MAX)`. Times at or above `LTIME_MAX` are +/// rejected at every ingress gate as out-of-range. +/// +/// Applied as the WHOLE-MESSAGE DROP gate at the entry of every Lamport +/// ingress handler (join intent, leave intent, user event, query, push-pull +/// top-level clocks, load_snapshot clocks), before any state mutation. +#[inline] +pub(crate) fn ltime_is_acceptable(t: u64) -> bool { + t < LTIME_MAX +} + +/// Advance `clock` to at least `t + 1`. +/// +/// Witness semantics: if the observed time `t` is at or beyond the current +/// clock value, advance the clock past it so the next local event gets a +/// strictly greater timestamp. Older observations are no-ops (no regression). +/// Mirrors Go serf `types/clock.go` `Witness`. +/// +/// Only values that pass `ltime_is_acceptable` are ever witnessed; this +/// function is a defense-in-depth backstop that re-checks the gate so that a +/// site which bypasses the ingress check cannot drive the clock to `u64::MAX`. +/// `saturating_add` is the no-UB backstop: it parks a clock driven to +/// `u64::MAX` there rather than wrapping to 0. +#[inline] +pub(crate) fn witness(clock: &mut u64, t: u64) { + // Defense-in-depth: refuse unacceptable Lamport times even if the ingress + // gate was bypassed. The primary gate is `ltime_is_acceptable` at each + // ingress site; this check catches any site that was missed. + if !ltime_is_acceptable(t) { + return; + } + if t >= *clock { + *clock = t.saturating_add(1); + } +} + +/// Stamp the current Lamport time from `clock` and advance the clock. +/// +/// Reads the current clock value as the stamp, then advances the stored clock +/// by one using `saturating_add` (the no-UB backstop: a clock at `u64::MAX` +/// parks there rather than wrapping to 0). +/// +/// All five local-emission sites (leave, force_leave, user_event, query, +/// internal_query) route through this helper. Mirrors Go serf +/// `types/clock.go` `Time` + `Increment`. +#[inline] +fn next_ltime(clock: &mut u64) -> u64 { + let stamped = *clock; + *clock = stamped.saturating_add(1); + stamped +} + +// ── Endpoint ────────────────────────────────────────────────────────────────── + +/// The serf Sans-I/O super-machine. +/// +/// `I` is the node-id type; `A` is the (resolved) address type; `R` is the +/// random number generator injected at construction time (default: `SmallRng`). +/// serf's own selection draws (relay picks, reconnect probabilistic gate, +/// coordinate jitter, query id generation) use `self.rng`; the inner +/// memberlist `Endpoint`'s gossip uses its own independently-seeded `R`. +pub struct Endpoint +where + I: Eq + core::hash::Hash, +{ + /// The inner memberlist Endpoint (SWIM state machine + gossip transport). + inner: memberlist_proto::Endpoint, + /// serf configuration knobs. + opts: Options, + /// Member (SWIM membership) Lamport clock — plain `u64`, no atomics. + /// Single-threaded machine; no concurrent writers. + clock: u64, + /// User-event Lamport clock. + event_clock: u64, + /// Query Lamport clock. + query_clock: u64, + /// Serf membership store (states, intents, left/failed lists). + members: Members, + /// Local lifecycle state of this serf endpoint. + state: SerfState, + /// serf's own injected RNG — separate from the inner Endpoint's `R`. + /// Used for: query id generation, relay / k-random member picks, + /// reconnect probabilistic gate, coordinate jitter. + rng: R, + /// Next deadline at which the reaper should run. + next_reap: Option, + /// Next deadline at which the reconnector should attempt a dial. + next_reconnect: Option, + /// Next deadline at which the broadcast queue depth is checked. + next_queue_check: Option, + /// Deadline after which we stop waiting for the leave-intent broadcast to + /// propagate and proceed to call the inner `leave()`. + /// + /// Armed in `leave()` at `now + broadcast_timeout`. The driver may + /// short-circuit by polling `user_broadcast_queue_len()` reaching zero; + /// the machine itself fires unconditionally when the deadline elapses. + /// `None` when not in the middle of a graceful leave. + leave_broadcast_deadline: Option, + /// Deadline after which the `Leaving → Left` transition fires. + /// + /// Armed in the `LeftCluster` sieve arm at `now + leave_propagate_delay`. + /// When this deadline elapses in `handle_timeout`, the state transitions to + /// `Left` (unless already `Shutdown`) and `Event::LeftCluster` is emitted. + /// `None` when not waiting for the propagation delay. + leave_complete_deadline: Option, + /// Ring-buffer for user-event deduplication. + /// + /// Sized by `opts.event_buffer_size()` at construction time. + /// Each slot holds the set of `(name, payload)` pairs seen at that lamport + /// time so exact duplicates are suppressed. + event_buffer: EventBuffer, + /// Ring-buffer for query deduplication keyed by `(ltime, id)`. + /// + /// Sized by `opts.query_buffer_size()` at construction time. + /// Each slot holds all `id` values seen at that lamport time. + query_buffer: QueryBuffer, + /// Registry of in-flight queries originated by this node. + /// + /// Keyed by `QueryId` (= `(ltime, id)` composite). Entries are created in + /// `query()` and removed when the `deadline` elapses in `handle_timeout`. + pending_queries: Vec>, + /// Registry of queries received by this node and surfaced to the app. + /// + /// Keyed by `QueryId` (= `(ltime, id)` composite). Entries are created when + /// a query passes filters and is emitted as `Event::Query`. `respond()` looks + /// up the entry to enforce the three guards (size, once-only, deadline) and + /// to obtain the originator address for the directed send. + received_queries: std::collections::HashMap>, + /// serf-level events queued for the driver to drain via `poll_event`. + pending_events: VecDeque>, + /// The most recent directed-send (address, bytes) produced by + /// `handle_relay` or `relay_response`. + /// + /// Captured only in test builds so assertions can inspect the relay path + /// without a live socket layer. + #[cfg(test)] + last_directed_send: Option<(A, Bytes)>, + /// Addresses dialled by `load_snapshot` for rejoin (G10, skip self). + /// + /// Populated only in test builds so assertions can verify which peers were + /// dialled after snapshot replay without a live socket layer. + #[cfg(test)] + rejoin_dials: Vec, + /// Vivaldi network-coordinate client (Vivaldi algorithm engine). + /// + /// `Some` when the `coordinates` feature is compiled in and + /// `opts.disable_coordinates()` was `false` at construction. + /// `None` when the feature is disabled at compile time or coordinates + /// were explicitly disabled via `Options::with_disable_coordinates(true)`. + #[cfg(feature = "coordinates")] + coord_client: Option>, + /// Cache of the most-recently-seen coordinate for each known peer. + /// + /// Updated on every successful `PingCompleted` RTT feed. Keyed by node id. + /// Removed when a node is reaped from membership (G13). + #[cfg(feature = "coordinates")] + coord_cache: std::collections::HashMap, + /// The `now` instant threaded into the most recent poll/handle call. + /// + /// The inner `poll_event` loop (drain_inner) fires synchronously from + /// within `handle_packet`/`handle_timeout`/etc., and inner events like + /// `NodeLeft` need `now` for `leave_time`. Rather than thread `now` + /// through every inner-sieve dispatch, we latch it here before each drain. + /// Callers that do not have a meaningful `now` (e.g., `poll_event` called + /// after a prior handle call) use the last latched value. + drain_now: Instant, + /// Dirty flag for the push-pull local-state snapshot (H6). + /// + /// Set whenever any of the three Lamport clocks, member status-ltimes, + /// `left_members`, or the event buffer changes. `drain_inner` calls + /// `resync_local_state()` lazily when this is `true` so the inner + /// `Endpoint` always ships a current serf snapshot on the next push-pull + /// egress — not the snapshot from the last explicit sync. + local_state_dirty: bool, + /// When `true`, a push-pull join bumps `event_buffer.min_time` to the + /// remote `event_ltime` (H8/G4 / Go serf `eventJoinIgnore`). + /// + /// Set by the driver via `set_event_join_ignore(true)` before issuing a + /// join that should suppress replay of the peer's buffered events. The + /// ingress `merge_remote_state` checks this on `is_join == true`. + event_join_ignore: bool, + /// The address of the most recently reconnect-dialled peer. + /// + /// Set by `fire_reconnect` each time a dial is initiated. Test helpers + /// expose this via `test_last_dial_addr` to assert which peer was chosen. + /// Only meaningful in test builds; production code ignores this field. + #[cfg(test)] + last_dial_addr: Option, + /// All directed sends produced by `relay_response` in insertion order. + /// + /// Accumulated across repeated calls so tests can assert the full set of + /// relay peers chosen across a single relay invocation (relay_factor > 1). + /// Reset to empty by `test_clear_relay_directed_sends`. Only present in + /// test builds; production code does not carry this allocation. + #[cfg(test)] + relay_all_directed_sends: Vec<(A, Bytes)>, +} + +// ── construction + cheap accessors ──────────────────────────────────────────── + +impl Endpoint +where + I: Clone + Eq + core::hash::Hash, + R: SeedableRng, +{ + /// Construct a serf `Endpoint` wrapping `inner`, using `opts` for serf-level + /// knobs, and `rng` as serf's own injected selection entropy. + /// + /// The inner `Endpoint` must already have been configured with + /// `EndpointOptions::with_user_broadcast_tiers(NonZeroU8::new(3))` so that + /// serf's three broadcast tiers (intent=0, query=1, event=2) are available. + /// + /// `rng` is **separate** from the inner Endpoint's `R`. Seed it from the + /// driver's own entropy source; do not share the same `R` instance. + pub fn new_with_rng(inner: memberlist_proto::Endpoint, opts: Options, rng: R) -> Self { + // Arm the first reap/reconnect/queue-check deadlines relative to the ORIGIN instant. + // The driver calls handle_timeout(now) and the deadlines fire when now >= deadline. + let first_reap = Instant::ORIGIN + opts.reap_interval(); + let first_reconnect = Instant::ORIGIN + opts.reconnect_interval(); + let first_queue_check = Instant::ORIGIN + opts.queue_check_interval(); + let event_buf_size = opts.event_buffer_size(); + let query_buf_size = opts.query_buffer_size(); + + #[cfg(feature = "coordinates")] + let coord_client: Option> = + if opts.disable_coordinates() { + None + } else { + Some(crate::coordinate_client::CoordinateClient::new( + crate::coordinate_client::CoordinateOptions::default(), + )) + }; + + Self { + inner, + opts, + clock: 0, + event_clock: 0, + query_clock: 0, + members: Members::default(), + state: SerfState::Alive, + rng, + next_reap: Some(first_reap), + next_reconnect: Some(first_reconnect), + next_queue_check: Some(first_queue_check), + leave_broadcast_deadline: None, + leave_complete_deadline: None, + event_buffer: EventBuffer::new(event_buf_size), + query_buffer: QueryBuffer::new(query_buf_size), + pending_queries: Vec::new(), + received_queries: std::collections::HashMap::new(), + pending_events: VecDeque::new(), + drain_now: Instant::ORIGIN, + // The snapshot starts dirty so the first push-pull always ships a fresh + // body even if no explicit API call has been made yet. + local_state_dirty: true, + event_join_ignore: false, + #[cfg(feature = "coordinates")] + coord_client, + #[cfg(feature = "coordinates")] + coord_cache: std::collections::HashMap::new(), + #[cfg(test)] + last_dial_addr: None, + #[cfg(test)] + last_directed_send: None, + #[cfg(test)] + rejoin_dials: Vec::new(), + #[cfg(test)] + relay_all_directed_sends: Vec::new(), + } + } + + /// Convenience constructor that seed serf's `R` with a zero seed. + /// + /// Suitable for tests and environments where determinism or an explicit seed + /// is acceptable. Production drivers should use `new_with_rng` and seed from + /// a cryptographically-secure source. + pub fn new(inner: memberlist_proto::Endpoint, opts: Options) -> Self { + Self::new_with_rng(inner, opts, R::seed_from_u64(0)) + } + + // ── read accessors ──────────────────────────────────────────────────────── + + /// The current lifecycle state of this serf endpoint. + pub const fn state(&self) -> SerfState { + self.state + } + + /// The current member (SWIM membership) Lamport clock value. + pub const fn member_time(&self) -> u64 { + self.clock + } + + /// The current user-event Lamport clock value. + pub const fn event_time(&self) -> u64 { + self.event_clock + } + + /// The current query Lamport clock value. + pub const fn query_time(&self) -> u64 { + self.query_clock + } + + /// Number of nodes currently tracked in the membership store. + pub fn num_members(&self) -> usize { + self.members.states.len() + } +} + +// ── poll API (requires full Id + Data bounds for inner delegation) ───────────── + +impl Endpoint +where + I: Id + Clone, + A: CheapClone + Data + PartialEq + Clone + 'static, + R: Rng + SeedableRng, +{ + /// Drain one serf event. + /// + /// Pumps the inner `Endpoint` to exhaustion, sieving each inner `Event` + /// into serf state, then returns the next queued serf `Event`. Call in a + /// loop until `None` before blocking. + pub fn poll_event(&mut self) -> Option> { + self.drain_inner(); + self.pending_events.pop_front() + } + + /// Drain one outgoing transmit. + /// + /// Passes through to the inner `Endpoint`; serf adds no additional + /// transmit framing at this layer. + pub fn poll_transmit(&mut self) -> Option> { + self.inner.poll_transmit() + } + + /// The earliest deadline requiring a `handle_timeout` call. + /// + /// Returns the minimum of the inner Endpoint's deadline and serf's own + /// periodic deadlines (reap, reconnect, queue-check, leave-broadcast, + /// leave-complete). + pub fn poll_timeout(&self) -> Option { + let inner = self.inner.poll_timeout(); + let query_min = self.pending_queries.iter().map(|pq| pq.deadline).min(); + let serf = [ + self.next_reap, + self.next_reconnect, + self.next_queue_check, + self.leave_broadcast_deadline, + self.leave_complete_deadline, + query_min, + ] + .into_iter() + .flatten() + .min(); + match (inner, serf) { + (Some(a), Some(b)) => Some(a.min(b)), + (Some(a), None) => Some(a), + (None, Some(b)) => Some(b), + (None, None) => None, + } + } + + /// Deliver an inbound datagram or packet to the machine. + /// + /// Decodes the memberlist wire `Message` from `data` and dispatches it + /// to the inner `Endpoint`. SWIM messages (Ping, Alive, Dead, Suspect, …) + /// are handled by the inner machine; when the inner encounters a + /// `Message::UserData` payload it emits `Event::UserPacket`, which the serf + /// sieve (`on_inner_event`) then decodes as a serf `AnyMessage` and + /// routes to the appropriate serf handler (join/leave intent, user event, + /// query, relay). + /// + /// Malformed or unrecognised bytes are silently dropped — the machine must + /// not panic on bad input from the network. + pub fn handle_packet(&mut self, from: A, data: Bytes, now: Instant) { + // Malformed frame or unrecognised tag: drop silently. The inner endpoint + // logs its own decode errors; serf takes no serf-level action here. + self.drain_now = now; + if let Ok(msg) = parse_message::(data) { + self.inner.handle_packet(from, msg, now); + } + self.drain_inner(); + } + + /// Deliver a stream event to the machine. + /// + /// Forwards to the inner `Endpoint`, then drains resulting inner events. + pub fn handle_stream_event( + &mut self, + ev: EndpointEvent, + now: Instant, + ) -> Option> { + self.drain_now = now; + let cmd = self.inner.handle_stream_event(ev, now); + self.drain_inner(); + cmd + } + + /// Accept an inbound stream connection. + /// + /// Passes directly to the inner `Endpoint`. + pub fn accept_stream(&mut self, from: A, now: Instant) -> Option> { + self.inner.accept_stream(from, now) + } + + /// Advance time and fire any expired serf or inner deadlines. + /// + /// Tick order (H1b / decision 5 step 4): + /// + /// 1. Resync the push-pull local-state snapshot if dirty (H6) so the inner + /// echoes a fresh serf-clock / member-status snapshot on this tick's + /// anti-entropy exchange — not a snapshot from a previous tick. + /// 2. Run `inner.handle_timeout(now)` first — the inner gossip scheduler + /// produces any pending SWIM transitions and SWIM piggyback gossip. + /// 3. Drain all inner events produced by the inner tick via `drain_inner()`, + /// processing NodeJoined / NodeLeft / UserPacket / etc. through the serf + /// sieve before any serf deadline fires. + /// 4. Then fire serf's own deadlines: reap → reconnect → queue-check → + /// query-closes → leave-complete. + /// + /// **Why this order:** firing serf deadlines after draining the inner's events + /// prevents tombstoning a member whose `NodeJoined` event is still queued in + /// the inner machine at the time the reap deadline would otherwise fire. + /// Without this ordering, a member that reconnects exactly at the reap + /// boundary could be incorrectly pruned. Decision 5 step 4 / pin H1b. + pub fn handle_timeout(&mut self, now: Instant) + where + I: Clone, + A: Clone, + { + self.drain_now = now; + + // H6: resync the push-pull snapshot before the inner tick so the next + // anti-entropy exchange carries the current serf state (not a stale one + // from a previous mutation burst). The dirty flag is cleared inside + // resync_local_state on success. + if self.local_state_dirty { + self.resync_local_state(); + } + + // Step 2: drive the inner memberlist machine (SWIM gossip + probe + push-pull). + // This may produce NodeJoined / NodeLeft / UserPacket inner events. + self.inner.handle_timeout(now); + + // Step 3: drain all inner events produced by the tick through the serf sieve. + // NodeJoined / NodeLeft / etc. are processed NOW, before any serf deadline fires. + self.drain_inner(); + + // Step 4: fire serf's own deadlines in deterministic order. + + // Reaper: remove tombstoned left/failed nodes and stale intents. + if let Some(dl) = self.next_reap { + if now >= dl { + self.fire_reap(now); + self.next_reap = Some(now + self.opts.reap_interval()); + } + } + + // Reconnector: probabilistically re-dial a random failed peer. + if let Some(dl) = self.next_reconnect { + if now >= dl { + self.fire_reconnect(now); + self.next_reconnect = Some(now + self.opts.reconnect_interval()); + } + } + + // Queue-check: re-arm the deadline. The depth gate in rebroadcast() prevents + // enqueue past queue_max; this periodic tick is retained for future telemetry. + if let Some(dl) = self.next_queue_check { + if now >= dl { + self.next_queue_check = Some(now + self.opts.queue_check_interval()); + } + } + + // Query-close: tally and expire any pending queries whose deadline elapsed. + self.fire_due_query_closes(now); + + // Prune expired received-query tokens. Entries for which respond() succeeded + // are removed there; this loop catches those whose deadline elapsed without a + // respond() call (driver missed the response window). + self.received_queries.retain(|_, rq| now < rq.deadline); + + // Leave-complete: Leaving → Left after inner LeftCluster + leave_propagate_delay. + if let Some(dl) = self.leave_complete_deadline { + if now >= dl { + self.leave_complete_deadline = None; + if self.state == SerfState::Leaving { + self.state = SerfState::Left; + self.pending_events.push_back(Event::LeftCluster); + } + // If already Shutdown: the leave chain was interrupted; no transition or event. + } + } + + // Leave-broadcast: clear the deadline once it has elapsed so poll_timeout + // does not keep returning it and causing driver spin. + if let Some(dl) = self.leave_broadcast_deadline { + if now >= dl { + self.leave_broadcast_deadline = None; + } + } + } + + // ── inner-event sieve ───────────────────────────────────────────────────── + + /// Pump the inner Endpoint to exhaustion, routing each event through the + /// serf sieve. + /// + /// After all inner events are drained, if the local-state snapshot is dirty + /// (H6), `resync_local_state` is called so the next push-pull egress ships + /// the current serf clock / member-status state, not a stale snapshot. + fn drain_inner(&mut self) { + while let Some(ev) = self.inner.poll_event() { + self.on_inner_event(ev); + } + if self.local_state_dirty { + self.resync_local_state(); + } + } + + /// Dispatch a single inner `memberlist_proto::Event` to the matching serf + /// handler. + /// + /// Every variant of the inner `Event` enum is covered (totality / H4). + /// Unimplemented handlers are stubs that will be filled in by later + /// sub-stages. + fn on_inner_event(&mut self, ev: memberlist_proto::Event) { + use memberlist_proto::Event as IE; + match ev { + // ── membership ─────────────────────────────────────────────────────── + IE::NodeJoined(node) => { + let now = self.drain_now; + self.handle_node_join(&node, now); + } + IE::NodeLeft(node) => { + // now is not available here; use the stored now from the most-recent + // handle_timeout/handle_packet call. For correctness the sieve callers + // thread `now` through drain_inner; at this sub-stage we carry it via + // a field set before the drain. See `drain_inner_at`. + self.handle_node_leave(&node, self.drain_now); + } + IE::NodeUpdated(node) => { + self.handle_node_update(&node); + } + IE::NodeConflict(_c) => { + if self.opts.enable_id_conflict_resolution() { + let now = self.drain_now; + self.resolve_node_conflict(now); + } + } + + // ── user gossip ─────────────────────────────────────────────────────── + IE::UserPacket(p) => { + // `drain_now` was latched by handle_packet / handle_stream_event / + // handle_timeout before drain_inner was called, so it is always a + // fresh `now` for the current call site. + let now = self.drain_now; + let (from, data, _reliability) = p.into_parts(); + self.handle_user_packet(from, data, now); + } + IE::RemoteStateReceived(r) => { + let (_peer, user_data, is_join) = r.into_parts(); + if !user_data.is_empty() { + self.merge_remote_state(user_data, is_join); + } + } + + // ── lifecycle ──────────────────────────────────────────────────────── + IE::LeftCluster => { + // The inner memberlist has finished broadcasting the dead-self fan-out. + // If we are in the middle of a graceful leave, arm the propagation + // delay deadline. The actual Leaving → Left transition fires in + // handle_timeout when the deadline elapses. + // + // Use the minimum of any already-armed deadline and the new one so + // that a repeated LeftCluster (e.g. from inner.handle_timeout during + // the propagation window) does not push the deadline forward — the + // first-fired deadline is the earliest and therefore the binding one. + if self.state == SerfState::Leaving { + let delay = self.opts.leave_propagate_delay(); + let deadline = self.drain_now + delay; + self.leave_complete_deadline = Some(match self.leave_complete_deadline { + Some(existing) => existing.min(deadline), + None => deadline, + }); + } + } + + // ── coordinates ─────────────────────────────────────────────────────── + IE::PingCompleted(p) => { + // G9: on every successful probe round-trip, feed the RTT and the + // remote peer's piggybacked coordinate into the local Vivaldi model, + // then refresh the ack payload so the next ack ships the updated + // local coordinate. No-op when the `coordinates` feature is disabled + // or when coordinates were disabled at construction. + #[cfg(feature = "coordinates")] + { + let node_id = p.node_ref().id_ref().clone(); + let rtt = p.rtt(); + let payload = p.payload_ref().clone(); + self.handle_ping_completed(&node_id, rtt, &payload); + } + // When the feature is disabled, suppress the unused-variable warning. + #[cfg(not(feature = "coordinates"))] + let _ = p; + } + + // ── drop with no serf-level action ─────────────────────────────────── + // PingFailed: no serf action; the inner already records the failure. + IE::PingFailed(_) => {} + // ExchangeCompleted: driver bookkeeping only; serf takes no action. + IE::ExchangeCompleted(_) => {} + // DecodeError: the inner has already logged / tracked the error; + // serf takes no action on undecodable inner messages. + IE::DecodeError(_) => {} + + // ── reconnect / dial passthrough (H3) ──────────────────────────────── + // The inner emits DialRequested when serf calls inner.start_push_pull() + // (from fire_reconnect) or when the inner initiates its own anti-entropy + // push/pull. The driver must dial the given peer and report back via + // dial_succeeded / dial_failed. serf passes this through to the driver + // as Event::DialRequested so the driver's event loop can perform the dial. + // + // H3: the reconnector is machine OUTPUT — serf does NO I/O itself. + IE::DialRequested(d) => { + // Re-emit as a serf-level event so the driver sees it from poll_event. + // DialPassthrough carries the StreamId and peer address; the driver uses + // StreamId to call inner.dial_succeeded / inner.dial_failed. + let (stream_id, peer, _deadline) = d.into_parts(); + // Track the dialled address for test assertions. + #[cfg(test)] + { + self.last_dial_addr = Some(peer.clone()); + } + self + .pending_events + .push_back(Event::DialRequested(DialPassthrough::new(stream_id, peer))); + } + } + } + + // ── push-pull local-state synthesis (H6) ──────────────────────────────── + + /// Mark the push-pull local-state snapshot as needing re-synthesis. + /// + /// Called at every site that mutates the three Lamport clocks, member + /// status-ltimes, `left_members`, or the event buffer. The actual + /// re-synthesis is deferred to the end of `drain_inner` so that a burst + /// of mutations within one call produces only one encode+push cycle. + #[inline] + fn mark_local_state_dirty(&mut self) { + self.local_state_dirty = true; + } + + /// Synthesise the push-pull body and push it to the inner Endpoint. + /// + /// Builds a `PushPullMessage` from the current machine state — three + /// Lamport clocks, per-member `status_ltime`, `left_members` id list, and + /// the event ring buffer — then encodes it via `AnyMessage::encode` and + /// calls `inner.set_local_state_snapshot`. On encode or snapshot-cap + /// errors the dirty flag is left set so the next drain attempt retries + /// (a transient alloc failure should not permanently corrupt the snapshot). + /// + /// Mirrors Go serf `delegate.go` `local_state` (~line 386). + pub fn resync_local_state(&mut self) + where + I: Clone + Data, + A: Data, + { + // Gather status_ltimes from the membership store. + // HashMap iteration order is arbitrary, so collect first then sort by the + // stable encoded id bytes so two machines with identical membership always + // produce byte-identical PushPullMessage wire output. + let mut status_ltimes: Vec<(I, LamportTime)> = self + .members + .states + .iter() + .map(|(id, ms)| (id.clone(), ms.status_time())) + .collect(); + status_ltimes.sort_unstable_by(|(a, _), (b, _)| { + let a_bytes = a.encode_to_vec().unwrap_or_default(); + let b_bytes = b.encode_to_vec().unwrap_or_default(); + a_bytes.cmp(&b_bytes) + }); + + // Gather left_members id list (the oracle sends node ids, not MemberState). + let left_members: Vec = self + .members + .left_members + .iter() + .filter_map(|id| self.members.states.get(id).map(|_| id.clone())) + .collect(); + + // Collect the event ring buffer (non-None slots). + let events: Vec = self + .event_buffer + .buffer + .iter() + .filter_map(|slot| slot.clone()) + .collect(); + + let pp = PushPullMessage::new( + LamportTime(self.clock), + status_ltimes, + left_members, + LamportTime(self.event_clock), + events, + LamportTime(self.query_clock), + ); + + // Encode via AnyMessage so the wire framing is applied consistently. + // AnyMessage::PushPull does not depend on `A`, but the type parameter is + // required for the enum variant; we supply `A` from the impl bound. + let encoded = match AnyMessage::::PushPull(pp).encode() { + Ok(b) => b, + Err(_) => { + // Encoding failure: leave dirty so the next drain retries. + // A snapshot that cannot be encoded should not crash the machine. + return; + } + }; + + // Push to the inner Endpoint. On cap-exceeded errors keep dirty for retry; + // the operator must raise max_stream_frame_size if the serf state is too large + // to fit in one push-pull frame. The snapshot is stale but the machine + // continues operating — the next drain attempt will retry. + if self.inner.set_local_state_snapshot(encoded).is_ok() { + self.local_state_dirty = false; + } + } + + /// Set the `event_join_ignore` flag. + /// + /// When `true`, an incoming push-pull marked as a join bumps + /// `event_buffer.min_time` to the remote `event_ltime` so that + /// pre-join events from that peer are not replayed locally (H8/G4). + /// + /// The driver sets this before calling `inner.join` to suppress historic + /// user events from the seed peers. + pub fn set_event_join_ignore(&mut self, v: bool) { + self.event_join_ignore = v; + } + + // ── push-pull ingress replay (H8/G2, G3, G4) ──────────────────────────── + + /// Replay a remote push-pull body received via `RemoteStateReceived`. + /// + /// Mirrors Go serf `delegate.go` `merge_remote_state` (~line 427). + /// + /// **G2 — witness all three clocks at `ltime - 1`, guarded `> 0`:** + /// Each of the remote's three clocks is witnessed at `value - 1`. The + /// subtraction ensures that no message stamped at the remote's current + /// value is treated as *already seen* — it would need to be received + /// first before the local clock passes it. Guard: only if `value > 0` + /// (witnessing at `LamportTime::MAX` would overflow). + /// + /// **G3 — process `left_members` FIRST, then join intents, skipping lefts:** + /// Left nodes must be installed before the join pass so that a node that + /// appears in both `status_ltimes` and `left_members` ends up as `Left`, + /// not `Alive`. The synthetic leave ltime for a left node is + /// `status_ltimes[id] + 1` (the leave is necessarily after the last known + /// join time; using `+1` avoids a stale-intent rejection while keeping + /// the causal ordering intact). + /// + /// **G4 — `eventJoinIgnore` bumps `event_buffer.min_time`:** + /// If `is_join` is `true` AND `self.event_join_ignore` is set, the + /// `event_buffer.min_time` is raised to `max(min_time, event_ltime)`. + /// This suppresses all buffered user events from the remote peer that + /// pre-date the current event-clock (prevents re-emitting stale events + /// on a fresh join exchange). + /// + /// After the intent passes, every buffered user event in the push-pull + /// body is replayed via `handle_user_event`. Events older than + /// `min_time` or already deduped in the ring are silently dropped + /// (handled inside `handle_user_event` / `EventBuffer::witness_event`). + /// + /// Decoding errors in the `user_data` bytes are silently dropped — + /// the machine must not panic on bad network input. + fn merge_remote_state(&mut self, user_data: Bytes, is_join: bool) + where + I: Clone + Data, + A: Data, + { + // Exact-consumption decode: the buffer must hold exactly one serf frame. + // A valid-prefix payload with trailing bytes (e.g. a PushPull frame followed + // by junk) must be dropped before any state mutation — trailing junk would + // otherwise allow clock witnessing, intent application, and dirty-marking + // while bypassing the semantic validity of the whole frame. + let pp = match AnyMessage::::decode_with_consumed(&user_data) { + Ok((AnyMessage::PushPull(pp), consumed)) if consumed == user_data.len() => pp, + _ => return, + }; + + let ltime = pp.ltime.0; + let event_ltime = pp.event_ltime.0; + let query_ltime = pp.query_ltime.0; + + // Whole-message drop gate for push-pull: if ANY of the three top-level + // Lamport clocks is unacceptable (u64::MAX or u64::MAX-1), drop the ENTIRE + // push-pull message — no mark_local_state_dirty, no status_ltimes + // processing, no left_members processing, no event replay. A single + // adversarial clock in a push-pull must not let the rest of the body apply. + // + // G2 witness uses `value - 1` so we need `value > 0` as an additional guard + // to avoid underflow; non-zero is checked inside the witness blocks below. + if !ltime_is_acceptable(ltime) + || !ltime_is_acceptable(event_ltime) + || !ltime_is_acceptable(query_ltime) + { + return; + } + + // G2: witness all three clocks at remote_value - 1, guarded > 0. + // Capture clock values before witnessing to detect whether any advanced. + let clock_before = self.clock; + let event_clock_before = self.event_clock; + let query_clock_before = self.query_clock; + if ltime > 0 { + witness(&mut self.clock, ltime - 1); + } + if event_ltime > 0 { + witness(&mut self.event_clock, event_ltime - 1); + } + if query_ltime > 0 { + witness(&mut self.query_clock, query_ltime - 1); + } + // Mark dirty only if a clock actually advanced (stale push-pulls whose three + // clocks are all <= current are no-ops and must not force a resync). + if self.clock != clock_before + || self.event_clock != event_clock_before + || self.query_clock != query_clock_before + { + self.mark_local_state_dirty(); + } + + // Build a fast lookup map from the status_ltimes Vec so we can look up a + // node's ltime when processing left_members (Vec is the wire type; HashMap + // is the lookup we need — O(n) build, O(1) lookup per left entry). + // Entries whose ltime is not acceptable are silently excluded — they would + // write a permanent status_time tombstone that no finite join intent can + // ever outrank. + let status_map: std::collections::HashMap<&I, LamportTime> = pp + .status_ltimes + .iter() + .filter(|(_, lt)| ltime_is_acceptable(lt.0)) + .map(|(id, lt)| (id, *lt)) + .collect(); + + // Build a fast lookup set for left_members so the join pass can skip them. + let left_set: std::collections::HashSet<&I> = pp.left_members.iter().collect(); + + let now = self.drain_now; + + // G3a: process left_members first as synthetic leave intents. + // The synthetic ltime = status_ltimes[id] + 1 (leave is causally after join). + // Nodes absent from status_map (because their ltime was u64::MAX or missing) + // are silently skipped — the oracle logs an error and skips; we do the same. + // handle_node_leave_intent calls mark_local_state_dirty() when it applies a change. + for node_id in &pp.left_members { + if let Some(&status_ltime) = status_map.get(node_id) { + let leave_ltime = LamportTime(status_ltime.0.saturating_add(1)); + self.handle_node_leave_intent(leave_ltime, node_id, false, now); + } + } + + // G3b: process status_ltimes as join intents, SKIPPING left nodes. + // status_map already excludes u64::MAX entries, so handle_node_join_intent's + // own u64::MAX gate is a redundant safety net here. + // handle_node_join_intent calls mark_local_state_dirty() when it applies a change. + for (node_id, ltime) in &pp.status_ltimes { + if left_set.contains(node_id) { + continue; + } + self.handle_node_join_intent(*ltime, node_id, now); + } + + // G4: if is_join && event_join_ignore, bump event_buffer.min_time to + // max(min_time, remote event_ltime). This prevents pre-join events + // from being replayed on a fresh join exchange. + if is_join && self.event_join_ignore && event_ltime > self.event_buffer.min_time { + self.event_buffer.min_time = event_ltime; + self.mark_local_state_dirty(); + } + + // Replay every buffered user event from the remote push-pull body. + // handle_user_event applies the min_time gate and dedup ring check, + // so stale or already-seen events are silently discarded. + // handle_user_event calls mark_local_state_dirty() when it accepts a new event. + for user_events in pp.events { + for ev in user_events.events { + let msg = UserEventMessage { + ltime: user_events.ltime, + cc: false, + name: ev.name, + payload: ev.payload, + }; + self.handle_user_event(msg); + } + } + } + + // ── member-status FSM handlers ─────────────────────────────────────────── + + /// Handle an inner `NodeJoined` event. + /// + /// If the node is already known and was previously Failed or Left, clears it + /// from those lists, resets `leave_time`, and sets status to Alive. For a + /// brand-new node, consults the recent-intent buffer: a pending Leave intent + /// sets initial status to `Leaving`; a Join intent (only) or no intent gives + /// `Alive`. Emits `Event::Member(Join)` in all cases. + /// + /// Mirrors Go serf `base.go` `handleNodeJoin` (lines 1213-1341). + fn handle_node_join(&mut self, node: &NodeState, _now: Instant) + where + I: Clone, + A: Clone, + { + // Decode tags from the node's meta bytes. Go serf's oracle returns early + // on a tag-decode failure, silently dropping the join — a footgun for + // callers who injected a node they expect to appear. The Sans-I/O machine + // cannot skip the join without emitting the event, so on decode failure we + // fall back to empty tags and proceed. + let tags = if node.meta_ref().is_empty() { + Tags::new() + } else { + // Tags is a HashMap in serf-proto. The memberlist + // meta field carries the serialised tags; at this stage we decode them + // via the protobuf framing helpers. If decoding fails, fall back to + // empty tags (forward-compat). + decode_tags_from_meta(node.meta_ref().as_bytes()).unwrap_or_default() + }; + + let id = node.id_ref(); + let n = node.node(); + + let (old_status, new_state) = if let Some(ms) = self.members.states.get_mut(id) { + let old = ms.status(); + // Keep existing status_time when re-joining. + let st = ms.status_time(); + ms.set_status(MemberStatus::Alive); + ms.set_leave_time(None); + *ms.member_mut() = Member::new(n.clone(), tags, MemberStatus::Alive); + ms.set_status_time(st); + (old, None) + } else { + // New node: check for a pending intent in recent_intents. + // The Leave intent takes priority over Join if both somehow coexist + // (the oracle checks Join then Leave, with Leave winning because it + // is checked second and overwrites status_ltime). + let (status, status_ltime) = { + let mut s = MemberStatus::Alive; + let mut lt = LamportTime::ZERO; + if let Some(t) = self.members.recent_intent(id, IntentKind::Join) { + lt = t; + } + if let Some(t) = self.members.recent_intent(id, IntentKind::Leave) { + lt = t; + s = MemberStatus::Leaving; + } + (s, lt) + }; + let ms = MemberState::new(Member::new(n.clone(), tags, status), status_ltime, None); + (MemberStatus::None, Some(ms)) + }; + + if let Some(ms) = new_state { + self.members.states.insert(id.clone(), ms); + } + + // Clear from failed/left lists when re-joining after failure or leave. + if matches!(old_status, MemberStatus::Failed | MemberStatus::Left) { + self.members.failed_members.retain(|i| i != id); + self.members.left_members.retain(|i| i != id); + } + + // Membership changed — snapshot is stale. + self.mark_local_state_dirty(); + + // Always emit Member(Join). + let member = self.members.states[id].member().clone(); + self + .pending_events + .push_back(Event::Member(MemberEvent::new( + MemberEventKind::Join, + vec![member], + ))); + } + + /// Handle an inner `NodeLeft` event. + /// + /// `Leaving` → `Left` (pushes to `left_members`, emits Leave). + /// `Alive` → `Failed` (pushes to `failed_members`, emits Failed). + /// Any other status is a no-op (mirrors the oracle's `_` arm at base.go:1410). + /// + /// Mirrors Go serf `base.go` `handleNodeLeave` (lines 1382-1447). + fn handle_node_leave(&mut self, node: &NodeState, now: Instant) + where + I: Clone, + { + let id = node.id_ref(); + let Some(ms) = self.members.states.get_mut(id) else { + return; + }; + + let current = ms.status(); + let (new_status, event_kind) = match current { + MemberStatus::Leaving => (MemberStatus::Left, MemberEventKind::Leave), + MemberStatus::Alive => (MemberStatus::Failed, MemberEventKind::Failed), + // Any other status (None, Left, Failed) is a no-op per the oracle. + _ => return, + }; + + ms.set_status(new_status); + ms.set_leave_time(Some(now)); + + let member = ms.member().clone(); + let id_clone = id.clone(); + + match new_status { + MemberStatus::Left => self.members.left_members.push(id_clone), + MemberStatus::Failed => self.members.failed_members.push(id_clone), + _ => {} + } + + // Membership changed — snapshot is stale. + self.mark_local_state_dirty(); + + self + .pending_events + .push_back(Event::Member(MemberEvent::new(event_kind, vec![member]))); + } + + /// Handle an inner `NodeUpdated` event. + /// + /// Refreshes the member's tags. If the node is not tracked, this is a + /// no-op (matches the oracle's `if let Some(ms) = members.states.get_mut`). + /// Emits `Event::Member(Update)`. + /// + /// Mirrors Go serf `base.go` `handleNodeUpdate` (lines 1583-1631). + fn handle_node_update(&mut self, node: &NodeState) + where + I: Clone, + A: Clone, + { + let id = node.id_ref(); + let Some(ms) = self.members.states.get_mut(id) else { + return; + }; + + let tags = if node.meta_ref().is_empty() { + Tags::new() + } else { + decode_tags_from_meta(node.meta_ref().as_bytes()).unwrap_or_default() + }; + + let n = node.node(); + let status = ms.status(); + *ms.member_mut() = Member::new(n, tags, status); + let member = ms.member().clone(); + // Drop the mutable borrow on ms before calling mark_local_state_dirty. + + // Membership changed — snapshot is stale. + self.mark_local_state_dirty(); + + self + .pending_events + .push_back(Event::Member(MemberEvent::new( + MemberEventKind::Update, + vec![member], + ))); + } + + /// Handle a gossiped join intent (`JoinMessage`). + /// + /// Witnesses the member clock. If the node is already in the membership + /// store and the intent is stale (`ltime <= status_time`), returns `false`. + /// Otherwise, updates `status_time` and, if the member is currently + /// `Leaving`, clears it back to `Alive`. If the node is absent, buffers + /// the intent via `upsert_intent`. + /// + /// Returns `true` if the intent should be rebroadcast. + /// + /// Mirrors Go serf `base.go` `handleNodeJoinIntent` (lines 1345-1380). + pub(crate) fn handle_node_join_intent(&mut self, ltime: LamportTime, id: &I, now: Instant) -> bool + where + I: Clone, + { + // Whole-message drop gate: reject unacceptable Lamport times before any + // state mutation, clock witness, dirty flag, or event emission. + if !ltime_is_acceptable(ltime.0) { + return false; + } + // Witness a potentially newer member clock. + witness(&mut self.clock, ltime.0); + + if let Some(ms) = self.members.states.get_mut(id) { + // Stale intent: ltime <= status_time — no state change, no dirty mark. + if ltime <= ms.status_time() { + return false; + } + ms.set_status_time(ltime); + // If we are Leaving, a newer join intent clears it back to Alive + // (the leave must have been for an older time). + if ms.status() == MemberStatus::Leaving { + ms.set_status(MemberStatus::Alive); + *ms.member_mut() = { + let m = ms.member(); + Member::new(m.node().clone(), m.tags().clone(), MemberStatus::Alive) + }; + } + self.mark_local_state_dirty(); + true + } else { + // Node not yet seen — buffer the intent. + let buffered = self.members.upsert_intent(id, IntentKind::Join, ltime, now); + if buffered { + self.mark_local_state_dirty(); + } + buffered + } + } + + /// Handle a gossiped leave intent (`LeaveMessage`). + /// + /// Witnesses the member clock. If the node is absent, buffers via + /// `upsert_intent`. If the node is present and the intent is stale + /// (`ltime <= status_time`), returns `false`. + /// + /// **Self-refute:** if the intent targets the local node id and the local + /// `SerfState` is `Alive`, the node refutes the leave by enqueuing a join + /// broadcast and returns `false`. + /// + /// **`status_time` update (consul#8179 / consul#7960):** `status_time` is + /// updated unconditionally even when the current status is already `Leaving` + /// or `Left`, preventing the infinite-rebroadcast bug. + /// + /// State transitions: + /// - `Alive` → `Leaving` + /// - `Failed` → `Left` (move to `left_members`, emit `Member(Leave)`) + /// - `Leaving | Left` → stay (status_time updated) + /// - `None` → `false` + /// + /// Returns `true` if the intent should be rebroadcast. + /// + /// Mirrors Go serf `base.go` `handleNodeLeaveIntent` (lines 1449-1579). + pub(crate) fn handle_node_leave_intent( + &mut self, + ltime: LamportTime, + id: &I, + prune: bool, + now: Instant, + ) -> bool + where + I: Clone, + { + // Whole-message drop gate: reject unacceptable Lamport times before any + // state mutation, clock witness, dirty flag, or event emission. + if !ltime_is_acceptable(ltime.0) { + return false; + } + // Witness a potentially newer member clock. + witness(&mut self.clock, ltime.0); + + if !self.members.states.contains_key(id) { + let buffered = self + .members + .upsert_intent(id, IntentKind::Leave, ltime, now); + if buffered { + self.mark_local_state_dirty(); + } + return buffered; + } + + // Stale intent: ltime <= status_time → no transition, no rebroadcast, no dirty. + // This guard fires for ALL nodes, including the local node, before the + // self-refute path. A stale leave for self must not trigger broadcast_join. + let ms = self.members.states.get(id).unwrap(); + if ltime <= ms.status_time() { + return false; + } + + // Self-refute: if this is a FRESH leave intent for the local node and the local + // endpoint is Alive, push back with a join broadcast and suppress the rebroadcast. + // We check this AFTER the stale guard so stale self-leaves do not trigger refutes. + // broadcast_join calls handle_node_join_intent which marks dirty when it buffers. + let is_local = id == self.inner.local_id_ref(); + if is_local && self.state == SerfState::Alive { + // Refute the leave by re-announcing our own Join intent so every peer + // clears the spurious Leaving status; the rebroadcast of the leave itself + // is suppressed. + self.broadcast_join(LamportTime(self.clock)); + return false; + } + + // Always update status_time even if the status doesn't change; the + // infinite-rebroadcast bug (consul#8179 / consul#7960) fires when the + // time is left stale on a repeated intent. + let ms = self.members.states.get_mut(id).unwrap(); + ms.set_status_time(ltime); + + let current = ms.status(); + match current { + MemberStatus::None => false, + MemberStatus::Alive => { + ms.set_status(MemberStatus::Leaving); + *ms.member_mut() = { + let m = ms.member(); + Member::new(m.node().clone(), m.tags().clone(), MemberStatus::Leaving) + }; + self.mark_local_state_dirty(); + // prune forgets the member outright instead of leaving the Leaving + // tombstone for the reaper. + if prune { + self.prune_member(id); + } + true + } + MemberStatus::Leaving | MemberStatus::Left => { + self.mark_local_state_dirty(); + if prune { + self.prune_member(id); + } + true + } + MemberStatus::Failed => { + let id_clone = id.clone(); + ms.set_status(MemberStatus::Left); + *ms.member_mut() = { + let m = ms.member(); + Member::new(m.node().clone(), m.tags().clone(), MemberStatus::Left) + }; + let member = ms.member().clone(); + // Move from failed_members to left_members. + self.members.failed_members.retain(|i| i != &id_clone); + self.members.left_members.push(id_clone); + self + .pending_events + .push_back(Event::Member(MemberEvent::new( + MemberEventKind::Leave, + vec![member], + ))); + self.mark_local_state_dirty(); + if prune { + self.prune_member(id); + } + true + } + } + } + + /// Forget a member entirely (the `prune` path of a forced leave). + /// + /// Removes the member from the state store, the left/failed index lists, and + /// the recent-intent buffer, drops any cached Vivaldi coordinate, and emits a + /// `Member(Reap)` event. Unlike the reaper, this fires immediately rather + /// than after the tombstone timeout — a `force_leave(prune = true)` purges the + /// node now. Mirrors serf-core `base.rs` `handle_prune` + `erase_node!`. + /// + /// Go serf's `handle_prune` only removes a `Leaving`/`Left` node from + /// `left_members` (relying on the invariant that a node is never in both + /// lists) and sleeps `broadcast_timeout + leave_propagate_delay` for a + /// `Leaving` member before erasing. The Sans-I/O machine cannot sleep, so + /// the prune is immediate; it also scrubs every index list and the + /// recent-intent entry so no stale reference to the forgotten node survives + /// in any structure. + fn prune_member(&mut self, id: &I) { + self.members.left_members.retain(|i| i != id); + self.members.failed_members.retain(|i| i != id); + self.members.recent_intents.remove(id); + + #[cfg(feature = "coordinates")] + { + if let Some(ref mut cc) = self.coord_client { + cc.forget_node(id); + } + self.coord_cache.remove(id); + } + + if let Some(ms) = self.members.states.remove(id) { + self + .pending_events + .push_back(Event::Member(MemberEvent::new( + MemberEventKind::Reap, + vec![ms.member().clone()], + ))); + } + self.mark_local_state_dirty(); + } + + // ── reaper ──────────────────────────────────────────────────────────────── + + /// Reap stale left/failed members and expired intents. + /// + /// Run order (mirrors Go serf `base.go` `Reaper.run`): + /// 1. `reap_failed` — remove `Failed` members whose `leave_time` elapsed + /// past `reconnect_timeout` (24 h default). + /// 2. `reap_left` — remove `Left` (tombstone) members whose `leave_time` + /// elapsed past `tombstone_timeout` (24 h default). + /// 3. `reap_intents` — purge `recent_intents` entries older than + /// `recent_intent_timeout` (10 min default). + /// + /// Each reaped member is erased from `states` and its id-list, and a + /// `Event::Member(Reap)` is emitted. + /// + /// `now` is the driver-threaded instant; no wall-clock reads occur here. + fn fire_reap(&mut self, now: Instant) + where + I: Clone, + A: Clone, + { + let reconnect_timeout = self.opts.reconnect_timeout(); + let tombstone_timeout = self.opts.tombstone_timeout(); + let intent_timeout = self.opts.recent_intent_timeout(); + + // Reap failed members whose leave_time > reconnect_timeout. + let mut i = 0; + while i < self.members.failed_members.len() { + let id = self.members.failed_members[i].clone(); + let expired = self + .members + .states + .get(&id) + .and_then(|ms| ms.leave_time()) + .is_some_and(|lt| now.duration_since(lt) > reconnect_timeout); + if expired { + self.members.failed_members.swap_remove(i); + if let Some(ms) = self.members.states.remove(&id) { + // G13: purge stale latency-filter history so a rejoining peer starts + // with a clean Vivaldi model rather than inheriting stale samples. + #[cfg(feature = "coordinates")] + { + if let Some(ref mut cc) = self.coord_client { + cc.forget_node(&id); + } + self.coord_cache.remove(&id); + } + self + .pending_events + .push_back(Event::Member(MemberEvent::new( + MemberEventKind::Reap, + vec![ms.member().clone()], + ))); + } + // Do not increment i — swap_remove moved the last element here. + } else { + i += 1; + } + } + + // Reap left (tombstone) members whose leave_time > tombstone_timeout. + let mut i = 0; + while i < self.members.left_members.len() { + let id = self.members.left_members[i].clone(); + let expired = self + .members + .states + .get(&id) + .and_then(|ms| ms.leave_time()) + .is_some_and(|lt| now.duration_since(lt) > tombstone_timeout); + if expired { + self.members.left_members.swap_remove(i); + if let Some(ms) = self.members.states.remove(&id) { + // G13: purge stale latency-filter history so a rejoining peer starts + // with a clean Vivaldi model rather than inheriting stale samples. + #[cfg(feature = "coordinates")] + { + if let Some(ref mut cc) = self.coord_client { + cc.forget_node(&id); + } + self.coord_cache.remove(&id); + } + self + .pending_events + .push_back(Event::Member(MemberEvent::new( + MemberEventKind::Reap, + vec![ms.member().clone()], + ))); + } + } else { + i += 1; + } + } + + // Reap stale recent intents. + self + .members + .recent_intents + .retain(|_, intent| now.duration_since(intent.wall_time()) <= intent_timeout); + + // Membership may have changed — snapshot is stale. + self.mark_local_state_dirty(); + } + + // ── reconnector ─────────────────────────────────────────────────────────── + + /// Probabilistically re-dial a random failed peer (reconnect attempt). + /// + /// Mirrors Go serf `base.go` `Reconnector.run` (lines 628-680): + /// + /// 1. If `failed_members` is empty, return immediately (no-op). + /// 2. Compute `prob = num_failed / max(num_alive, 1)` where `num_alive` is + /// the number of members that are neither failed nor left. + /// 3. Draw `r: f32` from `self.rng`; if `r > prob`, skip (probabilistic + /// throttle so that each failed peer is retried roughly once per + /// `reconnect_interval` across the whole cluster). + /// 4. Pick a random index into `failed_members` (uniform; draw from `self.rng`). + /// 5. Call `inner.start_push_pull(addr, PushPullKind::Join, now)`. + /// The inner queues `Event::DialRequested` which the sieve passes through + /// to the driver as `Event::DialRequested(DialPassthrough { .. })`. + /// + /// H3: serf emits a dial request; it does NO dial itself. + fn fire_reconnect(&mut self, now: Instant) + where + A: Clone, + { + let num_failed = self.members.failed_members.len(); + if num_failed == 0 { + return; + } + + // num_alive = all members that are neither failed nor left (Go serf: + // `(mu.states.len() - num_failed - mu.left_members.len()).max(1)`). + let num_alive = self + .members + .states + .len() + .saturating_sub(num_failed) + .saturating_sub(self.members.left_members.len()) + .max(1); + + let prob = num_failed as f32 / num_alive as f32; + let r: f32 = self.rng.random(); + if r > prob { + // Probabilistic throttle: skip this interval. + return; + } + + // Pick a random failed member (uniform distribution). + let idx: usize = self.rng.random_range(0..num_failed); + let id = self.members.failed_members[idx].clone(); + let Some(ms) = self.members.states.get(&id) else { + return; + }; + let addr = ms.member().node().addr_ref().clone(); + + // Call inner.start_push_pull; the inner queues Event::DialRequested. + // The sieve (on_inner_event) passes it through as Event::DialRequested. + self.inner.start_push_pull(addr, PushPullKind::Join, now); + self.drain_inner(); + } + + // ── join ───────────────────────────────────────────────────────────────── + + /// Announce the local node's join intent to the cluster. + /// + /// The driver owns the inner-memberlist join (it dials the seed nodes and + /// drives the push-pull exchange); once that succeeds it calls this method so + /// serf gossips its own `Join` intent and peers learn the local node's join + /// ltime without waiting for the next anti-entropy round. Push-pull only + /// backstops it. Mirrors serf-core `api.rs` `join()`'s + /// `broadcast_join(self.clock.time())` call on inner-join success. + /// + /// State gate: only `Alive` announces a join; any other lifecycle state + /// returns [`Error::BadJoinState`]. + pub fn join(&mut self) -> Result<(), Error> + where + I: Clone, + { + if self.state != SerfState::Alive { + return Err(Error::BadJoinState(self.state)); + } + self.broadcast_join(LamportTime(self.clock)); + Ok(()) + } + + /// Broadcast a `Join` intent for the local node at `ltime`. + /// + /// Used both on join (announce) and to refute a stale leave intent targeting + /// the local node. Witnesses the member clock at `ltime`, applies the intent + /// locally via `handle_node_join_intent`, then enqueues the encoded + /// `JoinMessage` on the intent tier (rank 0, highest priority). Mirrors + /// serf-core `base.rs` `broadcast_join`. + fn broadcast_join(&mut self, ltime: LamportTime) + where + I: Clone, + { + let local_id = self.inner.local_id_ref().clone(); + + // Witness the member clock, then apply the intent locally so the local node + // is recorded at this ltime (handle_node_join_intent also witnesses, but the + // explicit witness here mirrors the oracle and is a no-op on the second pass). + witness(&mut self.clock, ltime.0); + let now = self.drain_now; + self.handle_node_join_intent(ltime, &local_id, now); + + // Encode and enqueue on the intent tier. Ignoring Err: a `Join` carrying a + // single id never approaches the gossip MTU, so the only error path is an + // encode failure on a degenerate id type, which the driver surfaces at + // construction; a dropped intent is re-announced by the next push-pull. + let jm = JoinMessage::new(ltime, local_id); + if let Ok(encoded) = AnyMessage::::Join(jm).encode() { + let _ = self.inner.queue_user_broadcast_ranked(0, encoded); + } + } + + // ── leave chain ────────────────────────────────────────────────────────── + + /// Begin a graceful leave. + /// + /// State-machine gate (mirrors Go serf `api.go` `Leave()`): + /// - `Left` → `Ok(())` (idempotent; leave already completed). + /// - `Leaving` | `Shutdown` → `Err(BadLeaveState)`. + /// - `Alive` → proceeds with the leave chain below. + /// + /// Leave chain (decision 5 / oracle `api.go` leave()): + /// 1. Set `state = Leaving`. + /// 2. Handle the local leave intent (`handle_node_leave_intent` for the + /// local id), which marks the local node as `Leaving` in the membership + /// store and queues a join-refute suppression. + /// 3. Increment the member clock. + /// 4. Enqueue a leave-intent broadcast on the intent tier (rank 0) so + /// peers learn about the leave. (`FIX`: no `on_finished` callback — + /// flushing is bounded by `broadcast_timeout` deadline instead.) + /// 5. Call inner `leave(now)` to begin the memberlist dead-self fan-out. + /// The inner will eventually emit `Event::LeftCluster` once all dead-self + /// packets are drained via `poll_transmit`. + /// 6. Arm `leave_broadcast_deadline = now + broadcast_timeout`. The + /// driver can short-circuit by watching `user_broadcast_queue_len()`. + /// + /// The `Leaving → Left` transition happens later in `handle_timeout` when + /// `leave_complete_deadline` (armed on inner `LeftCluster` + `leave_propagate_delay`) + /// elapses. `Event::LeftCluster` is emitted at that point. + pub fn leave(&mut self, now: Instant) -> Result<(), Error> + where + I: Clone, + A: Clone, + { + match self.state { + SerfState::Left => return Ok(()), // idempotent + SerfState::Leaving | SerfState::Shutdown => { + return Err(Error::BadLeaveState(self.state)); + } + SerfState::Alive => {} + } + + // 1. Transition to Leaving. + self.state = SerfState::Leaving; + + // 2. Local leave intent — marks the local node as Leaving in the store + // and witnesses the member clock. + let local_id = self.inner.local_id_ref().clone(); + // next_ltime stamps the current clock value (clamped to < LTIME_MAX) and + // advances the clock in one atomic step, closing the local-emission hole. + let ltime = LamportTime(next_ltime(&mut self.clock)); + self.mark_local_state_dirty(); + // We are setting state = Leaving above so the self-refute guard in + // handle_node_leave_intent will NOT fire (it only fires when state == Alive). + self.handle_node_leave_intent(ltime, &local_id, false, now); + + // 4. Broadcast the leave intent on the intent tier (rank 0) so peers learn + // the local node is leaving without waiting for anti-entropy. The + // driver bounds the flush via the broadcast deadline below. + self.broadcast_leave(ltime, local_id, false); + + // 5. Arm the broadcast-timeout deadline so the driver always has a finite + // wait; it can short-circuit by watching `user_broadcast_queue_len()`. + self.leave_broadcast_deadline = Some(now + self.opts.broadcast_timeout()); + + // 5. Call inner leave; this queues the dead-self fan-out packets. + self.inner.leave(now)?; + + Ok(()) + } + + /// Forcibly remove another node from the cluster by broadcasting a leave intent. + /// + /// Mirrors Go serf `api.go` `force_leave()` / `remove_failed_node_prune()`. + /// Broadcasts a `LeaveMessage` with `prune` set to `prune`; this causes + /// peers to immediately remove the node from the membership store rather than + /// waiting for tombstone timeout. + /// + /// Does not require the local endpoint to be `Alive` (callers may want to + /// clean up failed nodes before leaving themselves), but rejects `Shutdown`. + pub fn force_leave(&mut self, id: I, prune: bool, now: Instant) -> Result<(), Error> + where + I: Clone, + A: Clone, + { + if self.state == SerfState::Shutdown { + return Err(Error::BadLeaveState(self.state)); + } + + // Advance first, then stamp the new value so the forced leave outranks any + // prior status the target node has (post-increment semantics, unlike leave() + // which stamps pre-increment). saturating_add is the no-UB backstop. + self.clock = self.clock.saturating_add(1); + let ltime = LamportTime(self.clock); + self.mark_local_state_dirty(); + self.handle_node_leave_intent(ltime, &id, prune, now); + + // Broadcast the leave intent (carrying the prune flag) so peers apply the + // same forced removal. + self.broadcast_leave(ltime, id, prune); + + // Arm the broadcast deadline so the driver knows how long to wait. + self.leave_broadcast_deadline = Some(now + self.opts.broadcast_timeout()); + + Ok(()) + } + + /// Encode and enqueue a `Leave` intent for `id` at `ltime` on the intent tier. + /// + /// Shared by the graceful `leave()` (local id, `prune = false`) and + /// `force_leave()` (remote id, caller's `prune`). Mirrors the + /// `encode_message_to_bytes(&LeaveMessage) + broadcast` step in serf-core + /// `api.rs` `leave()` and `base.rs` `force_leave()`. + /// + /// Go serf gates the broadcast on `has_alive_members()` (skipping the enqueue + /// when the cluster is a singleton). The Sans-I/O machine enqueues + /// unconditionally — the inner gossip layer only transmits when peers exist, + /// so an enqueue against an empty cluster is a harmless no-op rather than a + /// special case. + fn broadcast_leave(&mut self, ltime: LamportTime, id: I, prune: bool) { + let lm = LeaveMessage::new(ltime, id, prune); + // Ignoring Err: a `Leave` carrying a single id never approaches the gossip + // MTU, so the only error path is an encode failure on a degenerate id type + // (a construction-time concern the driver surfaces); a dropped intent is + // re-announced by the next anti-entropy round. + if let Ok(encoded) = AnyMessage::::Leave(lm).encode() { + let _ = self.inner.queue_user_broadcast_ranked(0, encoded); + } + } + + // ── user-event API ─────────────────────────────────────────────────────── + + /// Fire a user event (mirrors Go serf `api.go` `UserEvent`). + /// + /// Sequence (oracle `api.go` `user_event`): + /// 1. Pre-encode size check against `max_user_event_size` (name.len + payload.len). + /// 2. Stamp `ltime = self.event_clock` (pre-increment value). + /// 3. Build `UserEventMessage { ltime, name, payload, cc: coalesce }`. + /// 4. Encode to bytes; post-encode size check. + /// 5. Increment the event clock (`self.event_clock += 1`). + /// 6. Process locally via `handle_user_event` (dedup + emit `Event::User`). + /// 7. Enqueue encoded bytes on the **event tier** (rank 2) via + /// `inner.queue_user_broadcast_ranked(2, bytes)`. + /// + /// Go serf checks `max_user_event_size` in three places; this port + /// consolidates to two (pre-name+payload-len, post-encoded-len), matching + /// the oracle's intent without the redundant intermediate check. + pub fn user_event( + &mut self, + name: impl Into, + payload: bytes::Bytes, + coalesce: bool, + ) -> Result<(), Error> { + let name: smol_str::SmolStr = name.into(); + let max_size = self.opts.max_user_event_size(); + + // Pre-encode size check: name.len() + payload.len() must not exceed the limit. + let pre_len = name.len() + payload.len(); + if pre_len > max_size { + return Err(Error::UserEventTooLarge(pre_len, max_size)); + } + + // Stamp ltime via next_ltime: clamps to < LTIME_MAX and advances the clock. + let ltime = LamportTime(next_ltime(&mut self.event_clock)); + + let msg = UserEventMessage { + ltime, + cc: coalesce, + name: name.clone(), + payload: payload.clone(), + }; + + // Encode to bytes for the broadcast queue. + // UserEvent wire encoding does not depend on I or A — encode directly via + // the bridge + framing helpers to avoid a phantom type parameter. + let pb = user_event_to_pb(&msg); + let encoded: bytes::Bytes = encode_message(MessageType::UserEvent, &pb) + .map_err(EncodeError::Frame) + .map(|v| v.into())?; + + // Post-encode size check. + if encoded.len() > max_size { + return Err(Error::UserEventTooLarge(encoded.len(), max_size)); + } + + self.mark_local_state_dirty(); + + // Process locally (dedup + emit Event::User if first sight). + // For a locally-originated event the dedup always accepts it (it was + // not in the ring yet), so we can safely ignore the bool return. + let _ = self.handle_user_event(msg); + + // Enqueue on the event tier (rank 2 = bottom priority after intent=0, query=1). + // Ignoring Err: the inner rejects oversized frames; the pre-encode size + // check above already guarantees we are within max_user_event_size, so + // the only remaining path to an error is a frame larger than the inner's + // gossip MTU — a configuration mismatch the driver should detect at + // startup. We propagate it back rather than silently drop. + self + .inner + .queue_user_broadcast_ranked(2, encoded) + .map_err(Error::InnerLeave)?; + + Ok(()) + } + + /// Ingress handler for a received `UserEventMessage`. + /// + /// Called both on locally-originated events (from `user_event`) and on + /// gossiped events decoded from `UserPacket`. + /// + /// Mirrors Go serf `base.go` `handleUserEvent`: + /// 1. Witness the event clock. + /// 2. Drop if `ltime < min_time`. + /// 3. Drop if the event is too old relative to the ring size + /// (`cur_time > ring_len && ltime < cur_time - ring_len`). + /// 4. Dedup against the ring slot for this ltime. + /// 5. If new: emit `Event::User(msg)`. + /// + /// Returns `true` if the event was new (should be rebroadcast by the caller). + pub(crate) fn handle_user_event(&mut self, msg: UserEventMessage) -> bool { + let ltime = msg.ltime.0; + + // Whole-message drop gate: reject unacceptable Lamport times before any + // state mutation, clock witness, dirty flag, or event emission. + if !ltime_is_acceptable(ltime) { + return false; + } + + // Reject inbound events that exceed the configured size limit. + // Mirrors the pre-encode check in user_event() for locally-originated events, + // applied here on the ingress path to bound rebroadcast amplification. + let pre_len = msg.name.len() + msg.payload.len(); + if pre_len > self.opts.max_user_event_size() { + return false; + } + + // Witness a potentially newer event clock. + witness(&mut self.event_clock, ltime); + let cur_time = self.event_clock; + + let ev = UserEvent { + name: msg.name.clone(), + payload: msg.payload.clone(), + }; + + // Dedup ring: returns false for duplicates and stale events — no state change. + // Mark dirty only when witness_event confirms this is a new event. + if !self.event_buffer.witness_event(cur_time, ltime, ev) { + return false; + } + + self.mark_local_state_dirty(); + // First sight — emit to the driver. + self.pending_events.push_back(Event::User(msg)); + true + } + + /// Decode a serf `AnyMessage` off a `UserPacket` and dispatch to the + /// matching handler. + /// + /// The original `Bytes` buffer is retained (refcount-shared) for + /// re-broadcast — the relay-retain invariant: no re-encode on the + /// re-broadcast path. + /// + /// Dispatch table (oracle: `delegate.rs` `NotifyMsg`): + /// - `UserEvent` → `handle_user_event`; if new, `rebroadcast` on event tier. + /// - `Join` → `handle_node_join_intent`; if rebroadcast, intent tier. + /// - `Leave` → `handle_node_leave_intent`; if rebroadcast, intent tier. + /// - `Query` → `handle_query`; if rebroadcast, query tier. + /// - `QueryResponse` → `handle_query_response`: fold into `PendingQuery`. + /// - `Relay` → `handle_relay`: verbatim forward to destination. + /// - `ConflictResponse`→ dropped (conflict responses arrive as `QueryResponse` payloads). + /// - `KeyRequest`/`KeyResponse` → dropped (key responses arrive as `QueryResponse` payloads). + /// - `PushPull` → not valid on a UserPacket; dropped. + /// - Decode failure → silent drop (never panic on bad network input). + /// + /// H4: both `Reliability::Reliable` and `Reliability::Unreliable` dispatch + /// identically; the reliability value affects delivery guarantees at the + /// memberlist layer but not the serf handler logic. + fn handle_user_packet(&mut self, _from: A, data: Bytes, now: Instant) + where + I: Clone + Data, + A: Clone + Data, + { + // Pre-decode size fence: peek the message-type tag and the total frame + // length from the raw bytes before allocating or bridge-converting the body. + // For size-bounded types (UserEvent, Query) reject frames whose encoded + // length exceeds the configured limit before any decode work — memory cost + // scales with the attacker-controlled body size, so the fence must come + // first. `data.len()` is the frame length the exact-consumption gate will + // also enforce below; peeking from the raw bytes avoids body allocation. + // + // Relay and QueryResponse carry no configured size limit — the fence is a + // no-op for them; their payloads are already limited by the inner + // Endpoint's per-packet MTU contract and the query_size_limit applied at + // the originating peer before the query was sent. + match peek_frame_header(data.as_ref()) { + Ok((MessageType::UserEvent, _)) if data.len() > self.opts.max_user_event_size() => { + return; + } + Ok((MessageType::Query, _)) if data.len() > self.opts.query_size_limit() => { + return; + } + // Incomplete / empty / varint-overflow frames are caught by the + // decode_with_consumed call below; don't double-drop here. + _ => {} + } + + // Exact-consumption decode: the buffer must hold exactly one serf frame. + // A packet whose decoded frame does not consume the entire buffer is + // malformed (e.g. a valid small message followed by trailing junk bytes); + // drop before any state mutation, clock witness, event emission, or + // rebroadcast to prevent junk amplification and budget bypass. + let msg = match AnyMessage::::decode_with_consumed(&data) { + Ok((m, consumed)) if consumed == data.len() => m, + // Decode failure or trailing junk — drop silently; bad bytes must never + // panic the machine. + _ => return, + }; + + match msg { + AnyMessage::UserEvent(ue) => { + // Relay-retain: pass the original `data` bytes to rebroadcast; the + // decoded `ue` is consumed by handle_user_event for dedup + emission. + let is_new = self.handle_user_event(ue); + if is_new { + self.rebroadcast(MessageType::UserEvent, data); + } + } + AnyMessage::Join(join) => { + // handle_node_join_intent takes ltime + id reference. + let rebroadcast = self.handle_node_join_intent(join.ltime, &join.id.clone(), now); + if rebroadcast { + self.rebroadcast(MessageType::Join, data); + } + } + AnyMessage::Leave(leave) => { + let id = leave.id.clone(); + let rebroadcast = self.handle_node_leave_intent(leave.ltime, &id, leave.prune, now); + if rebroadcast { + self.rebroadcast(MessageType::Leave, data); + } + } + // Sub-stage 3: push-pull is not valid on a UserPacket; drop. + AnyMessage::PushPull(_) => {} + // Query handling: witness clock, dedup, filter, emit Event::Query. + AnyMessage::Query(q) => { + // Defense-in-depth: post-decode semantic size gate kept as a second + // check after the pre-decode fence above already enforces the limit. + if data.len() > self.opts.query_size_limit() { + return; + } + let rebroadcast = self.handle_query(q, QueryOrigin::Inbound); + if rebroadcast { + self.rebroadcast(MessageType::Query, data); + } + } + // Fold the query response into the matching PendingQuery. + AnyMessage::QueryResponse(resp) => { + self.handle_query_response(resp); + } + // Relay: forward the inner payload verbatim to the destination (decision 4). + AnyMessage::Relay(relay) => { + self.handle_relay(relay); + } + // ConflictResponse and Key* bare packets arrive only as payloads inside + // QueryResponseMessage; bare arrivals here are unexpected — drop silently. + AnyMessage::ConflictResponse(_) => {} + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + AnyMessage::KeyRequest(_) => {} + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + AnyMessage::KeyResponse(_) => {} + } + } + + /// Re-queue the original packet `Bytes` on the appropriate broadcast tier. + /// + /// The relay-retain rule: re-broadcast keeps the **original** `Bytes` + /// (a cheap `Bytes::clone` — just a refcount bump, no copy). No re-encode. + /// + /// Tier assignments (mirrors oracle `delegate.rs` `rebroadcast_queue`): + /// - `Join` / `Leave` intents → intent tier (rank 0, highest priority). + /// - `UserEvent` → event tier (rank 2, lowest priority). + /// - All other types → no-op (queries handled via their own tier + /// in sub-stage 4). + /// + /// Errors from `queue_user_broadcast_ranked` are silently dropped — the + /// inner already applies its own back-pressure and queue-depth limits. + fn rebroadcast(&mut self, ty: MessageType, original: Bytes) { + let rank: u8 = match ty { + MessageType::Join | MessageType::Leave => 0, // intent tier + MessageType::UserEvent => 2, // event tier + MessageType::Query => 1, // query tier (sub-stage 4) + _ => return, + }; + // Depth gate: skip enqueue if the queue is already at or over the effective + // cap. This prevents unbounded memory growth under a flood of unique first-seen + // messages. The gate mirrors Go serf's `getQueueMax` / `checkQueueDepth` + // logic applied inline at the rebroadcast site. + if self.inner.user_broadcast_queue_len() >= self.queue_max() { + return; + } + // Ignoring Err: the inner applies its own MTU back-pressure; a rejected + // broadcast is a flow-control decision, not a fatal error. + let _ = self.inner.queue_user_broadcast_ranked(rank, original); + } + + /// Compute the effective broadcast queue depth cap (mirrors Go serf `getQueueMax`). + /// + /// When `min_queue_depth > 0`, the cap is `max(min_queue_depth, 2 * num_members)`, + /// scaling with the cluster so larger clusters get a proportionally larger budget. + /// Otherwise the flat `max_queue_depth` applies. + fn queue_max(&self) -> usize { + let min = self.opts.min_queue_depth(); + if min > 0 { + min.max(2 * self.members.states.len()) + } else { + self.opts.max_queue_depth() + } + } + + /// Returns the number of unsent items in the user broadcast queue. + /// + /// The driver may poll this during a graceful leave to detect when the + /// leave-intent broadcast has been flushed without waiting the full + /// `broadcast_timeout`. + pub fn user_broadcast_queue_len(&self) -> usize { + self.inner.user_broadcast_queue_len() + } + + // ── Query issue + ingress (G8: read-not-increment) ──────────────────────── + + /// Issue an application query. + /// + /// Steps (oracle: `base.rs` `query_in`): + /// 1. Compute the default timeout if `params.timeout == 0`. + /// 2. Stamp `ltime = query_clock` (**read, not incremented** — G8/H8). + /// 3. Draw a random `id` from `self.rng`. + /// 4. Size-check the encoded `QueryMessage` against `query_size_limit`. + /// 5. Register a `PendingQuery` keyed by `(ltime, id)` with `deadline = now + timeout`. + /// 6. Process the query locally via `handle_query` (the machine is always a + /// potential responder to its own queries, matching the oracle's "process + /// locally first" order). + /// 7. Encode and enqueue on the **query tier** (rank 1). + /// + /// Returns the `QueryId` so the caller can correlate responses. + pub fn query( + &mut self, + name: impl Into, + payload: Bytes, + params: QueryParams, + now: Instant, + ) -> Result + where + I: Clone + Data, + A: Clone + Data, + { + // Tag-regex pre-validation: compile-check every Filter::Tag pattern FIRST, + // before any RNG draw, clock read, or state mutation. A broken pattern + // returns Err with zero side effects — no RNG advance, no ltime stamp, no + // PendingQuery, no broadcast, no event. Exact-string matching (no-regex + // path) has no compile step and cannot produce an invalid pattern. + #[cfg(feature = "tag-regex")] + for filter in ¶ms.filters { + if let Filter::Tag(tf) = filter { + if let Some(expr) = &tf.expr { + if regex::Regex::new(expr.as_str()).is_err() { + return Err(Error::InvalidQueryFilter); + } + } + } + } + + let name: SmolStr = name.into(); + + // G8 / H8: stamp from the query clock; the clock is not incremented for + // outbound queries — reads only, per G8. + let ltime = LamportTime(self.query_clock); + + // Draw a random id from serf's own injected RNG. + let id: u32 = self.rng.next_u32(); + + // Resolve the timeout: use provided value or fall back to the oracle's + // `gossip_interval * query_timeout_mult * log10(n + 1)` heuristic. + // The inner Endpoint does not expose `gossip_interval` via a public + // accessor, so when timeout is zero this uses `query_timeout_mult * 200ms` + // (a rough constant). Drivers that need precise timing should supply an + // explicit timeout via `QueryParams::timeout`. + let timeout = if params.timeout.is_zero() { + let n = self.members.states.len(); + let mult = self.opts.query_timeout_mult(); + let log_factor = ((n as f64 + 1.0).log10().ceil() as u32).max(1); + std::time::Duration::from_millis(200) * mult as u32 * log_factor + } else { + params.timeout + }; + + // Build the flags. + let flags = if params.request_ack { + QueryFlag::ACK + } else { + QueryFlag::empty() + }; + + let q = QueryMessage { + ltime, + id, + from: memberlist_proto::Node::new( + self.inner.local_id_ref().clone(), + self.inner.advertise_ref().clone(), + ), + filters: params.filters, + flags, + relay_factor: params.relay_factor, + timeout, + name, + payload, + }; + + // Size-check before encoding. + let encoded = AnyMessage::::Query(q.clone()) + .encode() + .map_err(Error::UserEventEncode)?; + if encoded.len() > self.opts.query_size_limit() { + return Err(Error::QueryTooLarge( + encoded.len(), + self.opts.query_size_limit(), + )); + } + + let query_id = QueryId { ltime, id }; + let deadline = now + timeout; + + // Register the pending query BEFORE processing locally so that if the + // local handle_query immediately generates a response (e.g. the local node + // passes its own filter), the response fold path can find the pending entry. + self.pending_queries.push(PendingQuery { + kind: QueryPurpose::App, + deadline, + responses: std::collections::HashMap::new(), + acks: std::collections::HashMap::new(), + query_id, + request_ack: params.request_ack, + conflict_matching: 0, + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + num_nodes: 0, + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + key_tally: None, + }); + + // Process the query locally (oracle: "Process query locally" before broadcast). + // QueryOrigin::Local bypasses the inbound cap so the initiating node always + // self-processes its own query. + self.drain_now = now; + self.handle_query(q, QueryOrigin::Local); + + // Enqueue on the query tier (rank 1). + // Ignoring Err: the inner applies queue-depth / MTU back-pressure; a rejected + // broadcast is a flow-control decision, not a fatal error. + let _ = self.inner.queue_user_broadcast_ranked(1, encoded); + + Ok(query_id) + } + + /// Handle a query, either received from a peer or locally originated. + /// + /// `origin` distinguishes the two call sites: + /// - `QueryOrigin::Inbound`: called from `handle_user_packet` for a + /// peer-broadcast query. The inbound overflow cap (`MAX_RECEIVED_QUERIES`) + /// applies here as a DoS defence against peer flooding. + /// - `QueryOrigin::Local`: called from `query()` and `internal_query()` for + /// queries the local node originates. The cap is bypassed — local ops are + /// app-rate-limited, never adversarial, and the initiating node MUST always + /// self-process its own query. + /// + /// Mirrors Go serf `base.go` `handleQuery`. + /// + /// Steps: + /// 1. **Witness the query clock** at `msg.ltime`. + /// 2. **Dedup** via `query_buffer.witness_query(cur_time, ltime, id)`. + /// Returns `false` (no rebroadcast, no emission) on duplicate or too-old. + /// 3. **`NO_BROADCAST` flag**: if set, suppress rebroadcast. + /// 4. **Filter check** (`should_process_query`): if the local node does not + /// match the filters, return `true` (G6 — still rebroadcast!), but do NOT + /// emit a local `Event::Query`. + /// 5. **Emit** `Event::Query(QueryEvent { … })` for the driver/app to respond. + /// + /// Returns `true` if the query should be rebroadcast (i.e., first sight AND + /// not `NO_BROADCAST`), even when the filter rejects local processing (G6). + fn handle_query(&mut self, msg: QueryMessage, origin: QueryOrigin) -> bool + where + I: Clone + Data, + A: Clone, + { + let ltime = msg.ltime.0; + + // Whole-message drop gate: reject unacceptable Lamport times before any + // state mutation, clock witness, dirty flag, or event emission. + if !ltime_is_acceptable(ltime) { + return false; + } + + // Internal-query payload gate: for `_serf_conflict`, decode and + // exact-consumption-validate the payload BEFORE any state mutation (before + // the clock witness, dedup insert, received_queries insert, ACK send, and + // rebroadcast). A payload whose decoded id does not consume ALL bytes is + // malformed: a valid-prefix id followed by trailing junk passes the outer + // `AnyMessage::decode_with_consumed` gate but must be rejected here. + // Dropping the whole query on malformed payload means: no clock advance, no + // dedup entry, no received_queries entry, no ConflictResponse send. + let pre_decoded_conflict_id: Option = if msg.name.as_str() == "_serf_conflict" { + match I::decode(msg.payload.as_ref()) { + Ok((consumed, id)) if consumed == msg.payload.len() => Some(id), + _ => return false, // Malformed payload: drop before any state mutation. + } + } else { + None + }; + + // Key-management query gate: decode and exact-consumption-validate before + // any state mutation. Mirrors the _serf_conflict gate above. + // + // Op-shape enforcement: `_serf_install_key`, `_serf_use_key`, and + // `_serf_remove_key` MUST carry `key = Some`; `_serf_list_keys` MUST carry + // `key = None`. A mismatch is treated as malformed and dropped before any + // clock witness, dedup, received_queries insert, ACK, event, or rebroadcast. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + let pre_decoded_key_op: Option<(KeyRequestOperation, Option)> = { + match msg.name.as_str() { + "_serf_install_key" | "_serf_use_key" | "_serf_remove_key" | "_serf_list_keys" => { + let op = match msg.name.as_str() { + "_serf_install_key" => KeyRequestOperation::Install, + "_serf_use_key" => KeyRequestOperation::Use, + "_serf_remove_key" => KeyRequestOperation::Remove, + _ => KeyRequestOperation::List, + }; + match AnyMessage::::decode_with_consumed(&msg.payload) { + Ok((AnyMessage::KeyRequest(m), consumed)) if consumed == msg.payload.len() => { + // Op-shape check: key presence must match the operation's expectation. + // Install/Use/Remove require a key; List must have none. + if op.has_key() != m.key.is_some() { + return false; // Shape mismatch: drop before any state mutation. + } + Some((op, m.key)) + } + _ => return false, // Malformed payload: drop before any state mutation. + } + } + _ => None, + } + }; + + // Tag-regex pre-validation gate: reject a query that carries any + // `Filter::Tag` with an uncompilable regex BEFORE any side effect. An + // uninterpretable filter is malformed input: the query must be dropped with + // zero side effects (no clock witness, no dedup insert, no + // received_queries entry, no ACK, no event, no rebroadcast). This is + // distinct from a valid-but-non-matching filter, which still propagates + // (witnesses the clock, inserts the dedup entry, rebroadcasts) while + // suppressing the local `Event::Query` (G6). + // + // The gate runs only when `tag-regex` is enabled because exact-string + // matching (the no-regex path) has no compile step and cannot produce an + // invalid pattern. + #[cfg(feature = "tag-regex")] + for filter in &msg.filters { + if let Filter::Tag(tf) = filter { + if let Some(expr) = &tf.expr { + if regex::Regex::new(expr.as_str()).is_err() { + return false; // Uncompilable regex: drop before any state mutation. + } + } + } + } + + // Hard-cap (INBOUND only): when `received_queries` is already at + // MAX_RECEIVED_QUERIES, drop the inbound query BEFORE any state mutation — + // no clock witness, no dedup write, no ACK, no event emission, no + // rebroadcast. Every already-inserted entry was surfaced to the driver + // (Event::Query or Event::KeyRequest) and its token must remain answerable + // via respond / respond_key until the deadline or until the driver responds. + // The deadline-pruning in handle_timeout regularly reclaims expired entries; + // the cap is only hit under peer flood. + // + // Local queries (QueryOrigin::Local) bypass this cap: the initiating node + // MUST always self-process its own query regardless of inbound saturation. + // Local query volume is app-controlled and not an adversarial flood vector. + if origin == QueryOrigin::Inbound && self.received_queries.len() >= MAX_RECEIVED_QUERIES { + return false; + } + + // Witness a potentially newer query clock. + witness(&mut self.query_clock, ltime); + let cur_time = self.query_clock; + + // Dedup by (ltime, id): returns false for duplicate (ltime, id) pairs. + // Mark dirty only when witness_query confirms this is a new query. + if !self.query_buffer.witness_query(cur_time, ltime, msg.id) { + return false; + } + + self.mark_local_state_dirty(); + + // Check the NO_BROADCAST flag. + let mut rebroadcast = true; + if msg.no_broadcast() { + rebroadcast = false; + } + + // Filter check (G6): even if the local node is not targeted, still rebroadcast. + if !self.should_process_query(&msg.filters) { + return rebroadcast; + } + + // Clamp the peer-supplied timeout to MAX_QUERY_TIMEOUT before computing the + // deadline. An unclamped timeout lets a flooder pin received_queries entries + // open for an arbitrarily long time; the clamp bounds the worst-case TTL. + let clamped_timeout = msg.timeout.min(MAX_QUERY_TIMEOUT); + let deadline = self.drain_now + clamped_timeout; + + // Register this received query so respond() can enforce the three guards + // (G7: size, once-only, deadline) and look up the originator address. + let query_id = QueryId { + ltime: msg.ltime, + id: msg.id, + }; + let from_addr = msg.from.addr_ref().clone(); + self.received_queries.insert( + query_id, + ReceivedQuery { + from: from_addr, + deadline, + }, + ); + + // If the querier requested an acknowledgement, send an immediate ACK before + // emitting Event::Query. The ACK carries no payload; it signals receipt. + if msg.ack() { + let local_node = memberlist_proto::Node::new( + self.inner.local_id_ref().clone(), + self.inner.advertise_ref().clone(), + ); + let ack_resp = QueryResponseMessage { + ltime: msg.ltime, + id: msg.id, + from: local_node, + flags: QueryFlag::ACK, + payload: Bytes::new(), + }; + if let Ok(ack_encoded) = AnyMessage::::QueryResponse(ack_resp).encode() { + let from_addr = msg.from.addr_ref().clone(); + // Ignoring Err: directed ACK sends are best-effort; a missing ACK is + // handled by the querier's timeout on its acks set. + let _ = self + .inner + .send_user_packet(from_addr.clone(), ack_encoded.clone()); + #[cfg(test)] + { + self.last_directed_send = Some((from_addr, ack_encoded.clone())); + } + // Relay the ACK through relay_factor random intermediary nodes when requested. + if msg.relay_factor > 0 { + self.relay_response(msg.from.clone(), ack_encoded, msg.relay_factor); + } + } + } + + // Internal conflict query: respond autonomously without surfacing to the driver. + // `pre_decoded_conflict_id` is always `Some` here because the + // internal-query payload gate at the top returned `false` for any + // `_serf_conflict` with a malformed payload; execution only reaches this + // point when the id was already decoded and exactly consumed. + if let Some(conflict_id) = pre_decoded_conflict_id { + self.received_queries.remove(&query_id); + self.handle_conflict_query(&msg, conflict_id); + return rebroadcast; + } + + // Key-management query: emit Event::KeyRequest and return without surfacing + // to the app as Event::Query. The received_queries entry is kept so + // respond_key can enforce the G7 guards and look up the originator address. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + if let Some((op, key)) = pre_decoded_key_op { + self + .pending_events + .push_back(Event::KeyRequest(KeyRequestEvent { + op, + key, + id: msg.id, + ltime: msg.ltime, + from: msg.from, + relay_factor: msg.relay_factor, + deadline, + })); + return rebroadcast; + } + + let ev = QueryEvent { + id: msg.id, + ltime: msg.ltime, + from: msg.from, + name: msg.name, + payload: msg.payload, + relay_factor: msg.relay_factor, + deadline, + }; + self.pending_events.push_back(Event::Query(ev)); + + rebroadcast + } + + /// Check whether the local node passes the query's filter list. + /// + /// Mirrors Go serf `query.go` `shouldProcessQuery`. + /// + /// - An empty filter list means "broadcast to all" → always `true`. + /// - `Filter::Id(ids)`: the local id must be present in `ids`. + /// - `Filter::Tag(tag_filter)`: the local node's tags must contain the + /// key and the value must match the expression. + /// + /// When `tag-regex` is enabled, the expression is compiled as a regex and + /// matched with partial (anywhere-in-value) semantics, mirroring Go serf + /// `query.go` `regexp.MatchString(filt.Expr, tags[filt.Tag])`. When + /// `tag-regex` is disabled, the expression is compared by exact string + /// equality. + /// + /// All `Filter::Tag` patterns with `tag-regex` enabled are guaranteed to be + /// valid by the pre-witness validation gate in `handle_query`; this function + /// is only called on queries that have already passed that gate. + /// + /// Returns `true` if the node should respond locally, `false` if filtered out. + /// A `false` return suppresses `Event::Query` but does NOT affect rebroadcast + /// (rebroadcast is decided by the caller, not by this function). + fn should_process_query(&self, filters: &[Filter]) -> bool + where + I: Clone, + { + let local_id = self.inner.local_id_ref(); + for filter in filters { + match filter { + Filter::Id(ids) => { + // The local node must appear in the id list. + if !ids.iter().any(|n| n == local_id) { + return false; + } + } + Filter::Tag(tag_filter) => { + // The local node's tags live in `members.states` under the local id. + // If the local node is not yet in the store (before the first join + // event has been processed), we conservatively return `false`. + let local_id = self.inner.local_id_ref(); + let empty_tags = Tags::new(); + let tags = self + .members + .states + .get(local_id) + .map(|ms| ms.member().tags()) + .unwrap_or(&empty_tags); + match tags.0.get(tag_filter.tag.as_str()) { + Some(val) => { + if let Some(expr) = &tag_filter.expr { + #[cfg(feature = "tag-regex")] + { + // The pre-validation gate in handle_query ensures every + // pattern reaching here is a valid regex; unwrap is safe. + let re = regex::Regex::new(expr.as_str()) + .expect("regex validated by handle_query pre-validation gate"); + if !re.is_match(val.as_str()) { + return false; + } + } + #[cfg(not(feature = "tag-regex"))] + { + if val.as_str() != expr.as_str() { + return false; + } + } + } + // No expr → key presence is sufficient. + } + None => return false, + } + } + } + } + true + } + + // ── respond() + handle_query_response() ───────────────────────────────────── + + /// Respond to a received query (G7 / oracle: `event.go` `QueryContext.respond`). + /// + /// Three guards applied in order: + /// 1. **Size** (`query_response_size_limit`): `payload.len() > limit` → `Err(RespondTooLarge)`. + /// 2. **Once-only**: already responded to this token → `Err(AlreadyResponded)`. + /// 3. **Deadline**: `now > deadline` → `Err(RespondAfterDeadline)`. + /// + /// On success: encodes a `QueryResponseMessage`, sends it via `send_user_packet` + /// to the querier's address (directed send, never broadcast), marks the token + /// as responded. The relay path is a TODO stub. + /// + /// Go serf's `event.go` checks size first, then uses a combined + /// already-responded+deadline guard (the mutex holding the span doubles as + /// both checks). This port uses three distinct guards in order — size, + /// already-responded, deadline — which is strictly more informative to + /// callers that need to distinguish the error cases. + pub fn respond( + &mut self, + token: &QueryEvent, + payload: Bytes, + now: Instant, + ) -> Result<(), Error> + where + I: Clone + Data, + A: Clone + Data, + { + // Look up the received-query entry for this token. + let query_id = QueryId { + ltime: token.ltime(), + id: token.id(), + }; + let entry = self + .received_queries + .get(&query_id) + .ok_or(Error::AlreadyResponded)?; + let deadline = entry.deadline; + let to = entry.from.clone(); + let relay_factor = token.relay_factor(); + let relay_querier = token.from().clone(); + + // Build and encode the `QueryResponseMessage`. + let local_node = memberlist_proto::Node::new( + self.inner.local_id_ref().clone(), + self.inner.advertise_ref().clone(), + ); + let resp = QueryResponseMessage { + ltime: token.ltime(), + id: token.id(), + from: local_node, + flags: QueryFlag::empty(), + payload, + }; + let encoded = AnyMessage::::QueryResponse(resp) + .encode() + .map_err(Error::RespondEncode)?; + + self.respond_inner( + query_id, + to, + relay_factor, + relay_querier, + encoded, + deadline, + now, + ) + } + + /// Shared directed-send logic for `respond` and `respond_key`. + /// + /// Applies the G7 size guard, sends `encoded` directed to `to`, removes the + /// `received_queries` entry on success, and relays when `relay_factor > 0`. + /// + /// The once-only guard is implemented by entry removal: the caller's + /// `.ok_or(AlreadyResponded)` at lookup acts as the gate; `respond_inner` + /// removes the entry on success so a subsequent lookup returns `None`. + // All parameters are distinct routing / payload values with no natural sub-grouping; + // a wrapper struct would add churn without clarity. + #[allow(clippy::too_many_arguments)] + fn respond_inner( + &mut self, + query_id: QueryId, + to: A, + relay_factor: u8, + relay_querier: Node, + encoded: Bytes, + deadline: Instant, + now: Instant, + ) -> Result<(), Error> + where + I: Clone, + A: Clone, + { + // Guard 3: deadline. + if now > deadline { + return Err(Error::RespondAfterDeadline); + } + + // Guard 1: encoded frame size against the limit. + let limit = self.opts.query_response_size_limit(); + if encoded.len() > limit { + return Err(Error::RespondTooLarge(encoded.len(), limit)); + } + + // Directed send — never broadcast. + self + .inner + .send_user_packet(to.clone(), encoded.clone()) + .map_err(Error::RespondSend)?; + + #[cfg(test)] + { + self.last_directed_send = Some((to, encoded.clone())); + } + + // Send succeeded: remove the entry. + self.received_queries.remove(&query_id); + + // Relay when requested. + if relay_factor > 0 { + self.relay_response(relay_querier, encoded, relay_factor); + } + + Ok(()) + } + + /// Respond to a received key-management query. + /// + /// Builds the sealed wire `KeyResponseMessage` from `resp`, wraps it in a + /// `QueryResponseMessage`, and sends it directed to the query originator via + /// `respond_inner` (which enforces the G7 size, once-only, and deadline guards). + /// + /// The caller (driver) is responsible for applying the key operation to its + /// keyring before calling this method. The machine never mutates key material. + /// + /// Requires the `aes-gcm` or `chacha20-poly1305` feature. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn respond_key( + &mut self, + req: &crate::event::KeyRequest, + resp: KeyResponseArgs, + now: Instant, + ) -> Result<(), Error> + where + I: Clone + Data, + A: Clone + Data, + { + let query_id = QueryId { + ltime: req.ltime, + id: req.id, + }; + let entry = self + .received_queries + .get(&query_id) + .ok_or(Error::AlreadyResponded)?; + let deadline = entry.deadline; + let to = entry.from.clone(); + let relay_factor = req.relay_factor; + let relay_querier = req.from.clone(); + + let key_resp = crate::KeyResponseMessage { + result: resp.result, + message: resp.message, + keys: resp.keys, + primary_key: resp.primary_key, + }; + let inner_payload = AnyMessage::::KeyResponse(key_resp) + .encode() + .map_err(Error::RespondEncode)?; + + let local_node = memberlist_proto::Node::new( + self.inner.local_id_ref().clone(), + self.inner.advertise_ref().clone(), + ); + let qresp = QueryResponseMessage { + ltime: req.ltime, + id: req.id, + from: local_node, + flags: QueryFlag::empty(), + payload: inner_payload, + }; + let encoded = AnyMessage::::QueryResponse(qresp) + .encode() + .map_err(Error::RespondEncode)?; + + self.respond_inner( + query_id, + to, + relay_factor, + relay_querier, + encoded, + deadline, + now, + ) + } + + /// Fold an incoming `QueryResponseMessage` into the matching `PendingQuery`. + /// + /// Mirrors Go serf `base.go` `handleQueryResponse` + `query.go` + /// `handle_query_response`. + /// + /// Steps: + /// 1. Look up `PendingQuery` by `(ltime, id)`. If not found (stale / already + /// expired), silently drop — the oracle logs a warn, the Sans-I/O machine + /// has no logging layer. + /// 2. Check if the `deadline` has elapsed; if so, silently drop. + /// 3. **Ack** (`msg.ack()` set): dedup by responder id in the separate `acks` + /// set; on first sight emit `Event::QueryAck`. An ack is a bare + /// delivery confirmation (no payload) and does not consume the responder's + /// response slot — the same peer may later send a real response. Mirrors + /// the oracle's per-query `ack_ch` send. + /// 4. **Response** (no ack flag): dedup by responder id in `responses`; on + /// first sight dispatch on `PendingQuery.kind`: + /// - `App` → emit `Event::QueryResponse { id, from, payload }`. + /// - `Conflict` → tally: decode `ConflictResponseMessage`, compare addr to local advertise. + /// - `Key` → tally: decode `KeyResponseMessage`, fold into the `KeyResponseTally`. + fn handle_query_response(&mut self, msg: QueryResponseMessage) + where + I: Clone, + A: Clone, + { + let query_id = QueryId { + ltime: msg.ltime, + id: msg.id, + }; + + let drain_now = self.drain_now; + + // Steps 1–3: validate, then for acks dedup + emit and return. The pending + // borrow is scoped; for the response path we capture the kind and fall + // through to dispatch after the borrow ends. + let kind = { + // Step 1: find matching PendingQuery. + let pending = match self + .pending_queries + .iter_mut() + .find(|pq| pq.query_id == query_id) + { + Some(pq) => pq, + // Stale response (query already expired or was never registered): drop. + None => return, + }; + + // Step 2: deadline elapsed — drop. + if drain_now > pending.deadline { + return; + } + + // Step 3: ack path — only emit if the originating query requested acks + // (`request_ack`); drop acks for non-ack queries (they may arrive as + // stale retransmits when the ACK flag was set by a buggy peer). + if msg.ack() { + if !pending.request_ack { + return; // This query did not request acks; drop the ack silently. + } + if pending.acks.contains_key(msg.from.id_ref()) { + return; + } + pending.acks.insert(msg.from.id_ref().clone(), ()); + self.pending_events.push_back(Event::QueryAck(QueryAck { + id: msg.id, + from: msg.from, + })); + return; + } + + // Step 4: response path — membership validation (internal queries only), + // then dedup by responder id. + // + // For internal queries (Conflict, Key) the responder id MUST be a known + // cluster member before counting the response in the tally denominator. + // An unknown / forged id could inflate the conflict majority or key-op + // success count and trigger false shutdown or false key-op confirmation. + // For App queries the response is forwarded verbatim to the driver, which + // makes its own trust decisions; the machine does not filter by membership. + // + // Transport-origin authentication is driver-side and out of scope for the + // machine; this check is the membership-plausibility gate the machine CAN + // enforce for its own internal tallies. + let is_internal = !matches!(pending.kind, QueryPurpose::App); + if is_internal && !self.members.states.contains_key(msg.from.id_ref()) { + return; + } + + // Dedup by responder id: the same responder cannot be counted twice. + if pending.responses.contains_key(msg.from.id_ref()) { + return; + } + + // Validate the payload before counting in the denominator. For every + // internal-query purpose (Conflict, Key) a malformed or wrong-type + // response must NOT increment `num_resp` — an inflated denominator skews + // the majority/success threshold. + // + // `App` responses are surfaced verbatim to the driver (no internal decode + // needed here) and are always counted. + match pending.kind { + QueryPurpose::Conflict => { + // Exact-consumption decode: trailing bytes in an internal response + // payload must reject the response before counting, not after. + match AnyMessage::::decode_with_consumed(&msg.payload) { + Ok((AnyMessage::ConflictResponse(_), consumed)) if consumed == msg.payload.len() => {} + _ => return, // Malformed, unexpected type, or trailing junk: drop without counting. + } + } + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + QueryPurpose::Key => { + // Exact-consumption decode: trailing bytes in an internal response + // payload must reject the response before counting, not after. + match AnyMessage::::decode_with_consumed(&msg.payload) { + Ok((AnyMessage::KeyResponse(_), consumed)) if consumed == msg.payload.len() => {} + _ => return, // Malformed, unexpected type, or trailing junk: drop without counting. + } + } + QueryPurpose::App => {} + } + + pending.responses.insert(msg.from.id_ref().clone(), ()); + + pending.kind + }; + + // Step 5: dispatch (pending_queries borrow has ended). + match kind { + QueryPurpose::App => { + let ev = QueryResponseEvent { + id: msg.id, + from: msg.from, + payload: msg.payload, + }; + self.pending_events.push_back(Event::QueryResponse(ev)); + } + QueryPurpose::Conflict => { + self.handle_conflict_response_fold(query_id, msg.payload); + } + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + QueryPurpose::Key => { + let from_id = msg.from.id_ref().clone(); + self.handle_key_response_fold(query_id, from_id, msg.payload); + } + } + } + + // ── Responder-side relay ───────────────────────────────────────────────────── + + /// Relay a query response through up to `relay_factor` random Alive members. + /// + /// Mirrors Go serf `query.rs` `relay_response` (lines 523-601): + /// + /// 1. **Early exit when `relay_factor == 0`**: no relay needed. + /// 2. **Count guard**: requires at least `relay_factor + 1` total members in the + /// membership store (so there are enough other nodes to relay through). + /// If the cluster is too small, **silently return** — no event, no error. + /// This is a deliberate improvement over the oracle (which returns an `Ok(())`): + /// the machine cannot surface the skip as an error since it is not a failure. + /// 3. **Size check**: if `relay_frame.len() > query_response_size_limit`, emit + /// `Event::RelayDropped` per destination (the response is too large to relay). + /// 4. **Build the relay wrapper**: one `RelayMessage { destination: querier, + /// payload: relay_frame }` — the same frame is sent to every relay peer. + /// 5. **Pick `relay_factor` random Alive non-self peers** from `self.rng` (the + /// oracle uses `random_members`; we implement the equivalent inline). + /// 6. **Directed-send** the relay frame to each chosen peer via + /// `inner.send_user_packet`. On send failure emit `Event::RelayDropped`. + /// + /// **Decision-4 FIX notes:** + /// - Count-guard failure (`num_members < k + 1`) is a **silent no-op** (no + /// event), not an error. The oracle also returns `Ok(())` here. + /// - Size-overflow is an explicit `Event::RelayDropped`. + /// - The destination querier IS reselectable as a relay peer (the relay dedup + /// is the query-response dedup at the querier, not here). + /// - Only `Alive` non-self members are eligible relay peers. + fn relay_response(&mut self, querier: Node, relay_frame: Bytes, relay_factor: u8) + where + I: Clone + Data, + A: Clone + Data, + { + if relay_factor == 0 { + return; + } + + // Count guard: need at least relay_factor + 1 members (silent no-op if too few). + let num_members = self.members.states.len(); + if num_members < relay_factor as usize + 1 { + return; + } + + // Size check: the relay wraps the full QueryResponse frame; its encoded size + // must not exceed the response size limit. + let limit = self.opts.query_response_size_limit(); + if relay_frame.len() > limit { + // The frame is too large to relay; emit RelayDropped for each would-be + // relay target (we don't know the exact addresses, so emit once for the + // destination to surface the failure to the driver). + self + .pending_events + .push_back(Event::RelayDropped(crate::event::RelayDropped { + destination: querier.addr_ref().clone(), + })); + return; + } + + // Build the relay wrapper. The inner payload is the verbatim QueryResponse + // frame — relay-retain: no re-encode. + let relay_msg = RelayMessage::new(querier, relay_frame); + let relay_encoded = match AnyMessage::::Relay(relay_msg).encode() { + Ok(b) => b, + Err(_) => return, // Encode failure: silently drop (best-effort relay). + }; + + // Collect Alive non-self members as eligible relay candidates. + // + // Collected as (id_bytes, addr) pairs so we can sort by the encoded id + // before the Fisher-Yates shuffle. The sort is required for determinism: + // HashMap::iter() returns keys in an arbitrary, per-instance order, so two + // endpoints with identical membership and identical RNG seed would select + // different relay peers without it. Sorting by the id's encoded bytes + // establishes a total, stable input order so the shuffle is a deterministic + // function of the RNG state (Go serf `random_members` uses a slice with a + // stable iteration order for the same reason). + let local_id = self.inner.local_id_ref(); + let mut candidates: Vec<(Vec, A)> = self + .members + .states + .iter() + .filter(|(id, ms)| *id != local_id && ms.status() == MemberStatus::Alive) + .map(|(id, ms)| { + let id_bytes = id.encode_to_vec().unwrap_or_default(); + (id_bytes, ms.member().node().addr_ref().clone()) + }) + .collect(); + + if candidates.is_empty() { + return; + } + + // Sort by encoded id bytes for a stable, deterministic input ordering. + candidates.sort_unstable_by(|(a, _), (b, _)| a.cmp(b)); + + // Pick relay_factor random peers using serf's own injected RNG (A2). + // Reservoir-shuffle the first relay_factor positions (Fisher-Yates partial). + let k = (relay_factor as usize).min(candidates.len()); + for i in 0..k { + let j: usize = self.rng.random_range(i..candidates.len()); + candidates.swap(i, j); + } + candidates.truncate(k); + + // Directed-send to each chosen relay peer. On failure emit RelayDropped. + for (_, peer_addr) in candidates { + let result = self + .inner + .send_user_packet(peer_addr.clone(), relay_encoded.clone()); + #[cfg(test)] + { + // Track the last directed send for test assertions. + self.last_directed_send = Some((peer_addr.clone(), relay_encoded.clone())); + self + .relay_all_directed_sends + .push((peer_addr.clone(), relay_encoded.clone())); + } + if result.is_err() { + self + .pending_events + .push_back(Event::RelayDropped(crate::event::RelayDropped { + destination: peer_addr, + })); + } + } + } + + /// Handle a received `RelayMessage` (relay node B). + /// + /// The local node is acting as an intermediary: forward the inner payload + /// verbatim to the wrapped destination via a single directed `send_user_packet`. + /// + /// Mirrors Go serf `delegate.rs` Relay arm (lines 264-310): + /// 1. Decode the destination address from `relay.destination`. + /// 2. **Self-destination guard**: if the destination id equals the local node id, + /// drop with `Event::RelayDropped` (forwarding to self is a no-op that hides + /// a configuration error; surfacing it helps debugging). + /// 3. Forward `relay.payload` verbatim via `send_user_packet` — NON-recursive: + /// we NEVER parse the inner payload or re-relay. + /// 4. On send failure emit `Event::RelayDropped`. + /// + /// **Global constraint:** the relay loop retains the original bytes — the inner + /// payload is carried opaque, never re-encoded or re-parsed. This function is + /// the FINAL hop; it does NOT check whether the inner payload is itself a Relay + /// message (the loop-guard is documented as an additive hardening that Go omits; + /// we omit it as well to keep faithful oracle correspondence — if a relay chain + /// were constructed, the recipient would decode an AnyMessage::Relay and call + /// handle_relay again, naturally bounding by TTL at the network layer). + fn handle_relay(&mut self, relay: RelayMessage) + where + I: Clone, + A: Clone, + { + let dest_id = relay.destination.id_ref(); + let local_id = self.inner.local_id_ref(); + + // Self-destination guard: relay to self is always a no-op failure. + if dest_id == local_id { + self + .pending_events + .push_back(Event::RelayDropped(crate::event::RelayDropped { + destination: relay.destination.addr_ref().clone(), + })); + return; + } + + let dest_addr = relay.destination.addr_ref().clone(); + let payload = relay.payload; + + let result = self + .inner + .send_user_packet(dest_addr.clone(), payload.clone()); + #[cfg(test)] + { + self.last_directed_send = Some((dest_addr.clone(), payload.clone())); + } + if result.is_err() { + self + .pending_events + .push_back(Event::RelayDropped(crate::event::RelayDropped { + destination: dest_addr, + })); + } + } + + /// Returns the `leave_broadcast_deadline`, if armed. + /// + /// `None` when not in the middle of a graceful leave or force-leave. + pub const fn leave_broadcast_deadline(&self) -> Option { + self.leave_broadcast_deadline + } + + /// Returns the `leave_complete_deadline`, if armed. + /// + /// `None` until the inner `LeftCluster` event has been received. + pub const fn leave_complete_deadline(&self) -> Option { + self.leave_complete_deadline + } + + // ── test helpers (test-only) ────────────────────────────────────────────── + + /// Return the current status of member `id`, or `None` if unknown. + #[cfg(test)] + pub(crate) fn test_member_status(&self, id: I) -> Option + where + I: Clone, + { + self.members.states.get(&id).map(|ms| ms.status()) + } + + /// Return the `status_time` of member `id`, or `None` if unknown. + #[cfg(test)] + pub(crate) fn test_member_status_time(&self, id: I) -> Option + where + I: Clone, + { + self.members.states.get(&id).map(|ms| ms.status_time()) + } + + /// Seed a member directly into the membership store (test fixture). + /// + /// Uses a zero socket address for the node. The caller is `Endpoint` + /// in all current tests, so we hard-code the sentinel address here. + #[cfg(test)] + pub(crate) fn test_seed_member(&mut self, id: I, status: MemberStatus, status_time: LamportTime) + where + I: Clone, + A: Clone + From, + { + let addr: A = std::net::SocketAddr::from(([127, 0, 0, 1], 0u16)).into(); + let node = memberlist_proto::Node::new(id.clone(), addr); + let member = Member::new(node, Tags::new(), status); + self + .members + .states + .insert(id, MemberState::new(member, status_time, None)); + } + + /// Seed a member with explicit tags into the membership store (test fixture). + /// + /// Like `test_seed_member` but lets the caller supply a `Tags` map, enabling + /// tag-filter unit tests to place a known value under a known key. + #[cfg(test)] + pub(crate) fn test_seed_member_with_tags( + &mut self, + id: I, + tags: Tags, + status: MemberStatus, + status_time: LamportTime, + ) where + I: Clone, + A: Clone + From, + { + let addr: A = std::net::SocketAddr::from(([127, 0, 0, 1], 0u16)).into(); + let node = memberlist_proto::Node::new(id.clone(), addr); + let member = Member::new(node, tags, status); + self + .members + .states + .insert(id, MemberState::new(member, status_time, None)); + } + + /// Seed a member as `Failed` into both `states` and `failed_members`. + #[cfg(test)] + pub(crate) fn test_seed_failed_member_by_status( + &mut self, + id: I, + status_time: LamportTime, + now: Instant, + ) where + I: Clone, + A: Clone + From, + { + let addr: A = std::net::SocketAddr::from(([127, 0, 0, 1], 0u16)).into(); + let node = memberlist_proto::Node::new(id.clone(), addr); + let member = Member::new(node, Tags::new(), MemberStatus::Failed); + self + .members + .states + .insert(id.clone(), MemberState::new(member, status_time, Some(now))); + self.members.failed_members.push(id); + } + + /// Seed a member as `Left` into both `states` and `left_members`. + #[cfg(test)] + pub(crate) fn test_seed_left_member_by_status( + &mut self, + id: I, + status_time: LamportTime, + now: Instant, + ) where + I: Clone, + A: Clone + From, + { + let addr: A = std::net::SocketAddr::from(([127, 0, 0, 1], 0u16)).into(); + let node = memberlist_proto::Node::new(id.clone(), addr); + let member = Member::new(node, Tags::new(), MemberStatus::Left); + self + .members + .states + .insert(id.clone(), MemberState::new(member, status_time, Some(now))); + self.members.left_members.push(id); + } + + /// Invoke `handle_node_join_intent` with bare parameters (test adapter). + #[cfg(test)] + pub(crate) fn test_handle_join_intent(&mut self, id: I, ltime: LamportTime, now: Instant) -> bool + where + I: Clone, + { + self.handle_node_join_intent(ltime, &id, now) + } + + /// Invoke `handle_node_leave_intent` with bare parameters (test adapter). + #[cfg(test)] + pub(crate) fn test_handle_leave_intent(&mut self, id: I, ltime: LamportTime, now: Instant) -> bool + where + I: Clone, + { + self.handle_node_leave_intent(ltime, &id, false, now) + } + + /// Synthesise an inner `NodeJoined` event for node `id` and drive it + /// through the sieve (test adapter). + #[cfg(test)] + pub(crate) fn test_inner_node_joined(&mut self, id: I, now: Instant) + where + I: Clone, + A: Clone + From, + { + use std::sync::Arc; + let addr: A = std::net::SocketAddr::from(([127, 0, 0, 1], 0u16)).into(); + let ns = Arc::new(memberlist_proto::typed::NodeState::new( + id, + addr, + memberlist_proto::typed::State::Alive, + )); + self.drain_now = now; + self.handle_node_join(&ns, now); + } + + /// Synthesise an inner `NodeLeft` event for node `id`. + #[cfg(test)] + pub(crate) fn test_inner_node_left(&mut self, id: I, now: Instant) + where + I: Clone, + A: Clone + From, + { + use std::sync::Arc; + let addr: A = std::net::SocketAddr::from(([127, 0, 0, 1], 0u16)).into(); + let ns = Arc::new(memberlist_proto::typed::NodeState::new( + id, + addr, + memberlist_proto::typed::State::Dead, + )); + self.drain_now = now; + self.handle_node_leave(&ns, now); + } + + /// Synthesise an inner `NodeUpdated` event for node `id`. + #[cfg(test)] + pub(crate) fn test_inner_node_updated(&mut self, id: I, now: Instant) + where + I: Clone, + A: Clone + From, + { + use std::sync::Arc; + let addr: A = std::net::SocketAddr::from(([127, 0, 0, 1], 0u16)).into(); + let ns = Arc::new(memberlist_proto::typed::NodeState::new( + id, + addr, + memberlist_proto::typed::State::Alive, + )); + self.drain_now = now; + self.handle_node_update(&ns); + } + + /// Check if `id` is in `failed_members`. + #[cfg(test)] + pub(crate) fn test_in_failed_members(&self, id: I) -> bool + where + I: PartialEq, + { + self.members.failed_members.contains(&id) + } + + /// Check if `id` is in `left_members`. + #[cfg(test)] + pub(crate) fn test_in_left_members(&self, id: I) -> bool + where + I: PartialEq, + { + self.members.left_members.contains(&id) + } + + /// Simulate the inner `LeftCluster` event arriving (test adapter). + /// + /// Drives `on_inner_event(LeftCluster)` directly, as if the inner memberlist + /// finished its dead-self fan-out. Used by leave-chain tests that do not + /// have a live inner endpoint to drive. + #[cfg(test)] + pub(crate) fn test_inner_left_cluster(&mut self) { + self.on_inner_event(memberlist_proto::Event::LeftCluster); + } + + /// Seed a `Failed` member with an explicit address into both `states` and + /// `failed_members` (test fixture for reconnect assertions). + #[cfg(test)] + pub(crate) fn test_seed_failed_member(&mut self, id: I, addr: A, now: Instant) + where + I: Clone, + A: Clone, + { + let node = memberlist_proto::Node::new(id.clone(), addr); + let member = Member::new(node, Tags::new(), MemberStatus::Failed); + self.members.states.insert( + id.clone(), + MemberState::new(member, LamportTime::ZERO, Some(now)), + ); + self.members.failed_members.push(id); + } + + /// Directly invoke `fire_reconnect` (test adapter). + #[cfg(test)] + pub(crate) fn test_fire_reconnect(&mut self, now: Instant) + where + A: Clone, + { + self.fire_reconnect(now); + } + + /// Directly invoke `fire_reap` (test adapter). + #[cfg(test)] + pub(crate) fn test_fire_reap(&mut self, now: Instant) + where + I: Clone, + A: Clone, + { + self.fire_reap(now); + } + + /// Return the address of the most recently reconnect-dialled peer. + /// + /// `None` if `fire_reconnect` has not yet produced a dial attempt. + #[cfg(test)] + pub(crate) fn test_last_dial_addr(&self) -> Option + where + A: Clone, + { + self.last_dial_addr.clone() + } + + // ── User-event test helpers ─────────────────────────────────────────────── + + /// Directly invoke `handle_user_event` with a `UserEventMessage` (test adapter). + /// + /// Returns the same bool that `handle_user_event` returns (true = first sight). + #[cfg(test)] + pub(crate) fn test_handle_user_event(&mut self, msg: UserEventMessage) -> bool { + self.handle_user_event(msg) + } + + /// Set the event ring-buffer's `min_time` floor directly (test adapter). + /// + /// Used to simulate snapshot-recovery or `eventJoinIgnore` bump without + /// driving a full push-pull. + #[cfg(test)] + pub(crate) fn test_set_event_min_time(&mut self, t: u64) { + self.event_buffer.min_time = t; + } + + /// Directly set the event clock value (test adapter for "too-old" ring checks). + #[cfg(test)] + pub(crate) fn test_set_event_clock(&mut self, t: u64) { + self.event_clock = t; + } + + /// Return the number of events recorded in the event ring slot for `ltime`. + /// + /// Used to verify the per-ltime cap (`MAX_EVENTS_PER_LTIME`) is enforced. + #[cfg(test)] + pub(crate) fn test_event_slot_len(&self, ltime: u64) -> usize { + let bltime = self.event_buffer.buffer.len() as u64; + if bltime == 0 { + return 0; + } + let idx = (ltime % bltime) as usize; + match &self.event_buffer.buffer[idx] { + Some(slot) if slot.ltime.0 == ltime => slot.events.len(), + _ => 0, + } + } + + /// Return the `num_nodes` stored on the most recent pending query (test adapter). + #[cfg(all(test, any(feature = "aes-gcm", feature = "chacha20-poly1305")))] + pub(crate) fn test_last_pending_query_num_nodes(&self) -> Option { + self.pending_queries.last().map(|pq| pq.num_nodes) + } + + // ── Packet-ingress test helpers ─────────────────────────────────────────── + + /// Inject a `UserPacket` with an explicit `from` address directly into the + /// inner-event sieve (test adapter). + /// + /// Bypasses the memberlist packet-framing layer so tests can inject + /// serf-encoded `Bytes` without wrapping them in a memberlist frame. + /// Sets `drain_now = now` then calls `on_inner_event` with a synthetic + /// `Event::UserPacket` carrying the given bytes and `Unreliable` reliability + /// (mirrors the gossip-plane delivery path). + #[cfg(test)] + pub(crate) fn test_inject_user_packet(&mut self, from: A, data: Bytes, now: Instant) + where + I: Clone + Data, + A: Clone + Data, + { + use memberlist_proto::{Reliability, UserPacket}; + self.drain_now = now; + self.on_inner_event(memberlist_proto::Event::UserPacket(UserPacket::new( + from, + data, + Reliability::Unreliable, + ))); + } + + // ── Clock + intent test helpers ─────────────────────────────────────────── + + /// Set all three Lamport clocks in one call (test fixture). + #[cfg(test)] + pub(crate) fn test_set_clocks(&mut self, member: u64, event: u64, query: u64) { + self.clock = member; + self.event_clock = event; + self.query_clock = query; + self.mark_local_state_dirty(); + } + + /// Seed a member as `Left` into `states` and `left_members` using just + /// a `status_time` (no wall-clock `now`). The `leave_time` is set to + /// `Instant::ORIGIN` so reaper tests that need a concrete timestamp can + /// adjust independently. + #[cfg(test)] + pub(crate) fn test_seed_left_member(&mut self, id: I, status_time: LamportTime) + where + I: Clone, + A: Clone + From, + { + let addr: A = std::net::SocketAddr::from(([127, 0, 0, 1], 0u16)).into(); + let node = memberlist_proto::Node::new(id.clone(), addr); + let member = Member::new(node, Tags::new(), MemberStatus::Left); + self.members.states.insert( + id.clone(), + MemberState::new(member, status_time, Some(Instant::ORIGIN)), + ); + self.members.left_members.push(id); + self.mark_local_state_dirty(); + } + + /// Read back the bytes currently stored in the inner Endpoint's + /// `local_state_snapshot` (test adapter). + #[cfg(test)] + pub(crate) fn test_inner_local_state_snapshot(&self) -> Bytes { + self.inner.local_state_snapshot_bytes() + } + + /// Decode `bytes` as a `PushPullMessage` for assertion (test adapter). + /// + /// Decodes via the same `AnyMessage::decode` path that a remote peer would + /// use, so the round-trip is exercised. + #[cfg(test)] + pub(crate) fn test_decode_pushpull(&self, bytes: &Bytes) -> crate::typed::PushPullMessage + where + I: Clone + Data, + A: Data, + { + match AnyMessage::::decode(bytes).expect("decode should succeed") { + AnyMessage::PushPull(pp) => pp, + other => panic!("expected PushPull, got {:?}", other.message_type()), + } + } + + /// Clear the dirty flag (test adapter for dirty-flag unit assertions). + #[cfg(test)] + pub(crate) fn test_clear_dirty(&mut self) { + self.local_state_dirty = false; + } + + /// Return the current dirty flag (test adapter). + #[cfg(test)] + pub(crate) fn test_is_dirty(&self) -> bool { + self.local_state_dirty + } + + /// Set the `event_join_ignore` flag (test adapter). + #[cfg(test)] + pub(crate) fn test_set_event_join_ignore(&mut self, v: bool) { + self.event_join_ignore = v; + } + + /// Read the current `event_buffer.min_time` (test adapter for G4 assertions). + #[cfg(test)] + pub(crate) fn test_event_min_time(&self) -> u64 { + self.event_buffer.min_time + } + + // ── Push-pull / merge test helpers ─────────────────────────────────────── + + /// Directly invoke `merge_remote_state` with raw `user_data` bytes (test adapter). + /// + /// Sets `drain_now = Instant::ORIGIN` before the call so intent handlers + /// receive a stable `now`. Use `test_set_drain_now` to override the + /// timestamp when wall-clock values matter. + #[cfg(test)] + pub(crate) fn test_merge_remote_state(&mut self, user_data: Bytes, is_join: bool) + where + I: Clone + Data, + A: Data, + { + self.merge_remote_state(user_data, is_join); + } + + /// Return the `ltime` of the most recently buffered intent for `id` of `kind`, + /// or `None` if no such intent exists (test adapter for G3 assertions). + #[cfg(test)] + pub(crate) fn test_intent_ltime(&self, id: I, kind: IntentKind) -> Option + where + I: Clone, + { + self.members.recent_intent(&id, kind) + } + + // ── Query ingress test helpers ──────────────────────────────────────────── + + /// Directly invoke `handle_query` as an inbound peer query and return whether + /// it should rebroadcast (test adapter). Always passes `QueryOrigin::Inbound` + /// so the inbound cap and all inbound semantics are exercised. + #[cfg(test)] + pub(crate) fn test_handle_query(&mut self, msg: QueryMessage) -> bool + where + I: Clone + Data, + A: Clone + Data, + { + self.handle_query(msg, QueryOrigin::Inbound) + } + + /// Return the `QueryId` of the last pending query entry (test adapter). + #[cfg(test)] + pub(crate) fn test_last_query_id(&self) -> Option { + self.pending_queries.last().map(|pq| pq.query_id) + } + + /// Return the number of pending queries (test adapter). + #[cfg(test)] + pub(crate) fn test_pending_query_count(&self) -> usize { + self.pending_queries.len() + } + + /// Return the number of entries in `received_queries` (test adapter). + #[cfg(test)] + pub(crate) fn test_received_queries_len(&self) -> usize { + self.received_queries.len() + } + + /// Return all deadline values from `received_queries` (test adapter). + /// + /// Used to verify that inbound query timeouts are clamped to MAX_QUERY_TIMEOUT. + #[cfg(test)] + pub(crate) fn test_peek_received_query_deadlines(&self) -> Vec { + self + .received_queries + .values() + .map(|rq| rq.deadline) + .collect() + } + + /// Return the current `query_buffer.min_time` (test adapter). + #[cfg(test)] + pub(crate) fn test_query_min_time(&self) -> u64 { + self.query_buffer.min_time + } + + /// Register a synthetic received-query entry and return a matching `QueryEvent` + /// token (test adapter for `respond()` tests). + /// + /// Inserts a `ReceivedQuery` keyed by `query_id` with `from = querier` and + /// `deadline = deadline`, `responded = false`. Returns a `QueryEvent` + /// with matching `id`/`ltime` so tests can call `respond(&token, ...)`. + #[cfg(test)] + pub(crate) fn test_register_received_query( + &mut self, + query_id: QueryId, + querier: A, + deadline: Instant, + ) -> QueryEvent + where + I: Default + Clone, + A: Clone, + { + self.received_queries.insert( + query_id, + ReceivedQuery { + from: querier.clone(), + deadline, + }, + ); + QueryEvent { + id: query_id.id, + ltime: query_id.ltime, + from: memberlist_proto::Node::new(I::default(), querier), + name: SmolStr::new("test"), + payload: Bytes::new(), + relay_factor: 0, + deadline, + } + } + + /// Call `handle_query_response` directly (test adapter). + #[cfg(test)] + pub(crate) fn test_handle_query_response(&mut self, msg: QueryResponseMessage) + where + I: Clone, + A: Clone, + { + self.handle_query_response(msg); + } + + /// Return `true` if the received-query entry for `query_id` has been + /// responded to (test adapter). + /// + /// `respond()` removes the entry on success, so this returns `true` when + /// the entry is absent (either successfully responded or pruned by + /// `handle_timeout`) and `false` while the entry is still present and + /// awaiting a response. + #[cfg(test)] + pub(crate) fn test_is_responded(&self, query_id: QueryId) -> bool { + !self.received_queries.contains_key(&query_id) + } + + /// Return the number of entries currently in `members.recent_intents` (test adapter). + #[cfg(test)] + pub(crate) fn test_recent_intents_len(&self) -> usize { + self.members.recent_intents.len() + } + + /// Return the conflict_matching counter for the pending query matching + /// `query_id`, or `None` if the query is not found (test adapter). + #[cfg(test)] + pub(crate) fn test_pending_query_conflict_matching(&self, query_id: QueryId) -> Option { + self + .pending_queries + .iter() + .find(|pq| pq.query_id == query_id) + .map(|pq| pq.conflict_matching) + } + + // ── Relay test helpers ──────────────────────────────────────────────────── + + /// Directly invoke `relay_response` (test adapter). + /// + /// Seeds a live Alive member into the store at `relay_port` so the count + /// guard can pass when `relay_factor == 1`. The caller is responsible for + /// ensuring the membership store is populated to satisfy the guard. + #[cfg(test)] + pub(crate) fn test_relay_response(&mut self, querier: Node, frame: Bytes, relay_factor: u8) + where + I: Clone + Data, + A: Clone + Data, + { + self.relay_response(querier, frame, relay_factor); + } + + /// Directly invoke `handle_relay` (test adapter). + #[cfg(test)] + pub(crate) fn test_handle_relay(&mut self, relay: RelayMessage) + where + I: Clone, + A: Clone, + { + self.handle_relay(relay); + } + + /// Return the most recent `(address, bytes)` pair sent via a directed + /// `send_user_packet` from the relay path (test adapter). + /// + /// Returns `None` if no directed send has occurred yet. + #[cfg(test)] + pub(crate) fn test_last_directed_send(&self) -> Option<(A, Bytes)> + where + A: Clone, + { + self.last_directed_send.clone() + } + + /// Return all `(address, bytes)` pairs accumulated by `relay_response` since + /// construction or the last `test_clear_relay_directed_sends` call. + /// + /// Unlike `test_last_directed_send`, this captures every send produced across + /// a single `relay_response` invocation, enabling determinism assertions when + /// `relay_factor > 1`. + #[cfg(test)] + pub(crate) fn test_relay_all_directed_sends(&self) -> &[(A, Bytes)] { + &self.relay_all_directed_sends + } + + /// Clear the accumulated relay directed-send log (test adapter). + /// + /// Call between invocations of `test_relay_response` to isolate per-call + /// assertions. + #[cfg(test)] + #[allow(dead_code)] + pub(crate) fn test_clear_relay_directed_sends(&mut self) { + self.relay_all_directed_sends.clear(); + } + + /// Seed a member at an explicit socket address into the membership store (test fixture). + /// + /// Like `test_seed_member` but lets the caller supply an explicit address, + /// enabling relay-determinism tests where members must have distinct addresses + /// so relay-peer selection is observable. + #[cfg(test)] + pub(crate) fn test_seed_member_at( + &mut self, + id: I, + addr: A, + status: MemberStatus, + status_time: LamportTime, + ) where + I: Clone, + A: Clone, + { + let node = memberlist_proto::Node::new(id.clone(), addr); + let member = Member::new(node, Tags::new(), status); + self + .members + .states + .insert(id, MemberState::new(member, status_time, None)); + } + + // ── Conflict-resolution + key-management internal queries ───────────────── + + /// Issue an internal (non-app) query using `purpose` as the `PendingQuery.kind`. + /// + /// Mirrors `query()` but skips the App-specific flag and timeout handling; + /// uses the oracle's `defaultQueryTimeout` heuristic unconditionally. The + /// registered `PendingQuery` will have `kind = purpose` so responses are + /// routed to the appropriate fold path. + fn internal_query( + &mut self, + name: SmolStr, + payload: Bytes, + purpose: QueryPurpose, + now: Instant, + ) -> Result + where + I: Clone + Data, + A: Clone + Data, + { + // G8 / H8: stamp from the query clock; queries read the clock but do not + // increment it. + let ltime = LamportTime(self.query_clock); + let id: u32 = self.rng.next_u32(); + + let n = self.members.states.len(); + let mult = self.opts.query_timeout_mult(); + let log_factor = ((n as f64 + 1.0).log10().ceil() as u32).max(1); + let timeout = std::time::Duration::from_millis(200) * mult as u32 * log_factor; + + let q = QueryMessage { + ltime, + id, + from: memberlist_proto::Node::new( + self.inner.local_id_ref().clone(), + self.inner.advertise_ref().clone(), + ), + filters: vec![], + flags: QueryFlag::empty(), + relay_factor: 0, + timeout, + name, + payload, + }; + + let encoded = AnyMessage::::Query(q.clone()) + .encode() + .map_err(Error::UserEventEncode)?; + if encoded.len() > self.opts.query_size_limit() { + return Err(Error::QueryTooLarge( + encoded.len(), + self.opts.query_size_limit(), + )); + } + + let query_id = QueryId { ltime, id }; + let deadline = now + timeout; + + // Capture the current member count before we push the pending query. + // For Key queries this becomes KeyResponse.num_nodes (mirrors Go serf + // key_manager.go `streamKeyResponse` which reads `this.num_members()` + // at the moment the query is issued). + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + let num_nodes = self.members.states.len(); + + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + let key_tally = if purpose == QueryPurpose::Key { + Some(KeyResponseTally { + num_err: 0, + keys: std::collections::HashMap::new(), + primary_keys: std::collections::HashMap::new(), + messages: std::collections::HashMap::new(), + }) + } else { + None + }; + + self.pending_queries.push(PendingQuery { + kind: purpose, + deadline, + responses: std::collections::HashMap::new(), + acks: std::collections::HashMap::new(), + query_id, + request_ack: false, // Internal queries never request per-hop acks. + conflict_matching: 0, + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + num_nodes, + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + key_tally, + }); + + // QueryOrigin::Local bypasses the inbound cap so the initiating node always + // self-processes its own internal query. + self.drain_now = now; + self.handle_query(q, QueryOrigin::Local); + + // Ignoring Err: queue back-pressure is a flow-control decision, not fatal. + let _ = self.inner.queue_user_broadcast_ranked(1, encoded); + + Ok(query_id) + } + + /// Issue a conflict-resolution query for the local node id. + /// + /// Called from `on_inner_event` when `IE::NodeConflict` fires and + /// `enable_id_conflict_resolution` is set. Broadcasts a `_serf_conflict` + /// query carrying the local id; peers respond with their view of that id's + /// address. When the deadline fires, `close_conflict_query` tallies votes and + /// emits `Event::Shutdown` if the local node lost the majority. + fn resolve_node_conflict(&mut self, now: Instant) + where + I: Clone + Data, + A: Clone + Data, + { + let local_id = self.inner.local_id_ref().clone(); + let payload = match local_id.encode_to_bytes() { + Ok(b) => b, + Err(_) => return, + }; + // Ignoring Err: encoding or queue failures are best-effort; if we cannot + // broadcast the conflict query the cluster will simply time out the conflict. + let _ = self.internal_query( + SmolStr::new("_serf_conflict"), + payload, + QueryPurpose::Conflict, + now, + ); + } + + /// Respond autonomously to a received `_serf_conflict` query. + /// + /// The local node looks up the conflicting id in its membership store and + /// sends a directed `QueryResponseMessage` carrying a `ConflictResponseMessage` + /// back to the originator. If the conflicting id is the local id itself, no + /// response is sent (the originator does not vote in its own conflict). + /// + /// `conflict_id` is pre-decoded and exact-consumption-validated by the + /// internal-query payload gate in `handle_query`; it is NOT re-decoded here. + fn handle_conflict_query(&mut self, msg: &QueryMessage, conflict_id: I) + where + I: Clone + Data, + A: Clone + Data, + { + // The originator does not respond to its own conflict query. + if &conflict_id == self.inner.local_id_ref() { + return; + } + + // Look up the conflicting node's info in the membership store. + let member_node = match self.members.states.get(&conflict_id) { + Some(ms) => ms.member().node().clone(), + None => return, // Unknown node: no response. + }; + + // Build and encode the ConflictResponseMessage. + let resp_msg = ConflictResponseMessage::new(member_node); + let conflict_resp_bytes = match AnyMessage::::ConflictResponse(resp_msg).encode() { + Ok(b) => b, + Err(_) => return, + }; + + // Build and encode the QueryResponseMessage to send to the originator. + let local_node = memberlist_proto::Node::new( + self.inner.local_id_ref().clone(), + self.inner.advertise_ref().clone(), + ); + let qresp = QueryResponseMessage { + ltime: msg.ltime, + id: msg.id, + from: local_node, + flags: QueryFlag::empty(), + payload: conflict_resp_bytes, + }; + let qresp_encoded = match AnyMessage::::QueryResponse(qresp).encode() { + Ok(b) => b, + Err(_) => return, + }; + + // Directed send to the originator (no relay for internal queries). + let dest_addr = msg.from.addr_ref().clone(); + // Ignoring Err: directed-send failure on the conflict-response path is + // best-effort; the originator will simply count this node as non-responding. + let _ = self.inner.send_user_packet(dest_addr, qresp_encoded); + } + + /// Fold a conflict-resolution response into the matching `PendingQuery`. + /// + /// The payload is a serf-framed `ConflictResponseMessage`. If the reported + /// member's address matches the local advertise address, `conflict_matching` + /// is incremented. + fn handle_conflict_response_fold(&mut self, query_id: QueryId, payload: Bytes) + where + A: PartialEq, + { + // Exact-consumption decode: the validation gate in handle_query_response + // already enforced exact consumption before counting; re-check here for + // defence-in-depth (the fold is called only after the gate passed, so this + // is a redundant safety net that adds no overhead on the hot path). + let msg = match AnyMessage::::decode_with_consumed(&payload) { + Ok((AnyMessage::ConflictResponse(m), consumed)) if consumed == payload.len() => m, + _ => return, + }; + + let local_addr = self.inner.advertise_ref().clone(); + let pending = match self + .pending_queries + .iter_mut() + .find(|pq| pq.query_id == query_id) + { + Some(p) => p, + None => return, + }; + + if msg.member.addr_ref() == &local_addr { + pending.conflict_matching += 1; + } + } + + /// Fold a key-management response into the matching `PendingQuery`. + /// + /// The payload is a serf-framed `KeyResponseMessage`. Errors, keys, and + /// primary-key reports are merged into `key_tally`. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + fn handle_key_response_fold(&mut self, query_id: QueryId, responder_id: I, payload: Bytes) + where + I: Eq + core::hash::Hash, + { + // Exact-consumption decode: the validation gate in handle_query_response + // already enforced exact consumption before counting; re-check here for + // defence-in-depth. + let key_msg = match AnyMessage::::decode_with_consumed(&payload) { + Ok((AnyMessage::KeyResponse(m), consumed)) if consumed == payload.len() => m, + _ => return, + }; + + let pending = match self + .pending_queries + .iter_mut() + .find(|pq| pq.query_id == query_id) + { + Some(p) => p, + None => return, + }; + + let tally = match &mut pending.key_tally { + Some(t) => t, + None => return, // Not a Key query: ignore. + }; + + if !key_msg.result { + tally.num_err += 1; + if !key_msg.message.is_empty() { + tally.messages.insert(responder_id, key_msg.message); + } + } + + for k in key_msg.keys { + *tally.keys.entry(k).or_insert(0) += 1; + } + + if let Some(pk) = key_msg.primary_key { + *tally.primary_keys.entry(pk).or_insert(0) += 1; + } + } + + /// Close all pending queries whose deadline has elapsed. + /// + /// `App` queries expire silently; `Conflict` queries trigger vote tallying; + /// `Key` queries (encryption-gated) emit `Event::KeyResponse`. + fn fire_due_query_closes(&mut self, now: Instant) + where + I: Clone, + A: Clone, + { + let mut i = 0; + while i < self.pending_queries.len() { + if now >= self.pending_queries[i].deadline { + let pq = self.pending_queries.swap_remove(i); + match pq.kind { + QueryPurpose::App => { + // App queries expire silently — no event emitted. + } + QueryPurpose::Conflict => { + self.close_conflict_query(&pq); + } + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + QueryPurpose::Key => { + self.close_key_query(pq); + } + } + // Do not advance i: swap_remove replaced index i with the last element. + } else { + i += 1; + } + } + } + + /// Tally a closed conflict-resolution query and emit `Event::Shutdown` if + /// the local node lost the majority vote. + /// + /// "Lost" means strictly fewer than `(num_responses / 2) + 1` respondents + /// reported our local advertise address as the canonical address for our id. + /// + /// **FIX (zero-response guard):** when `num_resp == 0` (all responses were + /// malformed and dropped before counting, or no peers responded at all), the + /// outcome is inconclusive — we cannot compute a meaningful majority. The + /// local node keeps its name; do NOT emit `Event::Shutdown`. Requiring + /// `num_resp > 0` before comparing prevents the `majority = 1, matching = 0` + /// false-shutdown that would otherwise occur. + fn close_conflict_query(&mut self, pq: &PendingQuery) { + let num_resp = pq.responses.len(); + // Zero valid responses → inconclusive; the local node keeps its name. + if num_resp == 0 { + return; + } + let matching = pq.conflict_matching; + let majority = (num_resp / 2) + 1; + if matching >= majority { + // Won — the local node is the canonical holder. + return; + } + // We lost — the driver must shut this node down. + self.pending_events.push_back(Event::Shutdown); + } + + /// Materialize a closed key-management query into `Event::KeyResponse`. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + fn close_key_query(&mut self, pq: PendingQuery) + where + I: Clone, + { + use crate::event::KeyResponse; + let tally = pq.key_tally.unwrap_or_else(|| KeyResponseTally { + num_err: 0, + keys: std::collections::HashMap::new(), + primary_keys: std::collections::HashMap::new(), + messages: std::collections::HashMap::new(), + }); + let num_resp = pq.responses.len(); + // num_nodes was captured at query-issue time from members.states.len() + // (mirrors Go serf key_manager.go `streamKeyResponse` num_nodes init). + let kr = KeyResponse { + num_nodes: pq.num_nodes, + num_resp, + num_err: tally.num_err, + keys: tally.keys, + primary_keys: tally.primary_keys, + messages: tally.messages, + }; + self.pending_events.push_back(Event::KeyResponse(kr)); + } + + /// Issue a cluster-wide `install_key` query for `key`. + /// + /// Requires the `aes-gcm` or `chacha20-poly1305` feature. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn install_key( + &mut self, + key: memberlist_proto::SecretKey, + now: Instant, + ) -> Result + where + I: Clone + Data, + A: Clone + Data, + { + let payload = self.encode_key_request(Some(key))?; + self.internal_query( + SmolStr::new("_serf_install_key"), + payload, + QueryPurpose::Key, + now, + ) + } + + /// Issue a cluster-wide `use_key` query to promote `key` to primary. + /// + /// Requires the `aes-gcm` or `chacha20-poly1305` feature. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn use_key( + &mut self, + key: memberlist_proto::SecretKey, + now: Instant, + ) -> Result + where + I: Clone + Data, + A: Clone + Data, + { + let payload = self.encode_key_request(Some(key))?; + self.internal_query( + SmolStr::new("_serf_use_key"), + payload, + QueryPurpose::Key, + now, + ) + } + + /// Issue a cluster-wide `remove_key` query to remove `key` from all nodes. + /// + /// Requires the `aes-gcm` or `chacha20-poly1305` feature. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn remove_key( + &mut self, + key: memberlist_proto::SecretKey, + now: Instant, + ) -> Result + where + I: Clone + Data, + A: Clone + Data, + { + let payload = self.encode_key_request(Some(key))?; + self.internal_query( + SmolStr::new("_serf_remove_key"), + payload, + QueryPurpose::Key, + now, + ) + } + + /// Issue a cluster-wide `list_keys` query to enumerate installed keys. + /// + /// Requires the `aes-gcm` or `chacha20-poly1305` feature. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn list_keys(&mut self, now: Instant) -> Result + where + I: Clone + Data, + A: Clone + Data, + { + let payload = self.encode_key_request(None)?; + self.internal_query( + SmolStr::new("_serf_list_keys"), + payload, + QueryPurpose::Key, + now, + ) + } + + /// Encode a `KeyRequestMessage` as serf-framed bytes. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + fn encode_key_request(&self, key: Option) -> Result + where + I: Data, + A: Data, + { + let req = KeyRequestMessage::new(key); + AnyMessage::::KeyRequest(req) + .encode() + .map_err(Error::UserEventEncode) + } + + // ── Conflict-resolution + key-management test helpers ───────────────────── + + /// Insert a synthetic `Conflict` `PendingQuery` into `pending_queries`. + /// + /// Returns the `QueryId` so the test can fold responses and fire the close. + #[cfg(test)] + pub(crate) fn test_register_conflict_query(&mut self, deadline: Instant) -> QueryId + where + I: Clone, + A: Clone, + { + let ltime = LamportTime(self.query_clock); + let id: u32 = 99; + let query_id = QueryId { ltime, id }; + self.pending_queries.push(PendingQuery { + kind: QueryPurpose::Conflict, + deadline, + responses: std::collections::HashMap::new(), + acks: std::collections::HashMap::new(), + query_id, + request_ack: false, + conflict_matching: 0, + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + num_nodes: 0, + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + key_tally: None, + }); + query_id + } + + /// Insert a synthetic `Key` `PendingQuery` into `pending_queries`. + /// + /// Returns the `QueryId` so the test can inject responses and fire the close. + /// Requires the `aes-gcm` or `chacha20-poly1305` feature. + #[cfg(all(test, any(feature = "aes-gcm", feature = "chacha20-poly1305")))] + pub(crate) fn test_register_key_query(&mut self, deadline: Instant) -> QueryId + where + I: Clone, + A: Clone, + { + let ltime = LamportTime(self.query_clock); + let id: u32 = 77; + let query_id = QueryId { ltime, id }; + let num_nodes = self.members.states.len(); + self.pending_queries.push(PendingQuery { + kind: QueryPurpose::Key, + deadline, + responses: std::collections::HashMap::new(), + acks: std::collections::HashMap::new(), + query_id, + request_ack: false, + conflict_matching: 0, + num_nodes, + key_tally: Some(KeyResponseTally { + num_err: 0, + keys: std::collections::HashMap::new(), + primary_keys: std::collections::HashMap::new(), + messages: std::collections::HashMap::new(), + }), + }); + query_id + } + + /// Fold a synthetic conflict response for test purposes. + /// + /// `responder_id` is used as the dedup key; `agrees` controls whether the + /// `conflict_matching` counter is bumped. + #[cfg(test)] + pub(crate) fn test_fold_conflict_response( + &mut self, + query_id: QueryId, + responder_id: I, + agrees: bool, + ) where + I: Clone, + { + let pending = match self + .pending_queries + .iter_mut() + .find(|pq| pq.query_id == query_id) + { + Some(p) => p, + None => return, + }; + pending.responses.insert(responder_id, ()); + if agrees { + pending.conflict_matching += 1; + } + } + + /// Return the number of valid responses counted for a pending query (test adapter). + /// + /// This is `pq.responses.len()` — the deduplicated set that forms the + /// denominator in the majority calculation. + #[cfg(test)] + pub(crate) fn test_pending_query_response_count(&self, query_id: QueryId) -> usize { + self + .pending_queries + .iter() + .find(|pq| pq.query_id == query_id) + .map_or(0, |pq| pq.responses.len()) + } + + /// Return the number of ids in the query-buffer slot for `ltime` (test adapter). + #[cfg(test)] + pub(crate) fn test_query_slot_len(&self, ltime: u64) -> usize { + let bltime = self.query_buffer.buffer.len() as u64; + let idx = (ltime % bltime) as usize; + match &self.query_buffer.buffer[idx] { + Some(q) if q.ltime.0 == ltime => q.query_ids.len(), + _ => 0, + } + } + + /// Directly invoke `fire_due_query_closes` (test adapter). + #[cfg(test)] + pub(crate) fn test_fire_due_query_closes(&mut self, now: Instant) + where + I: Clone, + A: Clone, + { + self.fire_due_query_closes(now); + } + + // ── Timeout tick-order test helpers ────────────────────────────────────── + + /// Inject a synthetic `NodeJoined` event for `id` directly through the sieve. + /// + /// Sets `drain_now = now` then calls `handle_node_join` on a synthetic + /// `NodeState`, simulating an inner event that arrives in the same tick as + /// a serf deadline. Used to verify the H1b drain-before-deadline ordering. + #[cfg(test)] + pub(crate) fn test_inject_inner_joined(&mut self, id: I, now: Instant) + where + I: Clone, + A: Clone + From, + { + use std::sync::Arc; + let addr: A = std::net::SocketAddr::from(([127, 0, 0, 1], 0u16)).into(); + let ns = Arc::new(memberlist_proto::typed::NodeState::new( + id, + addr, + memberlist_proto::typed::State::Alive, + )); + self.drain_now = now; + self.handle_node_join(&ns, now); + } + + /// Enqueue raw `bytes` on the intent broadcast tier (rank 0, highest priority). + #[cfg(test)] + pub(crate) fn test_enqueue_intent_broadcast(&mut self, bytes: Bytes) { + // Ignoring Err: test helper; queue back-pressure is not exercised here. + let _ = self.inner.queue_user_broadcast_ranked(0, bytes); + } + + /// Enqueue raw `bytes` on the query broadcast tier (rank 1). + #[cfg(test)] + pub(crate) fn test_enqueue_query_broadcast(&mut self, bytes: Bytes) { + // Ignoring Err: test helper; queue back-pressure is not exercised here. + let _ = self.inner.queue_user_broadcast_ranked(1, bytes); + } + + // ── Snapshot replay → Endpoint load (G5 + G10) ─────────────────────────── + + /// Apply a [`ReplayResult`] to this endpoint after restart. + /// + /// Mirrors Go serf `serf.go` `handleRejoin` + the clock-recovery section of + /// `open_and_replay_snapshot` (~lines 130-160). + /// + /// **G5 — clock floors:** + /// - The member clock is advanced to at least `replay.last_clock`. + /// - `event_buffer.min_time` is set to `last_event_clock + 1` so that any + /// buffered user events from before the snapshot are not replayed + /// (prevents duplicate `Event::User` deliveries after restart). + /// - `query_buffer.min_time` is set to `last_query_clock + 1` for the same + /// reason on the query dedup path. + /// + /// **G10 — rejoin dials (skip self):** + /// For each node in `replay.alive_nodes` whose id is NOT the local node id, + /// the machine calls `inner.start_push_pull(addr, Join, now)`. The inner + /// then emits `Event::DialRequested` which the sieve passes through to the + /// driver as `Event::DialRequested(DialPassthrough { … })`. This mirrors + /// Go serf `handleRejoin` which shuffles `AliveNodes` and skips + /// `node.Name == s.config.NodeName` before dialling each peer. + /// + /// The self-skip check compares node ids (not addresses), consistent with + /// the rest of serf's membership logic. + /// + /// The local state is marked dirty so the next push-pull egress ships the + /// recovered clock state. + pub fn load_snapshot(&mut self, replay: crate::snapshot::ReplayResult, now: Instant) + where + A: Clone, + { + // G5: advance the member clock to at least last_clock. + // Whole-message drop gate: a corrupt or adversarially-crafted snapshot with + // an unacceptable ltime must not advance the local clock. + if ltime_is_acceptable(replay.last_clock.0) { + witness(&mut self.clock, replay.last_clock.0); + } + self.mark_local_state_dirty(); + + // G5: set event min_time so pre-snapshot events are not replayed. + if ltime_is_acceptable(replay.last_event_clock.0) { + let event_min = replay.last_event_clock.0.saturating_add(1); + if event_min > self.event_buffer.min_time { + self.event_buffer.min_time = event_min; + } + // Advance event_clock to at least the snapshot floor so that user_event() + // stamps a ltime >= min_time. Without this, event_clock stays at 0 and + // every new event is dropped as "too old" by handle_user_event. + witness(&mut self.event_clock, replay.last_event_clock.0); + } + + // G5: set query min_time so pre-snapshot queries are not replayed. + if ltime_is_acceptable(replay.last_query_clock.0) { + let query_min = replay.last_query_clock.0.saturating_add(1); + if query_min > self.query_buffer.min_time { + self.query_buffer.min_time = query_min; + } + // Advance query_clock to at least the snapshot floor so that query() + // stamps a ltime >= min_time. + witness(&mut self.query_clock, replay.last_query_clock.0); + } + + // G10: dial each alive peer (skip self) so the node re-joins the cluster. + // The inner emits Event::DialRequested; the sieve passes it through to the + // driver. The driver owns the actual network dial. + let local_id = self.inner.local_id_ref().clone(); + for node in replay.alive_nodes { + if node.id_ref() == &local_id { + // Self-skip: the local node is already "alive" by definition. + continue; + } + let addr = node.addr_ref().clone(); + // Capture for test assertions before calling start_push_pull. + #[cfg(test)] + self.rejoin_dials.push(addr.clone()); + self.inner.start_push_pull(addr, PushPullKind::Join, now); + self.drain_inner(); + } + } + + // ── PingCompleted handler (G9 both halves) ─────────────────────────────── + + /// Handle a `PingCompleted` event from the inner memberlist Endpoint. + /// + /// Implements G9 (both halves): + /// + /// **Half 1 — update local Vivaldi model**: Decodes the remote peer's + /// coordinate from `payload[1..]` (byte 0 is `PING_VERSION = 1`), then + /// calls `coord_client.update(node_id, &remote_coord, rtt, &mut self.rng)`. + /// Stores the remote peer's updated coordinate in `coord_cache[node_id]`. + /// + /// **Half 2 — refresh own ack payload**: After updating the local model, + /// re-encodes the new local coordinate as `[PING_VERSION] ++ pb_bytes` and + /// calls `inner.set_ack_payload` so the next probe ack piggybacks the fresh + /// coordinate automatically. + /// + /// A `payload` that is empty, has an unexpected version byte, or fails to + /// decode is dropped silently (mirrors Go serf `delegate.go` `notify_ping_complete`). + /// + /// No-op when `coord_client` is `None` (coordinates disabled at construction). + #[cfg(feature = "coordinates")] + fn handle_ping_completed(&mut self, node_id: &I, rtt: std::time::Duration, payload: &Bytes) { + use crate::{bridge::coordinate_from_pb, messages::serf::v1 as pb}; + use buffa::Message as _; + + // Guard: no-op when coordinates are disabled at construction. + let cc = match self.coord_client.as_mut() { + Some(c) => c, + None => return, + }; + + // Validate payload version byte (mirrors delegate.go notify_ping_complete). + if payload.is_empty() || payload[0] != PING_VERSION { + return; + } + + // Decode the remote peer's coordinate from payload[1..]. + let remote_coord = match pb::Coordinate::decode_from_slice(&payload[1..]) { + Ok(pb_coord) => coordinate_from_pb(&pb_coord), + Err(_) => return, + }; + + // Half 1: update the local Vivaldi model with (peer, remote_coord, rtt). + let new_local_coord = match cc.update(node_id, &remote_coord, rtt, &mut self.rng) { + Ok(c) => c, + Err(_) => return, + }; + + // Cache the remote peer's coordinate for `cached_coordinate()` queries. + self.coord_cache.insert(node_id.clone(), remote_coord); + + // Half 2: refresh own ack payload so future probe acks piggyback the + // updated local coordinate. + let ack_payload = coord_ack_payload(new_local_coord); + // Ignoring Err: set_ack_payload only fails when Leaving/Left/Shutdown. + // PingCompleted arrives on the probe path, active only while Alive. + let _ = self.inner.set_ack_payload(ack_payload); + } + + // ── Snapshot replay test helpers ───────────────────────────────────────── + + /// Return the list of addresses dialled by `load_snapshot` for rejoin + /// (test adapter for G10 assertions). + /// + /// Each call to `load_snapshot` appends to this list, so the test can + /// inspect which peers were targeted for the rejoin dial without needing + /// a live transport layer. + #[cfg(test)] + pub(crate) fn test_rejoin_dials(&self) -> Vec + where + A: Clone, + { + self.rejoin_dials.clone() + } + + /// Synthesise a `PingCompleted` update and drive it through + /// `handle_ping_completed` (test adapter). + /// + /// Sets `drain_now = Instant::ORIGIN` before the call. + #[cfg(all(feature = "coordinates", test))] + pub(crate) fn test_ping_completed( + &mut self, + node_id: I, + rtt: std::time::Duration, + payload: Bytes, + ) { + self.drain_now = memberlist_proto::Instant::ORIGIN; + self.handle_ping_completed(&node_id, rtt, &payload); + } +} + +// ── Tags helper ────────────────────────────────────────────────────────────── + +/// Attempt to decode serf `Tags` from raw meta bytes. +/// +/// The meta field in a memberlist `NodeState` carries buffa-encoded `pb::Tags` +/// (a protobuf map of string→string entries). On decode failure the caller +/// falls back to empty tags — FIX over the oracle: the oracle returns early +/// on tag-failure and silently skips the join/update, which is a footgun; +/// we prefer to join/update with empty tags instead. +/// +/// Mirrors Go serf `types.go` `Tags` decode path: +/// `Tags.Decode(n.Meta)` (oracle: `base.go` line ~1227, `~1587`). +fn decode_tags_from_meta(bytes: &[u8]) -> Option { + use crate::messages::serf::v1 as pb; + use buffa::Message as _; + + let pb_tags = pb::Tags::decode_from_slice(bytes).ok()?; + Some(tags_from_pb(&pb_tags)) +} + +// ── Coordinates (G9 / G13 / A5) ────────────────────────────────────────────── + +/// Wire version byte that prefixes every ack payload carrying a Vivaldi coordinate. +/// +/// Mirrors Go serf `delegate.go` `PingVersion` constant (= 1). +/// The payload layout is `[PING_VERSION] ++ pb::Coordinate`. +#[cfg(feature = "coordinates")] +const PING_VERSION: u8 = 1; + +/// Build the ack-payload bytes `[PING_VERSION] ++ encode(coord)`. +/// +/// Called at construction and after each successful coordinate update so the +/// inner Endpoint's ack piggybacks the current local coordinate on every probe. +#[cfg(feature = "coordinates")] +pub(crate) fn coord_ack_payload(coord: crate::typed::Coordinate) -> Bytes { + use crate::bridge::coordinate_to_pb; + use buffa::Message as _; + let pb = coordinate_to_pb(&coord); + let encoded = pb.encode_to_vec(); + let mut buf = Vec::with_capacity(1 + encoded.len()); + buf.push(PING_VERSION); + buf.extend_from_slice(&encoded); + Bytes::from(buf) +} + +// ── Coordinate public accessors ─────────────────────────────────────────────── + +impl Endpoint +where + I: Clone + Eq + core::hash::Hash, +{ + /// Return the local node's current Vivaldi coordinate. + /// + /// `None` when the `coordinates` feature is compiled out or when + /// `Options::with_disable_coordinates(true)` was set at construction. + #[cfg(feature = "coordinates")] + #[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] + pub fn get_coordinate(&self) -> Option { + self.coord_client.as_ref().map(|cc| cc.get_coordinate()) + } + + /// Return the most-recently-observed Vivaldi coordinate of `node`. + /// + /// Updated on each successful `PingCompleted` RTT feed from that peer. + /// Returns `None` when coordinates are disabled or when no RTT sample + /// has been received from `node` yet. + #[cfg(feature = "coordinates")] + #[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] + pub fn cached_coordinate(&self, node: &I) -> Option { + self.coord_cache.get(node).cloned() + } +} + +#[cfg(test)] +mod serf_parity_tests; +#[cfg(test)] +mod tests; diff --git a/serf-proto/src/endpoint/serf_parity_tests.rs b/serf-proto/src/endpoint/serf_parity_tests.rs new file mode 100644 index 00000000..bf756c44 --- /dev/null +++ b/serf-proto/src/endpoint/serf_parity_tests.rs @@ -0,0 +1,520 @@ +//! Oracle-conformance tests for the serf member-status FSM and push-pull replay. +//! +//! Verifies that the Sans-I/O `Endpoint` FSM matches the behaviour specified by +//! `legacy/serf-core/src/serf/base.rs` (`handle_node_join`, `handle_node_leave`, +//! `handle_node_join_intent`, `handle_node_leave_intent`, `upsert_intent`) and +//! `legacy/serf-core/src/serf/delegate.rs` (`merge_remote_state`). +//! Each test is named after the invariant it checks. + +use bytes::Bytes; +use memberlist_proto::{EndpointOptions, Instant, SeedableRng, SmallRng}; + +use crate::{ + AnyMessage, LamportTime, + endpoint::Endpoint, + event::{Event, MemberEventKind}, + members::{IntentKind, MemberStatus}, + options::Options, + typed::{PushPullMessage, UserEvent, UserEvents}, +}; + +fn ep() -> Endpoint { + let inner_opts = EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); + let inner = + memberlist_proto::Endpoint::new_at(inner_opts, Instant::ORIGIN, SmallRng::seed_from_u64(0)); + Endpoint::new(inner, Options::new()) +} + +// ── base.rs handle_node_join invariants ─────────────────────────────────────── + +/// base.rs:1286-1311 — new node with a pending Leave intent starts as Leaving. +#[test] +fn handle_node_join_with_pending_leave_intent_starts_leaving() { + let mut e = ep(); + // Leave intent arrives before the inner NodeJoined. + e.test_handle_leave_intent(2, LamportTime::new(4), Instant::ORIGIN); + e.test_inner_node_joined(2, Instant::ORIGIN); + assert_eq!(e.test_member_status(2), Some(MemberStatus::Leaving)); +} + +/// base.rs:1286-1311 — new node with only a Join intent starts as Alive. +#[test] +fn handle_node_join_with_pending_join_intent_starts_alive() { + let mut e = ep(); + e.test_handle_join_intent(2, LamportTime::new(4), Instant::ORIGIN); + e.test_inner_node_joined(2, Instant::ORIGIN); + assert_eq!(e.test_member_status(2), Some(MemberStatus::Alive)); +} + +/// base.rs:1324-1327 — re-joining after Failed clears the failed_members list. +#[test] +fn handle_node_join_clears_failed_and_left_lists() { + let mut e = ep(); + // Insert as Left first. + e.test_seed_left_member_by_status(2, LamportTime::new(5), Instant::ORIGIN); + assert!(e.test_in_left_members(2)); + e.test_inner_node_joined(2, Instant::ORIGIN); + assert!( + !e.test_in_left_members(2), + "re-join must clear left_members" + ); + assert_eq!(e.test_member_status(2), Some(MemberStatus::Alive)); +} + +/// base.rs:1274 — a Member(Join) event is always emitted on re-join. +#[test] +fn handle_node_join_always_emits_join_event() { + let mut e = ep(); + e.test_seed_failed_member_by_status(2, LamportTime::new(3), Instant::ORIGIN); + e.test_inner_node_joined(2, Instant::ORIGIN); + let ev = e.poll_event(); + assert!( + matches!(ev, Some(Event::Member(ref me)) if me.kind() == MemberEventKind::Join), + "expected Member(Join), got {ev:?}" + ); +} + +// ── base.rs handle_node_leave invariants ───────────────────────────────────── + +/// base.rs:1382 — Leaving → Left, emits Leave. +#[test] +fn handle_node_leave_leaving_to_left() { + let mut e = ep(); + e.test_seed_member(2, MemberStatus::Leaving, LamportTime::new(5)); + e.test_inner_node_left(2, Instant::ORIGIN); + assert_eq!(e.test_member_status(2), Some(MemberStatus::Left)); + assert!(e.test_in_left_members(2)); + let ev = e.poll_event(); + assert!(matches!(ev, Some(Event::Member(ref me)) if me.kind() == MemberEventKind::Leave)); +} + +/// base.rs:1400 — Alive → Failed, emits Failed. +#[test] +fn handle_node_leave_alive_to_failed() { + let mut e = ep(); + e.test_seed_member(2, MemberStatus::Alive, LamportTime::new(5)); + e.test_inner_node_left(2, Instant::ORIGIN); + assert_eq!(e.test_member_status(2), Some(MemberStatus::Failed)); + assert!(e.test_in_failed_members(2)); + let ev = e.poll_event(); + assert!(matches!(ev, Some(Event::Member(ref me)) if me.kind() == MemberEventKind::Failed)); +} + +/// base.rs:1410 — other statuses are a no-op. +#[test] +fn handle_node_leave_non_alive_leaving_is_noop() { + let mut e = ep(); + e.test_seed_member(2, MemberStatus::Failed, LamportTime::new(5)); + e.test_inner_node_left(2, Instant::ORIGIN); + // The test_inner_node_left path for Failed should not change status (no-op). + // Verify no event was emitted (the oracle ignores other statuses). + // NOTE: Failed→Left transition is triggered via handle_node_leave_intent (not inner_node_left). + // handle_node_leave in the machine only handles Alive/Leaving, so Failed stays. + // This is oracle-faithful: the oracle's `match ms` at base.rs:1390 returns from `_`. + let ev = e.poll_event(); + assert!( + ev.is_none(), + "no-op statuses should not emit events; got {ev:?}" + ); +} + +// ── base.rs handle_node_join_intent invariants ─────────────────────────────── + +/// base.rs:1345-1380 — witness the member clock on join intent. +#[test] +fn handle_node_join_intent_witnesses_member_clock() { + let mut e = ep(); + assert_eq!(e.member_time(), 0); + e.test_handle_join_intent(2, LamportTime::new(10), Instant::ORIGIN); + assert!( + e.member_time() >= 11, + "member clock must be witnessed past ltime=10" + ); +} + +/// base.rs:1353-1355 — stale join intent (ltime <= status_time) returns false. +#[test] +fn handle_node_join_intent_stale_returns_false() { + let mut e = ep(); + e.test_seed_member(2, MemberStatus::Alive, LamportTime::new(10)); + let rebroadcast = e.test_handle_join_intent(2, LamportTime::new(10), Instant::ORIGIN); + assert!(!rebroadcast); +} + +/// base.rs:1363-1365 — Leaving member moves back to Alive on fresh join intent. +#[test] +fn handle_node_join_intent_leaving_returns_to_alive() { + let mut e = ep(); + e.test_seed_member(2, MemberStatus::Leaving, LamportTime::new(5)); + let rebroadcast = e.test_handle_join_intent(2, LamportTime::new(8), Instant::ORIGIN); + assert!(rebroadcast); + assert_eq!(e.test_member_status(2), Some(MemberStatus::Alive)); +} + +// ── base.rs handle_node_leave_intent invariants ────────────────────────────── + +/// base.rs:1449 — always witness member clock. +#[test] +fn handle_node_leave_intent_witnesses_clock() { + let mut e = ep(); + e.test_handle_leave_intent(99, LamportTime::new(15), Instant::ORIGIN); + assert!(e.member_time() >= 16); +} + +/// base.rs:1471-1473 — stale intent (ltime <= status_time) → false, no transition. +#[test] +fn handle_node_leave_intent_stale_for_existing_member() { + let mut e = ep(); + e.test_seed_member(2, MemberStatus::Alive, LamportTime::new(10)); + let rb = e.test_handle_leave_intent(2, LamportTime::new(9), Instant::ORIGIN); + assert!(!rb); + assert_eq!(e.test_member_status(2), Some(MemberStatus::Alive)); +} + +/// base.rs:1489-1504 — status_time is updated even when already Leaving/Left +/// to prevent the infinite-rebroadcast bug (consul#8179 / consul#7960). +#[test] +fn handle_node_leave_intent_updates_status_time_for_leaving() { + let mut e = ep(); + e.test_seed_member(2, MemberStatus::Leaving, LamportTime::new(5)); + let rb = e.test_handle_leave_intent(2, LamportTime::new(8), Instant::ORIGIN); + assert!(rb, "Leaving node: fresh leave intent should rebroadcast"); + // Status unchanged (already Leaving), but status_time updated. + assert_eq!(e.test_member_status(2), Some(MemberStatus::Leaving)); + assert_eq!( + e.test_member_status_time(2), + Some(LamportTime::new(8)), + "status_time must be updated unconditionally" + ); +} + +/// base.rs:1527-1565 — Failed → Left, move to left_members, emit Leave. +#[test] +fn handle_node_leave_intent_failed_to_left() { + let mut e = ep(); + e.test_seed_failed_member_by_status(2, LamportTime::new(3), Instant::ORIGIN); + let rb = e.test_handle_leave_intent(2, LamportTime::new(7), Instant::ORIGIN); + assert!(rb); + assert_eq!(e.test_member_status(2), Some(MemberStatus::Left)); + assert!( + !e.test_in_failed_members(2), + "should be removed from failed_members" + ); + assert!(e.test_in_left_members(2), "should be in left_members"); + let ev = e.poll_event(); + assert!(matches!(ev, Some(Event::Member(ref me)) if me.kind() == MemberEventKind::Leave)); +} + +// ── merge_remote_state (G2 witness, G3 left-first, G4 eventJoinIgnore) ─────── + +/// Helper: build a `PushPullMessage` body as encoded `Bytes`. +fn push_pull_body( + ltime: u64, + status_ltimes: Vec<(u32, u64)>, + left_members: Vec, + event_ltime: u64, + events: Vec, + query_ltime: u64, +) -> Bytes { + let pp = PushPullMessage::::new( + LamportTime::new(ltime), + status_ltimes + .into_iter() + .map(|(id, lt)| (id, LamportTime::new(lt))) + .collect(), + left_members, + LamportTime::new(event_ltime), + events, + LamportTime::new(query_ltime), + ); + AnyMessage::::PushPull(pp) + .encode() + .expect("encode must succeed in test") +} + +/// delegate.rs:466-480 — G2: all three clocks are witnessed at `remote_value - 1`. +/// +/// Remote sends ltime=10, event_ltime=8, query_ltime=6. +/// After merge: member_clock >= 10 (witness at 9 → 10), event_clock >= 8, +/// query_clock >= 6. +#[test] +fn merge_witnesses_clocks_at_ltime_minus_one() { + let mut e = ep(); + let body = push_pull_body(10, vec![], vec![], 8, vec![], 6); + e.test_merge_remote_state(body, false); + // witness(9) → clock = 10; witness(7) → event_clock = 8; witness(5) → query_clock = 6 + assert_eq!(e.member_time(), 10, "member clock: witness(9) → 10"); + assert_eq!(e.event_time(), 8, "event clock: witness(7) → 8"); + assert_eq!(e.query_time(), 6, "query clock: witness(5) → 6"); +} + +/// delegate.rs:466 — G2 guard: `ltime = 0` must NOT witness (would underflow/wrap). +/// +/// All three clocks stay at 0 when the remote sends 0. +#[test] +fn merge_zero_clocks_are_not_witnessed() { + let mut e = ep(); + let body = push_pull_body(0, vec![], vec![], 0, vec![], 0); + e.test_merge_remote_state(body, false); + assert_eq!(e.member_time(), 0, "zero ltime must not witness"); + assert_eq!(e.event_time(), 0, "zero event_ltime must not witness"); + assert_eq!(e.query_time(), 0, "zero query_ltime must not witness"); +} + +/// delegate.rs:495-523 — G3: left_members are processed as leave intents BEFORE +/// the join pass, so a node in both lists ends up as Left, not Alive. +/// +/// Node 2 appears in both `status_ltimes` (ltime=4) and `left_members`. +/// Expected result: a synthetic leave intent at ltime=5 (=4+1) is processed +/// before the join pass, and the join pass skips node 2. The intent buffer +/// carries ltime=5. +#[test] +fn merge_processes_left_members_before_joins() { + let mut e = ep(); + // Node 2 in status_ltimes (ltime=4) AND in left_members. + let body = push_pull_body(5, vec![(2u32, 4)], vec![2u32], 0, vec![], 0); + e.test_merge_remote_state(body, false); + // The join pass must have skipped node 2 (left_set guard). + // The leave intent at synthetic ltime=5 must be buffered (node 2 not in states yet). + assert_eq!( + e.test_intent_ltime(2, IntentKind::Leave), + Some(LamportTime::new(5)), + "node in left_members must have a synthetic Leave intent at status_ltime + 1" + ); + // No Join intent should have been buffered (join pass skipped it). + assert_eq!( + e.test_intent_ltime(2, IntentKind::Join), + None, + "join pass must skip nodes in left_members" + ); +} + +/// delegate.rs:513-523 — G3: nodes that appear ONLY in status_ltimes (not left) +/// receive a synthetic join intent. +#[test] +fn merge_join_intent_buffered_for_non_left_node() { + let mut e = ep(); + // Node 3 appears only in status_ltimes, NOT in left_members. + let body = push_pull_body(5, vec![(3u32, 7)], vec![], 0, vec![], 0); + e.test_merge_remote_state(body, false); + // A join intent must be buffered for node 3 (it is not in states yet). + assert_eq!( + e.test_intent_ltime(3, IntentKind::Join), + Some(LamportTime::new(7)), + "non-left node must receive a join intent at its status_ltime" + ); +} + +/// delegate.rs:528-534 — G4: `eventJoinIgnore` + `is_join` bumps `event_buffer.min_time` +/// to max(min_time, event_ltime). +#[test] +fn join_with_event_join_ignore_bumps_event_min_time() { + let mut e = ep(); + e.test_set_event_join_ignore(true); + let body = push_pull_body(0, vec![], vec![], 42, vec![], 0); + e.test_merge_remote_state(body, /*is_join*/ true); + assert_eq!( + e.test_event_min_time(), + 42, + "event_join_ignore + is_join must set min_time to remote event_ltime" + ); +} + +/// G4 inverse: `is_join = false` must NOT bump event_buffer.min_time. +#[test] +fn refresh_exchange_does_not_bump_event_min_time() { + let mut e = ep(); + e.test_set_event_join_ignore(true); + let body = push_pull_body(0, vec![], vec![], 99, vec![], 0); + e.test_merge_remote_state(body, /*is_join*/ false); + assert_eq!( + e.test_event_min_time(), + 0, + "non-join exchange must not bump event_buffer.min_time" + ); +} + +/// G4 inverse: flag not set even with is_join must NOT bump min_time. +#[test] +fn join_without_event_join_ignore_does_not_bump_min_time() { + let mut e = ep(); + // event_join_ignore is false (default) + let body = push_pull_body(0, vec![], vec![], 99, vec![], 0); + e.test_merge_remote_state(body, /*is_join*/ true); + assert_eq!( + e.test_event_min_time(), + 0, + "event_join_ignore=false: min_time must not be bumped" + ); +} + +/// delegate.rs:536-548 — buffered user events in the push-pull body are replayed +/// via handle_user_event (dedup + emit). +#[test] +fn merge_replays_buffered_user_events() { + let mut e = ep(); + let events = vec![UserEvents { + ltime: LamportTime::new(3), + events: vec![UserEvent { + name: "deploy".into(), + payload: Bytes::from_static(b"v1"), + }], + }]; + let body = push_pull_body(4, vec![], vec![], 4, events, 0); + e.test_merge_remote_state(body, false); + // The replayed user event must surface. + let ev = e.poll_event().expect("replayed user event must surface"); + assert!( + matches!(ev, Event::User(ref u) if u.name == "deploy"), + "expected Event::User(deploy), got: {ev:?}" + ); +} + +/// G4 + replay: when event_join_ignore bumps min_time to the remote event_ltime, +/// events strictly below the new min_time must be suppressed on replay. +/// +/// G4 sets `min_time = event_ltime`. An event at `ltime=4` with `min_time=5` +/// satisfies `ltime < min_time`, so it is suppressed. +#[test] +fn join_with_event_join_ignore_suppresses_event_replay() { + let mut e = ep(); + e.test_set_event_join_ignore(true); + // event_ltime = 5 → G4 bumps min_time to 5. Event at ltime=4 < 5 must be dropped. + let events = vec![UserEvents { + ltime: LamportTime::new(4), + events: vec![UserEvent { + name: "really-old".into(), + payload: Bytes::from_static(b"y"), + }], + }]; + let body = push_pull_body(0, vec![], vec![], 5, events, 0); + e.test_merge_remote_state(body, /*is_join*/ true); + // min_time = 5; event at ltime=4 must be suppressed. + assert!( + e.poll_event().is_none(), + "event at ltime < min_time must be suppressed after G4 bump" + ); +} + +/// Empty user_data must be silently ignored without panicking. +#[test] +fn merge_empty_user_data_is_silently_dropped() { + let mut e = ep(); + // Simulate the RemoteStateReceived path: empty bytes are dropped before + // merge_remote_state is called (the sieve arm guards `!user_data.is_empty()`). + // Here we test that a zero-byte body passed directly does not panic. + e.test_merge_remote_state(Bytes::new(), false); + assert!(e.poll_event().is_none()); +} + +/// Malformed (non-PushPull) bytes are silently dropped. +#[test] +fn merge_malformed_body_is_silently_dropped() { + let mut e = ep(); + e.test_merge_remote_state(Bytes::from_static(b"\xff\xfe\xfd"), false); + assert_eq!( + e.member_time(), + 0, + "clocks must be unchanged on decode error" + ); + assert!(e.poll_event().is_none()); +} + +// ── handle_timeout tick order (H1b) + cross-tier priority ──────────────────── + +/// H1b / decision 5 step 4 — inner events are drained BEFORE serf's own +/// deadlines fire. +/// +/// Scenario: member 2 is `Failed` with an elapsed `reconnect_timeout` (would be +/// reaped). Before `handle_timeout` fires, an inner `NodeJoined` for the same +/// node is injected directly through the sieve (simulating an inner event that +/// arrives in the same tick as the reap deadline). Because `drain_inner` runs +/// BEFORE `fire_reap`, the join must win: member 2 ends up `Alive`, not reaped. +/// +/// This test verifies the ordering invariant without needing to enqueue inside +/// the inner machine's queue (which is not externally addressable): the join is +/// applied via the sieve before `fire_reap` runs, matching the production path +/// where inner events queued during `inner.handle_timeout` are drained first. +#[test] +fn tick_drains_inner_before_firing_reap() { + let mut e = ep(); + let t0 = Instant::ORIGIN; + + // Seed member 2 as Failed with leave_time = t0 (reconnect_timeout = 24h by default). + e.test_seed_failed_member(2, "127.0.0.1:1002".parse().unwrap(), t0); + + // Advance to a time far past reconnect_timeout (24h + 1h) so the reaper + // would normally remove member 2. + let past_timeout = t0 + std::time::Duration::from_secs(3600 * 25); + + // Inject a NodeJoined event for member 2 directly through the sieve + // (simulates an inner event produced during inner.handle_timeout but before + // serf's deadlines fire). We set drain_now to past_timeout so the sieve + // sees a consistent "now". + e.test_inject_inner_joined(2, past_timeout); + + // The join must be processed first; drain clears the pending_events. + while e.poll_event().is_some() {} + + // Now call handle_timeout at past_timeout — the reaper fires but member 2 + // is Alive after the join, so it must NOT be reaped. + e.handle_timeout(past_timeout); + + assert_eq!( + e.test_member_status(2), + Some(MemberStatus::Alive), + "inner NodeJoined must win over the concurrent reap deadline (H1b: drain-before-reap)" + ); +} + +/// Three broadcast tiers are populated at the correct ranks. +/// +/// serf configures the inner Endpoint with 3 broadcast tiers +/// (intent=0 highest, query=1, event=2 lowest) and enqueues: +/// - join/leave intents on tier 0 (rank 0), +/// - query broadcasts on tier 1 (rank 1), +/// - user-event broadcasts on tier 2 (rank 2). +/// +/// This test verifies the queue depth after serf enqueues one message on each +/// tier (via the intent-broadcast helper, the query-broadcast helper, and +/// `user_event`), confirming that all three tiers are populated. The inner +/// Endpoint's priority drain order (rank 0 before 1 before 2) is exercised by +/// memberlist-proto's own test suite; serf's responsibility is correct tier +/// assignment, not internal drain scheduling. +/// +/// H1b corollary: broadcasts pushed to the inner tiers before `handle_timeout` +/// are available for the inner gossip scheduler on the next tick — there is no +/// extra serf-side drain step needed between enqueue and tick. +#[test] +fn three_tiers_drain_intent_then_query_then_event() { + let mut e = ep(); + + // Enqueue one broadcast on each of the three tiers. + // Tier 0 = intent (highest), tier 1 = query, tier 2 = event (lowest). + e.test_enqueue_intent_broadcast(Bytes::from_static(b"intent-bytes")); + e.test_enqueue_query_broadcast(Bytes::from_static(b"query-bytes")); + + // user_event enqueues on the event tier (rank 2). + e.user_event("ev", Bytes::from_static(b"event"), false) + .expect("user_event must succeed"); + + // All three tiers should be populated (user_broadcast_queue_len = total across all tiers). + // The intent broadcast at rank 0 and query broadcast at rank 1 were enqueued via + // test helpers; user_event enqueues at rank 2. + let total = e.user_broadcast_queue_len(); + assert!( + total >= 3, + "all three broadcast tiers must be populated (intent + query + event); got queue_len = {total}" + ); + + // Verify: enqueue one more intent-tier item then check total grew, confirming + // rank-0 enqueue hits a different internal tier slot from rank-2 enqueue. + e.test_enqueue_intent_broadcast(Bytes::from_static(b"intent2")); + let total2 = e.user_broadcast_queue_len(); + assert_eq!( + total2, + total + 1, + "second intent-tier enqueue must increment total queue len" + ); +} diff --git a/serf-proto/src/endpoint/tests.rs b/serf-proto/src/endpoint/tests.rs new file mode 100644 index 00000000..d19106d9 --- /dev/null +++ b/serf-proto/src/endpoint/tests.rs @@ -0,0 +1,6621 @@ +use super::*; +use crate::{ + AnyMessage, JoinMessage, LamportTime, LeaveMessage, + event::{Event, MemberEventKind}, + members::{MemberStatus, SerfState}, + typed::{Filter, QueryFlag, QueryMessage, RelayMessage, UserEventMessage}, +}; +#[cfg(feature = "coordinates")] +use bytes::Bytes; +use memberlist_proto::{EndpointOptions, SeedableRng, SmallRng}; + +/// Build a minimal serf `Endpoint` suitable for unit tests. +/// +/// Uses `u32` node ids and `SocketAddr` addresses with a deterministically +/// seeded `SmallRng` so tests are reproducible. +fn ep() -> Endpoint { + let inner_opts = EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); + let inner = memberlist_proto::Endpoint::new_at( + inner_opts, + memberlist_proto::Instant::ORIGIN, + SmallRng::seed_from_u64(0), + ); + Endpoint::new(inner, Options::new()) +} + +/// Build a serf `Endpoint` with coordinates enabled (for coordinate-gated tests). +#[cfg(feature = "coordinates")] +fn ep_with_coords() -> Endpoint { + let inner_opts = EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); + let inner = memberlist_proto::Endpoint::new_at( + inner_opts, + memberlist_proto::Instant::ORIGIN, + SmallRng::seed_from_u64(0), + ); + let opts = Options::new().with_disable_coordinates(false); + Endpoint::new(inner, opts) +} + +#[test] +fn new_endpoint_starts_alive_with_zero_clocks() { + let e = ep(); + assert!(e.state().is_alive()); + assert_eq!(e.member_time(), 0); + assert_eq!(e.event_time(), 0); + assert_eq!(e.query_time(), 0); + assert_eq!(e.num_members(), 0); +} + +#[test] +fn witness_advances_to_at_least() { + let mut c = 0u64; + witness(&mut c, 5); + assert_eq!(c, 6); // advance past the witnessed time + witness(&mut c, 3); + assert_eq!(c, 6); // older time does not regress + witness(&mut c, 6); + assert_eq!(c, 7); // equal triggers advance +} + +#[test] +fn poll_event_drains_when_inner_empty() { + let mut e = ep(); + assert!(e.poll_event().is_none()); +} + +#[test] +fn poll_timeout_is_none_on_idle_alive_endpoint() { + // No serf deadlines armed yet; inner scheduler idle at ORIGIN-relative new. + // Either None or Some — assert it does not panic. + let e = ep(); + let _ = e.poll_timeout(); +} + +#[test] +fn handle_packet_with_garbage_bytes_is_a_noop() { + let mut e = ep(); + e.handle_packet( + "127.0.0.1:9999".parse().unwrap(), + Bytes::from_static(b"\xff\xff"), + memberlist_proto::Instant::ORIGIN, + ); + // Undecodable inner message -> no serf event, no panic. + assert!(e.poll_event().is_none()); +} + +#[test] +fn poll_transmit_delegates_to_inner() { + let mut e = ep(); + // No transmits queued at construction time; must not panic. + assert!(e.poll_transmit().is_none()); +} + +// ── Task 1.4: member-status FSM + intent reconciliation + clock witnessing ──── + +#[test] +fn leave_intent_for_known_alive_self_refutes_and_does_not_rebroadcast() { + // local_id = 1, state Alive, seeded in states. A FRESH leave intent for the + // local node is refuted: the node re-announces its join and returns false. + // The stale check fires first (for known nodes); this ltime=5 exceeds + // status_time=0, so it reaches the self-refute path. + let mut e = ep(); + e.test_seed_member(1u32, MemberStatus::Alive, LamportTime::new(0)); + let rebroadcast = + e.test_handle_leave_intent(1, LamportTime::new(5), memberlist_proto::Instant::ORIGIN); + assert!( + !rebroadcast, + "fresh self-leave while Alive and known must be refuted, not rebroadcast" + ); + // The member clock must have witnessed ltime=5, so clock >= 6. + assert!( + e.member_time() >= 6, + "member clock should have been witnessed" + ); +} + +#[test] +fn stale_leave_intent_is_dropped() { + let mut e = ep(); + // Seed member 2 at status_time=10; a leave intent at ltime=3 is stale. + e.test_seed_member(2, MemberStatus::Alive, LamportTime::new(10)); + let rebroadcast = + e.test_handle_leave_intent(2, LamportTime::new(3), memberlist_proto::Instant::ORIGIN); + assert!(!rebroadcast, "stale leave intent must not be rebroadcast"); + assert_eq!( + e.test_member_status(2), + Some(MemberStatus::Alive), + "stale intent must not change member status" + ); +} + +#[test] +fn live_leave_intent_transitions_alive_to_leaving() { + let mut e = ep(); + e.test_seed_member(2, MemberStatus::Alive, LamportTime::new(5)); + let rebroadcast = + e.test_handle_leave_intent(2, LamportTime::new(8), memberlist_proto::Instant::ORIGIN); + assert!( + rebroadcast, + "fresh leave intent for Alive should rebroadcast" + ); + assert_eq!(e.test_member_status(2), Some(MemberStatus::Leaving)); + assert!( + e.member_time() >= 9, + "member clock must have been witnessed" + ); +} + +#[test] +fn leave_intent_for_failed_transitions_to_left_and_emits_leave_event() { + let mut e = ep(); + // Seed a Failed member in failed_members list. + e.test_seed_failed_member_by_status(2, LamportTime::new(5), memberlist_proto::Instant::ORIGIN); + let rebroadcast = + e.test_handle_leave_intent(2, LamportTime::new(9), memberlist_proto::Instant::ORIGIN); + assert!(rebroadcast, "leave intent for Failed should rebroadcast"); + assert_eq!( + e.test_member_status(2), + Some(MemberStatus::Left), + "Failed + leave intent should transition to Left" + ); + // A Member(Leave) event should be pending. + let ev = e.poll_event(); + assert!( + matches!(ev, Some(Event::Member(ref me)) if me.kind() == MemberEventKind::Leave), + "expected Member(Leave) event, got: {ev:?}" + ); +} + +#[test] +fn leave_intent_for_unknown_node_is_buffered_as_intent() { + let mut e = ep(); + // Node 99 is not in states yet — intent should be upserted. + let rebroadcast = + e.test_handle_leave_intent(99, LamportTime::new(3), memberlist_proto::Instant::ORIGIN); + assert!(rebroadcast, "unknown node leave intent should be buffered"); + assert_eq!( + e.test_member_status(99), + None, + "no member should be created" + ); +} + +#[test] +fn join_intent_for_existing_leaving_member_clears_to_alive() { + let mut e = ep(); + // Seed member 2 as Leaving at status_time=5. + e.test_seed_member(2, MemberStatus::Leaving, LamportTime::new(5)); + // A fresh join intent at ltime=8 should move it back to Alive. + let rebroadcast = + e.test_handle_join_intent(2, LamportTime::new(8), memberlist_proto::Instant::ORIGIN); + assert!( + rebroadcast, + "fresh join intent for Leaving should rebroadcast" + ); + assert_eq!(e.test_member_status(2), Some(MemberStatus::Alive)); + assert!( + e.member_time() >= 9, + "member clock must have been witnessed" + ); +} + +#[test] +fn stale_join_intent_for_existing_member_is_ignored() { + let mut e = ep(); + e.test_seed_member(2, MemberStatus::Alive, LamportTime::new(10)); + let rebroadcast = + e.test_handle_join_intent(2, LamportTime::new(3), memberlist_proto::Instant::ORIGIN); + assert!(!rebroadcast, "stale join intent must not rebroadcast"); +} + +#[test] +fn join_intent_for_unknown_node_is_buffered() { + let mut e = ep(); + assert!( + e.test_handle_join_intent(3, LamportTime::new(7), memberlist_proto::Instant::ORIGIN), + "first join intent for unknown node should be buffered" + ); + assert_eq!(e.test_member_status(3), None); +} + +#[test] +fn inner_node_joined_with_pending_leave_intent_creates_leaving_member() { + let mut e = ep(); + // Leave intent buffered before the inner NodeJoined fires. + e.test_handle_leave_intent(2, LamportTime::new(7), memberlist_proto::Instant::ORIGIN); + // Now the inner NodeJoined arrives. + e.test_inner_node_joined(2, memberlist_proto::Instant::ORIGIN); + // Should be Leaving (intent applied), not Alive. + assert_eq!(e.test_member_status(2), Some(MemberStatus::Leaving)); + // A Member(Join) event must have been emitted regardless. + let ev = e.poll_event(); + assert!( + matches!(ev, Some(Event::Member(ref me)) if me.kind() == MemberEventKind::Join), + "expected Member(Join), got: {ev:?}" + ); +} + +#[test] +fn inner_node_joined_without_intents_creates_alive_member() { + let mut e = ep(); + e.test_inner_node_joined(2, memberlist_proto::Instant::ORIGIN); + assert_eq!(e.test_member_status(2), Some(MemberStatus::Alive)); + let ev = e.poll_event(); + assert!(matches!(ev, Some(Event::Member(ref me)) if me.kind() == MemberEventKind::Join)); +} + +#[test] +fn inner_node_left_alive_transitions_to_failed() { + let mut e = ep(); + e.test_seed_member(2, MemberStatus::Alive, LamportTime::new(5)); + e.test_inner_node_left(2, memberlist_proto::Instant::ORIGIN); + assert_eq!(e.test_member_status(2), Some(MemberStatus::Failed)); + let ev = e.poll_event(); + assert!(matches!(ev, Some(Event::Member(ref me)) if me.kind() == MemberEventKind::Failed)); +} + +#[test] +fn inner_node_left_leaving_transitions_to_left() { + let mut e = ep(); + e.test_seed_member(2, MemberStatus::Leaving, LamportTime::new(5)); + e.test_inner_node_left(2, memberlist_proto::Instant::ORIGIN); + assert_eq!(e.test_member_status(2), Some(MemberStatus::Left)); + let ev = e.poll_event(); + assert!(matches!(ev, Some(Event::Member(ref me)) if me.kind() == MemberEventKind::Leave)); +} + +#[test] +fn inner_node_left_unknown_is_a_noop() { + let mut e = ep(); + // Node not in states — should not panic. + e.test_inner_node_left(99, memberlist_proto::Instant::ORIGIN); + assert_eq!(e.test_member_status(99), None); + assert!(e.poll_event().is_none()); +} + +#[test] +fn inner_node_updated_refreshes_member_and_emits_update_event() { + let mut e = ep(); + e.test_seed_member(2, MemberStatus::Alive, LamportTime::new(3)); + e.test_inner_node_updated(2, memberlist_proto::Instant::ORIGIN); + assert_eq!(e.test_member_status(2), Some(MemberStatus::Alive)); + let ev = e.poll_event(); + assert!(matches!(ev, Some(Event::Member(ref me)) if me.kind() == MemberEventKind::Update)); +} + +#[test] +fn inner_node_updated_for_unknown_is_a_noop() { + let mut e = ep(); + e.test_inner_node_updated(99, memberlist_proto::Instant::ORIGIN); + assert!(e.poll_event().is_none()); +} + +#[test] +fn handle_node_join_re_joining_failed_clears_lists() { + let mut e = ep(); + // Add node 2 as Failed in failed_members. + e.test_seed_failed_member_by_status(2, LamportTime::new(5), memberlist_proto::Instant::ORIGIN); + assert!(e.test_in_failed_members(2), "should be in failed_members"); + // Inner NodeJoined re-joins it. + e.test_inner_node_joined(2, memberlist_proto::Instant::ORIGIN); + assert_eq!(e.test_member_status(2), Some(MemberStatus::Alive)); + assert!( + !e.test_in_failed_members(2), + "should be cleared from failed_members" + ); +} + +#[test] +fn leave_intent_no_op_on_none_status() { + let mut e = ep(); + // Seed a member with None status. + e.test_seed_member(2, MemberStatus::None, LamportTime::new(5)); + let rebroadcast = + e.test_handle_leave_intent(2, LamportTime::new(8), memberlist_proto::Instant::ORIGIN); + // The status_time IS updated (the FIX), but FSM does not transition from None. + assert!(!rebroadcast, "None-status leave intent returns false"); + assert_eq!(e.test_member_status(2), Some(MemberStatus::None)); +} + +// ── Task 1.5: SerfState lifecycle FSM + leave chain ─────────────────────────── + +#[test] +fn leave_from_alive_transitions_to_leaving() { + let mut e = ep(); + e.leave(memberlist_proto::Instant::ORIGIN).unwrap(); + assert!(e.state().is_leaving(), "leave() must set state to Leaving"); +} + +#[test] +fn double_leave_is_rejected() { + let mut e = ep(); + e.leave(memberlist_proto::Instant::ORIGIN).unwrap(); + let err = e + .leave(memberlist_proto::Instant::ORIGIN) + .expect_err("second leave from Leaving must fail"); + assert!(matches!(err, Error::BadLeaveState(SerfState::Leaving))); +} + +#[test] +fn leave_from_already_left_is_idempotent() { + let mut e = ep(); + // Manually drive to Left state to test the idempotent path. + e.leave(memberlist_proto::Instant::ORIGIN).unwrap(); + // Simulate the leave chain completing: inner LeftCluster + delay elapses. + e.test_inner_left_cluster(); + let delay = std::time::Duration::from_secs(2); // > leave_propagate_delay (1s) + e.handle_timeout(memberlist_proto::Instant::ORIGIN + delay); + assert!(e.state().is_left(), "should have transitioned to Left"); + // A second leave from Left is Ok(()). + assert!(e.leave(memberlist_proto::Instant::ORIGIN).is_ok()); +} + +#[test] +fn leave_from_shutdown_is_rejected() { + let mut e = ep(); + // Force state to Shutdown. + e.state = SerfState::Shutdown; + let err = e + .leave(memberlist_proto::Instant::ORIGIN) + .expect_err("leave from Shutdown must fail"); + assert!(matches!(err, Error::BadLeaveState(SerfState::Shutdown))); +} + +#[test] +fn inner_left_cluster_arms_leave_complete_deadline() { + let mut e = ep(); + e.leave(memberlist_proto::Instant::ORIGIN).unwrap(); + assert!( + e.leave_complete_deadline().is_none(), + "deadline not armed until inner LeftCluster arrives" + ); + // Simulate inner LeftCluster. + e.test_inner_left_cluster(); + assert!( + e.leave_complete_deadline().is_some(), + "deadline must be armed after inner LeftCluster" + ); +} + +#[test] +fn inner_left_cluster_not_in_leaving_state_is_ignored() { + // If the inner emits LeftCluster while serf is still Alive (unexpected but + // must not panic or set a spurious deadline). + let mut e = ep(); + assert!(e.state().is_alive()); + e.test_inner_left_cluster(); + assert!( + e.leave_complete_deadline().is_none(), + "LeftCluster while Alive must not arm leave_complete_deadline" + ); +} + +#[test] +fn inner_left_cluster_drives_serf_to_left_and_emits_left_cluster_event() { + let mut e = ep(); + e.leave(memberlist_proto::Instant::ORIGIN).unwrap(); + // Simulate the inner emitting LeftCluster (no live peers → immediate). + e.test_inner_left_cluster(); + // Advance time past leave_propagate_delay (default 1s). + let after_delay = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(2); + e.handle_timeout(after_delay); + // State must be Left. + assert!( + e.state().is_left(), + "state must be Left after propagation delay" + ); + // The next poll_event must yield Event::LeftCluster. + let ev = e.poll_event().expect("LeftCluster event must be pending"); + assert!( + matches!(ev, Event::LeftCluster), + "expected Event::LeftCluster, got {ev:?}" + ); +} + +#[test] +fn leave_complete_deadline_not_fired_before_delay() { + let mut e = ep(); + e.leave(memberlist_proto::Instant::ORIGIN).unwrap(); + e.test_inner_left_cluster(); + // Tick to just before the propagation deadline (< 1s). + let before_delay = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_millis(500); + e.handle_timeout(before_delay); + // Still Leaving; no event yet. + assert!(e.state().is_leaving(), "must still be Leaving before delay"); + assert!(e.poll_event().is_none(), "no LeftCluster before deadline"); +} + +#[test] +fn shutdown_prevents_leaving_to_left_transition() { + // If state is set to Shutdown before leave_complete_deadline fires, the + // transition to Left is skipped (mirrors oracle: Shutdown wins over Left). + let mut e = ep(); + e.leave(memberlist_proto::Instant::ORIGIN).unwrap(); + e.test_inner_left_cluster(); + // Force Shutdown before the deadline fires. + e.state = SerfState::Shutdown; + let after_delay = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(2); + e.handle_timeout(after_delay); + // Must remain Shutdown, not Left. + assert!( + e.state().is_shutdown(), + "Shutdown should not transition to Left" + ); + // No LeftCluster event. + assert!(e.poll_event().is_none()); +} + +#[test] +fn leave_arms_broadcast_deadline() { + let mut e = ep(); + assert!(e.leave_broadcast_deadline().is_none()); + e.leave(memberlist_proto::Instant::ORIGIN).unwrap(); + let dl = e + .leave_broadcast_deadline() + .expect("broadcast deadline must be armed"); + // Default broadcast_timeout is 5s; deadline = ORIGIN + 5s. + assert_eq!( + dl, + memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(5) + ); +} + +#[test] +fn force_leave_from_shutdown_is_rejected() { + let mut e = ep(); + e.state = SerfState::Shutdown; + let err = e + .force_leave(2u32, false, memberlist_proto::Instant::ORIGIN) + .expect_err("force_leave from Shutdown must fail"); + assert!(matches!(err, Error::BadLeaveState(SerfState::Shutdown))); +} + +#[test] +fn force_leave_transitions_alive_member_to_leaving() { + let mut e = ep(); + e.test_seed_member(2, MemberStatus::Alive, LamportTime::new(0)); + e.force_leave(2u32, false, memberlist_proto::Instant::ORIGIN) + .unwrap(); + assert_eq!( + e.test_member_status(2), + Some(MemberStatus::Leaving), + "force_leave must transition Alive → Leaving" + ); +} + +#[test] +fn leave_witnesses_and_advances_member_clock() { + let mut e = ep(); + let clock_before = e.member_time(); + e.leave(memberlist_proto::Instant::ORIGIN).unwrap(); + assert!( + e.member_time() > clock_before, + "leave() must advance the member clock" + ); +} + +#[test] +fn poll_timeout_includes_leave_deadlines_when_armed() { + let mut e = ep(); + // Initially no serf deadlines; poll_timeout may be None (inner idle) or + // Some from inner's own schedule — just confirm it does not panic. + let _ = e.poll_timeout(); + + e.leave(memberlist_proto::Instant::ORIGIN).unwrap(); + // After leave(), leave_broadcast_deadline is armed. + let timeout = e + .poll_timeout() + .expect("must have a deadline after leave()"); + let expected = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(5); + assert!( + timeout <= expected, + "poll_timeout must be ≤ leave_broadcast_deadline ({expected:?}), got {timeout:?}" + ); +} + +// ── Task 1.6: reconnector dial-output + reaper deadlines ───────────────────── + +// Helper: seed a failed member with an explicit address so we can assert what +// addr is dialled by the reconnector. +fn seed_failed( + e: &mut Endpoint, + id: u32, + addr: std::net::SocketAddr, + leave_time: memberlist_proto::Instant, +) { + e.test_seed_failed_member(id, addr, leave_time); +} + +// Helper: seed the endpoint's one alive member (the local node) explicitly so +// the probability computation has a stable num_alive value. +fn seed_alive(e: &mut Endpoint, id: u32) { + e.test_seed_member(id, MemberStatus::Alive, LamportTime::new(0)); +} + +#[test] +fn reap_failed_removes_after_reconnect_timeout() { + let mut e = ep(); + let t0 = memberlist_proto::Instant::ORIGIN; + seed_failed(&mut e, 2, "127.0.0.1:1002".parse().unwrap(), t0); + // 25 hours > reconnect_timeout (24h) + let past_timeout = t0 + std::time::Duration::from_secs(3600 * 25); + e.test_fire_reap(past_timeout); + assert_eq!( + e.test_member_status(2), + None, + "failed member should be reaped after reconnect_timeout" + ); + let reaped = std::iter::from_fn(|| e.poll_event()) + .any(|ev| matches!(ev, Event::Member(ref me) if me.kind() == MemberEventKind::Reap)); + assert!(reaped, "a Member(Reap) event should have been emitted"); +} + +#[test] +fn reap_failed_keeps_member_before_reconnect_timeout() { + let mut e = ep(); + let t0 = memberlist_proto::Instant::ORIGIN; + seed_failed(&mut e, 2, "127.0.0.1:1002".parse().unwrap(), t0); + // 1 hour < reconnect_timeout (24h) — should NOT reap + let before_timeout = t0 + std::time::Duration::from_secs(3600); + e.test_fire_reap(before_timeout); + assert_eq!( + e.test_member_status(2), + Some(MemberStatus::Failed), + "member not yet past reconnect_timeout must not be reaped" + ); + assert!(e.poll_event().is_none(), "no Reap event before timeout"); +} + +#[test] +fn reap_left_removes_after_tombstone_timeout() { + let mut e = ep(); + let t0 = memberlist_proto::Instant::ORIGIN; + e.test_seed_left_member_by_status(2, LamportTime::new(3), t0); + // 25 hours > tombstone_timeout (24h) + let past_timeout = t0 + std::time::Duration::from_secs(3600 * 25); + e.test_fire_reap(past_timeout); + assert_eq!( + e.test_member_status(2), + None, + "left member should be reaped after tombstone_timeout" + ); + let reaped = std::iter::from_fn(|| e.poll_event()) + .any(|ev| matches!(ev, Event::Member(ref me) if me.kind() == MemberEventKind::Reap)); + assert!(reaped, "a Member(Reap) event should have been emitted"); +} + +#[test] +fn reap_intents_removes_stale_intents() { + let mut e = ep(); + let t0 = memberlist_proto::Instant::ORIGIN; + // Buffer a leave intent for an unknown node at t0. + e.test_handle_leave_intent(99, LamportTime::new(3), t0); + // recent_intent_timeout is 600s (10 min). At t0 + 700s the intent is stale. + let past_intent_timeout = t0 + std::time::Duration::from_secs(700); + e.test_fire_reap(past_intent_timeout); + // The intent buffer should be empty now (no state was created, the node was unknown). + // Verify by checking that the intent is no longer present — seeded as Leave intent for 99. + // We confirm indirectly: a second identical intent would be accepted (it would + // insert fresh), which it should be regardless. Instead we check members directly. + assert_eq!(e.test_member_status(99), None, "no member state expected"); + // No reap event should have been emitted (intents are not members). + assert!( + e.poll_event().is_none(), + "reap_intents must not emit Member events" + ); +} + +#[test] +fn reconnect_gate_skips_when_no_failed_members() { + let mut e = ep(); + // No failed members: reconnect should be a no-op. + e.test_fire_reconnect(memberlist_proto::Instant::ORIGIN); + // No DialRequested passthrough in pending events. + assert!(e.poll_event().is_none()); +} + +#[test] +fn reconnect_picks_a_failed_member_and_emits_dial_requested() { + // Use a fixed seed so the probability gate (1/1 = 1.0) always fires. + // ep() seeds SmallRng from 0, so draws are deterministic. + let mut e = ep(); + seed_alive(&mut e, 1); // local node alive, num_alive=1 + seed_failed( + &mut e, + 2, + "127.0.0.1:1002".parse().unwrap(), + memberlist_proto::Instant::ORIGIN, + ); + // With num_failed=1, num_alive=1, prob=1.0 → gate always fires. + e.test_fire_reconnect(memberlist_proto::Instant::ORIGIN); + // The inner start_push_pull emits a DialRequested which the sieve passes through. + let dialled = e.test_last_dial_addr(); + assert!( + dialled.is_some(), + "reconnect should have triggered a dial request" + ); +} + +#[test] +fn reconnect_deadline_armed_and_polls_in_poll_timeout() { + let mut e = ep(); + seed_alive(&mut e, 1); + seed_failed( + &mut e, + 2, + "127.0.0.1:1002".parse().unwrap(), + memberlist_proto::Instant::ORIGIN, + ); + // Before arming: next_reconnect is None internally but poll_timeout may still + // return Some from the inner. After handle_timeout fires and re-arms: + // drive handle_timeout past the first reconnect_interval (30s). + let after_interval = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(31); + e.handle_timeout(after_interval); + // After the tick the reconnect deadline is re-armed, so poll_timeout is Some. + let _ = e.poll_timeout(); // must not panic +} + +#[test] +fn reap_deadline_fires_via_handle_timeout() { + let mut e = ep(); + let t0 = memberlist_proto::Instant::ORIGIN; + seed_failed(&mut e, 2, "127.0.0.1:1002".parse().unwrap(), t0); + // Drive handle_timeout well past reconnect_timeout (24h) + reap_interval (15s). + let far_future = t0 + std::time::Duration::from_secs(3600 * 25 + 16); + e.handle_timeout(far_future); + // Member should be reaped. + assert_eq!( + e.test_member_status(2), + None, + "failed member should be reaped after handle_timeout fires the reap deadline" + ); +} + +// ── Task 2.1: event ring-buffer dedup + event-clock + event broadcast tier ──── + +#[test] +fn user_event_increments_event_clock_and_emits_locally() { + let mut e = ep(); + e.user_event("deploy", bytes::Bytes::from_static(b"v2"), false) + .unwrap(); + // Clock is incremented after stamping; local event at ltime=0 → clock now 1. + assert_eq!(e.event_time(), 1); + let ev = e.poll_event().expect("local user event must be pending"); + match ev { + Event::User(u) => { + assert_eq!(u.name.as_str(), "deploy"); + assert_eq!(u.payload.as_ref(), b"v2"); + } + other => panic!("expected Event::User, got {other:?}"), + } +} + +#[test] +fn duplicate_user_event_is_deduped() { + let mut e = ep(); + let m = crate::typed::UserEventMessage { + ltime: 4.into(), + cc: false, + name: "x".into(), + payload: bytes::Bytes::from_static(b"p"), + }; + // First sight → rebroadcast=true, event emitted. + assert!( + e.test_handle_user_event(m.clone()), + "first sight should return true" + ); + // Drain the event. + let _ = e.poll_event(); + // Duplicate → dropped (rebroadcast=false, no second event). + assert!( + !e.test_handle_user_event(m), + "duplicate should return false" + ); + assert!(e.poll_event().is_none(), "no second event for duplicate"); +} + +#[test] +fn user_event_below_min_time_is_dropped() { + let mut e = ep(); + e.test_set_event_min_time(10); + let m = crate::typed::UserEventMessage { + ltime: 3.into(), + cc: false, + name: "old".into(), + payload: bytes::Bytes::new(), + }; + assert!( + !e.test_handle_user_event(m), + "event below min_time must return false" + ); + assert!( + e.poll_event().is_none(), + "no event emitted for below-min-time message" + ); +} + +#[test] +fn oversized_user_event_is_rejected() { + let mut e = ep(); // max_user_event_size = 512 + // 1024 bytes payload, well over the 512-byte limit. + let big = bytes::Bytes::from(vec![0u8; 1024]); + assert!( + e.user_event("big", big, false).is_err(), + "oversized user event must return Err" + ); +} + +#[test] +fn user_event_broadcast_is_queued_at_event_tier() { + let mut e = ep(); + e.user_event("ship", bytes::Bytes::from_static(b"ok"), false) + .unwrap(); + // After queuing, the user-broadcast queue must be non-empty (event tier = rank 2). + assert!( + e.user_broadcast_queue_len() > 0, + "user broadcast queue must be non-empty after user_event" + ); +} + +#[test] +fn user_event_too_old_relative_to_ring_is_dropped() { + let mut e = ep(); + // Fire enough events to advance the clock well past the ring size (512). + // Then try to inject a very old event (ltime=0 while clock is at 600+). + // Shortcut: use the min_time setter to simulate the effect. + // event_buffer_size = 512; clock = 600; ltime=0: cur_time(600) > bltime(512) + // and ltime(0) < cur_time - bltime = 88 → too old. + e.test_set_event_clock(600); + let m = crate::typed::UserEventMessage { + ltime: 0.into(), + cc: false, + name: "stale".into(), + payload: bytes::Bytes::new(), + }; + assert!( + !e.test_handle_user_event(m), + "too-old event must be dropped" + ); + assert!(e.poll_event().is_none()); +} + +// ── Task 2.2: UserPacket decode + dispatch + relay-retain ───────────────────── + +#[test] +fn user_event_arrives_over_user_packet_and_surfaces() { + // A gossiped UserEvent encoded as an AnyMessage is injected via the + // UserPacket sieve arm. The machine must decode it, dedup, emit + // Event::User, and re-queue the ORIGINAL bytes on the event broadcast + // tier (relay-retain: no re-encode). + let mut e = ep(); + let serf_bytes = AnyMessage::::UserEvent(UserEventMessage { + ltime: 1.into(), + cc: false, + name: "deploy".into(), + payload: bytes::Bytes::from_static(b"v3"), + }) + .encode() + .unwrap(); + let from: std::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); + e.test_inject_user_packet(from, serf_bytes, memberlist_proto::Instant::ORIGIN); + + let ev = e.poll_event().expect("user event must surface"); + assert!( + matches!(ev, Event::User(ref u) if u.name == "deploy"), + "expected Event::User with name 'deploy', got: {ev:?}" + ); +} + +#[test] +fn duplicate_user_event_over_user_packet_is_deduped() { + // Same UserEvent injected twice → second one is silently dropped. + // The relay-retain property: first-sight re-queues original bytes; second + // sight is dropped without touching the broadcast queue. + let mut e = ep(); + let serf_bytes = AnyMessage::::UserEvent(UserEventMessage { + ltime: 3.into(), + cc: false, + name: "once".into(), + payload: bytes::Bytes::from_static(b"x"), + }) + .encode() + .unwrap(); + + let from: std::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); + e.test_inject_user_packet(from, serf_bytes.clone(), memberlist_proto::Instant::ORIGIN); + let first = e.poll_event(); + assert!( + matches!(first, Some(Event::User(_))), + "first packet must surface Event::User" + ); + + // Deliver the identical encoded bytes again. + e.test_inject_user_packet(from, serf_bytes, memberlist_proto::Instant::ORIGIN); + assert!( + e.poll_event().is_none(), + "duplicate must be silently dropped, not emitted again" + ); +} + +#[test] +fn join_intent_over_user_packet_buffers_and_requeues_on_intent_tier() { + // A Join intent for an unknown node arrives via UserPacket. It is buffered + // in the intent store and the original bytes are re-queued on the intent + // broadcast tier (rank 0 = highest priority). + let mut e = ep(); + let serf_bytes = + AnyMessage::::Join(JoinMessage::new(LamportTime::new(7), 2u32)) + .encode() + .unwrap(); + let before = e.user_broadcast_queue_len(); + let from: std::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); + e.test_inject_user_packet(from, serf_bytes, memberlist_proto::Instant::ORIGIN); + + assert!( + e.user_broadcast_queue_len() > before, + "intent broadcast queue must grow after a new join-intent over UserPacket" + ); +} + +#[test] +fn leave_intent_over_user_packet_dispatches_and_requeues() { + // A Leave intent for a known Alive member arrives via UserPacket. + // The member transitions to Leaving and the bytes are re-queued on the + // intent broadcast tier. + let mut e = ep(); + e.test_seed_member(2u32, MemberStatus::Alive, LamportTime::new(3)); + let serf_bytes = AnyMessage::::Leave(LeaveMessage::new( + LamportTime::new(8), + 2u32, + false, + )) + .encode() + .unwrap(); + let before = e.user_broadcast_queue_len(); + let from: std::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); + e.test_inject_user_packet(from, serf_bytes, memberlist_proto::Instant::ORIGIN); + + assert_eq!( + e.test_member_status(2), + Some(MemberStatus::Leaving), + "Alive member must transition to Leaving on a fresh leave intent" + ); + assert!( + e.user_broadcast_queue_len() > before, + "leave intent must be re-queued on the intent tier" + ); +} + +#[test] +fn stale_join_intent_over_user_packet_is_not_requeued() { + // A stale join intent (ltime <= current status_time) must be dropped without + // growing the broadcast queue (relay-retain: only first-sight is re-queued). + let mut e = ep(); + e.test_seed_member(2u32, MemberStatus::Alive, LamportTime::new(10)); + let serf_bytes = + AnyMessage::::Join(JoinMessage::new(LamportTime::new(3), 2u32)) + .encode() + .unwrap(); + let before = e.user_broadcast_queue_len(); + let from: std::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); + e.test_inject_user_packet(from, serf_bytes, memberlist_proto::Instant::ORIGIN); + + assert_eq!( + e.user_broadcast_queue_len(), + before, + "stale intent must not grow the broadcast queue" + ); +} + +#[test] +fn user_event_over_user_packet_witnesses_event_clock() { + // H4: The event clock must be witnessed when a UserEvent arrives via + // UserPacket. Both Reliable and Unreliable paths run through the same + // handler; this test verifies the clock-witness side-effect. + let mut e = ep(); + let serf_bytes = AnyMessage::::UserEvent(UserEventMessage { + ltime: 5.into(), + cc: false, + name: "ping".into(), + payload: bytes::Bytes::from_static(b"ok"), + }) + .encode() + .unwrap(); + let from: std::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); + e.test_inject_user_packet(from, serf_bytes, memberlist_proto::Instant::ORIGIN); + + let ev = e.poll_event().expect("event must surface"); + assert!( + matches!(ev, Event::User(ref u) if u.name == "ping"), + "expected Event::User(ping), got {ev:?}" + ); + // After witnessing ltime=5, clock must be at least 6. + assert!( + e.event_time() >= 6, + "event_time must be >= 6 after witnessing ltime=5" + ); +} + +#[test] +fn malformed_bytes_in_user_packet_are_silently_dropped() { + // A UserPacket carrying garbage bytes must not panic or emit events. + let mut e = ep(); + let from: std::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); + e.test_inject_user_packet( + from, + bytes::Bytes::from_static(b"\xff\xff\xfe"), + memberlist_proto::Instant::ORIGIN, + ); + assert!( + e.poll_event().is_none(), + "malformed bytes must not produce any serf event" + ); +} + +#[test] +fn user_event_rebroadcast_uses_original_bytes() { + // Relay-retain: the bytes put on the broadcast queue after a first-sight + // UserEvent must be the same encoding that arrived (refcount-shared, + // not re-encoded). We verify this by checking that the queue grows by + // exactly the original encoding. + let mut e = ep(); + let original_bytes = AnyMessage::::UserEvent(UserEventMessage { + ltime: 2.into(), + cc: false, + name: "ship".into(), + payload: bytes::Bytes::from_static(b"payload"), + }) + .encode() + .unwrap(); + + let queue_before = e.user_broadcast_queue_len(); + let from: std::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); + e.test_inject_user_packet( + from, + original_bytes.clone(), + memberlist_proto::Instant::ORIGIN, + ); + + // The event must have surfaced. + assert!(matches!(e.poll_event(), Some(Event::User(_)))); + // The broadcast queue must have grown (re-queue happened). + assert!( + e.user_broadcast_queue_len() > queue_before, + "relay-retain: user event must be re-queued after first sight" + ); +} + +// ── Task 3.1: local-state synthesis + H6 dirty-flag re-push ────────────────── + +#[test] +fn local_state_carries_three_clocks_and_members() { + // Seed an endpoint with known clock values and members, call resync_local_state, + // then decode the snapshot from the inner and verify the fields. + let mut e = ep(); + e.test_set_clocks(11, 22, 33); + e.test_seed_member(2, MemberStatus::Alive, LamportTime::new(9)); + e.test_seed_left_member(3, LamportTime::new(7)); + e.resync_local_state(); + + let snap = e.test_inner_local_state_snapshot(); + assert!(!snap.is_empty(), "snapshot must be non-empty after resync"); + + let pp = e.test_decode_pushpull(&snap); + assert_eq!(u64::from(pp.ltime), 11, "member clock must match"); + assert_eq!(u64::from(pp.event_ltime), 22, "event clock must match"); + assert_eq!(u64::from(pp.query_ltime), 33, "query clock must match"); + + // member 2 must appear in status_ltimes + assert!( + pp.status_ltimes.iter().any(|(id, _)| *id == 2u32), + "status_ltimes must contain member 2" + ); + // member 3 must appear in left_members + assert!( + pp.left_members.contains(&3u32), + "left_members must contain id 3" + ); +} + +#[test] +fn resync_clears_dirty_flag() { + let mut e = ep(); + e.test_set_clocks(1, 2, 3); + // After set_clocks, dirty = true. + assert!(e.test_is_dirty(), "clock mutation must mark dirty"); + e.resync_local_state(); + assert!( + !e.test_is_dirty(), + "resync_local_state must clear dirty flag" + ); +} + +#[test] +fn mutating_event_clock_via_user_event_marks_state_dirty() { + let mut e = ep(); + e.test_clear_dirty(); + // user_event increments the event clock → must mark dirty. + e.user_event("x", bytes::Bytes::new(), false).unwrap(); + assert!(e.test_is_dirty(), "user_event must mark local_state dirty"); +} + +#[test] +fn drain_inner_calls_resync_when_dirty() { + // After a mutation, the first poll_event call (which drives drain_inner) + // must sync the snapshot into the inner endpoint. + let mut e = ep(); + e.test_set_clocks(5, 0, 0); + assert!(e.test_is_dirty()); + // A poll_event call drives drain_inner which calls resync_local_state when dirty. + let _ = e.poll_event(); + assert!( + !e.test_is_dirty(), + "drain_inner must have called resync_local_state, clearing dirty" + ); + // The snapshot must be non-empty and carry the clock we set. + let snap = e.test_inner_local_state_snapshot(); + assert!( + !snap.is_empty(), + "snapshot must be present after drain_inner" + ); + let pp = e.test_decode_pushpull(&snap); + assert_eq!( + u64::from(pp.ltime), + 5, + "snapshot must carry the member clock" + ); +} + +#[test] +fn event_join_ignore_flag_can_be_set() { + // Accessor smoke-test: set_event_join_ignore and test_set_event_join_ignore + // both write the same field. + let mut e = ep(); + assert!(!e.test_is_dirty() || true); // just ensure no panic + e.set_event_join_ignore(true); + // test_set_event_join_ignore is also wired, confirm symmetry. + e.test_set_event_join_ignore(false); + // Confirm the flag can be toggled (no panic, no side-effects without a merge). +} + +#[test] +fn snapshot_left_members_only_includes_known_ids() { + // test_seed_left_member inserts into both states and left_members. + // resync must include only ids that also exist in states. + let mut e = ep(); + e.test_seed_left_member(3u32, LamportTime::new(2)); + e.test_set_clocks(1, 0, 0); + e.resync_local_state(); + let snap = e.test_inner_local_state_snapshot(); + assert!(!snap.is_empty()); + let pp = e.test_decode_pushpull(&snap); + // Member 3 should appear in left_members. + assert!( + pp.left_members.contains(&3u32), + "seeded left member must appear in snapshot left_members" + ); +} + +// ── Determinism: push-pull local-state bytes ────────────────────────────────── + +#[test] +fn push_pull_local_state_bytes_deterministic() { + // Two endpoints with the same members inserted in opposite id orders must + // produce byte-identical push-pull wire output after resync_local_state. + // This verifies that HashMap iteration order in `members.states` does NOT + // leak into the encoded PushPullMessage. + fn build_ep_asc() -> Endpoint { + let mut e = ep(); + e.test_set_clocks(5, 10, 15); + // Insert members in ascending id order: 1, 2, 3, 4, 5. + for id in [1u32, 2, 3, 4, 5] { + e.test_seed_member(id, MemberStatus::Alive, LamportTime::new(id as u64)); + } + e.resync_local_state(); + e + } + fn build_ep_desc() -> Endpoint { + let mut e = ep(); + e.test_set_clocks(5, 10, 15); + // Insert members in descending id order: 5, 4, 3, 2, 1. + for id in [5u32, 4, 3, 2, 1] { + e.test_seed_member(id, MemberStatus::Alive, LamportTime::new(id as u64)); + } + e.resync_local_state(); + e + } + + let snap_asc = build_ep_asc().test_inner_local_state_snapshot(); + let snap_desc = build_ep_desc().test_inner_local_state_snapshot(); + + assert!( + !snap_asc.is_empty(), + "ascending-order snapshot must be non-empty" + ); + assert_eq!( + snap_asc, snap_desc, + "push-pull bytes must be identical regardless of member insertion order" + ); +} + +// ── Task 4.1: query ring-buffer + query() + handle_query + Event::Query ─────── + +/// Helper: build a minimal `QueryMessage` for tests. +/// +/// Uses `ltime`, `id`, no filters, no flags, no relay, 5 s timeout, +/// and a sentinel from-address. +fn test_query(ltime: LamportTime, id: u32) -> QueryMessage { + QueryMessage { + ltime, + id, + from: memberlist_proto::Node::new(99u32, "127.0.0.1:9999".parse().unwrap()), + filters: vec![], + flags: QueryFlag::empty(), + relay_factor: 0, + timeout: std::time::Duration::from_secs(5), + name: "ping".into(), + payload: bytes::Bytes::new(), + } +} + +#[test] +fn query_reads_not_explicitly_increments_query_clock() { + // G8 / H8: `query()` stamps ltime = query_clock (READ), not query_clock += 1. + // Unlike user_event (which explicitly calls event_clock += 1 then stamps), + // query() does NOT contain an explicit increment. The clock may still advance + // via the witness inside handle_query (witnessing ltime=N at clock=N → N+1). + // The invariant tested here: QueryId.ltime equals the clock value AT ISSUE TIME, + // and the clock after the call is >= that value (witness, not explicit increment). + let mut e = ep(); + e.test_set_clocks(0, 0, 5); + let id = e + .query( + "ping", + bytes::Bytes::new(), + QueryParams::default(), + memberlist_proto::Instant::ORIGIN, + ) + .unwrap(); + // The ltime stamped on the QueryId is the clock value READ before any witness. + assert_eq!( + id.ltime.0, 5, + "QueryId.ltime must equal the clock at issue time" + ); + // The clock may advance to 6 via witness(ltime=5, clock=5) in handle_query + // (oracle-correct; contrast user_event which explicitly does clock += 1 BEFORE stamp). + assert!( + e.query_time() >= 5, + "query clock must not regress below the issued ltime" + ); +} + +#[test] +fn query_registers_pending_entry() { + // After `query()`, one pending query must be registered. + let mut e = ep(); + let id = e + .query( + "test", + bytes::Bytes::new(), + QueryParams::default(), + memberlist_proto::Instant::ORIGIN, + ) + .unwrap(); + assert_eq!(e.test_pending_query_count(), 1); + assert_eq!(e.test_last_query_id(), Some(id)); +} + +#[test] +fn query_processes_locally_and_emits_event_query() { + // `query()` calls `handle_query(self)` before broadcasting, + // so the local node sees Event::Query if no filters reject it. + let mut e = ep(); + e.query( + "local-ping", + bytes::Bytes::new(), + QueryParams::default(), + memberlist_proto::Instant::ORIGIN, + ) + .unwrap(); + // The local process step emits an Event::Query. + let ev = e.poll_event().expect("must emit a local Event::Query"); + assert!( + matches!(ev, Event::Query(_)), + "expected Event::Query, got {:?}", + std::mem::discriminant(&ev) + ); +} + +#[test] +fn handle_query_witnesses_query_clock() { + let mut e = ep(); + assert_eq!(e.query_time(), 0); + let q = test_query(LamportTime::new(7), 1); + e.test_handle_query(q); + // witness(7) → clock becomes 8 + assert_eq!( + e.query_time(), + 8, + "query clock must be witnessed on ingress" + ); +} + +#[test] +fn duplicate_query_id_is_deduped_but_distinct_ids_at_same_ltime_pass() { + // G6 / dedup: (ltime, id) is the composite key. + // Same ltime + same id → dropped. Same ltime + different id → first sight. + let mut e = ep(); + let a = test_query(LamportTime::new(3), 100); + let b = QueryMessage { + id: 200, + ..a.clone() + }; // same ltime, different random id + + assert!( + e.test_handle_query(a.clone()), + "first sight of (3, 100) must return true" + ); + assert!( + !e.test_handle_query(a), + "exact dup (3, 100) must return false" + ); + assert!( + e.test_handle_query(b), + "distinct id (3, 200) at same ltime must return true" + ); +} + +#[test] +fn filter_rejected_query_still_rebroadcasts_but_does_not_surface() { + // G6: a filter-rejected query must STILL return rebroadcast=true. + // The local node (id=1) is NOT in the id filter [999]. + let mut e = ep(); + let q = QueryMessage { + filters: vec![Filter::Id(vec![999u32])], + ..test_query(LamportTime::new(1), 42) + }; + assert!( + e.test_handle_query(q), + "filter-rejected query must still rebroadcast (G6)" + ); + // But no Event::Query must be emitted locally. + assert!( + e.poll_event().is_none(), + "filter-rejected query must not surface as Event::Query" + ); +} + +#[test] +fn query_too_old_is_dropped() { + // A query whose ltime is older than the entire ring is dropped. + let mut e = ep(); + // Set the query clock high enough that ltime=1 is "too old" relative to the ring. + // query_buffer_size = 512 (default). cur_time > 512 && ltime < cur_time - 512. + e.test_set_clocks(0, 0, 600); // query_clock = 600 + + // Deliver a query at ltime=1 (much older than cur_time - 512 = 88). + let q = test_query(LamportTime::new(1), 77); + // handle_query witnesses the clock first; after witness(1) clock stays 600 (>1). + // Then dedup check: cur_time=600, bltime=512, 1 < 600 - 512 = 88 → too old. + assert!( + !e.test_handle_query(q), + "query older than the ring must be dropped (not rebroadcast)" + ); + assert!( + e.poll_event().is_none(), + "too-old query must not emit an event" + ); +} + +#[test] +fn no_broadcast_flag_suppresses_rebroadcast() { + // When the NO_BROADCAST flag is set, rebroadcast must be false even on first sight. + let mut e = ep(); + let q = QueryMessage { + flags: QueryFlag::NO_BROADCAST, + ..test_query(LamportTime::new(2), 11) + }; + assert!( + !e.test_handle_query(q), + "NO_BROADCAST flag must suppress rebroadcast" + ); + // But the query is still processed locally (no filter), so Event::Query is emitted. + assert!( + matches!(e.poll_event(), Some(Event::Query(_))), + "NO_BROADCAST query must still surface locally" + ); +} + +#[test] +fn query_id_has_correct_ltime_and_nonzero_id() { + // The QueryId returned from query() must carry the clock value at issue and a non- + // deterministic (but seed-reproducible) random id. The random id must be u32 (any). + let mut e = ep(); + e.test_set_clocks(0, 0, 9); + let qid = e + .query( + "check", + bytes::Bytes::new(), + QueryParams::default(), + memberlist_proto::Instant::ORIGIN, + ) + .unwrap(); + assert_eq!(qid.ltime.0, 9, "ltime must equal query_clock at issue"); + // id is random but deterministically seeded; just assert it is a u32 (no panic). + let _ = qid.id; +} + +#[test] +fn query_size_limit_is_enforced() { + // A query whose encoded size exceeds query_size_limit must return Err. + let mut e = ep(); + // Default query_size_limit = 1024. A 1500-byte payload will exceed it. + let big = bytes::Bytes::from(vec![0u8; 1500]); + let result = e.query( + "big", + big, + QueryParams::default(), + memberlist_proto::Instant::ORIGIN, + ); + assert!(result.is_err(), "oversized query must be rejected"); +} + +#[test] +fn query_buffer_min_time_is_zero_at_start() { + let e = ep(); + assert_eq!( + e.test_query_min_time(), + 0, + "initial query min_time must be zero" + ); +} + +#[test] +fn query_emits_on_query_tier_broadcast_queue() { + // After query(), the encoded query must be in the user broadcast queue (rank 1). + let mut e = ep(); + let _id = e + .query( + "broadcast-test", + bytes::Bytes::new(), + QueryParams::default(), + memberlist_proto::Instant::ORIGIN, + ) + .unwrap(); + assert!( + e.user_broadcast_queue_len() >= 1, + "query must enqueue on the broadcast queue" + ); +} + +// Regression: invalid tag-regex must not advance the RNG. +// +// Two endpoints seeded identically must produce the same QueryId for the same +// valid query, even when endpoint A first receives a rejected invalid-regex +// query. The zero-side-effect contract requires that a failed query() leaves +// the RNG stream unmoved. +#[cfg(feature = "tag-regex")] +#[test] +fn invalid_tag_regex_does_not_advance_rng() { + // Helper that builds a serf Endpoint with a specified u64 seed so both + // endpoints start with exactly the same RNG state. + let make_ep = |seed: u64| { + let inner_opts = EndpointOptions::new( + 1u32, + "127.0.0.1:7946".parse::().unwrap(), + ) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); + let inner = memberlist_proto::Endpoint::new_at( + inner_opts, + memberlist_proto::Instant::ORIGIN, + SmallRng::seed_from_u64(0), + ); + Endpoint::new_with_rng(inner, Options::new(), SmallRng::seed_from_u64(seed)) + }; + + let mut ep_a = make_ep(42); + let mut ep_b = make_ep(42); + + // Invalid regex: unbalanced bracket. + let bad_filter = QueryParams { + filters: vec![Filter::Tag(crate::typed::TagFilter { + tag: "role".into(), + expr: Some("[invalid-regex".into()), + })], + ..Default::default() + }; + + // Endpoint A: rejected call — must return Err and leave RNG untouched. + let bad_result = ep_a.query( + "ping", + bytes::Bytes::new(), + bad_filter, + memberlist_proto::Instant::ORIGIN, + ); + assert!( + matches!(bad_result, Err(Error::InvalidQueryFilter)), + "invalid tag regex must return Err(InvalidQueryFilter)" + ); + + // Valid filter (no tag filter at all). + let good_params = QueryParams::default(); + + // Endpoint A: valid call after the failed one. + let qid_a = ep_a + .query( + "ping", + bytes::Bytes::new(), + good_params.clone(), + memberlist_proto::Instant::ORIGIN, + ) + .unwrap(); + + // Endpoint B: only the valid call — no prior failed call. + let qid_b = ep_b + .query( + "ping", + bytes::Bytes::new(), + good_params, + memberlist_proto::Instant::ORIGIN, + ) + .unwrap(); + + // The random ids must be equal; if the failed query advanced the RNG on A + // but not on B, they would diverge. + assert_eq!( + qid_a.id, qid_b.id, + "rejected invalid-regex query must not advance the RNG (ids diverged: A={}, B={})", + qid_a.id, qid_b.id + ); + // Lamport ltimes must also match (both endpoints share the same clock state). + assert_eq!( + qid_a.ltime, qid_b.ltime, + "ltime must match between the two identically-seeded endpoints" + ); +} + +// ── Task 4.2: respond() three guards + query-response fold ─────────────────── + +fn addr(port: u16) -> std::net::SocketAddr { + format!("127.0.0.1:{port}").parse().unwrap() +} + +fn t_secs(s: u64) -> memberlist_proto::Instant { + memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(s) +} + +fn qresp( + ltime: LamportTime, + id: u32, + from_port: u16, +) -> crate::typed::QueryResponseMessage { + crate::typed::QueryResponseMessage { + ltime, + id, + from: memberlist_proto::Node::new(from_port as u32, addr(from_port)), + flags: QueryFlag::empty(), + payload: bytes::Bytes::new(), + } +} + +#[test] +fn respond_once_succeeds() { + // A valid respond() within the deadline must succeed. + let mut e = ep(); + let deadline = t_secs(10); + let token = e.test_register_received_query( + QueryId { + ltime: LamportTime::new(1), + id: 5, + }, + addr(1002), + deadline, + ); + assert!( + e.respond( + &token, + bytes::Bytes::from_static(b"ok"), + memberlist_proto::Instant::ORIGIN + ) + .is_ok(), + "first respond() must succeed" + ); +} + +#[test] +fn respond_twice_is_rejected() { + // G7 guard 2: a second respond() on the same token must return AlreadyResponded. + let mut e = ep(); + let deadline = t_secs(10); + let token = e.test_register_received_query( + QueryId { + ltime: LamportTime::new(1), + id: 5, + }, + addr(1002), + deadline, + ); + assert!( + e.respond( + &token, + bytes::Bytes::from_static(b"ok"), + memberlist_proto::Instant::ORIGIN + ) + .is_ok() + ); + let err = e + .respond( + &token, + bytes::Bytes::from_static(b"ok2"), + memberlist_proto::Instant::ORIGIN, + ) + .unwrap_err(); + assert!( + matches!(err, Error::AlreadyResponded), + "second respond() must return AlreadyResponded" + ); +} + +#[test] +fn respond_after_deadline_is_rejected() { + // G7 guard 3: respond() with now > deadline must return RespondAfterDeadline. + let mut e = ep(); + let deadline = t_secs(1); + let token = e.test_register_received_query( + QueryId { + ltime: LamportTime::new(1), + id: 5, + }, + addr(1002), + deadline, + ); + // now = t_secs(5) > deadline = t_secs(1) + let err = e + .respond(&token, bytes::Bytes::from_static(b"late"), t_secs(5)) + .unwrap_err(); + assert!( + matches!(err, Error::RespondAfterDeadline), + "respond() past deadline must return RespondAfterDeadline" + ); +} + +#[test] +fn respond_with_oversized_payload_is_rejected() { + // G7 guard 1: payload.len() > query_response_size_limit (default 1024) must error. + let mut e = ep(); + let deadline = t_secs(10); + let token = e.test_register_received_query( + QueryId { + ltime: LamportTime::new(1), + id: 5, + }, + addr(1002), + deadline, + ); + let big = bytes::Bytes::from(vec![0u8; 2048]); + let err = e + .respond(&token, big, memberlist_proto::Instant::ORIGIN) + .unwrap_err(); + assert!( + matches!(err, Error::RespondTooLarge(_, _)), + "oversized respond() must return RespondTooLarge" + ); +} + +#[test] +fn respond_succeeds_and_prevents_second_call() { + // After a successful respond(), the entry is removed from received_queries. + // A second respond() must return AlreadyResponded (via the .ok_or guard). + let mut e = ep(); + let qid = QueryId { + ltime: LamportTime::new(2), + id: 9, + }; + let deadline = t_secs(10); + let token = e.test_register_received_query(qid, addr(1002), deadline); + assert!(!e.test_is_responded(qid), "initially not responded"); + e.respond( + &token, + bytes::Bytes::new(), + memberlist_proto::Instant::ORIGIN, + ) + .unwrap(); + // Entry removed on success: second call returns AlreadyResponded. + let err = e + .respond( + &token, + bytes::Bytes::new(), + memberlist_proto::Instant::ORIGIN, + ) + .unwrap_err(); + assert!( + matches!(err, Error::AlreadyResponded), + "second respond() after success must return AlreadyResponded, got {err:?}" + ); +} + +#[test] +fn app_query_response_surfaces_as_event() { + // A QueryResponseMessage arriving for an App-kind PendingQuery must emit + // Event::QueryResponse with the matching id. + let mut e = ep(); + let id = e + .query( + "ping", + bytes::Bytes::new(), + QueryParams::default(), + memberlist_proto::Instant::ORIGIN, + ) + .unwrap(); + // Drain the locally-emitted Event::Query. + let _ = e.poll_event(); + + // A responder (node 2) replies. + let resp = qresp(id.ltime, id.id, 2); + e.test_handle_query_response(resp); + + // The response must surface as Event::QueryResponse. + let ev = e.poll_event().expect("Event::QueryResponse expected"); + assert!( + matches!(&ev, Event::QueryResponse(qr) if qr.id() == id.id), + "response must carry the query id" + ); +} + +#[test] +fn duplicate_query_response_is_deduped() { + // A second response from the same node for the same query must be dropped. + let mut e = ep(); + let id = e + .query( + "ping", + bytes::Bytes::new(), + QueryParams::default(), + memberlist_proto::Instant::ORIGIN, + ) + .unwrap(); + // Drain locally-emitted Event::Query. + let _ = e.poll_event(); + + // First response from node 2. + let resp = qresp(id.ltime, id.id, 2); + e.test_handle_query_response(resp.clone()); + let ev1 = e.poll_event(); + assert!( + matches!(ev1, Some(Event::QueryResponse(_))), + "first response must surface" + ); + + // Duplicate from the same node. + e.test_handle_query_response(resp); + assert!( + e.poll_event().is_none(), + "duplicate responder must be deduped and not surface" + ); +} + +#[test] +fn multiple_responders_each_surface_independently() { + // Distinct responders for the same query must each produce an Event::QueryResponse. + let mut e = ep(); + let id = e + .query( + "ping", + bytes::Bytes::new(), + QueryParams::default(), + memberlist_proto::Instant::ORIGIN, + ) + .unwrap(); + // Drain locally-emitted Event::Query. + let _ = e.poll_event(); + + // Two distinct responders. + e.test_handle_query_response(qresp(id.ltime, id.id, 2)); + e.test_handle_query_response(qresp(id.ltime, id.id, 3)); + + let ev1 = e.poll_event().expect("response from node 2"); + let ev2 = e.poll_event().expect("response from node 3"); + assert!(matches!(ev1, Event::QueryResponse(_))); + assert!(matches!(ev2, Event::QueryResponse(_))); + assert!(e.poll_event().is_none(), "no further events"); +} + +#[test] +fn stale_query_response_is_dropped() { + // A response for an unknown/expired query id must be silently discarded. + let mut e = ep(); + // Deliver a response for a query that was never registered. + let resp = qresp(LamportTime::new(99), 0xdead, 2); + e.test_handle_query_response(resp); + assert!( + e.poll_event().is_none(), + "response for unknown query must not emit an event" + ); +} + +#[test] +fn query_response_via_user_packet_wire_path_surfaces_event() { + // A QueryResponseMessage arriving via the gossip UserPacket path (injected as + // serf-level bytes via test_inject_user_packet) must fold into the matching + // PendingQuery and emit Event::QueryResponse. + let mut e = ep(); + let id = e + .query( + "wire-test", + bytes::Bytes::new(), + QueryParams::default(), + memberlist_proto::Instant::ORIGIN, + ) + .unwrap(); + // Drain locally-emitted Event::Query. + let _ = e.poll_event(); + + // Encode a QueryResponseMessage as serf-level bytes and inject via the + // UserPacket path (mimicking the gossip plane delivery). + let resp = crate::typed::QueryResponseMessage:: { + ltime: id.ltime, + id: id.id, + from: memberlist_proto::Node::new(2u32, addr(1002)), + flags: QueryFlag::empty(), + payload: bytes::Bytes::new(), + }; + let serf_bytes = AnyMessage::::QueryResponse(resp) + .encode() + .unwrap(); + e.test_inject_user_packet(addr(1002), serf_bytes, memberlist_proto::Instant::ORIGIN); + + let ev = e + .poll_event() + .expect("Event::QueryResponse from UserPacket path"); + assert!( + matches!(ev, Event::QueryResponse(_)), + "UserPacket-path response must surface as Event::QueryResponse" + ); +} + +// ── Task 4.3: responder-side relay ─────────────────────────────────────────── + +/// Build a `Node` at `127.0.0.1:` with `id = port as u32`. +fn relay_node(port: u16) -> memberlist_proto::Node { + memberlist_proto::Node::new(port as u32, addr(port)) +} + +#[test] +fn relay_response_is_silent_noop_when_too_few_members() { + // With 0 non-self members, relay_factor=2 requires at least 3 total members + // (relay_factor + 1 = 3) but we have 0 → silent no-op: no RelayDropped event, + // no directed send. + let mut e = ep(); + let querier = relay_node(2000); + e.test_relay_response(querier, bytes::Bytes::from_static(b"frame"), 2); + assert!( + e.poll_event().is_none(), + "relay with too few members must not emit RelayDropped" + ); + assert!( + e.test_last_directed_send().is_none(), + "no directed send should have occurred" + ); +} + +#[test] +fn relay_response_with_zero_factor_is_noop() { + // relay_factor == 0 must be a fast-path no-op: no sends, no events. + let mut e = ep(); + let querier = relay_node(2000); + e.test_relay_response(querier, bytes::Bytes::from_static(b"frame"), 0); + assert!(e.poll_event().is_none()); + assert!(e.test_last_directed_send().is_none()); +} + +#[test] +fn relay_response_picks_alive_non_self_member_and_sends() { + // Seed 2 Alive members (ids 10, 11). With relay_factor=1 the count guard + // requires at least 2 members. One directed send must occur. + let mut e = ep(); + // Seed Alive members with ports 1010 and 1011. + e.test_seed_member(10u32, MemberStatus::Alive, LamportTime::new(1)); + // test_seed_member uses port 0; override by using test_seed_failed_member + // approach for an explicit address: + { + let node = memberlist_proto::Node::new(11u32, addr(1011)); + let member = crate::members::Member::new(node, crate::typed::Tags::new(), MemberStatus::Alive); + e.members.states.insert( + 11u32, + crate::members::MemberState::new(member, LamportTime::new(1), None), + ); + } + + let querier = relay_node(2000); + let frame = bytes::Bytes::from_static(b"\x06relay-payload"); + e.test_relay_response(querier, frame.clone(), 1); + + // A directed send must have happened to one of the Alive peers. + let (dest_addr, sent_bytes) = e + .test_last_directed_send() + .expect("relay_response must produce a directed send when members >= k+1"); + // The sent bytes are the relay-wrapped frame, not the raw frame. + // Verify that the destination is one of the seeded alive peers (not self port 7946). + assert_ne!( + dest_addr.port(), + 7946, + "relay must not target the local node" + ); + // The relay wrapper is non-empty (the inner frame is embedded in it). + assert!(!sent_bytes.is_empty(), "relay frame must be non-empty"); + assert!(e.poll_event().is_none(), "no RelayDropped on success"); +} + +#[test] +fn relay_node_b_forwards_verbatim_to_destination() { + // Node B (this node) receives a RelayMessage and must forward the inner + // payload verbatim to the destination via send_user_packet. + // The destination is node 2 (not self = 1). + let mut e = ep(); + let inner_payload = bytes::Bytes::from_static(b"\x06inner-qresp"); + let relay = RelayMessage::new(relay_node(1002), inner_payload.clone()); + e.test_handle_relay(relay); + + // The directed send must carry the verbatim inner payload to node 2's address. + let (dest_addr, sent_bytes) = e + .test_last_directed_send() + .expect("handle_relay must produce a directed send to the destination"); + assert_eq!( + dest_addr, + addr(1002), + "relay must forward to the wrapped destination address" + ); + assert_eq!( + sent_bytes, inner_payload, + "relay payload must be forwarded verbatim, not re-encoded" + ); + assert!( + e.poll_event().is_none(), + "successful handle_relay must not emit any event" + ); +} + +#[test] +fn relay_to_self_emits_relay_dropped() { + // A Relay with destination == local_id (1) must emit Event::RelayDropped + // and must NOT attempt a directed send (self-relay is a no-op failure). + let mut e = ep(); // local id = 1 + let relay = RelayMessage::new( + relay_node(1), // id=1 matches the local endpoint + bytes::Bytes::from_static(b"x"), + ); + e.test_handle_relay(relay); + // The test_last_directed_send must be None (no real send attempted). + // Note: port 7946 is the local bind port, not the relay_node(1) port; + // the id-equality guard fires before any send. The event must be present. + let ev = e.poll_event().expect("relay to self must emit an event"); + assert!( + matches!(ev, Event::RelayDropped(_)), + "relay to self must surface as Event::RelayDropped" + ); +} + +#[test] +fn relay_sieve_arm_decodes_relay_message_from_user_packet() { + // A RelayMessage arriving via the gossip UserPacket path must be decoded + // and handled (the Relay sieve arm must call handle_relay). + // We inject a Relay wrapping a tiny inner payload targeting a non-self node. + let mut e = ep(); + let inner_payload = bytes::Bytes::from_static(b"\x06fake-resp"); + let relay = RelayMessage::new(relay_node(1002), inner_payload.clone()); + let serf_bytes = AnyMessage::::Relay(relay) + .encode() + .unwrap(); + + e.test_inject_user_packet(addr(1002), serf_bytes, memberlist_proto::Instant::ORIGIN); + + // The inner payload must have been directed-sent to the destination. + let (dest, sent) = e + .test_last_directed_send() + .expect("relay sieve arm must forward to destination"); + assert_eq!(dest, addr(1002)); + assert_eq!(sent, inner_payload); +} + +/// Build a serf `Endpoint` with an explicit RNG seed for the +/// serf-level RNG (the relay/reconnect draws). The inner Endpoint uses a fixed +/// seed 0; the serf-level seed is the caller-supplied `serf_seed`. +fn ep_with_serf_seed(serf_seed: u64) -> Endpoint { + let inner_opts = memberlist_proto::EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); + let inner = memberlist_proto::Endpoint::new_at( + inner_opts, + memberlist_proto::Instant::ORIGIN, + SmallRng::seed_from_u64(0), + ); + Endpoint::new_with_rng(inner, Options::new(), SmallRng::seed_from_u64(serf_seed)) +} + +#[test] +fn relay_response_candidate_selection_is_deterministic() { + // Regression test for HashMap-iteration-order nondeterminism in relay candidate + // selection. + // + // Two endpoints with identical RNG seeds and identical Alive membership MUST + // select the same relay peer set regardless of HashMap iteration order. + // Pre-fix: the candidates Vec was collected directly from HashMap::iter() — + // which Rust randomises per map instance — so a Fisher-Yates shuffle on a + // fixed RNG seed produced different selections when the input ordering differed. + // Post-fix: candidates are sorted by the encoded node id before the shuffle so + // the shuffle input is always the same total order, making the selection a pure + // function of the RNG state. + // + // The two endpoints intentionally insert the same members in REVERSE id order + // to maximise the chance that HashMap iteration diverges (hash-map bucket + // assignment is seed-randomised, but inserting in different orders can shift + // collision chains and iteration position). With relay_factor=2 and five + // candidates the probability that both happen to produce the same two-element + // selection by chance is at most (2/5)^2 = 4% — negligible for a + // determinism gate, and zero after the sort fix. + + // Five Alive members with distinct ids and distinct addresses. + // Local id = 1 (from ep_with_serf_seed), so ids 10..14 are non-self. + let members: Vec<(u32, u16)> = vec![(10, 1010), (11, 1011), (12, 1012), (13, 1013), (14, 1014)]; + + let seed = 0xdeadbeef_cafebabe_u64; + + // Endpoint A: members inserted in ascending id order. + let mut ep_a = ep_with_serf_seed(seed); + for &(id, port) in &members { + ep_a.test_seed_member_at(id, addr(port), MemberStatus::Alive, LamportTime::new(1)); + } + + // Endpoint B: members inserted in DESCENDING id order (maximum input-order + // difference from A, exercising the HashMap-order divergence). + let mut ep_b = ep_with_serf_seed(seed); + for &(id, port) in members.iter().rev() { + ep_b.test_seed_member_at(id, addr(port), MemberStatus::Alive, LamportTime::new(1)); + } + + let querier = relay_node(2000); + let frame = bytes::Bytes::from_static(b"\x06determinism-test-frame"); + + // relay_factor=2 — select two relay peers from the five candidates. + // The count guard requires relay_factor + 1 = 3 total members; we have 5. + ep_a.test_relay_response(querier, frame.clone(), 2); + ep_b.test_relay_response(relay_node(2000), frame, 2); + + let sends_a = ep_a.test_relay_all_directed_sends().to_vec(); + let sends_b = ep_b.test_relay_all_directed_sends().to_vec(); + + // Both must have produced exactly relay_factor=2 directed sends. + assert_eq!(sends_a.len(), 2, "endpoint A must relay to exactly 2 peers"); + assert_eq!(sends_b.len(), 2, "endpoint B must relay to exactly 2 peers"); + + // The selected peer addresses must be identical in both order and identity. + // Any divergence here indicates the candidate ordering was not stabilised + // before the Fisher-Yates shuffle. + let addrs_a: Vec = sends_a.iter().map(|(a, _)| *a).collect(); + let addrs_b: Vec = sends_b.iter().map(|(a, _)| *a).collect(); + assert_eq!( + addrs_a, addrs_b, + "relay peer selection must be identical across endpoints with the same RNG seed \ + regardless of HashMap insertion order; got A={addrs_a:?}, B={addrs_b:?}" + ); +} + +// ── Task 4.4: conflict-resolution and key-management queries ───────────────── + +fn far_future() -> memberlist_proto::Instant { + memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(3600) +} + +#[test] +fn conflict_win_does_not_shut_down() { + // Majority of responses agree → local node won → no Event::Shutdown. + let mut e = ep(); + let deadline = far_future(); + let qid = e.test_register_conflict_query(deadline); + + // 3 responses: 2 agree (matching), 1 disagrees. + e.test_fold_conflict_response(qid, 100u32, true); + e.test_fold_conflict_response(qid, 101u32, true); + e.test_fold_conflict_response(qid, 102u32, false); + + let past = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(3601); + e.test_fire_due_query_closes(past); + + assert!( + e.poll_event().is_none(), + "winning conflict must not emit Shutdown" + ); +} + +#[test] +fn conflict_loss_emits_shutdown() { + // Minority of responses agree → local node lost → Event::Shutdown. + let mut e = ep(); + let deadline = far_future(); + let qid = e.test_register_conflict_query(deadline); + + // 3 responses: 1 agrees, 2 disagree → majority = 2, matching = 1 < 2 → lost. + e.test_fold_conflict_response(qid, 200u32, true); + e.test_fold_conflict_response(qid, 201u32, false); + e.test_fold_conflict_response(qid, 202u32, false); + + let past = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(3601); + e.test_fire_due_query_closes(past); + + let ev = e.poll_event().expect("conflict loss must emit an event"); + assert!( + matches!(ev, Event::Shutdown), + "conflict loss must emit Event::Shutdown, got {:?}", + ev + ); +} + +#[test] +fn app_query_close_is_silent() { + // An App query closing (deadline elapsed) must not emit any serf event. + let mut e = ep(); + let now = memberlist_proto::Instant::ORIGIN; + let params = QueryParams { + timeout: std::time::Duration::from_millis(1), + ..Default::default() + }; + let _ = e.query("test", Bytes::new(), params, now); + // Drain any Event::Query emitted by handle_query (the local node sees its own query). + while e.poll_event().is_some() {} + + // Advance time past the deadline. + let past = now + std::time::Duration::from_secs(1); + e.test_fire_due_query_closes(past); + + // No serf event should be emitted (App queries close silently). + assert!( + e.poll_event().is_none(), + "App query close must not emit any event" + ); +} + +// ── Internal-query payload exact-consumption gate ──────────────────────────── + +#[test] +fn conflict_query_with_trailing_junk_is_dropped_entirely() { + // Regression: a `_serf_conflict` Query whose payload is a valid encoded id + // followed by trailing junk bytes must cause NO state mutation — no + // query_clock advance, no dedup entry, no received_queries entry, and no + // directed ConflictResponse send. + // + // Before the fix, `handle_conflict_query` decoded the id with `I::decode` + // which ignores the returned byte count, allowing the malformed payload to + // pass AFTER the clock witness, dedup insert, and received_queries insert had + // already mutated state. + let mut e = ep(); + + // Build a valid-prefix payload for id=42 followed by trailing junk. + // u32 is varint-encoded; 42u32 encodes to a single byte (0x2a). + let id_bytes = (42u32).encode_to_bytes().unwrap(); + let mut payload_with_junk = id_bytes.to_vec(); + payload_with_junk.extend_from_slice(&[0xaa, 0xbb]); // 2 junk bytes + + let q = QueryMessage { + ltime: LamportTime::new(1), + id: 55, + from: memberlist_proto::Node::new(99u32, "127.0.0.1:9999".parse().unwrap()), + filters: vec![], + flags: QueryFlag::empty(), + relay_factor: 0, + timeout: std::time::Duration::from_secs(5), + name: "_serf_conflict".into(), + payload: Bytes::from(payload_with_junk), + }; + + // Preconditions: all state counters at zero. + assert_eq!(e.query_time(), 0, "query_clock must start at 0"); + assert_eq!( + e.test_received_queries_len(), + 0, + "no received_queries initially" + ); + + let rebroadcast = e.test_handle_query(q); + + // A malformed internal query must cause NO state mutation. + assert_eq!( + e.query_time(), + 0, + "query_clock must NOT advance for a malformed _serf_conflict payload" + ); + assert_eq!( + e.test_query_slot_len(1), + 0, + "dedup buffer must NOT record an entry for a malformed _serf_conflict payload" + ); + assert_eq!( + e.test_received_queries_len(), + 0, + "received_queries must NOT gain an entry for a malformed _serf_conflict payload" + ); + assert!( + e.test_last_directed_send().is_none(), + "no ConflictResponse must be sent for a malformed _serf_conflict payload" + ); + // A malformed internal query also must NOT trigger a rebroadcast. + assert!( + !rebroadcast, + "malformed _serf_conflict must not request rebroadcast" + ); +} + +#[test] +fn conflict_query_with_exact_payload_processes_normally() { + // Complement: a `_serf_conflict` Query whose payload is an exactly-encoded id + // (no trailing bytes) must still be processed: clock advances, dedup entry is + // recorded. The member is not in the local store, so no ConflictResponse is + // sent, but the clock and dedup state must have been updated. + let mut e = ep(); + + // Encode id=42 exactly (no junk). + let id_bytes = (42u32).encode_to_bytes().unwrap(); + + let q = QueryMessage { + ltime: LamportTime::new(3), + id: 77, + from: memberlist_proto::Node::new(99u32, "127.0.0.1:9999".parse().unwrap()), + filters: vec![], + flags: QueryFlag::empty(), + relay_factor: 0, + timeout: std::time::Duration::from_secs(5), + name: "_serf_conflict".into(), + payload: Bytes::from(id_bytes.to_vec()), + }; + + assert_eq!(e.query_time(), 0); + e.test_handle_query(q); + + // Clock must have advanced past the witnessed ltime of 3. + assert!( + e.query_time() > 3, + "query_clock must advance after a well-formed _serf_conflict query" + ); + // Dedup slot for ltime=3 must be non-empty. + assert!( + e.test_query_slot_len(3) > 0, + "dedup buffer must record the well-formed _serf_conflict query" + ); +} + +// ── Task 6.2: replay(records) -> ReplayResult + Endpoint::load_snapshot ────── + +use crate::snapshot::ReplayResult; + +fn snapshot_node(id: u32, port: u16) -> memberlist_proto::Node { + memberlist_proto::Node::new(id, format!("127.0.0.1:{port}").parse().unwrap()) +} + +#[test] +fn load_snapshot_sets_member_clock_to_last_clock() { + let mut e = ep(); + let r = ReplayResult { + alive_nodes: vec![], + last_clock: 5.into(), + last_event_clock: 0.into(), + last_query_clock: 0.into(), + }; + e.load_snapshot(r, memberlist_proto::Instant::ORIGIN); + // G5: member clock >= last_clock. + assert!( + e.member_time() >= 5, + "member_time must be at least last_clock after load_snapshot" + ); +} + +#[test] +fn load_snapshot_sets_event_min_time_to_last_event_clock_plus_one() { + let mut e = ep(); + let r = ReplayResult { + alive_nodes: vec![], + last_clock: 0.into(), + last_event_clock: 7.into(), + last_query_clock: 0.into(), + }; + e.load_snapshot(r, memberlist_proto::Instant::ORIGIN); + // G5: event_buffer.min_time = last_event_clock + 1 = 8. + assert_eq!( + e.test_event_min_time(), + 8, + "event min_time must be last_event_clock + 1" + ); +} + +#[test] +fn load_snapshot_sets_query_min_time_to_last_query_clock_plus_one() { + let mut e = ep(); + let r = ReplayResult { + alive_nodes: vec![], + last_clock: 0.into(), + last_event_clock: 0.into(), + last_query_clock: 9.into(), + }; + e.load_snapshot(r, memberlist_proto::Instant::ORIGIN); + // G5: query_buffer.min_time = last_query_clock + 1 = 10. + assert_eq!( + e.test_query_min_time(), + 10, + "query min_time must be last_query_clock + 1" + ); +} + +#[test] +fn load_snapshot_skips_self_on_rejoin() { + // The local endpoint has id=1 (see ep()). A ReplayResult containing id=1 + // plus id=2 must only emit a dial for id=2 (self is skipped per G10). + let mut e = ep(); + let r = ReplayResult { + alive_nodes: vec![ + snapshot_node(1, 7946), // self + snapshot_node(2, 1002), + ], + last_clock: 5.into(), + last_event_clock: 7.into(), + last_query_clock: 9.into(), + }; + e.load_snapshot(r, memberlist_proto::Instant::ORIGIN); + // Dials recorded via test_last_dial_addr: the reconnect should have been issued + // for node 2 only (and none for self). + let dialled = e.test_rejoin_dials(); + assert_eq!( + dialled.len(), + 1, + "exactly one rejoin dial expected (self is skipped)" + ); + assert_eq!( + dialled[0], + "127.0.0.1:1002".parse::().unwrap(), + "rejoin dial must target node 2's address" + ); +} + +#[test] +fn load_snapshot_empty_alive_nodes_emits_no_dials() { + let mut e = ep(); + let r = ReplayResult { + alive_nodes: vec![], + last_clock: 3.into(), + last_event_clock: 2.into(), + last_query_clock: 1.into(), + }; + e.load_snapshot(r, memberlist_proto::Instant::ORIGIN); + let dialled = e.test_rejoin_dials(); + assert!(dialled.is_empty(), "no dials for empty alive_nodes"); +} + +#[test] +fn load_snapshot_all_clocks_combined() { + // G5: all three clocks + min-times in one test. + let mut e = ep(); + let r = ReplayResult { + alive_nodes: vec![], + last_clock: 10.into(), + last_event_clock: 20.into(), + last_query_clock: 30.into(), + }; + e.load_snapshot(r, memberlist_proto::Instant::ORIGIN); + assert!(e.member_time() >= 10, "member_time >= last_clock"); + assert_eq!(e.test_event_min_time(), 21, "event min_time = 20 + 1"); + assert_eq!(e.test_query_min_time(), 31, "query min_time = 30 + 1"); +} + +#[test] +fn load_snapshot_marks_local_state_dirty() { + // After load_snapshot, the push-pull snapshot must reflect the recovered clocks. + let mut e = ep(); + e.test_clear_dirty(); + let r = ReplayResult { + alive_nodes: vec![], + last_clock: 3.into(), + last_event_clock: 0.into(), + last_query_clock: 0.into(), + }; + e.load_snapshot(r, memberlist_proto::Instant::ORIGIN); + // Dirty flag must be set so the first push-pull egress carries the recovered clocks. + assert!( + e.test_is_dirty(), + "load_snapshot must mark local state dirty" + ); +} + +// ── Task 5.2: CoordinateClient + PingCompleted RTT update ──────────────────── + +/// Build an ack-payload bytes `[1u8] ++ pb::Coordinate` for the given `Coordinate` +/// (test utility mirroring the production `coord_ack_payload` helper). +#[cfg(feature = "coordinates")] +fn make_coord_payload(coord: &crate::typed::Coordinate) -> Bytes { + use crate::bridge::coordinate_to_pb; + use buffa::Message as _; + let pb = coordinate_to_pb(coord); + let encoded = pb.encode_to_vec(); + let mut buf = Vec::with_capacity(1 + encoded.len()); + buf.push(1u8); // PING_VERSION + buf.extend_from_slice(&encoded); + Bytes::from(buf) +} + +#[cfg(feature = "coordinates")] +#[test] +fn ping_completed_updates_local_coordinate_and_caches_remote() { + // G9 both halves: a PingCompleted with a valid coordinate payload must + // (1) update the local Vivaldi model and (2) cache the remote coordinate. + let mut e = ep_with_coords(); + + // Build a synthetic peer coordinate and its wire payload. + let peer_coord = crate::typed::Coordinate { + vec: vec![5.0; 8], + error: 1.0, + adjustment: 0.0, + height: 0.0, + }; + let payload = make_coord_payload(&peer_coord); + + e.test_ping_completed(2u32, std::time::Duration::from_millis(40), payload); + + // Half 1: remote coord is cached under node id 2. + assert!( + e.cached_coordinate(&2u32).is_some(), + "remote coordinate should be cached after PingCompleted" + ); + // Half 2: local coordinate was updated (not None). + assert!( + e.get_coordinate().is_some(), + "local coordinate should be present after PingCompleted" + ); +} + +#[test] +fn ping_completed_is_noop_when_coordinates_disabled() { + // When the `coordinates` feature is absent or coordinates are runtime-disabled, + // a PingCompleted with any payload must not panic or emit any event. + let mut e = ep(); // opts.disable_coordinates() == true (default) + // Payload starts with PING_VERSION byte to ensure it is not rejected by the + // version guard; the test still expects a no-op. + let payload = Bytes::from_static(b"\x01garbage"); + // Must not panic; test_ping_completed is a no-op without the feature or when disabled. + #[cfg(feature = "coordinates")] + e.test_ping_completed(2u32, std::time::Duration::from_millis(40), payload); + #[cfg(not(feature = "coordinates"))] + let _ = payload; // consume without calling the cfg-gated adapter + assert!( + e.poll_event().is_none(), + "ping_completed must not emit any event" + ); +} + +#[cfg(feature = "coordinates")] +#[test] +fn ping_completed_bad_version_is_noop() { + // A PingCompleted payload with a wrong version byte must be silently dropped. + let mut e = ep_with_coords(); + let payload = Bytes::from_static(b"\x02garbage"); // version 2, not 1 + e.test_ping_completed(2u32, std::time::Duration::from_millis(10), payload); + assert!( + e.cached_coordinate(&2u32).is_none(), + "bad version byte must not update coord_cache" + ); +} + +#[cfg(feature = "coordinates")] +#[test] +fn ping_completed_empty_payload_is_noop() { + // An empty PingCompleted payload must be silently dropped. + let mut e = ep_with_coords(); + e.test_ping_completed(2u32, std::time::Duration::from_millis(10), Bytes::new()); + assert!( + e.cached_coordinate(&2u32).is_none(), + "empty payload must not update coord_cache" + ); +} + +#[cfg(feature = "coordinates")] +#[test] +fn reap_forgets_coordinate() { + // G13: after a member is reaped, its coordinate must be purged from the cache. + let mut e = ep_with_coords(); + + // Seed a valid coordinate for node 42. + let peer_coord = crate::typed::Coordinate { + vec: vec![1.0; 8], + error: 0.5, + adjustment: 0.0, + height: 0.0, + }; + let payload = make_coord_payload(&peer_coord); + e.test_ping_completed(42u32, std::time::Duration::from_millis(20), payload); + assert!( + e.cached_coordinate(&42u32).is_some(), + "coordinate should be cached before reap" + ); + + // Seed node 42 as a Failed member with leave_time = ORIGIN. + use memberlist_proto::Instant; + e.test_seed_failed_member(42u32, "127.0.0.1:7947".parse().unwrap(), Instant::ORIGIN); + + // Fire the reaper at a time past reconnect_timeout (default = 24 h = 86 400 s). + let past = Instant::ORIGIN + std::time::Duration::from_secs(90_000); + e.test_fire_reap(past); + + // The coordinate cache entry for 42 must be gone. + assert!( + e.cached_coordinate(&42u32).is_none(), + "coordinate must be removed from cache after reap" + ); +} + +// ── Bug-fix regression tests ────────────────────────────────────────────────── + +// Bug 1: EventBuffer wraparound corruption. +// +// Use a tiny buffer (size=4) so ltime=1 and ltime=5 map to the same ring index +// without needing a high clock that would make ltime=1 "too old". +fn ep_tiny_event_buf() -> Endpoint { + let inner_opts = memberlist_proto::EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); + let inner = memberlist_proto::Endpoint::new_at( + inner_opts, + memberlist_proto::Instant::ORIGIN, + SmallRng::seed_from_u64(0), + ); + Endpoint::new(inner, Options::new().with_event_buffer_size(4)) +} + +#[test] +fn event_ring_wraparound_delivers_new_event() { + // Buffer size = 4. ltime=1 and ltime=5 both map to ring index 1 (1%4=1, 5%4=1). + // With the bug the ltime=5 event is spuriously deduped against the ltime=1 slot. + // Use a small ring so neither ltime is "too old" at the time of delivery. + let mut e = ep_tiny_event_buf(); + let m1 = crate::typed::UserEventMessage { + ltime: 1.into(), + cc: false, + name: "dup_test".into(), + payload: bytes::Bytes::new(), + }; + let m2 = crate::typed::UserEventMessage { + ltime: 5.into(), + cc: false, + name: "dup_test".into(), + payload: bytes::Bytes::new(), + }; + // First delivery at ltime=1 → first sight → true. + assert!( + e.test_handle_user_event(m1), + "ltime=1 delivery must return true (first sight)" + ); + // Drain the event so it doesn't pollute the next assertion. + // Ignoring Err: test drain, we don't care about the value. + let _ = e.poll_event(); + // Second delivery at ltime=5, same event name — a different ltime, must return true. + // With the bug the ltime=1 slot is reused and the event is spuriously deduped → false. + assert!( + e.test_handle_user_event(m2), + "ltime=5 delivery must return true (different ltime, not a duplicate)" + ); +} + +// Bug 2: ACK queries never produce ACKs. +#[test] +fn ack_query_produces_immediate_ack_directed_send() { + // A QueryMessage with the ACK flag set must trigger an immediate directed + // ACK response to the querier before emitting Event::Query. + let mut e = ep(); + let q = QueryMessage { + flags: QueryFlag::ACK, + ..test_query(LamportTime::new(3), 77) + }; + e.test_handle_query(q); + + // A directed send must have occurred with the ACK response. + let (dest_addr, sent_bytes) = e + .test_last_directed_send() + .expect("ACK query must produce a directed send"); + + // The destination must be the querier's address (127.0.0.1:9999 from test_query). + assert_eq!( + dest_addr, + "127.0.0.1:9999".parse::().unwrap(), + "ACK must be directed to the querier" + ); + + // Decode the sent bytes and verify it is a QueryResponse with ACK flag. + let decoded = AnyMessage::::decode(&sent_bytes) + .expect("ACK bytes must decode as AnyMessage"); + match decoded { + AnyMessage::QueryResponse(resp) => { + assert!(resp.ack(), "decoded response must have ACK flag set"); + assert!(resp.payload.is_empty(), "ACK payload must be empty"); + } + other => panic!( + "expected QueryResponse(ACK), got {:?}", + other.message_type() + ), + } +} + +// Bug 3: Oversized query responses silently consumed. +fn ep_small_resp_limit() -> Endpoint { + let inner_opts = memberlist_proto::EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); + let inner = memberlist_proto::Endpoint::new_at( + inner_opts, + memberlist_proto::Instant::ORIGIN, + SmallRng::seed_from_u64(0), + ); + Endpoint::new(inner, Options::new().with_query_response_size_limit(10)) +} + +#[test] +fn respond_encoded_frame_size_check_rejects_small_payload_on_tight_limit() { + // Guard 1 must check the ENCODED frame size, not the raw payload size. + // With limit=10, even an empty payload produces an encoded frame >> 10 bytes. + let mut e = ep_small_resp_limit(); + let qid = QueryId { + ltime: LamportTime::new(1), + id: 42, + }; + let token = e.test_register_received_query(qid, addr(1002), t_secs(100)); + // Empty payload passes the OLD raw-size guard (0 <= 10) but must fail the + // new encoded-size guard. + let err = e + .respond( + &token, + bytes::Bytes::new(), + memberlist_proto::Instant::ORIGIN, + ) + .expect_err("encoded frame exceeds limit=10, must return RespondTooLarge"); + assert!( + matches!(err, Error::RespondTooLarge(_, _)), + "expected RespondTooLarge, got {:?}", + err + ); + // The responded flag must NOT be set (send was not attempted). + assert!( + !e.test_is_responded(qid), + "responded flag must remain false when RespondTooLarge is returned" + ); +} + +// Bug 4: Stale self-leave → unbounded refute. +#[test] +fn stale_self_leave_does_not_trigger_refute() { + // Seed local node (id=1) as Alive with status_time=10. + // A leave intent at ltime=3 is stale (3 <= 10) and must NOT fire broadcast_join. + let mut e = ep(); + e.test_seed_member(1u32, MemberStatus::Alive, LamportTime::new(10)); + + let result = + e.test_handle_leave_intent(1u32, LamportTime::new(3), memberlist_proto::Instant::ORIGIN); + + // Must return false (stale: no rebroadcast). + assert!(!result, "stale self-leave must return false"); + + // Clock was witnessed to 3 → clock = 4. broadcast_join would advance it further. + // broadcast_join(LamportTime(4)) calls witness(clock=4, 4) → clock=5. + // With the fix, the stale check fires before the self-refute, so clock stays at 4. + assert_eq!( + e.member_time(), + 4, + "stale self-leave must not advance the clock beyond witness(0, 3) = 4" + ); +} + +// Bug 5: Leave-broadcast deadline never retired. +#[test] +fn leave_broadcast_deadline_is_cleared_after_expiry() { + let mut e = ep(); + // leave() arms leave_broadcast_deadline = ORIGIN + broadcast_timeout (5s). + e.leave(memberlist_proto::Instant::ORIGIN).unwrap(); + assert!( + e.leave_broadcast_deadline().is_some(), + "leave_broadcast_deadline must be armed after leave()" + ); + + // Tick past the deadline. + let past = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(6); + e.handle_timeout(past); + + // The deadline must be cleared. + assert!( + e.leave_broadcast_deadline().is_none(), + "leave_broadcast_deadline must be cleared after expiry" + ); +} + +// ── Regression: 6 correctness fixes ────────────────────────────────────────── + +// ── Fix 1: load_snapshot must advance event_clock / query_clock ─────────────── + +#[test] +fn load_snapshot_advances_event_clock() { + // Bug: load_snapshot set event_buffer.min_time but left event_clock at 0. + // user_event() stamps ltime = event_clock (0) which is below min_time → dropped. + // Fix: witness(&mut event_clock, last_event_clock.0) after setting min_time. + let mut e = ep(); + let r = ReplayResult { + alive_nodes: vec![], + last_clock: 0.into(), + last_event_clock: 10.into(), + last_query_clock: 0.into(), + }; + e.load_snapshot(r, memberlist_proto::Instant::ORIGIN); + assert!( + e.event_time() >= 10, + "event_clock must be advanced to at least last_event_clock after load_snapshot, got {}", + e.event_time() + ); +} + +#[test] +fn load_snapshot_advances_query_clock() { + // Bug: load_snapshot set query_buffer.min_time but left query_clock at 0. + // Fix: witness(&mut query_clock, last_query_clock.0) after setting min_time. + let mut e = ep(); + let r = ReplayResult { + alive_nodes: vec![], + last_clock: 0.into(), + last_event_clock: 0.into(), + last_query_clock: 20.into(), + }; + e.load_snapshot(r, memberlist_proto::Instant::ORIGIN); + assert!( + e.query_time() >= 20, + "query_clock must be advanced to at least last_query_clock after load_snapshot, got {}", + e.query_time() + ); +} + +#[test] +fn load_snapshot_event_clock_allows_new_events_above_floor() { + // After load_snapshot with last_event_clock=10, a new user_event() stamps + // ltime = event_clock (>= 10), which is >= min_time floor, so it must be delivered. + let mut e = ep(); + let r = ReplayResult { + alive_nodes: vec![], + last_clock: 0.into(), + last_event_clock: 10.into(), + last_query_clock: 0.into(), + }; + e.load_snapshot(r, memberlist_proto::Instant::ORIGIN); + // Drain any pending events from load_snapshot. + while e.poll_event().is_some() {} + // Issue a new user event — must succeed and be delivered above the floor. + e.user_event("post-snap", bytes::Bytes::from_static(b"ok"), false) + .expect("user_event after load_snapshot must succeed"); + let ev = e + .poll_event() + .expect("user_event after load_snapshot must be delivered"); + assert!( + matches!(ev, Event::User(ref u) if u.name == "post-snap"), + "expected Event::User(post-snap), got: {ev:?}" + ); +} + +// ── Fix 2: malformed conflict response must not inflate the denominator ──────── + +#[test] +fn malformed_conflict_response_does_not_inflate_denominator() { + // Scenario: 1 malformed + 1 valid-agreeing response. + // Bug: malformed response counted in responses → num_resp=2, majority=2, + // matching=1 < 2 → false Shutdown. + // Fix: malformed dropped before insert → num_resp=1, majority=1, + // matching=1 >= 1 → WIN (no Shutdown). + let mut e = ep(); + let deadline = far_future(); + let qid = e.test_register_conflict_query(deadline); + + // Build a QueryResponseMessage carrying the wrong inner type (UserEvent bytes). + let bad_inner = AnyMessage::::UserEvent(UserEventMessage { + ltime: 1.into(), + cc: false, + name: "bad".into(), + payload: bytes::Bytes::new(), + }); + let bad_payload = bad_inner.encode().expect("encode must succeed"); + let bad_resp = crate::typed::QueryResponseMessage:: { + ltime: qid.ltime, + id: qid.id, + from: memberlist_proto::Node::new(200u32, addr(2000)), + flags: QueryFlag::empty(), + payload: bad_payload, + }; + e.test_handle_query_response(bad_resp); + + // 1 valid response that agrees (conflict_matching += 1). + e.test_fold_conflict_response(qid, 201u32, true); + + // Fire query close at a time past the deadline. + let past = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(3601); + e.test_fire_due_query_closes(past); + + // Fix: malformed not counted → num_resp=1, majority=1, matching=1 → WIN → no Shutdown. + assert!( + e.poll_event().is_none(), + "malformed conflict response must not inflate denominator and trigger false Shutdown" + ); +} + +// ── Fix 3: respond() must return Err and leave responded=false on send failure ─ + +#[test] +fn respond_send_failure_returns_err_and_leaves_responded_false() { + // Force the inner Endpoint's send_user_packet to fail by setting gossip_mtu + // to the minimum (512 bytes) and sending a 490-byte payload. The + // QueryResponseMessage encoding wraps the payload with framing overhead, + // pushing the total past 512. The serf query_response_size_limit is raised + // to 50_000 so our guard passes and the inner's MTU check is the one that fires. + let inner_opts = + EndpointOptions::::new(1u32, "127.0.0.1:7946".parse().unwrap()) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()) + .with_gossip_mtu(512); // minimum MTU: a 490-byte payload + framing exceeds it. + let inner = memberlist_proto::Endpoint::new_at( + inner_opts, + memberlist_proto::Instant::ORIGIN, + SmallRng::seed_from_u64(0), + ); + let opts = Options::new().with_query_response_size_limit(50_000); + let mut e = Endpoint::new(inner, opts); + + let qid = QueryId { + ltime: LamportTime::new(1), + id: 5, + }; + let token = e.test_register_received_query(qid, addr(1002), t_secs(10)); + + // 490-byte payload: encodes to > 512 bytes with QueryResponse + UserData framing. + let large_payload = bytes::Bytes::from(vec![0u8; 490]); + let result = e.respond(&token, large_payload, memberlist_proto::Instant::ORIGIN); + + assert!( + result.is_err(), + "respond() must return Err when the inner directed send fails" + ); + assert!( + matches!(result.unwrap_err(), Error::RespondSend(_)), + "send failure must surface as Error::RespondSend" + ); + // Entry must still be present (not removed on failure): test_is_responded returns false. + assert!( + !e.test_is_responded(qid), + "entry must not be removed when the directed send fails" + ); +} + +// ── Fix 4: ACKs must call relay_response when relay_factor > 0 ─────────────── + +#[test] +fn ack_query_with_relay_factor_relays_ack() { + // An ACK-flagged query with relay_factor=1 must relay the ACK response through + // a random Alive non-self peer after the direct send. + let mut e = ep(); + // Seed 2 Alive members so the relay count guard passes (relay_factor=1 needs >= 2). + e.test_seed_member(10u32, MemberStatus::Alive, LamportTime::new(1)); + e.test_seed_member(11u32, MemberStatus::Alive, LamportTime::new(1)); + + let q = QueryMessage { + flags: QueryFlag::ACK, + relay_factor: 1, + ..test_query(LamportTime::new(1), 42) + }; + e.test_handle_query(q); + + // The direct ACK sends to the querier (127.0.0.1:9999). + // The relay sends to one of the alive members (ports 1010 or 1011). + // last_directed_send is overwritten by relay_response, so it records the relay peer. + let (dest, _) = e + .test_last_directed_send() + .expect("at least one directed send must have occurred"); + + // The relay overwrites last_directed_send to a relay peer (not the querier at 9999). + assert_ne!( + dest.port(), + 9999, + "last directed send after relay must be to a relay peer, not the querier" + ); +} + +// ── Fix 5: received_queries growth and pruning ──────────────────────────────── + +#[test] +fn received_query_is_removed_after_successful_respond() { + // After a successful respond(), the entry must be removed from received_queries. + // A second respond() must return AlreadyResponded (not panic or succeed again). + let mut e = ep(); + let qid = QueryId { + ltime: LamportTime::new(1), + id: 5, + }; + let token = e.test_register_received_query(qid, addr(1002), t_secs(10)); + + e.respond( + &token, + bytes::Bytes::new(), + memberlist_proto::Instant::ORIGIN, + ) + .expect("first respond() must succeed"); + + // Entry removed: second call returns AlreadyResponded. + let err = e + .respond( + &token, + bytes::Bytes::new(), + memberlist_proto::Instant::ORIGIN, + ) + .unwrap_err(); + assert!( + matches!(err, Error::AlreadyResponded), + "second respond() after removal must return AlreadyResponded, got {err:?}" + ); +} + +#[test] +fn expired_received_queries_are_pruned_in_handle_timeout() { + // An expired received-query entry (deadline elapsed, never responded) must be + // pruned by handle_timeout so received_queries does not grow without bound. + let mut e = ep(); + let qid = QueryId { + ltime: LamportTime::new(2), + id: 8, + }; + let token = e.test_register_received_query(qid, addr(1003), t_secs(1)); + + // handle_timeout at t=10, past deadline=1. + e.handle_timeout(t_secs(10)); + + // Entry pruned: respond() returns AlreadyResponded (entry absent, .ok_or path). + let err = e + .respond(&token, bytes::Bytes::new(), t_secs(10)) + .unwrap_err(); + assert!( + matches!(err, Error::AlreadyResponded), + "expired received query must be pruned by handle_timeout, got {err:?}" + ); +} + +// ── Bug 1: zero valid conflict responses must not emit Event::Shutdown ──────── + +#[test] +fn zero_conflict_responses_does_not_shut_down() { + // Scenario: a conflict query receives ONLY malformed/wrong-type responses so + // num_resp stays 0. The prior code computed majority = 0/2 + 1 = 1 and then + // compared matching=0 < 1 → emitted Event::Shutdown. With the fix, + // num_resp == 0 is treated as inconclusive and no shutdown is emitted. + let mut e = ep(); + let deadline = far_future(); + let _qid = e.test_register_conflict_query(deadline); + + // Build a QueryResponseMessage carrying the wrong inner type (UserEvent bytes). + // The existing validate-before-count fix (Bug 2/R2) ensures this does NOT + // increment responses; we just confirm close_conflict_query also handles + // the num_resp==0 case correctly when even no malformed response is present. + // Fire query close: no responses at all (responses is empty). + let past = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(3601); + e.test_fire_due_query_closes(past); + + assert!( + e.poll_event().is_none(), + "zero conflict responses must not emit Event::Shutdown (inconclusive)" + ); +} + +#[test] +fn zero_valid_conflict_responses_all_malformed_does_not_shut_down() { + // Scenario: conflict query receives only wrong-type responses (all dropped + // before counting). Confirm the combined effect of Bug 1 + Bug 2 fixes: + // malformed dropped → num_resp=0 → no shutdown. + let mut e = ep(); + let deadline = far_future(); + let qid = e.test_register_conflict_query(deadline); + + // Inject a QueryResponseMessage carrying wrong inner type (dropped by Bug 2 fix). + let bad_inner = AnyMessage::::UserEvent(UserEventMessage { + ltime: 1.into(), + cc: false, + name: "bad".into(), + payload: bytes::Bytes::new(), + }); + let bad_payload = bad_inner.encode().expect("encode must succeed"); + let bad_resp = crate::typed::QueryResponseMessage:: { + ltime: qid.ltime, + id: qid.id, + from: memberlist_proto::Node::new(300u32, addr(3000)), + flags: QueryFlag::empty(), + payload: bad_payload, + }; + e.test_handle_query_response(bad_resp); + + // Confirm: zero responses counted. + assert_eq!( + e.test_pending_query_response_count(qid), + 0, + "malformed conflict response must not be counted" + ); + + let past = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(3601); + e.test_fire_due_query_closes(past); + + assert!( + e.poll_event().is_none(), + "zero valid conflict responses (all malformed) must not emit Event::Shutdown" + ); +} + +// ── Bug 2 (class sweep): malformed key response must not inflate num_resp ───── + +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn malformed_key_response_does_not_inflate_num_resp() { + // Scenario: a Key query receives a wrong-type response payload (UserEvent + // bytes where KeyResponse is expected). Without the fix the responder is + // inserted into pending.responses before the payload is checked in + // handle_key_response_fold, inflating num_resp. With the fix, the + // validate-before-count guard rejects it and num_resp stays 0. + let mut e = ep(); + let deadline = far_future(); + let qid = e.test_register_key_query(deadline); + + // Build a QueryResponseMessage carrying wrong inner type. + let bad_inner = AnyMessage::::UserEvent(UserEventMessage { + ltime: 1.into(), + cc: false, + name: "bad".into(), + payload: bytes::Bytes::new(), + }); + let bad_payload = bad_inner.encode().expect("encode must succeed"); + let bad_resp = crate::typed::QueryResponseMessage:: { + ltime: qid.ltime, + id: qid.id, + from: memberlist_proto::Node::new(400u32, addr(4000)), + flags: QueryFlag::empty(), + payload: bad_payload, + }; + e.test_handle_query_response(bad_resp); + + // Fix: malformed key response was not counted → num_resp stays 0. + assert_eq!( + e.test_pending_query_response_count(qid), + 0, + "malformed key response must not inflate num_resp" + ); +} + +// ── Bug 3 (class sweep): clock witness overflow ──────────────────────────────── + +#[test] +fn witness_max_u64_is_noop() { + // Witnessing u64::MAX must not panic and must not wrap the clock to 0. + let mut c = 5u64; + witness(&mut c, u64::MAX); + assert_eq!( + c, 5, + "witness(u64::MAX) must be a no-op (impossible Lamport time)" + ); +} + +#[test] +fn witness_max_u64_minus_one_is_rejected() { + // u64::MAX - 1 is now an unacceptable Lamport time (the two-value safety + // buffer: witnessing it would advance the clock to u64::MAX, a permanent + // tombstone). `ltime_is_acceptable` rejects it; `witness` is a no-op. + let mut c = 0u64; + witness(&mut c, u64::MAX - 1); + assert_eq!( + c, 0, + "witness(MAX-1) must be a no-op — unacceptable Lamport time" + ); +} + +#[test] +fn ingress_max_ltime_user_event_does_not_regress_clock() { + // An ingress UserEventMessage carrying ltime == u64::MAX must not panic + // (overflow-checked builds) and must not regress the clock to 0. + let mut e = ep(); + e.test_set_clocks(10, 10, 10); + let msg = UserEventMessage { + ltime: LamportTime::new(u64::MAX), + cc: false, + name: "flood".into(), + payload: bytes::Bytes::new(), + }; + // This must not panic; the event is dropped as a dedup no-op (slot is + // below min_time floor if we ever reach u64::MAX, and witness rejects MAX). + let _ = e.handle_user_event(msg); + // Clock must not have regressed to 0. + assert!( + e.event_time() >= 10, + "event clock must not regress after witnessing u64::MAX ltime" + ); +} + +#[test] +fn ingress_max_ltime_query_does_not_regress_clock() { + // A QueryMessage carrying ltime == u64::MAX must not panic and must not + // wrap the query_clock to 0. + use crate::typed::{QueryFlag, QueryMessage}; + let mut e = ep(); + e.test_set_clocks(0, 0, 20); + let msg = QueryMessage:: { + ltime: LamportTime::new(u64::MAX), + id: 42, + from: memberlist_proto::Node::new(1u32, addr(9000)), + filters: vec![], + flags: QueryFlag::empty(), + relay_factor: 0, + timeout: std::time::Duration::from_secs(1), + name: "flood".into(), + payload: bytes::Bytes::new(), + }; + let _ = e.test_handle_query(msg); + // Clock must not have regressed to 0. + assert!( + e.query_time() >= 20, + "query clock must not regress after witnessing u64::MAX ltime" + ); +} + +#[test] +fn ingress_max_ltime_join_intent_does_not_regress_member_clock() { + // A JoinMessage with ltime == u64::MAX must not wrap the member clock. + let mut e = ep(); + e.test_set_clocks(15, 0, 0); + let _ = e.handle_node_join_intent(LamportTime::new(u64::MAX), &999u32, t_secs(0)); + assert!( + e.member_time() >= 15, + "member clock must not regress after witnessing u64::MAX join intent" + ); +} + +#[test] +fn ingress_max_ltime_leave_intent_does_not_regress_member_clock() { + // A LeaveMessage with ltime == u64::MAX must not wrap the member clock. + // Seed the node first so the leave handler has a member to update. + let mut e = ep(); + e.test_set_clocks(15, 0, 0); + e.test_seed_left_member(888u32, LamportTime::new(1)); + let _ = e.handle_node_leave_intent(LamportTime::new(u64::MAX), &888u32, false, t_secs(0)); + assert!( + e.member_time() >= 15, + "member clock must not regress after witnessing u64::MAX leave intent" + ); +} + +// ── Bug 4: query dedup slot Vec must be capped ──────────────────────────────── + +#[test] +fn query_buffer_slot_capped_at_max_query_ids_per_ltime() { + // Inserting more than MAX_QUERY_IDS_PER_LTIME unique ids at the same ltime + // must not grow the slot beyond the cap; excess ids are treated as seen + // (return false) and the slot length stays at MAX_QUERY_IDS_PER_LTIME. + let mut buf = QueryBuffer::new(64); + let ltime = 1u64; + // Fill up to the cap. + for id in 0..MAX_QUERY_IDS_PER_LTIME as u32 { + let accepted = buf.witness_query(ltime + 1, ltime, id); + assert!( + accepted, + "id {id} at ltime {ltime} must be accepted before cap" + ); + } + // One more unique id past the cap: must be rejected. + let overflow_id = MAX_QUERY_IDS_PER_LTIME as u32; + let rejected = buf.witness_query(ltime + 1, ltime, overflow_id); + assert!( + !rejected, + "id past the per-slot cap must be rejected (treated as already-seen)" + ); + // Slot length must not exceed the cap. + let slot_len = match buf.buffer[(ltime % 64) as usize].as_ref() { + Some(q) if q.ltime.0 == ltime => q.query_ids.len(), + _ => panic!("expected a slot for ltime {ltime}"), + }; + assert_eq!( + slot_len, MAX_QUERY_IDS_PER_LTIME, + "slot must not grow past MAX_QUERY_IDS_PER_LTIME" + ); +} + +#[test] +fn endpoint_query_buffer_slot_capped_via_adapter() { + // Same check via the Endpoint adapter to confirm the cap is enforced in the + // full machine path (witness_query called from handle_query). + use crate::typed::{QueryFlag, QueryMessage}; + let mut e = ep(); + e.test_set_clocks(0, 0, 1); // query_clock = 1 + let ltime = 1u64; // queries at this ltime + + // Push unique ids up to the cap through handle_query. + for id in 0..MAX_QUERY_IDS_PER_LTIME as u32 { + let msg = QueryMessage:: { + ltime: LamportTime::new(ltime), + id, + from: memberlist_proto::Node::new(id + 10, addr(9000)), + filters: vec![], + flags: QueryFlag::NO_BROADCAST, + relay_factor: 0, + timeout: std::time::Duration::from_secs(1), + name: "flood".into(), + payload: bytes::Bytes::new(), + }; + let _ = e.test_handle_query(msg); + } + + let before_overflow = e.test_query_slot_len(ltime); + assert_eq!( + before_overflow, MAX_QUERY_IDS_PER_LTIME, + "slot must be exactly at the cap" + ); + + // One more unique id: must not grow the slot. + let overflow_msg = QueryMessage:: { + ltime: LamportTime::new(ltime), + id: MAX_QUERY_IDS_PER_LTIME as u32, + from: memberlist_proto::Node::new(9999u32, addr(9999)), + filters: vec![], + flags: QueryFlag::NO_BROADCAST, + relay_factor: 0, + timeout: std::time::Duration::from_secs(1), + name: "overflow".into(), + payload: bytes::Bytes::new(), + }; + let _ = e.test_handle_query(overflow_msg); + + assert_eq!( + e.test_query_slot_len(ltime), + MAX_QUERY_IDS_PER_LTIME, + "slot must not exceed the cap after overflow attempt" + ); +} + +// ── Fix 6: failed resync must not clear the dirty flag ─────────────────────── + +#[test] +fn resync_keeps_dirty_when_inner_snapshot_rejects() { + // When set_local_state_snapshot returns Err (snapshot exceeds the inner's + // max_stream_frame_size), local_state_dirty must stay true for retry. + // Construction validates the minimal push-pull (empty snapshot) and requires + // max_stream_frame_size >= its encoded length (~547 bytes for u32/SocketAddr). + // At 547 the construction preflight passes (547 >= 547), but validate_local_ + // state_snapshot uses budget = max_stream_frame_size - LOCAL_STATE_FRAME_BUDGET + // (1 MiB) which saturating_sub-underflows to 0, so any non-empty serf + // PushPull (encoded_len > 0) exceeds the budget and is rejected. + let inner_opts = + EndpointOptions::::new(1u32, "127.0.0.1:7946".parse().unwrap()) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()) + .with_max_stream_frame_size(547); // passes construction but rejects serf PushPull. + let inner = memberlist_proto::Endpoint::new_at( + inner_opts, + memberlist_proto::Instant::ORIGIN, + SmallRng::seed_from_u64(0), + ); + let mut e = Endpoint::new(inner, Options::new()); + + // Force dirty and call resync. + e.test_set_clocks(1, 2, 3); + e.resync_local_state(); + + // The inner rejected the snapshot: dirty flag must remain set. + assert!( + e.test_is_dirty(), + "dirty flag must remain true when inner set_local_state_snapshot fails" + ); +} + +// ── Bug 1: class sweep — reject u64::MAX Lamport times at all ingress sites ── + +fn sa(port: u16) -> std::net::SocketAddr { + format!("127.0.0.1:{port}").parse().unwrap() +} + +/// A join intent carrying ltime == u64::MAX must be dropped before any state +/// mutation: clock unchanged, status_time not written, no rebroadcast. +#[test] +fn join_intent_max_ltime_is_dropped_no_state_mutation() { + let mut e = ep(); + // Pre-seed node 42 as Alive with status_time = 1. + e.test_seed_member(42u32, MemberStatus::Alive, LamportTime::new(1)); + let clock_before = e.member_time(); + + let result = e.test_handle_join_intent( + 42u32, + LamportTime::new(u64::MAX), + memberlist_proto::Instant::ORIGIN, + ); + + assert!( + !result, + "join intent at u64::MAX must return false (no rebroadcast)" + ); + assert_eq!( + e.member_time(), + clock_before, + "member clock must not advance on u64::MAX join intent" + ); + assert_eq!( + e.test_member_status_time(42u32).map(|lt| lt.0), + Some(1), + "status_time must not be updated to u64::MAX" + ); + assert_eq!( + e.test_member_status(42u32), + Some(MemberStatus::Alive), + "status must remain Alive" + ); +} + +/// A leave intent carrying ltime == u64::MAX must be dropped before any state +/// mutation: clock unchanged, status_time not written, no rebroadcast. +#[test] +fn leave_intent_max_ltime_is_dropped_no_state_mutation() { + let mut e = ep(); + e.test_seed_member(42u32, MemberStatus::Alive, LamportTime::new(1)); + let clock_before = e.member_time(); + + let result = e.test_handle_leave_intent( + 42u32, + LamportTime::new(u64::MAX), + memberlist_proto::Instant::ORIGIN, + ); + + assert!( + !result, + "leave intent at u64::MAX must return false (no rebroadcast)" + ); + assert_eq!( + e.member_time(), + clock_before, + "member clock must not advance on u64::MAX leave intent" + ); + assert_eq!( + e.test_member_status_time(42u32).map(|lt| lt.0), + Some(1), + "status_time must not be updated to u64::MAX" + ); + assert_eq!( + e.test_member_status(42u32), + Some(MemberStatus::Alive), + "status must remain Alive" + ); +} + +/// A user event carrying ltime == u64::MAX must be dropped: event clock +/// unchanged, no Event::User emitted, returns false. +#[test] +fn user_event_max_ltime_is_dropped_no_state_mutation() { + let mut e = ep(); + let clock_before = e.event_time(); + + let msg = UserEventMessage { + ltime: LamportTime::new(u64::MAX), + cc: false, + name: "test".into(), + payload: bytes::Bytes::new(), + }; + let is_new = e.test_handle_user_event(msg); + + assert!(!is_new, "user event at u64::MAX must return false"); + assert_eq!( + e.event_time(), + clock_before, + "event clock must not advance on u64::MAX user event" + ); + assert!( + e.poll_event().is_none(), + "no event must be emitted for u64::MAX user event" + ); +} + +/// A query message carrying ltime == u64::MAX must be dropped: query clock +/// unchanged, no Event::Query emitted, returns false. +#[test] +fn query_max_ltime_is_dropped_no_state_mutation() { + let mut e = ep(); + let clock_before = e.query_time(); + + let msg = QueryMessage:: { + ltime: LamportTime::new(u64::MAX), + id: 1, + from: memberlist_proto::Node::new(99u32, sa(9001)), + filters: vec![], + flags: QueryFlag::NO_BROADCAST, + relay_factor: 0, + timeout: std::time::Duration::from_secs(1), + name: "test".into(), + payload: bytes::Bytes::new(), + }; + let rebroadcast = e.test_handle_query(msg); + + assert!( + !rebroadcast, + "query at u64::MAX must return false (no rebroadcast)" + ); + assert_eq!( + e.query_time(), + clock_before, + "query clock must not advance on u64::MAX query" + ); + assert!( + e.poll_event().is_none(), + "no event must be emitted for u64::MAX query" + ); +} + +/// merge_remote_state with a member clock of u64::MAX must not advance any +/// local clock and must not write a u64::MAX status_time on any member. +#[test] +fn merge_remote_state_max_member_clock_is_ignored() { + use crate::typed::PushPullMessage; + + let mut e = ep(); + e.test_seed_member(42u32, MemberStatus::Alive, LamportTime::new(1)); + let clock_before = e.member_time(); + let event_clock_before = e.event_time(); + let query_clock_before = e.query_time(); + + // Build a PushPull body with all three clocks at u64::MAX and a + // status_ltimes entry at u64::MAX for node 42. + let pp = PushPullMessage:: { + ltime: LamportTime::new(u64::MAX), + event_ltime: LamportTime::new(u64::MAX), + query_ltime: LamportTime::new(u64::MAX), + status_ltimes: vec![(42u32, LamportTime::new(u64::MAX))], + left_members: vec![], + events: vec![], + }; + let encoded = AnyMessage::::PushPull(pp) + .encode() + .expect("encode must succeed"); + e.test_merge_remote_state(encoded, false); + + assert_eq!( + e.member_time(), + clock_before, + "member clock must not advance on MAX push-pull" + ); + assert_eq!( + e.event_time(), + event_clock_before, + "event clock must not advance on MAX push-pull" + ); + assert_eq!( + e.query_time(), + query_clock_before, + "query clock must not advance on MAX push-pull" + ); + // The u64::MAX status_ltime entry must be filtered out; node 42 retains status_time = 1. + assert_eq!( + e.test_member_status_time(42u32).map(|lt| lt.0), + Some(1), + "status_time must not be overwritten with u64::MAX via push-pull" + ); +} + +/// load_snapshot with all clocks at u64::MAX must not advance any local clock +/// and must not set min_time to u64::MAX (which would drop all future events). +#[test] +fn load_snapshot_max_clocks_are_ignored() { + use crate::snapshot::ReplayResult; + let mut e = ep(); + let clock_before = e.member_time(); + let event_clock_before = e.event_time(); + let query_clock_before = e.query_time(); + let event_min_before = e.test_event_min_time(); + + let replay = ReplayResult { + last_clock: LamportTime::new(u64::MAX), + last_event_clock: LamportTime::new(u64::MAX), + last_query_clock: LamportTime::new(u64::MAX), + alive_nodes: vec![], + }; + e.load_snapshot(replay, memberlist_proto::Instant::ORIGIN); + + assert_eq!( + e.member_time(), + clock_before, + "member clock must not advance on MAX snapshot" + ); + assert_eq!( + e.event_time(), + event_clock_before, + "event clock must not advance on MAX snapshot" + ); + assert_eq!( + e.query_time(), + query_clock_before, + "query clock must not advance on MAX snapshot" + ); + assert_eq!( + e.test_event_min_time(), + event_min_before, + "event min_time must not be set to u64::MAX by snapshot" + ); + assert_ne!( + e.test_event_min_time(), + u64::MAX, + "event min_time == u64::MAX would drop all future events" + ); +} + +// ── Bug 2: class sweep — per-ltime event buffer cap ────────────────────────── + +/// Sending many unique (name, payload) events at the same ltime must not grow +/// the slot past MAX_EVENTS_PER_LTIME. +#[test] +fn event_buffer_slot_capped_at_max_events_per_ltime() { + let mut e = ep(); + e.test_set_event_clock(1); + let ltime = 1u64; + + // Fill the slot to the cap. + for i in 0..MAX_EVENTS_PER_LTIME as u32 { + let msg = UserEventMessage { + ltime: LamportTime::new(ltime), + cc: false, + name: format!("ev-{i}").into(), + payload: bytes::Bytes::new(), + }; + let _ = e.test_handle_user_event(msg); + } + assert_eq!( + e.test_event_slot_len(ltime), + MAX_EVENTS_PER_LTIME, + "slot must be exactly at the cap" + ); + + // One more unique event: slot must not grow. + let overflow = UserEventMessage { + ltime: LamportTime::new(ltime), + cc: false, + name: "overflow".into(), + payload: bytes::Bytes::new(), + }; + let is_new = e.test_handle_user_event(overflow); + assert!(!is_new, "event past the per-ltime cap must be rejected"); + assert_eq!( + e.test_event_slot_len(ltime), + MAX_EVENTS_PER_LTIME, + "slot must not grow past MAX_EVENTS_PER_LTIME" + ); +} + +/// Inbound events exceeding max_user_event_size must be dropped without +/// emitting an Event::User or growing the slot. +#[test] +fn inbound_user_event_too_large_is_dropped() { + let mut e = ep(); + e.test_set_event_clock(1); + let ltime = 1u64; + + // Default max_user_event_size is 512 bytes. Build a name+payload > 512. + let big_name: smol_str::SmolStr = "x".repeat(300).into(); + let big_payload = bytes::Bytes::from(vec![0u8; 300]); + let msg = UserEventMessage { + ltime: LamportTime::new(ltime), + cc: false, + name: big_name, + payload: big_payload, + }; + let is_new = e.test_handle_user_event(msg); + assert!(!is_new, "oversized inbound event must be rejected"); + assert!( + e.poll_event().is_none(), + "no Event::User for oversized inbound event" + ); + assert_eq!( + e.test_event_slot_len(ltime), + 0, + "slot must not grow for oversized event" + ); +} + +// ── Bug 3: key query num_nodes ──────────────────────────────────────────────── + +/// A key query issued with N members in the map must report num_nodes == N +/// in the emitted KeyResponse, both at timeout and after all responses. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn key_query_num_nodes_equals_member_count_at_issue_time() { + let mut e = ep(); + // Seed 3 distinct members so num_members() == 3 at issue time. + for id in [10u32, 11, 12] { + e.test_seed_member(id, MemberStatus::Alive, LamportTime::new(1)); + } + assert_eq!(e.num_members(), 3); + + let far_future = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(9999); + let _query_id = e.test_register_key_query(far_future); + + // num_nodes is captured at registration time. + let num_nodes = e + .test_last_pending_query_num_nodes() + .expect("pending query must exist"); + assert_eq!( + num_nodes, 3, + "num_nodes must equal the member count at issue time" + ); + + // Fire the timeout: close_key_query emits Event::KeyResponse. + let after_deadline = far_future + std::time::Duration::from_nanos(1); + e.test_fire_due_query_closes(after_deadline); + + let ev = e + .poll_event() + .expect("Event::KeyResponse must be emitted on timeout"); + match ev { + Event::KeyResponse(kr) => { + assert_eq!( + kr.num_nodes, 3, + "KeyResponse.num_nodes must equal the queried member count" + ); + assert_eq!(kr.num_resp, 0, "no responses were folded before timeout"); + } + other => panic!("expected Event::KeyResponse, got {other:?}"), + } +} + +/// A key query with num_nodes captured correctly reports the right count even +/// when members are added after the query is issued. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn key_query_num_nodes_is_captured_at_issue_not_at_close() { + let mut e = ep(); + e.test_seed_member(10u32, MemberStatus::Alive, LamportTime::new(1)); + assert_eq!(e.num_members(), 1); + + let far_future = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(9999); + let _query_id = e.test_register_key_query(far_future); + + // Now add 2 more members after the query was issued. + e.test_seed_member(11u32, MemberStatus::Alive, LamportTime::new(1)); + e.test_seed_member(12u32, MemberStatus::Alive, LamportTime::new(1)); + assert_eq!(e.num_members(), 3); + + // Close the query: num_nodes must reflect membership AT ISSUE TIME (1), not now (3). + let after_deadline = far_future + std::time::Duration::from_nanos(1); + e.test_fire_due_query_closes(after_deadline); + + let ev = e.poll_event().expect("Event::KeyResponse must be emitted"); + match ev { + Event::KeyResponse(kr) => { + assert_eq!( + kr.num_nodes, 1, + "num_nodes must be the count at issue time (1), not the current count (3)" + ); + } + other => panic!("expected Event::KeyResponse, got {other:?}"), + } +} + +// ── UNIFIED Lamport-validation redesign regressions (Finding 1) ─────────────── +// +// u64::MAX - 1 is the NEW threshold: witnessing it would advance the clock to +// u64::MAX, a permanent tombstone. Prior fixes only rejected u64::MAX; these +// regressions verify the two-value safety buffer. + +/// A join intent at u64::MAX - 1 must be dropped with NO clock change and +/// no state mutation — the same guarantee previously only tested for u64::MAX. +#[test] +fn join_intent_max_minus_one_ltime_is_dropped() { + let mut e = ep(); + e.test_seed_member(42u32, MemberStatus::Alive, LamportTime::new(1)); + let clock_before = e.member_time(); + + let result = e.test_handle_join_intent( + 42u32, + LamportTime::new(u64::MAX - 1), + memberlist_proto::Instant::ORIGIN, + ); + + assert!(!result, "join intent at u64::MAX-1 must return false"); + assert_eq!( + e.member_time(), + clock_before, + "member clock must not advance on u64::MAX-1 join intent" + ); + assert_eq!( + e.test_member_status_time(42u32).map(|lt| lt.0), + Some(1), + "status_time must not be updated to u64::MAX-1" + ); + // After the bad ingress a subsequent valid local user_event must still work. + e.user_event("ok", bytes::Bytes::new(), false) + .expect("user_event must succeed after rejected intent"); +} + +/// A leave intent at u64::MAX - 1 must be dropped with NO clock change and +/// no state mutation. +#[test] +fn leave_intent_max_minus_one_ltime_is_dropped() { + let mut e = ep(); + e.test_seed_member(42u32, MemberStatus::Alive, LamportTime::new(1)); + let clock_before = e.member_time(); + + let result = e.test_handle_leave_intent( + 42u32, + LamportTime::new(u64::MAX - 1), + memberlist_proto::Instant::ORIGIN, + ); + + assert!(!result, "leave intent at u64::MAX-1 must return false"); + assert_eq!( + e.member_time(), + clock_before, + "member clock must not advance on u64::MAX-1 leave intent" + ); + assert_eq!( + e.test_member_status_time(42u32).map(|lt| lt.0), + Some(1), + "status_time must not be updated to u64::MAX-1" + ); + // A subsequent valid join intent at a sane ltime must be accepted. + let ok = e.test_handle_join_intent( + 42u32, + LamportTime::new(5), + memberlist_proto::Instant::ORIGIN, + ); + assert!( + ok, + "valid join intent after rejected leave must be accepted" + ); +} + +/// A user event at u64::MAX - 1 must be dropped: event clock unchanged, no +/// Event::User emitted, subsequent valid user_event() must succeed. +#[test] +fn user_event_max_minus_one_ltime_is_dropped() { + let mut e = ep(); + let clock_before = e.event_time(); + + let msg = crate::typed::UserEventMessage { + ltime: LamportTime::new(u64::MAX - 1), + cc: false, + name: "bad".into(), + payload: bytes::Bytes::new(), + }; + let is_new = e.test_handle_user_event(msg); + + assert!(!is_new, "user event at u64::MAX-1 must return false"); + assert_eq!( + e.event_time(), + clock_before, + "event clock must not advance on u64::MAX-1 user event" + ); + assert!( + e.poll_event().is_none(), + "no event must be emitted for u64::MAX-1 user event" + ); + // A subsequent valid user_event() must still work. + e.user_event("ok", bytes::Bytes::new(), false) + .expect("user_event must succeed after rejected event"); +} + +/// A query at u64::MAX - 1 must be dropped: query clock unchanged, no +/// Event::Query, subsequent valid query() must succeed. +#[test] +fn query_max_minus_one_ltime_is_dropped() { + let mut e = ep(); + let clock_before = e.query_time(); + + let msg = crate::typed::QueryMessage:: { + ltime: LamportTime::new(u64::MAX - 1), + id: 1, + from: memberlist_proto::Node::new(99u32, sa(9001)), + filters: vec![], + flags: crate::typed::QueryFlag::NO_BROADCAST, + relay_factor: 0, + timeout: std::time::Duration::from_secs(1), + name: "bad".into(), + payload: bytes::Bytes::new(), + }; + let rebroadcast = e.test_handle_query(msg); + + assert!(!rebroadcast, "query at u64::MAX-1 must return false"); + assert_eq!( + e.query_time(), + clock_before, + "query clock must not advance on u64::MAX-1 query" + ); + assert!( + e.poll_event().is_none(), + "no event must be emitted for u64::MAX-1 query" + ); + // Subsequent valid query() must succeed. + let _qid = e + .query( + "ok", + bytes::Bytes::new(), + QueryParams::default(), + memberlist_proto::Instant::ORIGIN, + ) + .expect("query must succeed after rejected query"); +} + +/// A push-pull with an unacceptable top-level member clock (u64::MAX - 1) must +/// drop the ENTIRE message: no dirty flag, no status_ltimes applied, no events. +#[test] +fn merge_remote_state_max_minus_one_member_clock_drops_entire_message() { + use crate::typed::PushPullMessage; + + let mut e = ep(); + // Seed node 42 with status_time = 1. + e.test_seed_member(42u32, MemberStatus::Alive, LamportTime::new(1)); + e.test_clear_dirty(); + let clock_before = e.member_time(); + + // Push-pull carries member_ltime = u64::MAX-1 (unacceptable) but also + // a valid status_ltime = 100 for node 42 — the whole message must be dropped, + // not just the top-level clock. + let pp = PushPullMessage:: { + ltime: LamportTime::new(u64::MAX - 1), // unacceptable + event_ltime: LamportTime::new(5), + query_ltime: LamportTime::new(5), + status_ltimes: vec![(42u32, LamportTime::new(100))], + left_members: vec![], + events: vec![], + }; + let encoded = AnyMessage::::PushPull(pp) + .encode() + .expect("encode must succeed"); + e.test_merge_remote_state(encoded, false); + + // No state mutation: clock unchanged, dirty flag not set by this call. + assert_eq!( + e.member_time(), + clock_before, + "member clock must not advance when top-level clock is u64::MAX-1" + ); + assert!( + !e.test_is_dirty(), + "dirty flag must not be set when the entire push-pull is dropped" + ); + // The status_ltimes entry (ltime=100) must NOT have been applied. + assert_eq!( + e.test_member_status_time(42u32).map(|lt| lt.0), + Some(1), + "status_time must not be updated when the whole push-pull is dropped" + ); +} + +/// A push-pull with an unacceptable event clock drops the ENTIRE message — +/// the status_ltimes body is NOT applied even though member_ltime is valid. +#[test] +fn merge_remote_state_max_minus_one_event_clock_drops_entire_message() { + use crate::typed::PushPullMessage; + + let mut e = ep(); + e.test_seed_member(42u32, MemberStatus::Alive, LamportTime::new(1)); + e.test_clear_dirty(); + let event_clock_before = e.event_time(); + + let pp = PushPullMessage:: { + ltime: LamportTime::new(5), // valid + event_ltime: LamportTime::new(u64::MAX - 1), // unacceptable + query_ltime: LamportTime::new(5), + status_ltimes: vec![(42u32, LamportTime::new(100))], + left_members: vec![], + events: vec![], + }; + let encoded = AnyMessage::::PushPull(pp) + .encode() + .expect("encode must succeed"); + e.test_merge_remote_state(encoded, false); + + assert_eq!( + e.event_time(), + event_clock_before, + "event clock must not advance when event_ltime is u64::MAX-1" + ); + assert!( + !e.test_is_dirty(), + "dirty flag must not be set when the entire push-pull is dropped" + ); + assert_eq!( + e.test_member_status_time(42u32).map(|lt| lt.0), + Some(1), + "status_time must not be updated when push-pull is dropped on event_ltime" + ); +} + +// ── Responder identity validation regressions (Finding 2) ───────────────────── + +/// A query response from an unknown (non-member) responder id must NOT be +/// counted in the conflict denominator, so it cannot cause a false shutdown. +#[test] +fn conflict_response_from_unknown_responder_is_not_counted() { + let mut e = ep(); + // Seed one known member (id=10) so the machine has a real cluster. + e.test_seed_member(10u32, MemberStatus::Alive, LamportTime::new(1)); + + let far_future = t_secs(9999); + let qid = e.test_register_conflict_query(far_future); + + // Send a response claiming to be from id=999 — unknown, not in members.states. + let forged = crate::typed::QueryResponseMessage:: { + ltime: qid.ltime, + id: qid.id, + from: memberlist_proto::Node::new(999u32, addr(9999)), + flags: crate::typed::QueryFlag::empty(), + payload: bytes::Bytes::new(), // payload doesn't matter; membership check fires first + }; + e.test_handle_query_response(forged); + + // The response must NOT have been counted (responses map is still empty). + assert_eq!( + e.test_pending_query_response_count(qid), + 0, + "forged responder id must not be counted in conflict denominator" + ); + // conflict_matching must also be 0. + assert_eq!( + e.test_pending_query_conflict_matching(qid), + Some(0), + "conflict_matching must not increase for forged responder" + ); +} + +/// A flood of forged unknown responder ids for a conflict query must not +/// inflate the denominator, preventing a false shutdown. +#[test] +fn conflict_response_flood_of_unknown_ids_does_not_inflate_denominator() { + let mut e = ep(); + // One real member. + e.test_seed_member(10u32, MemberStatus::Alive, LamportTime::new(1)); + + let far_future = t_secs(9999); + let qid = e.test_register_conflict_query(far_future); + + // Send 1000 responses each from a different unknown id. + for forger_id in 1000u32..2000 { + let resp = crate::typed::QueryResponseMessage:: { + ltime: qid.ltime, + id: qid.id, + from: memberlist_proto::Node::new(forger_id, addr(9000)), + flags: crate::typed::QueryFlag::empty(), + payload: bytes::Bytes::new(), + }; + e.test_handle_query_response(resp); + } + + assert_eq!( + e.test_pending_query_response_count(qid), + 0, + "1000 forged responder ids must not inflate the denominator" + ); +} + +/// A response from a KNOWN member is still counted (the validation must not +/// over-reject). +#[test] +fn conflict_response_from_known_member_is_counted() { + let mut e = ep(); + e.test_seed_member(10u32, MemberStatus::Alive, LamportTime::new(1)); + + let far_future = t_secs(9999); + let qid = e.test_register_conflict_query(far_future); + + // Build a valid ConflictResponseMessage payload so the payload check passes. + let member_node = memberlist_proto::Node::::new(10u32, addr(7946)); + let conflict_resp = ConflictResponseMessage::new(member_node); + let payload = AnyMessage::::ConflictResponse(conflict_resp) + .encode() + .expect("encode conflict response"); + + let resp = crate::typed::QueryResponseMessage { + ltime: qid.ltime, + id: qid.id, + from: member_node, + flags: crate::typed::QueryFlag::empty(), + payload, + }; + e.test_handle_query_response(resp); + + assert_eq!( + e.test_pending_query_response_count(qid), + 1, + "response from known member must be counted" + ); +} + +// ── UNIFIED bounded-memory regressions (Finding 3) ─────────────────────────── + +/// A flood of unique unknown node ids via join intents must not grow +/// recent_intents past MAX_RECENT_INTENTS. +#[test] +fn recent_intents_capped_on_join_intent_flood() { + use crate::members::MAX_RECENT_INTENTS; + let mut e = ep(); + let now = memberlist_proto::Instant::ORIGIN; + + // Flood with 2 * MAX_RECENT_INTENTS unique unknown ids. + for unknown_id in 100u32..(100 + 2 * MAX_RECENT_INTENTS as u32) { + // These ids are NOT in members.states so they go into recent_intents. + e.test_handle_join_intent(unknown_id, LamportTime::new(1), now); + } + + assert!( + e.test_recent_intents_len() <= MAX_RECENT_INTENTS, + "recent_intents must not exceed MAX_RECENT_INTENTS after flood: got {}", + e.test_recent_intents_len() + ); +} + +/// A flood of unique unknown node ids via leave intents must not grow +/// recent_intents past MAX_RECENT_INTENTS. +#[test] +fn recent_intents_capped_on_leave_intent_flood() { + use crate::members::MAX_RECENT_INTENTS; + let mut e = ep(); + let now = memberlist_proto::Instant::ORIGIN; + + for unknown_id in 100u32..(100 + 2 * MAX_RECENT_INTENTS as u32) { + e.test_handle_leave_intent(unknown_id, LamportTime::new(1), now); + } + + assert!( + e.test_recent_intents_len() <= MAX_RECENT_INTENTS, + "recent_intents must not exceed MAX_RECENT_INTENTS after leave flood: got {}", + e.test_recent_intents_len() + ); +} + +// ── Zero event buffer panic regression (Finding 4) ──────────────────────────── + +/// An endpoint constructed with event_buffer_size = 0 must not panic when +/// a user event arrives (ltime % 0 would panic without the .max(1) guard). +#[test] +fn zero_event_buffer_size_does_not_panic_on_first_event() { + let inner_opts = + EndpointOptions::::new(1u32, "127.0.0.1:7946".parse().unwrap()) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); + let inner = memberlist_proto::Endpoint::new_at( + inner_opts, + memberlist_proto::Instant::ORIGIN, + SmallRng::seed_from_u64(0), + ); + // event_buffer_size = 0 should be clamped to 1 internally. + let opts = crate::options::Options::new().with_event_buffer_size(0); + let mut e = Endpoint::new(inner, opts); + + // Must NOT panic. + e.user_event("test", bytes::Bytes::new(), false) + .expect("user_event must not panic when event_buffer_size was 0"); +} + +/// A push-pull with event_buffer_size = 0 endpoint must not panic when +/// merge_remote_state processes events. +#[test] +fn zero_event_buffer_size_does_not_panic_on_merge_remote_state() { + use crate::typed::PushPullMessage; + + let inner_opts = + EndpointOptions::::new(1u32, "127.0.0.1:7946".parse().unwrap()) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); + let inner = memberlist_proto::Endpoint::new_at( + inner_opts, + memberlist_proto::Instant::ORIGIN, + SmallRng::seed_from_u64(0), + ); + let opts = crate::options::Options::new().with_event_buffer_size(0); + let mut e = Endpoint::new(inner, opts); + + let pp = PushPullMessage:: { + ltime: LamportTime::new(1), + event_ltime: LamportTime::new(1), + query_ltime: LamportTime::new(1), + status_ltimes: vec![], + left_members: vec![], + events: vec![crate::typed::UserEvents { + ltime: LamportTime::new(1), + events: vec![crate::typed::UserEvent { + name: "e".into(), + payload: bytes::Bytes::new(), + }], + }], + }; + let encoded = AnyMessage::::PushPull(pp) + .encode() + .expect("encode must succeed"); + + // Must NOT panic. + e.test_merge_remote_state(encoded, false); +} + +// ── Clock integrity + overflow regression tests ─────────────────────────────── + +/// An ingress ltime of LTIME_MAX is dropped on the join-intent path. +#[test] +fn ltime_max_join_intent_is_dropped() { + let mut e = ep(); + e.test_set_clocks(5, 0, 0); + let result = e.test_handle_join_intent( + 42u32, + LamportTime::new(LTIME_MAX), + memberlist_proto::Instant::ORIGIN, + ); + assert!(!result, "join intent at LTIME_MAX must be dropped"); + assert_eq!( + e.member_time(), + 5, + "clock must not advance on LTIME_MAX join" + ); +} + +/// An ingress ltime of LTIME_MAX + 5 is dropped on the user-event path. +#[test] +fn ltime_max_plus5_user_event_is_dropped() { + let mut e = ep(); + e.test_set_clocks(0, 3, 0); + let msg = UserEventMessage { + ltime: LamportTime::new(LTIME_MAX + 5), + cc: false, + name: "flood".into(), + payload: bytes::Bytes::new(), + }; + let is_new = e.test_handle_user_event(msg); + assert!(!is_new, "user event at LTIME_MAX+5 must be dropped"); + assert_eq!( + e.event_time(), + 3, + "event clock must not advance on LTIME_MAX+5 event" + ); + assert!(e.poll_event().is_none(), "no event must be emitted"); +} + +/// Integrity floor: next_ltime on a poisoned clock (LTIME_MAX) must not wrap +/// to 0 and must not produce u64::MAX via saturating_add-then-max-wrap. +/// +/// The no-UB contract: stamp = LTIME_MAX (the poisoned value as-is); stored +/// clock advances to LTIME_MAX.saturating_add(1) = LTIME_MAX + 1. Neither +/// the stamped value nor the stored clock may be 0 or u64::MAX. +#[test] +fn poisoned_clock_integrity_floor() { + // Simulate a hypothetically-poisoned clock set to LTIME_MAX. + let mut clock = LTIME_MAX; + let stamped = next_ltime(&mut clock); + // Stamped is LTIME_MAX (the poisoned value — peers will reject it via + // ltime_is_acceptable, which is the correct degraded behaviour). + // The integrity floor: no wrap to 0, no tombstone at u64::MAX. + assert_ne!(stamped, 0, "stamped must not wrap to 0; got {stamped}"); + assert_ne!( + stamped, + u64::MAX, + "stamped must not be u64::MAX tombstone; got {stamped}" + ); + // Stored clock must not wrap to 0 (no UB). + assert_ne!(clock, 0, "stored clock must not wrap to 0; got {clock}"); + assert_ne!( + clock, + u64::MAX, + "stored clock must not be u64::MAX tombstone; got {clock}" + ); +} + +/// After witnessing the max acceptable ltime, a subsequent local user_event still +/// succeeds and emits a finite sub-watermark ltime. +#[test] +fn after_high_clock_local_user_event_still_works() { + let mut e = ep(); + // Set event clock to LTIME_MAX - 2 (highest acceptable witness value). + e.test_set_event_clock(LTIME_MAX - 2); + // Witness LTIME_MAX - 2: event_clock advances to LTIME_MAX - 1. + let msg = UserEventMessage { + ltime: LamportTime::new(LTIME_MAX - 2), + cc: false, + name: "ok".into(), + payload: bytes::Bytes::new(), + }; + let _ = e.test_handle_user_event(msg); + // Now emit a local user_event via user_event() — next_ltime clamps and advances. + let result = e.user_event("local", bytes::Bytes::new(), false); + assert!(result.is_ok(), "user_event must succeed even at high clock"); +} + +/// A flood of distinct (ltime,id) queries injected via handle_query does not +/// grow received_queries past MAX_RECEIVED_QUERIES. +#[test] +fn received_queries_capped_at_max_via_handle_query() { + let mut e = ep(); + e.test_set_clocks(0, 0, 1); + + // Send MAX_RECEIVED_QUERIES + 20 unique (ltime=1, id=i) queries. + // Each passes filter (no filters = broadcast all) and unique id = first sight. + for i in 0..(MAX_RECEIVED_QUERIES + 20) as u32 { + let msg = crate::typed::QueryMessage:: { + ltime: LamportTime::new(1), + id: i, + from: memberlist_proto::Node::new(99u32, sa(9001)), + filters: vec![], + flags: QueryFlag::NO_BROADCAST, + relay_factor: 0, + timeout: std::time::Duration::from_secs(3600), // huge timeout + name: "flood".into(), + payload: bytes::Bytes::new(), + }; + let _ = e.test_handle_query(msg); + } + + assert!( + e.test_received_queries_len() <= MAX_RECEIVED_QUERIES, + "received_queries must not exceed cap after flood; got {}", + e.test_received_queries_len() + ); +} + +/// A peer-supplied huge timeout is clamped so the received_query deadline is at +/// most now + MAX_QUERY_TIMEOUT. +#[test] +fn inbound_query_timeout_clamped() { + let mut e = ep(); + e.test_set_clocks(0, 0, 1); + let now = memberlist_proto::Instant::ORIGIN; + + // Inject a query with a 1-hour timeout (far exceeds MAX_QUERY_TIMEOUT = 600s). + let msg = crate::typed::QueryMessage:: { + ltime: LamportTime::new(1), + id: 77, + from: memberlist_proto::Node::new(99u32, sa(9001)), + filters: vec![], + flags: QueryFlag::NO_BROADCAST, + relay_factor: 0, + timeout: std::time::Duration::from_secs(3600), + name: "test".into(), + payload: bytes::Bytes::new(), + }; + // drain_now is ORIGIN by default. + let _ = e.test_handle_query(msg); + + // The received_query entry must have a deadline <= now + MAX_QUERY_TIMEOUT. + let max_deadline = now + MAX_QUERY_TIMEOUT; + let all_within = e + .test_peek_received_query_deadlines() + .iter() + .all(|&dl| dl <= max_deadline); + assert!( + all_within, + "all received_query deadlines must be <= now + MAX_QUERY_TIMEOUT" + ); +} + +/// A flood of first-seen messages does not push the broadcast queue past queue_max. +#[test] +fn rebroadcast_queue_depth_capped() { + let mut e = ep(); + // Default max_queue_depth = 4096. Inject 5000 unique user events. + for i in 0u32..5000 { + let msg = UserEventMessage { + ltime: LamportTime::new(i as u64 + 1), + cc: false, + name: format!("ev{i}").into(), + payload: bytes::Bytes::new(), + }; + let _ = e.test_handle_user_event(msg); + } + let depth = e.user_broadcast_queue_len(); + let queue_max = Options::new().max_queue_depth(); + assert!( + depth <= queue_max, + "broadcast queue depth {depth} must not exceed queue_max {queue_max}" + ); +} + +// ── FIX: inbound query size limit enforced symmetrically ───────────────────── + +/// An inbound `Query` whose encoded wire size exceeds `query_size_limit` must +/// be rejected BEFORE any state mutation: no `Event::Query`, no entry in +/// `received_queries`, no rebroadcast, and no `query_clock` advance. +/// +/// Regression for the asymmetry where the local `query()` outbound path +/// enforced the limit but the inbound `UserPacket → AnyMessage::Query` path +/// did not. +#[test] +fn inbound_query_oversized_is_dropped_before_state_mutation() { + // Build an endpoint with a small query_size_limit (64 bytes) so that a + // query with a large payload reliably exceeds it. + let inner_opts = memberlist_proto::EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); + let inner = memberlist_proto::Endpoint::new_at( + inner_opts, + memberlist_proto::Instant::ORIGIN, + memberlist_proto::SmallRng::seed_from_u64(0), + ); + let opts = crate::options::Options::new().with_query_size_limit(64); + let mut e: Endpoint = Endpoint::new(inner, opts); + + // Encode a QueryMessage whose payload pushes the wire encoding over 64 bytes. + let oversized_payload = bytes::Bytes::from(vec![0u8; 100]); + let q = QueryMessage:: { + ltime: LamportTime::new(3), + id: 0xabcd_ef01, + from: memberlist_proto::Node::new(2u32, sa(9002)), + filters: vec![], + flags: QueryFlag::empty(), + relay_factor: 0, + timeout: std::time::Duration::from_secs(5), + name: "ping".into(), + payload: oversized_payload, + }; + let encoded = AnyMessage::::Query(q) + .encode() + .unwrap(); + assert!( + encoded.len() > 64, + "test precondition: encoded size ({}) must exceed limit (64)", + encoded.len() + ); + + let before_qclock = e.query_time(); + let before_rcv = e.test_received_queries_len(); + let before_queue = e.user_broadcast_queue_len(); + + e.test_inject_user_packet(sa(9002), encoded, memberlist_proto::Instant::ORIGIN); + + // No Event::Query must have been emitted. + assert!( + e.poll_event().is_none(), + "oversized inbound query must not emit Event::Query" + ); + // query_clock must not have advanced. + assert_eq!( + e.query_time(), + before_qclock, + "oversized inbound query must not advance query_clock" + ); + // received_queries must not have grown. + assert_eq!( + e.test_received_queries_len(), + before_rcv, + "oversized inbound query must not insert into received_queries" + ); + // No rebroadcast: broadcast queue must not have grown. + assert_eq!( + e.user_broadcast_queue_len(), + before_queue, + "oversized inbound query must not be rebroadcast" + ); +} + +/// A well-formed inbound `Query` within the size limit DOES advance +/// `query_clock`, insert into `received_queries`, and emit `Event::Query`. +/// +/// Regression twin: confirms the gate does not over-reject. +#[test] +fn inbound_query_within_size_limit_is_accepted() { + let mut e = ep(); // query_size_limit = 1024 (default) + + let q = QueryMessage:: { + ltime: LamportTime::new(1), + id: 0x1234_5678, + from: memberlist_proto::Node::new(2u32, sa(9002)), + filters: vec![], + flags: QueryFlag::empty(), + relay_factor: 0, + timeout: std::time::Duration::from_secs(5), + name: "ok".into(), + payload: bytes::Bytes::from_static(b"small"), + }; + let encoded = AnyMessage::::Query(q) + .encode() + .unwrap(); + assert!( + encoded.len() <= 1024, + "test precondition: encoded size ({}) must be within default limit (1024)", + encoded.len() + ); + + let before_qclock = e.query_time(); + + e.test_inject_user_packet(sa(9002), encoded, memberlist_proto::Instant::ORIGIN); + + // Event::Query must be emitted. + let ev = e.poll_event(); + assert!( + matches!(ev, Some(Event::Query(_))), + "in-limit inbound query must emit Event::Query; got {ev:?}" + ); + // query_clock must have advanced. + assert!( + e.query_time() > before_qclock, + "in-limit inbound query must advance query_clock" + ); + // received_queries must have grown. + assert_eq!( + e.test_received_queries_len(), + 1, + "in-limit inbound query must insert into received_queries" + ); +} + +/// An ACK for a non-ack PendingQuery is dropped silently (no QueryAck event emitted). +#[test] +fn ack_for_non_ack_query_is_dropped() { + let mut e = ep(); + let deadline = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(10); + + // Register a pending query WITHOUT request_ack. + let qid = QueryId { + ltime: LamportTime::new(1), + id: 42, + }; + e.pending_queries.push(PendingQuery { + kind: QueryPurpose::App, + deadline, + responses: std::collections::HashMap::new(), + acks: std::collections::HashMap::new(), + query_id: qid, + request_ack: false, + conflict_matching: 0, + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + num_nodes: 0, + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + key_tally: None, + }); + + // Inject an ACK response for that query. + let ack_msg = crate::typed::QueryResponseMessage:: { + ltime: LamportTime::new(1), + id: 42, + from: memberlist_proto::Node::new(99u32, sa(9001)), + flags: QueryFlag::ACK, + payload: bytes::Bytes::new(), + }; + e.test_handle_query_response(ack_msg); + + // No QueryAck event must have been emitted. + assert!( + e.poll_event().is_none(), + "ACK for a non-ack query must not emit Event::QueryAck" + ); +} + +/// An ACK for a request_ack=true PendingQuery IS delivered. +#[test] +fn ack_for_ack_query_is_delivered() { + let mut e = ep(); + let deadline = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(10); + + let qid = QueryId { + ltime: LamportTime::new(1), + id: 42, + }; + e.pending_queries.push(PendingQuery { + kind: QueryPurpose::App, + deadline, + responses: std::collections::HashMap::new(), + acks: std::collections::HashMap::new(), + query_id: qid, + request_ack: true, // <-- requesting acks + conflict_matching: 0, + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + num_nodes: 0, + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + key_tally: None, + }); + + // Inject an ACK response. + let ack_msg = crate::typed::QueryResponseMessage:: { + ltime: LamportTime::new(1), + id: 42, + from: memberlist_proto::Node::new(99u32, sa(9001)), + flags: QueryFlag::ACK, + payload: bytes::Bytes::new(), + }; + e.test_handle_query_response(ack_msg); + + // A QueryAck event must have been emitted. + let ev = e.poll_event(); + assert!( + matches!(ev, Some(Event::QueryAck(_))), + "ACK for a request_ack=true query must emit Event::QueryAck; got {ev:?}" + ); +} + +// ── FIX 1: exact-consumption decode gate on UserPacket ingress ──────────────── + +/// A valid UserEvent frame followed by trailing junk bytes must be dropped — +/// no event emitted, no clock advance, no rebroadcast. +/// +/// Without the exact-consumption gate the junk-padded packet passes the frame +/// decode (the framing decoder accepts the prefix), mutates clock / dedup state, +/// emits `Event::User`, and rebroadcasts the WHOLE original bytes (junk included). +/// The gate fires at the TOP of `handle_user_packet`, before any handler call. +#[test] +fn user_event_with_trailing_junk_is_dropped_no_event_no_clock_advance() { + let mut e = ep(); + + // Build a valid UserEvent frame. + let valid = AnyMessage::::UserEvent(UserEventMessage { + ltime: 7.into(), + cc: false, + name: "op".into(), + payload: bytes::Bytes::from_static(b"data"), + }) + .encode() + .unwrap(); + + // Append trailing junk so the total length exceeds the frame. + let mut padded = valid.to_vec(); + padded.extend_from_slice(b"\xde\xad\xbe\xef"); + let padded = bytes::Bytes::from(padded); + + let clock_before = e.event_time(); + let queue_before = e.user_broadcast_queue_len(); + let from: std::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); + + e.test_inject_user_packet(from, padded, memberlist_proto::Instant::ORIGIN); + + assert!( + e.poll_event().is_none(), + "trailing-junk packet must be dropped — no Event::User" + ); + assert_eq!( + e.event_time(), + clock_before, + "trailing-junk packet must not advance the event clock" + ); + assert_eq!( + e.user_broadcast_queue_len(), + queue_before, + "trailing-junk packet must not grow the rebroadcast queue" + ); +} + +/// A Join frame with trailing junk bytes is dropped — no intent buffered, no +/// member clock advance, no rebroadcast. +#[test] +fn join_intent_with_trailing_junk_is_dropped() { + let mut e = ep(); + + let valid = + AnyMessage::::Join(JoinMessage::new(LamportTime::new(5), 99u32)) + .encode() + .unwrap(); + let mut padded = valid.to_vec(); + padded.push(0xff); + let padded = bytes::Bytes::from(padded); + + let clock_before = e.member_time(); + let queue_before = e.user_broadcast_queue_len(); + let from: std::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); + + e.test_inject_user_packet(from, padded, memberlist_proto::Instant::ORIGIN); + + assert!( + e.poll_event().is_none(), + "trailing-junk join must be dropped" + ); + assert_eq!( + e.member_time(), + clock_before, + "trailing-junk join must not advance the member clock" + ); + assert_eq!( + e.user_broadcast_queue_len(), + queue_before, + "trailing-junk join must not rebroadcast" + ); + assert_eq!( + e.test_member_status(99u32), + None, + "trailing-junk join must not buffer an intent" + ); +} + +// ── FIX 1 regression: exact-consumption decode at every ingress decode site ── + +/// A PushPull frame with trailing junk bytes is dropped by `merge_remote_state` +/// — no clock witness, no intents applied, not marked dirty, no event. +#[test] +fn merge_remote_state_with_trailing_junk_is_dropped() { + use crate::typed::PushPullMessage; + + let mut e = ep(); + // Build a valid PushPull payload. + let pp = PushPullMessage:: { + ltime: LamportTime::new(7), + status_ltimes: vec![], + left_members: vec![], + event_ltime: LamportTime::new(3), + events: vec![], + query_ltime: LamportTime::new(2), + }; + let valid = AnyMessage::::PushPull(pp) + .encode() + .expect("encode must succeed"); + // Append trailing junk. + let mut padded = valid.to_vec(); + padded.push(0xde); + padded.push(0xad); + let padded = bytes::Bytes::from(padded); + + let clock_before = e.member_time(); + let dirty_before = e.test_is_dirty(); + e.test_merge_remote_state(padded, false); + + assert_eq!( + e.member_time(), + clock_before, + "trailing-junk PushPull must not advance the member clock" + ); + assert_eq!( + e.test_is_dirty(), + dirty_before, + "trailing-junk PushPull must not mark local state dirty" + ); + assert!( + e.poll_event().is_none(), + "trailing-junk PushPull must not emit any event" + ); +} + +/// A ConflictResponse payload with trailing junk bytes is NOT counted toward +/// `conflict_matching` — exact consumption required. +#[test] +fn conflict_response_with_trailing_junk_is_not_counted() { + use crate::typed::ConflictResponseMessage; + use memberlist_proto::Node; + + let mut e = ep(); + let local_addr: std::net::SocketAddr = "127.0.0.1:7946".parse().unwrap(); + let deadline = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(3600); + let qid = e.test_register_conflict_query(deadline); + + // Seed the responder as a known Alive member so the membership gate passes. + let responder_addr: std::net::SocketAddr = "127.0.0.1:2000".parse().unwrap(); + e.test_seed_member(200u32, MemberStatus::Alive, LamportTime::new(1)); + + // Encode a ConflictResponseMessage that points to the local address (would agree if counted). + let resp_msg = ConflictResponseMessage::new(Node::new(999u32, local_addr)); + let valid = AnyMessage::::ConflictResponse(resp_msg) + .encode() + .expect("encode must succeed"); + // Append trailing junk — must be dropped. + let mut padded = valid.to_vec(); + padded.push(0xff); + let padded = bytes::Bytes::from(padded); + + // Inject via handle_query_response (which routes to handle_conflict_response_fold). + let qresp = crate::typed::QueryResponseMessage:: { + ltime: qid.ltime, + id: qid.id, + from: Node::new(200u32, responder_addr), + flags: crate::typed::QueryFlag::empty(), + payload: padded, + }; + e.test_handle_query_response(qresp); + + assert_eq!( + e.test_pending_query_conflict_matching(qid), + Some(0), + "trailing-junk ConflictResponse must not increment conflict_matching" + ); +} + +/// A KeyResponse payload with trailing junk bytes is NOT folded into the tally +/// — exact consumption required. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn key_response_with_trailing_junk_is_not_tallied() { + use crate::typed::KeyResponseMessage; + use memberlist_proto::Node; + + let mut e = ep(); + let deadline = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(3600); + let qid = e.test_register_key_query(deadline); + + // Seed the responder as a known Alive member so the membership gate passes. + e.test_seed_member(300u32, MemberStatus::Alive, LamportTime::new(1)); + + // Encode a successful KeyResponseMessage. + let key_msg = KeyResponseMessage { + result: true, + message: "ok".into(), + keys: vec![], + primary_key: None, + }; + let valid = AnyMessage::::KeyResponse(key_msg) + .encode() + .expect("encode must succeed"); + // Append trailing junk — must be dropped. + let mut padded = valid.to_vec(); + padded.push(0xbe); + padded.push(0xef); + let padded = bytes::Bytes::from(padded); + + let qresp = crate::typed::QueryResponseMessage:: { + ltime: qid.ltime, + id: qid.id, + from: Node::new(300u32, "127.0.0.1:3000".parse().unwrap()), + flags: crate::typed::QueryFlag::empty(), + payload: padded, + }; + e.test_handle_query_response(qresp); + + // Trailing junk dropped → response NOT counted (num_resp stays 0). + assert_eq!( + e.test_pending_query_response_count(qid), + 0, + "trailing-junk KeyResponse must not be counted toward the tally" + ); +} + +// ── FIX 2 regression: zero-interval busy-loop ──────────────────────────────── + +/// With `reap_interval` set to zero, `handle_timeout` must advance +/// `next_reap` to a strictly later instant — no busy-loop. +#[test] +fn zero_reap_interval_advances_deadline() { + let inner_opts = + EndpointOptions::::new(1u32, "127.0.0.1:7946".parse().unwrap()) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); + let inner = memberlist_proto::Endpoint::new_at( + inner_opts, + memberlist_proto::Instant::ORIGIN, + SmallRng::seed_from_u64(0), + ); + let opts = Options::new().with_reap_interval(std::time::Duration::ZERO); + let mut e = Endpoint::new(inner, opts); + + // Advance time past the first reap deadline so handle_timeout fires it. + let now = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(1); + e.handle_timeout(now); + + // The deadline must be strictly after `now` — clamp prevents busy-loop. + if let Some(dl) = e.poll_timeout() { + assert!( + dl > now, + "zero reap_interval must still advance the deadline past now (got dl={:?}, now={:?})", + dl, + now + ); + } +} + +/// With `reconnect_interval` set to zero, `handle_timeout` must advance +/// `next_reconnect` to a strictly later instant — no busy-loop. +#[test] +fn zero_reconnect_interval_advances_deadline() { + let inner_opts = + EndpointOptions::::new(1u32, "127.0.0.1:7946".parse().unwrap()) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); + let inner = memberlist_proto::Endpoint::new_at( + inner_opts, + memberlist_proto::Instant::ORIGIN, + SmallRng::seed_from_u64(0), + ); + let opts = Options::new().with_reconnect_interval(std::time::Duration::ZERO); + let mut e = Endpoint::new(inner, opts); + + let now = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(1); + e.handle_timeout(now); + + if let Some(dl) = e.poll_timeout() { + assert!( + dl > now, + "zero reconnect_interval must still advance the deadline past now (got dl={:?}, now={:?})", + dl, + now + ); + } +} + +/// With `queue_check_interval` set to zero, `handle_timeout` must advance +/// `next_queue_check` to a strictly later instant — no busy-loop. +#[test] +fn zero_queue_check_interval_advances_deadline() { + let inner_opts = + EndpointOptions::::new(1u32, "127.0.0.1:7946".parse().unwrap()) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); + let inner = memberlist_proto::Endpoint::new_at( + inner_opts, + memberlist_proto::Instant::ORIGIN, + SmallRng::seed_from_u64(0), + ); + let opts = Options::new().with_queue_check_interval(std::time::Duration::ZERO); + let mut e = Endpoint::new(inner, opts); + + let now = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(1); + e.handle_timeout(now); + + if let Some(dl) = e.poll_timeout() { + assert!( + dl > now, + "zero queue_check_interval must still advance the deadline past now (got dl={:?}, now={:?})", + dl, + now + ); + } +} + +/// A valid UserEvent whose TOTAL packet (frame + junk) exceeds +/// `max_user_event_size` is dropped — the size budget must see the whole +/// packet, not just the decoded payload. +#[test] +fn user_event_total_packet_with_junk_exceeding_size_limit_is_dropped() { + // Set max_user_event_size to a small value so a padded packet exceeds it. + let inner_opts = memberlist_proto::EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); + let inner = memberlist_proto::Endpoint::new_at( + inner_opts, + memberlist_proto::Instant::ORIGIN, + memberlist_proto::SmallRng::seed_from_u64(0), + ); + // Set a tight size limit: 32 bytes. + let opts = crate::options::Options::new().with_max_user_event_size(32); + let mut e = super::Endpoint::new(inner, opts); + + // A small valid UserEvent that fits within 32 bytes on its own. + let valid = AnyMessage::::UserEvent(UserEventMessage { + ltime: 3.into(), + cc: false, + name: "x".into(), + payload: bytes::Bytes::from_static(b"y"), + }) + .encode() + .unwrap(); + // Pad to exceed the 32-byte limit with junk. + let mut padded = valid.to_vec(); + while padded.len() <= 32 { + padded.push(0x00); + } + let padded = bytes::Bytes::from(padded); + + let from: std::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); + e.test_inject_user_packet(from, padded, memberlist_proto::Instant::ORIGIN); + + assert!( + e.poll_event().is_none(), + "junk-padded packet exceeding size limit must be dropped" + ); +} + +// ── FIX: pre-decode size fence in handle_user_packet ───────────────────────── + +/// An over-limit but syntactically-valid `Query` frame is dropped at the +/// pre-decode size fence — before `AnyMessage::decode_with_consumed` is called. +/// +/// Asserts: no `Event::Query`, no `query_clock` advance, no insert into +/// `received_queries`, no rebroadcast. The frame is fully parseable by the +/// codec; the fence must fire on frame length alone. +#[test] +fn pre_decode_fence_drops_oversized_valid_query_frame() { + let inner_opts = memberlist_proto::EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); + let inner = memberlist_proto::Endpoint::new_at( + inner_opts, + memberlist_proto::Instant::ORIGIN, + memberlist_proto::SmallRng::seed_from_u64(0), + ); + // Small limit (64 bytes) so that a query with a 100-byte payload exceeds it. + let opts = crate::options::Options::new().with_query_size_limit(64); + let mut e: Endpoint = Endpoint::new(inner, opts); + + // Construct a syntactically valid Query whose encoded frame exceeds 64 bytes. + let q = QueryMessage:: { + ltime: LamportTime::new(7), + id: 0xdead_beef, + from: memberlist_proto::Node::new(2u32, sa(9002)), + filters: vec![], + flags: QueryFlag::empty(), + relay_factor: 0, + timeout: std::time::Duration::from_secs(5), + name: "oversized".into(), + payload: bytes::Bytes::from(vec![0xffu8; 100]), + }; + let frame = AnyMessage::::Query(q) + .encode() + .expect("encode must succeed"); + assert!( + frame.len() > 64, + "test precondition: frame ({} bytes) must exceed limit (64)", + frame.len() + ); + + let before_qclock = e.query_time(); + let before_rcv = e.test_received_queries_len(); + let before_queue = e.user_broadcast_queue_len(); + + e.test_inject_user_packet(sa(9002), frame, memberlist_proto::Instant::ORIGIN); + + assert!( + e.poll_event().is_none(), + "pre-decode fence must drop oversized valid Query — no Event::Query" + ); + assert_eq!( + e.query_time(), + before_qclock, + "pre-decode fence must not advance query_clock" + ); + assert_eq!( + e.test_received_queries_len(), + before_rcv, + "pre-decode fence must not insert into received_queries" + ); + assert_eq!( + e.user_broadcast_queue_len(), + before_queue, + "pre-decode fence must not rebroadcast" + ); +} + +/// An over-limit but syntactically-valid `UserEvent` frame is dropped at the +/// pre-decode size fence — before `AnyMessage::decode_with_consumed` is called. +/// +/// Asserts: no `Event::User`, no `event_clock` advance, no rebroadcast. +#[test] +fn pre_decode_fence_drops_oversized_valid_user_event_frame() { + let inner_opts = memberlist_proto::EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); + let inner = memberlist_proto::Endpoint::new_at( + inner_opts, + memberlist_proto::Instant::ORIGIN, + memberlist_proto::SmallRng::seed_from_u64(0), + ); + // Tight limit: 32 bytes. + let opts = crate::options::Options::new().with_max_user_event_size(32); + let mut e: Endpoint = Endpoint::new(inner, opts); + + // Construct a syntactically valid UserEvent whose encoded frame exceeds 32 bytes. + let frame = AnyMessage::::UserEvent(UserEventMessage { + ltime: 5.into(), + cc: false, + name: "big-event".into(), + payload: bytes::Bytes::from(vec![0xaau8; 60]), + }) + .encode() + .expect("encode must succeed"); + assert!( + frame.len() > 32, + "test precondition: frame ({} bytes) must exceed limit (32)", + frame.len() + ); + + let before_eclock = e.event_time(); + let before_queue = e.user_broadcast_queue_len(); + + e.test_inject_user_packet(sa(9001), frame, memberlist_proto::Instant::ORIGIN); + + assert!( + e.poll_event().is_none(), + "pre-decode fence must drop oversized valid UserEvent — no Event::User" + ); + assert_eq!( + e.event_time(), + before_eclock, + "pre-decode fence must not advance event_clock" + ); + assert_eq!( + e.user_broadcast_queue_len(), + before_queue, + "pre-decode fence must not rebroadcast" + ); +} + +// ── Pre-dedup dirty-mark regression tests ───────────────────────────────────── +// +// These guard that `local_state_dirty` is only set when a clock actually +// advanced or a buffer/member state actually changed — never on a duplicate, +// stale, or witness-only no-op. + +/// A duplicate UserEvent (same ltime + payload, second delivery) must not set +/// `local_state_dirty`. Only the first delivery should mark dirty. +#[test] +fn duplicate_user_event_does_not_set_local_state_dirty() { + let mut e = ep(); + + let msg = UserEventMessage { + ltime: LamportTime::new(3), + cc: false, + name: "ping".into(), + payload: bytes::Bytes::from_static(b"data"), + }; + + // First delivery: new event, should dirty. + let first = e.handle_user_event(msg.clone()); + assert!(first, "first delivery must be accepted as new"); + assert!( + e.test_is_dirty(), + "first delivery must set local_state_dirty" + ); + + // Clear the flag so we can observe whether the second delivery re-sets it. + e.test_clear_dirty(); + + // Second delivery of the exact same message: dedup ring returns false. + let second = e.handle_user_event(msg); + assert!(!second, "duplicate delivery must be dropped by dedup ring"); + assert!( + !e.test_is_dirty(), + "duplicate user event must NOT set local_state_dirty" + ); +} + +/// A stale JoinIntent (ltime <= status_time for a known member) must not set +/// `local_state_dirty`. +#[test] +fn stale_join_intent_does_not_set_local_state_dirty() { + let mut e = ep(); + // Seed member 2 at status_time=10. + e.test_seed_member(2u32, MemberStatus::Alive, LamportTime::new(10)); + e.test_clear_dirty(); + + // A join intent at ltime=3 is stale (3 <= 10). + let rebroadcast = + e.test_handle_join_intent(2, LamportTime::new(3), memberlist_proto::Instant::ORIGIN); + assert!(!rebroadcast, "stale join intent must not rebroadcast"); + assert!( + !e.test_is_dirty(), + "stale join intent must NOT set local_state_dirty" + ); +} + +/// A stale LeaveIntent (ltime <= status_time for a known member) must not set +/// `local_state_dirty`. +#[test] +fn stale_leave_intent_does_not_set_local_state_dirty() { + let mut e = ep(); + // Seed member 2 at status_time=10. + e.test_seed_member(2u32, MemberStatus::Alive, LamportTime::new(10)); + e.test_clear_dirty(); + + // A leave intent at ltime=3 is stale (3 <= 10). + let rebroadcast = + e.test_handle_leave_intent(2, LamportTime::new(3), memberlist_proto::Instant::ORIGIN); + assert!(!rebroadcast, "stale leave intent must not rebroadcast"); + assert!( + !e.test_is_dirty(), + "stale leave intent must NOT set local_state_dirty" + ); +} + +/// A duplicate Query (same ltime + id, second delivery) must not set +/// `local_state_dirty`. +#[test] +fn duplicate_query_does_not_set_local_state_dirty() { + let mut e = ep(); + + let msg = QueryMessage { + ltime: LamportTime::new(4), + id: 0xdead_beef, + from: memberlist_proto::Node::new( + 42u32, + "127.0.0.1:9999".parse::().unwrap(), + ), + filters: vec![], + flags: QueryFlag::empty(), + relay_factor: 0, + timeout: std::time::Duration::from_secs(1), + name: "test-query".into(), + payload: bytes::Bytes::new(), + }; + + // First delivery: new query, should dirty. + let first = e.test_handle_query(msg.clone()); + assert!(first, "first delivery must be accepted as new"); + assert!( + e.test_is_dirty(), + "first query delivery must set local_state_dirty" + ); + + // Clear the flag. + e.test_clear_dirty(); + + // Second delivery: dedup ring returns false. + let second = e.test_handle_query(msg); + assert!(!second, "duplicate query must be dropped by dedup ring"); + assert!( + !e.test_is_dirty(), + "duplicate query must NOT set local_state_dirty" + ); +} + +// ── Key-management regression tests ────────────────────────────────────────── + +/// When `received_queries` reaches `MAX_RECEIVED_QUERIES`, overflow queries are +/// DROPPED (not inserted and not surfaced) so that already-surfaced tokens +/// remain answerable via `respond_key`. +/// +/// The test injects `MAX_RECEIVED_QUERIES + 1` distinct key queries and verifies +/// that the first surfaced `Event::KeyRequest` can still be answered with +/// `respond_key` — its token was not evicted. Exactly `MAX_RECEIVED_QUERIES` +/// events are surfaced (the overflow query produces no event). +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn surfaced_token_survives_overflow_and_overflow_never_surfaces() { + use crate::{ + AnyMessage, KeyRequestMessage, + event::{Event, KeyRequestOperation, KeyResponseArgs}, + }; + use memberlist_proto::SecretKey; + + #[cfg(feature = "aes-gcm")] + let test_key = SecretKey::Aes128([0u8; 16]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let test_key = SecretKey::ChaCha20Poly1305([0u8; 32]); + + let mut e = ep(); + + // Inject MAX_RECEIVED_QUERIES + 1 well-formed key queries with distinct + // (ltime, id) pairs so each is a first-sight query (passes dedup). + // Use huge timeouts so none expire during the test. + let n = MAX_RECEIVED_QUERIES + 1; + for i in 0..n as u32 { + let req = KeyRequestMessage::new(Some(test_key)); + let payload = AnyMessage::::KeyRequest(req) + .encode() + .expect("encode KeyRequestMessage"); + let q = crate::typed::QueryMessage:: { + ltime: LamportTime::new(i as u64 + 1), + id: i, + from: memberlist_proto::Node::new(99u32, sa(9001)), + filters: vec![], + flags: QueryFlag::NO_BROADCAST, + relay_factor: 0, + timeout: std::time::Duration::from_secs(3600), + name: "_serf_install_key".into(), + payload, + }; + let _ = e.test_handle_query(q); + } + + // received_queries must be exactly at the cap (the overflow was dropped, not + // inserted). + assert_eq!( + e.test_received_queries_len(), + MAX_RECEIVED_QUERIES, + "received_queries must be exactly at cap after overflow; got {}", + e.test_received_queries_len() + ); + + // The first emitted Event::KeyRequest corresponds to the FIRST query. + // Its received_queries token must still be present (not evicted). + let first_ev = e + .poll_event() + .expect("first query must emit Event::KeyRequest"); + let first_req = match first_ev { + Event::KeyRequest(kr) => kr, + other => panic!( + "expected Event::KeyRequest for first query, got {:?}", + std::mem::discriminant(&other) + ), + }; + assert!( + matches!(first_req.op(), KeyRequestOperation::Install), + "first event must be Install" + ); + + // The first token must still be answerable (not evicted). + let now = memberlist_proto::Instant::ORIGIN; + e.respond_key( + &first_req, + KeyResponseArgs { + result: true, + message: smol_str::SmolStr::default(), + keys: vec![test_key], + primary_key: Some(test_key), + }, + now, + ) + .expect("respond_key must succeed — first surfaced token must not have been evicted"); + + // Drain remaining events: there must be exactly MAX_RECEIVED_QUERIES - 1 + // more (the overflow's event was never produced). + let mut remaining = 0usize; + while let Some(ev) = e.poll_event() { + assert!( + matches!(ev, Event::KeyRequest(_)), + "all remaining events must be KeyRequest, got {:?}", + std::mem::discriminant(&ev) + ); + remaining += 1; + } + assert_eq!( + remaining, + MAX_RECEIVED_QUERIES - 1, + "must have exactly MAX_RECEIVED_QUERIES - 1 remaining events; the overflow must not surface" + ); +} + +// ── FIX 1b regression: local key ops self-apply even when inbound cap is full ── + +/// A local `install_key` / `use_key` / `remove_key` MUST surface +/// `Event::KeyRequest` on the initiating node even when `received_queries` is +/// already at `MAX_RECEIVED_QUERIES` from inbound peer queries. +/// +/// The inbound overflow cap is a DoS defence against peer flooding; it MUST NOT +/// gate locally-originated processing (which is app-rate-limited, not +/// peer-controlled). Without this guard, a full `received_queries` caused +/// `handle_query(Local)` to return early, so the initiator never applied the key +/// op to its own keyring. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn local_key_op_self_applies_when_inbound_cap_is_full() { + use crate::{AnyMessage, KeyRequestMessage, event::KeyRequestOperation}; + use memberlist_proto::SecretKey; + + #[cfg(feature = "aes-gcm")] + let test_key = SecretKey::Aes128([1u8; 16]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let test_key = SecretKey::ChaCha20Poly1305([1u8; 32]); + + let mut e = ep(); + + // Saturate received_queries with MAX_RECEIVED_QUERIES inbound queries so + // the cap is exactly full. Use distinct (ltime, id) pairs and huge timeouts + // so none expire during the test. + for i in 0..MAX_RECEIVED_QUERIES as u32 { + let req = KeyRequestMessage::new(Some(test_key)); + let payload = AnyMessage::::KeyRequest(req) + .encode() + .expect("encode KeyRequestMessage for inbound flood"); + let q = crate::typed::QueryMessage:: { + ltime: LamportTime::new(i as u64 + 1), + id: i, + from: memberlist_proto::Node::new(99u32, sa(9001)), + filters: vec![], + flags: QueryFlag::NO_BROADCAST, + relay_factor: 0, + timeout: std::time::Duration::from_secs(3600), + name: "_serf_install_key".into(), + payload, + }; + let _ = e.test_handle_query(q); + } + + // Confirm the cap is exactly full. + assert_eq!( + e.test_received_queries_len(), + MAX_RECEIVED_QUERIES, + "received_queries must be exactly at cap before local key op" + ); + + // Drain the inbound events so we can observe the local one in isolation. + while e.poll_event().is_some() {} + + // Issue a local install_key: the initiating node MUST process its own query + // and surface Event::KeyRequest even though the inbound cap is full. + let now = memberlist_proto::Instant::ORIGIN; + e.install_key(test_key, now) + .expect("install_key must succeed"); + + let ev = e + .poll_event() + .expect("local install_key must surface Event::KeyRequest on the initiating node even when inbound cap is full"); + match ev { + Event::KeyRequest(kr) => { + assert!( + matches!(kr.op(), KeyRequestOperation::Install), + "local key op must be Install" + ); + } + other => panic!( + "expected Event::KeyRequest from local install_key, got {:?}", + std::mem::discriminant(&other) + ), + } +} + +// ── Key op-shape check ──────────────────────────────────────────────────────── + +/// `_serf_install_key`, `_serf_use_key`, `_serf_remove_key` with `key = None` +/// must be dropped before any state mutation (no `Event::KeyRequest`, no clock +/// advance, no `received_queries` entry). +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn key_must_ops_without_key_are_dropped() { + use crate::{AnyMessage, KeyRequestMessage}; + + for name in ["_serf_install_key", "_serf_use_key", "_serf_remove_key"] { + let mut e = ep(); + // Encode a KeyRequestMessage with key = None (shape mismatch for must-have-key ops). + let req = KeyRequestMessage::new(None); + let payload = AnyMessage::::KeyRequest(req) + .encode() + .expect("encode KeyRequestMessage"); + let q = crate::typed::QueryMessage:: { + ltime: LamportTime::new(1), + id: 42, + from: memberlist_proto::Node::new(99u32, sa(9001)), + filters: vec![], + flags: QueryFlag::NO_BROADCAST, + relay_factor: 0, + timeout: std::time::Duration::from_secs(5), + name: name.into(), + payload, + }; + let clock_before = e.query_time(); + let _ = e.test_handle_query(q); + assert_eq!( + e.query_time(), + clock_before, + "{name}: clock must NOT advance for key=None on a must-have-key op" + ); + assert_eq!( + e.test_received_queries_len(), + 0, + "{name}: no received_queries entry for key=None on a must-have-key op" + ); + assert!( + e.poll_event().is_none(), + "{name}: no Event::KeyRequest for key=None on a must-have-key op" + ); + } +} + +/// `_serf_list_keys` with `key = Some(..)` must be dropped before any state +/// mutation (no `Event::KeyRequest`, no clock advance, no `received_queries` +/// entry). +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn list_keys_with_key_is_dropped() { + use crate::{AnyMessage, KeyRequestMessage}; + use memberlist_proto::SecretKey; + + #[cfg(feature = "aes-gcm")] + let test_key = SecretKey::Aes128([0u8; 16]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let test_key = SecretKey::ChaCha20Poly1305([0u8; 32]); + + let mut e = ep(); + let req = KeyRequestMessage::new(Some(test_key)); + let payload = AnyMessage::::KeyRequest(req) + .encode() + .expect("encode KeyRequestMessage"); + let q = crate::typed::QueryMessage:: { + ltime: LamportTime::new(1), + id: 77, + from: memberlist_proto::Node::new(99u32, sa(9001)), + filters: vec![], + flags: QueryFlag::NO_BROADCAST, + relay_factor: 0, + timeout: std::time::Duration::from_secs(5), + name: "_serf_list_keys".into(), + payload, + }; + let clock_before = e.query_time(); + let _ = e.test_handle_query(q); + assert_eq!( + e.query_time(), + clock_before, + "_serf_list_keys with key=Some must NOT advance query clock" + ); + assert_eq!( + e.test_received_queries_len(), + 0, + "_serf_list_keys with key=Some must have no received_queries entry" + ); + assert!( + e.poll_event().is_none(), + "_serf_list_keys with key=Some must emit no Event::KeyRequest" + ); +} + +// ── merge_remote_state dirty-flag semantics ─────────────────────────────────── + +/// A push-pull whose three clocks are all <= current (no-op witness) and which +/// carries no new status_ltimes, no left_members, and no events must NOT set the +/// dirty flag. +#[test] +fn stale_push_pull_does_not_set_dirty() { + use crate::typed::PushPullMessage; + + let mut e = ep(); + // Advance the clocks above the values in the push-pull so witness is a no-op. + e.test_set_clocks(10, 10, 10); + e.test_clear_dirty(); + + // Push-pull with clocks <= current (no advance). + let pp = PushPullMessage:: { + ltime: LamportTime::new(5), // < 10 → no clock advance + event_ltime: LamportTime::new(3), // < 10 + query_ltime: LamportTime::new(2), // < 10 + status_ltimes: vec![], + left_members: vec![], + events: vec![], + }; + let encoded = AnyMessage::::PushPull(pp) + .encode() + .expect("encode PushPull"); + e.test_merge_remote_state(encoded, false); + + assert!( + !e.test_is_dirty(), + "stale push-pull (no clock advance, no intents, no events) must not set the dirty flag" + ); +} + +/// A push-pull that advances one of the three clocks DOES set the dirty flag. +#[test] +fn clock_advancing_push_pull_sets_dirty() { + use crate::typed::PushPullMessage; + + let mut e = ep(); + e.test_set_clocks(5, 5, 5); + e.test_clear_dirty(); + + // Push-pull with member clock > current (advances clock). + let pp = PushPullMessage:: { + ltime: LamportTime::new(20), // > 5 → advances member clock + event_ltime: LamportTime::new(3), // < 5 → no advance + query_ltime: LamportTime::new(2), // < 5 → no advance + status_ltimes: vec![], + left_members: vec![], + events: vec![], + }; + let encoded = AnyMessage::::PushPull(pp) + .encode() + .expect("encode PushPull"); + e.test_merge_remote_state(encoded, false); + + assert!( + e.test_is_dirty(), + "push-pull that advances the member clock must set the dirty flag" + ); +} + +// ── Tag-filter regex matching ───────────────────────────────────────────────── +// +// These tests verify that `should_process_query` honours the `Filter::Tag` +// predicate. The Go oracle (`query.go`) calls `regexp.MatchString(filt.Expr, +// tags[filt.Tag])` which is a PARTIAL (anywhere-in-value) match. With the +// `tag-regex` feature active the Rust machine matches identically; without the +// feature it degrades to exact-string equality. + +#[cfg(feature = "tag-regex")] +mod tag_filter_regex { + use super::*; + use crate::typed::{TagFilter, Tags}; + + /// Seed the local node (id=1) with the given tags and return an endpoint + /// ready for tag-filter query tests. + fn ep_with_tags(tags: Tags) -> Endpoint { + let mut e = ep(); + e.test_seed_member_with_tags(1u32, tags, MemberStatus::Alive, LamportTime::new(0)); + e + } + + fn tag_query(tag: &str, expr: Option<&str>) -> QueryMessage { + QueryMessage { + filters: vec![Filter::Tag(TagFilter { + tag: tag.into(), + expr: expr.map(Into::into), + })], + ..test_query(LamportTime::new(1), 100) + } + } + + #[test] + fn tag_filter_regex_prefix_matches() { + // A node tagged `role=webserver` must match the partial regex `^web`. + // Go: regexp.MatchString("^web", "webserver") == true. + let tags: Tags = [("role", "webserver")].into_iter().collect(); + let mut e = ep_with_tags(tags); + let q = tag_query("role", Some("^web")); + assert!( + e.test_handle_query(q), + "partial prefix regex `^web` must match `webserver`" + ); + assert!( + e.poll_event().is_some(), + "matching tag filter must surface Event::Query" + ); + } + + #[test] + fn tag_filter_regex_suffix_matches() { + // A node tagged `cloud=aws` must match the partial regex `aws$`. + let tags: Tags = [("cloud", "aws")].into_iter().collect(); + let mut e = ep_with_tags(tags); + let q = tag_query("cloud", Some("aws$")); + assert!( + e.test_handle_query(q), + "partial suffix regex `aws$` must match `aws`" + ); + assert!( + e.poll_event().is_some(), + "matching tag filter must surface Event::Query" + ); + } + + #[test] + fn tag_filter_regex_non_match_suppresses() { + // A node tagged `role=database` must NOT match `^web`. + let tags: Tags = [("role", "database")].into_iter().collect(); + let mut e = ep_with_tags(tags); + let q = tag_query("role", Some("^web")); + assert!( + e.test_handle_query(q), + "filtered query must still rebroadcast (G6)" + ); + assert!( + e.poll_event().is_none(), + "non-matching tag filter must not surface Event::Query" + ); + } + + #[test] + fn invalid_regex_tag_filter_drops_before_witness() { + // A `Filter::Tag` with an uncompilable regex pattern is malformed input. + // The query must be dropped with zero side effects — no clock advance, no + // dedup entry, no received_queries entry, no rebroadcast, no Event::Query. + // This is distinct from a valid-but-non-matching filter, which still + // rebroadcasts (see `valid_non_matching_tag_filter_still_rebroadcasts`). + let tags: Tags = [("role", "webserver")].into_iter().collect(); + let mut e = ep_with_tags(tags); + let q = tag_query("role", Some("(unclosed")); + + let clock_before = e.query_time(); + let dedup_before = e.test_query_slot_len(1); + let rcv_before = e.test_received_queries_len(); + + let rebroadcast = e.test_handle_query(q); + + assert!( + !rebroadcast, + "invalid regex: must NOT rebroadcast (dropped before any side effect)" + ); + assert_eq!( + e.query_time(), + clock_before, + "invalid regex: query_clock must NOT advance" + ); + assert_eq!( + e.test_query_slot_len(1), + dedup_before, + "invalid regex: dedup buffer must NOT gain an entry" + ); + assert_eq!( + e.test_received_queries_len(), + rcv_before, + "invalid regex: received_queries must NOT gain an entry" + ); + assert!( + e.poll_event().is_none(), + "invalid regex: no Event::Query must be emitted" + ); + } + + #[test] + fn valid_non_matching_tag_filter_still_rebroadcasts() { + // A `Filter::Tag` with a valid regex that does not match the local node's + // tag value is a filter-miss (G6): the query propagates (rebroadcast=true, + // clock witnessed, dedup entry recorded) but no Event::Query is surfaced. + let tags: Tags = [("role", "database")].into_iter().collect(); + let mut e = ep_with_tags(tags); + let q = tag_query("role", Some("^web")); // valid regex, does not match "database" + + let rebroadcast = e.test_handle_query(q); + + assert!( + rebroadcast, + "valid-but-non-matching tag filter must still rebroadcast (G6)" + ); + // Clock must have advanced (witness occurred). + assert!( + e.query_time() > 0, + "valid-but-non-matching filter: query_clock must advance" + ); + // Dedup entry must be present. + assert!( + e.test_query_slot_len(1) > 0, + "valid-but-non-matching filter: dedup buffer must record the query" + ); + // No local event. + assert!( + e.poll_event().is_none(), + "valid-but-non-matching filter: no Event::Query must be emitted" + ); + } + + #[test] + fn tag_filter_no_expr_matches_on_key_presence() { + // `expr = None` means "any node that has this tag key" regardless of value. + let tags: Tags = [("role", "anything")].into_iter().collect(); + let mut e = ep_with_tags(tags); + let q = tag_query("role", None); + assert!( + e.test_handle_query(q), + "absent expr with matching key must rebroadcast" + ); + assert!( + e.poll_event().is_some(), + "key-presence match (expr=None) must surface Event::Query" + ); + } + + // ── Local-origination invalid-regex gate ───────────────────────────────── + + #[test] + fn query_with_invalid_tag_regex_returns_err_with_no_side_effects() { + // A caller passing Filter::Tag with an uncompilable regex to query() must + // receive Err(Error::InvalidQueryFilter) before any side effect. + // Specifically: no PendingQuery is inserted, the broadcast queue does not + // grow, no Event::Query is emitted, and query_clock is unchanged. + let tags: Tags = [("role", "webserver")].into_iter().collect(); + let mut e = ep_with_tags(tags); + e.test_set_clocks(0, 0, 5); // set a known query clock + + let clock_before = e.query_time(); + let pending_before = e.test_pending_query_count(); + let queue_before = e.user_broadcast_queue_len(); + + let result = e.query( + "bad-filter", + bytes::Bytes::new(), + QueryParams { + filters: vec![Filter::Tag(TagFilter { + tag: "role".into(), + expr: Some("(unclosed".into()), + })], + ..QueryParams::default() + }, + memberlist_proto::Instant::ORIGIN, + ); + + assert!( + matches!(result, Err(Error::InvalidQueryFilter)), + "query() with invalid tag regex must return Err(Error::InvalidQueryFilter), got: {result:?}" + ); + assert_eq!( + e.query_time(), + clock_before, + "query_clock must NOT change after invalid-filter rejection" + ); + assert_eq!( + e.test_pending_query_count(), + pending_before, + "no PendingQuery must be inserted for an invalid-filter query" + ); + assert_eq!( + e.user_broadcast_queue_len(), + queue_before, + "broadcast queue must NOT grow after invalid-filter rejection" + ); + assert!( + e.poll_event().is_none(), + "no Event::Query must be emitted after invalid-filter rejection" + ); + } + + #[test] + fn query_with_valid_tag_regex_succeeds_and_broadcasts() { + // A valid tag-regex filter must not be rejected: query() succeeds, a + // PendingQuery is registered, and the encoded query appears on the broadcast + // queue. + let tags: Tags = [("role", "webserver")].into_iter().collect(); + let mut e = ep_with_tags(tags); + + let pending_before = e.test_pending_query_count(); + let queue_before = e.user_broadcast_queue_len(); + + let result = e.query( + "good-filter", + bytes::Bytes::new(), + QueryParams { + filters: vec![Filter::Tag(TagFilter { + tag: "role".into(), + expr: Some("^web".into()), + })], + ..QueryParams::default() + }, + memberlist_proto::Instant::ORIGIN, + ); + + assert!( + result.is_ok(), + "query() with valid tag regex must succeed, got: {result:?}" + ); + assert_eq!( + e.test_pending_query_count(), + pending_before + 1, + "a PendingQuery must be inserted for a valid-filter query" + ); + assert!( + e.user_broadcast_queue_len() > queue_before, + "broadcast queue must grow after a valid-filter query" + ); + } +} + +// ── Responder-side key-management (Stage 4 respond path) ───────────────────── + +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +mod key_request_responder { + use super::*; + use crate::event::{KeyRequest, KeyRequestOperation, KeyResponseArgs}; + use memberlist_proto::SecretKey; + + fn make_key_query( + name: &str, + key: Option, + ) -> crate::typed::QueryMessage { + use crate::{AnyMessage, KeyRequestMessage}; + let req = KeyRequestMessage::new(key); + let payload = AnyMessage::::KeyRequest(req) + .encode() + .expect("encode KeyRequestMessage"); + crate::typed::QueryMessage { + ltime: LamportTime::new(1), + id: 55, + from: memberlist_proto::Node::new(99u32, "127.0.0.1:9999".parse().unwrap()), + filters: vec![], + flags: crate::typed::QueryFlag::empty(), + relay_factor: 0, + timeout: std::time::Duration::from_secs(5), + name: name.into(), + payload, + } + } + + #[cfg(feature = "aes-gcm")] + fn test_key() -> SecretKey { + SecretKey::Aes128([0u8; 16]) + } + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + fn test_key() -> SecretKey { + SecretKey::ChaCha20Poly1305([0u8; 32]) + } + + #[test] + fn key_query_emits_key_request_event() { + let mut e = ep(); + // Register originator as a member so membership path is normal. + e.test_seed_member(99u32, MemberStatus::Alive, LamportTime::new(1)); + let key = test_key(); + let q = make_key_query("_serf_install_key", Some(key)); + let rebroadcast = e.test_handle_query(q); + // query_clock must have advanced. + assert!(e.query_time() > 0, "query_clock must advance"); + // A received_queries entry MUST exist (not removed — driver responds later). + assert_eq!( + e.test_received_queries_len(), + 1, + "received_queries entry must exist" + ); + // Must emit Event::KeyRequest. + let ev = e.poll_event().expect("must emit Event::KeyRequest"); + match ev { + Event::KeyRequest(kr) => { + assert!(matches!(kr.op(), KeyRequestOperation::Install)); + assert!(kr.key().is_some(), "install key request must carry a key"); + } + other => panic!( + "expected Event::KeyRequest, got {:?}", + std::mem::discriminant(&other) + ), + } + // Must not also emit Event::Query. + assert!(e.poll_event().is_none(), "no second event expected"); + let _ = rebroadcast; // rebroadcast value is valid but not the focus here + } + + #[test] + fn list_keys_query_emits_key_request_with_no_key() { + let mut e = ep(); + let q = make_key_query("_serf_list_keys", None); + e.test_handle_query(q); + let ev = e + .poll_event() + .expect("must emit Event::KeyRequest for list_keys"); + match ev { + Event::KeyRequest(kr) => { + assert!(matches!(kr.op(), KeyRequestOperation::List)); + assert!(kr.key().is_none(), "list_keys must carry no key"); + } + other => panic!( + "expected Event::KeyRequest, got {:?}", + std::mem::discriminant(&other) + ), + } + } + + #[test] + fn malformed_key_query_payload_is_dropped() { + let mut e = ep(); + // Build a query with a valid-prefix payload + trailing junk. + let key = test_key(); + let mut good_q = make_key_query("_serf_install_key", Some(key)); + let mut bad_payload = good_q.payload.to_vec(); + bad_payload.extend_from_slice(&[0xaa, 0xbb]); // trailing junk + good_q.payload = bytes::Bytes::from(bad_payload); + let clock_before = e.query_time(); + let _rb = e.test_handle_query(good_q); + assert_eq!( + e.query_time(), + clock_before, + "clock must NOT advance for malformed key query" + ); + assert_eq!( + e.test_received_queries_len(), + 0, + "no received_queries entry for malformed key query" + ); + assert!(e.poll_event().is_none(), "no event for malformed key query"); + } + + #[test] + fn key_query_never_surfaces_as_app_query() { + let mut e = ep(); + let q = make_key_query("_serf_use_key", Some(test_key())); + e.test_handle_query(q); + // Must emit KeyRequest but NOT Event::Query. + if let Some(ev) = e.poll_event() { + assert!( + !matches!(ev, Event::Query(_)), + "key query must never surface as Event::Query" + ); + } + } + + #[test] + fn respond_key_directed_sends_key_response() { + let mut e = ep(); + let key = test_key(); + let q = make_key_query("_serf_install_key", Some(key)); + e.test_handle_query(q); + // Drain the KeyRequest event to get the token. + let req = match e.poll_event().expect("must emit KeyRequest") { + Event::KeyRequest(kr) => kr, + other => panic!( + "expected KeyRequest, got {:?}", + std::mem::discriminant(&other) + ), + }; + // Verify received_queries entry exists before respond_key. + assert_eq!(e.test_received_queries_len(), 1); + // Call respond_key. + let now = memberlist_proto::Instant::ORIGIN; + e.respond_key( + &req, + KeyResponseArgs { + result: true, + message: smol_str::SmolStr::default(), + keys: vec![key], + primary_key: Some(key), + }, + now, + ) + .expect("respond_key must succeed"); + // received_queries entry must be removed after respond_key. + assert_eq!( + e.test_received_queries_len(), + 0, + "received_queries must be cleared after respond_key" + ); + // last_directed_send must be set (a QueryResponse to the originator address). + let (dest, _bytes) = e + .test_last_directed_send() + .expect("must have directed send"); + assert_eq!( + dest, + "127.0.0.1:9999".parse::().unwrap() + ); + } + + #[test] + fn key_request_event_debug_does_not_leak_key_bytes() { + let key = test_key(); + let req = KeyRequest:: { + op: KeyRequestOperation::Install, + key: Some(key), + id: 1, + ltime: LamportTime::new(1), + from: memberlist_proto::Node::new(99u32, "127.0.0.1:9999".parse().unwrap()), + relay_factor: 0, + deadline: memberlist_proto::Instant::ORIGIN, + }; + let ev = Event::::KeyRequest(req); + let debug_str = format!("{:?}", ev); + // SecretKey's Debug impl uses "" — check the raw bytes don't appear. + // For Aes128([0u8;16]) the raw bytes would be "0, 0, 0, 0". + assert!( + !debug_str.contains("0, 0, 0, 0, 0"), + "Debug output must not leak raw key bytes: {debug_str}" + ); + // The key field must contain the redaction marker, not raw bytes. + assert!( + debug_str.contains("redacted") || !debug_str.contains("key: Some("), + "Key field in Debug must show redacted: {debug_str}" + ); + } +} + +// ── Lamport watermark boundary: integrity floor ─────────────────────────────── +// +// The integrity floor: `witness` and `next_ltime` must never produce 0 (wrap) +// or u64::MAX (tombstone) from a near-watermark input. `load_snapshot` sets +// buffer floors via `saturating_add(1)`, which may produce LTIME_MAX when the +// snapshot clock is LTIME_MAX - 1; that floor is acceptable because the +// event_clock is also witnessed to LTIME_MAX - 1, so `next_ltime` stamps +// LTIME_MAX - 1 and the stamp matches or exceeds the floor. + +/// After witnessing `LTIME_MAX - 1`, `next_ltime` stamps `LTIME_MAX - 1` (the +/// current clock value) and advances the stored clock to `LTIME_MAX`. Neither +/// 0 nor u64::MAX may appear. A subsequent `resync_local_state` emits a +/// push-pull whose clocks are not 0 and not u64::MAX. +#[test] +fn next_ltime_integrity_floor_near_watermark() { + // Force all three clocks to LTIME_MAX - 1 (the largest acceptable witness). + let mut e = ep(); + e.test_set_clocks(LTIME_MAX - 1, LTIME_MAX - 1, LTIME_MAX - 1); + + // user_event() calls next_ltime(&mut self.event_clock). + // next_ltime stamps LTIME_MAX - 1 and stores LTIME_MAX. The user_event is + // emitted locally (it passes the min_time floor of 0); it then enters the + // event ring. + let result = e.user_event("probe", bytes::Bytes::new(), false); + assert!( + result.is_ok(), + "user_event must succeed with event_clock at LTIME_MAX-1: {result:?}" + ); + + // Integrity floor: stored event_clock must not be 0 (no wrap) or u64::MAX. + assert_ne!( + e.event_time(), + 0, + "stored event_clock must not wrap to 0 after next_ltime; got {}", + e.event_time() + ); + assert_ne!( + e.event_time(), + u64::MAX, + "stored event_clock must not be u64::MAX tombstone; got {}", + e.event_time() + ); + // After witnessing LTIME_MAX-1, the stored clock = LTIME_MAX (saturating). + assert_eq!( + e.event_time(), + LTIME_MAX, + "stored event_clock after witnessing LTIME_MAX-1 must be LTIME_MAX; got {}", + e.event_time() + ); + + // The leave path uses next_ltime(&mut self.clock). + let mut e2 = ep(); + e2.test_set_clocks(LTIME_MAX - 1, LTIME_MAX - 1, LTIME_MAX - 1); + let now = memberlist_proto::Instant::ORIGIN; + let _ = e2.leave(now); + // Integrity floor: no wrap, no tombstone. + assert_ne!( + e2.member_time(), + 0, + "stored member clock must not wrap to 0 after leave(); got {}", + e2.member_time() + ); + assert_ne!( + e2.member_time(), + u64::MAX, + "stored member clock must not be u64::MAX tombstone after leave(); got {}", + e2.member_time() + ); + + // resync_local_state must emit a push-pull with non-0 / non-u64::MAX clocks. + e.resync_local_state(); + let snap = e.test_inner_local_state_snapshot(); + let pp = e.test_decode_pushpull(&snap); + assert_ne!( + pp.ltime.0, 0, + "push-pull member ltime must not be 0; got {}", + pp.ltime.0 + ); + assert_ne!( + pp.event_ltime.0, 0, + "push-pull event ltime must not be 0; got {}", + pp.event_ltime.0 + ); +} + +/// `load_snapshot` with all three clocks at `LTIME_MAX - 1` must not panic, +/// not wrap any clock to 0, and not set any clock to `u64::MAX`. +/// +/// With the headroom apparatus removed, `event_buffer.min_time` is set to +/// `(LTIME_MAX - 1).saturating_add(1) == LTIME_MAX` and the stored event_clock +/// is `LTIME_MAX - 1` (from witness). A subsequent `user_event()` stamps +/// `LTIME_MAX - 1` via `next_ltime`; `handle_user_event` rejects it because +/// `!ltime_is_acceptable(LTIME_MAX - 1)` ... wait, `LTIME_MAX - 1 < LTIME_MAX` +/// is TRUE, so the event passes the ingress gate. The `next_ltime` stamp is +/// `LTIME_MAX - 1` (current event_clock = LTIME_MAX - 1), then event_clock +/// advances to LTIME_MAX. `handle_user_event` checks `ltime < min_time`: +/// `LTIME_MAX - 1 < LTIME_MAX` = true, so the event IS dropped by the floor. +/// This is the degraded-but-safe state: no panic, no crash, no UB. +/// Full functional recovery from a near-watermark snapshot is out of scope. +#[test] +fn load_snapshot_near_watermark_no_panic_integrity_floor() { + use crate::snapshot::ReplayResult; + + let mut e = ep(); + let replay = ReplayResult { + alive_nodes: vec![], + last_clock: LamportTime::new(LTIME_MAX - 1), + last_event_clock: LamportTime::new(LTIME_MAX - 1), + last_query_clock: LamportTime::new(LTIME_MAX - 1), + }; + let now = memberlist_proto::Instant::ORIGIN; + // Must not panic. + e.load_snapshot(replay, now); + + // Integrity floor: buffer floors are not 0 and not u64::MAX. + // (They will be LTIME_MAX = saturating_add(1) of LTIME_MAX - 1.) + assert_ne!( + e.test_event_min_time(), + 0, + "event_buffer.min_time must not be 0 after snapshot; got {}", + e.test_event_min_time() + ); + assert_ne!( + e.test_event_min_time(), + u64::MAX, + "event_buffer.min_time must not be u64::MAX; got {}", + e.test_event_min_time() + ); + assert_ne!( + e.test_query_min_time(), + 0, + "query_buffer.min_time must not be 0 after snapshot; got {}", + e.test_query_min_time() + ); + assert_ne!( + e.test_query_min_time(), + u64::MAX, + "query_buffer.min_time must not be u64::MAX; got {}", + e.test_query_min_time() + ); + + // Stored clocks: not 0, not u64::MAX (event_clock = LTIME_MAX - 1 from witness). + assert_ne!( + e.member_time(), + 0, + "member clock must not be 0; got {}", + e.member_time() + ); + assert_ne!( + e.member_time(), + u64::MAX, + "member clock must not be u64::MAX" + ); + assert_ne!( + e.event_time(), + 0, + "event clock must not be 0; got {}", + e.event_time() + ); + assert_ne!(e.event_time(), u64::MAX, "event clock must not be u64::MAX"); + assert_ne!( + e.query_time(), + 0, + "query clock must not be 0; got {}", + e.query_time() + ); + assert_ne!(e.query_time(), u64::MAX, "query clock must not be u64::MAX"); + + // user_event() must not panic (returns Ok even in the degraded state). + let ue_result = e.user_event("near-max", bytes::Bytes::new(), false); + assert!( + ue_result.is_ok(), + "user_event must not panic/error after near-watermark snapshot: {ue_result:?}" + ); + + // query() must not panic. + let query_result = e.query("probe", bytes::Bytes::new(), QueryParams::default(), now); + assert!( + query_result.is_ok(), + "query() must not panic/error after near-watermark snapshot: {query_result:?}" + ); +} + +/// Witnessing `LTIME_MAX - 1` (the largest acceptable ingress) must advance the +/// stored clock to `LTIME_MAX` — not 0 (no wrap) and not `u64::MAX` (no +/// tombstone). The resulting stored clock equals `LTIME_MAX` exactly +/// (`saturating_add(1)` of the witnessed value). +#[test] +fn witness_near_watermark_integrity_floor() { + let mut clock = 0u64; + // Largest acceptable ingress: LTIME_MAX - 1. + witness(&mut clock, LTIME_MAX - 1); + // The stored clock advances to LTIME_MAX (saturating_add(1) of LTIME_MAX - 1). + assert_eq!( + clock, LTIME_MAX, + "stored clock after witnessing LTIME_MAX-1 must be LTIME_MAX; got {clock}" + ); + // Integrity floor: no wrap to 0, no tombstone at u64::MAX. + assert_ne!(clock, 0, "stored clock must not wrap to 0; got {clock}"); + assert_ne!( + clock, + u64::MAX, + "stored clock must not be u64::MAX tombstone; got {clock}" + ); +} + +/// `load_snapshot` integrity floor: stored clocks must not be 0 or u64::MAX +/// for any near-watermark input; snapshot clocks at or above LTIME_MAX must be +/// rejected by the ingress gate and leave the stored clocks unchanged. +/// +/// Case 1 (LTIME_MAX - 1): `witness(LTIME_MAX - 1)` → stored clock = LTIME_MAX +/// (saturating_add(1)), which is the degraded-but-safe state. Emissions from +/// this node are rejected by peers' `ltime_is_acceptable` gate — consistent +/// with Go serf's undefined behavior at the extreme upper end of the range. +/// +/// Case 2 (LTIME_MAX - 2): stored clock = LTIME_MAX - 1; still acceptable to +/// peers. +/// +/// Case 3 (LTIME_MAX): the ingress gate rejects it; stored clocks unchanged. +#[test] +fn load_snapshot_near_watermark_integrity_floor() { + use crate::snapshot::ReplayResult; + + // ── case 1: snapshot clocks at LTIME_MAX - 1 + { + let mut e = ep(); + let replay = ReplayResult { + alive_nodes: vec![], + last_clock: LamportTime::new(LTIME_MAX - 1), + last_event_clock: LamportTime::new(LTIME_MAX - 1), + last_query_clock: LamportTime::new(LTIME_MAX - 1), + }; + e.load_snapshot(replay, memberlist_proto::Instant::ORIGIN); + + // Stored clocks = LTIME_MAX (degraded state); integrity floor: not 0, not u64::MAX. + assert_ne!( + e.member_time(), + 0, + "member clock must not be 0; got {}", + e.member_time() + ); + assert_ne!( + e.member_time(), + u64::MAX, + "member clock must not be u64::MAX" + ); + assert_ne!( + e.event_time(), + 0, + "event clock must not be 0; got {}", + e.event_time() + ); + assert_ne!(e.event_time(), u64::MAX, "event clock must not be u64::MAX"); + assert_ne!( + e.query_time(), + 0, + "query clock must not be 0; got {}", + e.query_time() + ); + assert_ne!(e.query_time(), u64::MAX, "query clock must not be u64::MAX"); + // Exact value: saturating_add(1) of LTIME_MAX - 1 = LTIME_MAX. + assert_eq!( + e.member_time(), + LTIME_MAX, + "member clock must be LTIME_MAX; got {}", + e.member_time() + ); + + // user_event must not panic. + let stamp_result = e.user_event("after-snapshot", bytes::Bytes::new(), false); + assert!( + stamp_result.is_ok(), + "user_event must not panic after LTIME_MAX-1 snapshot" + ); + } + + // ── case 2: snapshot clocks at LTIME_MAX - 2 (one step safe) + { + let mut e = ep(); + let replay = ReplayResult { + alive_nodes: vec![], + last_clock: LamportTime::new(LTIME_MAX - 2), + last_event_clock: LamportTime::new(LTIME_MAX - 2), + last_query_clock: LamportTime::new(LTIME_MAX - 2), + }; + e.load_snapshot(replay, memberlist_proto::Instant::ORIGIN); + + // stored clocks: witness(LTIME_MAX - 2) → LTIME_MAX - 1 (saturating_add(1)). + assert_eq!( + e.member_time(), + LTIME_MAX - 1, + "member clock must be LTIME_MAX-1; got {}", + e.member_time() + ); + assert_eq!( + e.event_time(), + LTIME_MAX - 1, + "event clock must be LTIME_MAX-1; got {}", + e.event_time() + ); + assert_eq!( + e.query_time(), + LTIME_MAX - 1, + "query clock must be LTIME_MAX-1; got {}", + e.query_time() + ); + // Integrity floor: not 0, not u64::MAX. + assert_ne!( + e.member_time(), + 0, + "member clock must not be 0; got {}", + e.member_time() + ); + assert_ne!( + e.member_time(), + u64::MAX, + "member clock must not be u64::MAX" + ); + } + + // ── case 3: snapshot clock at LTIME_MAX — the ingress gate rejects it. + { + let mut e = ep(); + e.test_set_clocks(5, 5, 5); + let replay = ReplayResult { + alive_nodes: vec![], + last_clock: LamportTime::new(LTIME_MAX), + last_event_clock: LamportTime::new(LTIME_MAX), + last_query_clock: LamportTime::new(LTIME_MAX), + }; + e.load_snapshot(replay, memberlist_proto::Instant::ORIGIN); + + // Clocks must not have advanced past their pre-snapshot values. + assert_eq!( + e.member_time(), + 5, + "member clock must not advance on LTIME_MAX snapshot clock" + ); + assert_eq!( + e.event_time(), + 5, + "event clock must not advance on LTIME_MAX snapshot clock" + ); + assert_eq!( + e.query_time(), + 5, + "query clock must not advance on LTIME_MAX snapshot clock" + ); + } +} + +// ── Watermark-boundary derivation: integrity floor ──────────────────────────── +// +// The G3a left-member replay derives `leave_ltime = status_ltime + 1`. When +// `status_ltime = LTIME_MAX - 1`, the derived value is `LTIME_MAX`, which is +// rejected by `handle_node_leave_intent`'s `ltime_is_acceptable` gate. The +// leave is silently skipped and the member stays Alive. This is the +// degraded-but-safe state: no panic, no wrap, no UB — consistent with Go +// serf's undefined behavior at the extreme upper end of the Lamport range. +// +// The integrity floor is: the member clock must not be 0 and not u64::MAX +// after the failed derivation. + +/// merge_remote_state with status_ltimes = [(id, LTIME_MAX - 1)] and +/// left_members = [id]: the derived leave ltime (LTIME_MAX) is rejected by +/// the ingress gate, so the leave is NOT applied and the member stays Alive. +/// No panic, no wrap — integrity floor holds. +#[test] +fn left_member_replay_at_watermark_boundary_skips_leave() { + use crate::typed::PushPullMessage; + + let mut e = ep(); + // Seed node 99 as Alive. + e.test_seed_member(99u32, MemberStatus::Alive, LamportTime::new(1)); + + // Push-pull with status_ltimes[99] = LTIME_MAX - 1 and left_members = [99]. + // The derived leave ltime = (LTIME_MAX - 1) + 1 = LTIME_MAX, rejected by + // handle_node_leave_intent's ltime_is_acceptable gate. + let pp = PushPullMessage:: { + ltime: LamportTime::new(10), + event_ltime: LamportTime::new(10), + query_ltime: LamportTime::new(10), + status_ltimes: vec![(99u32, LamportTime::new(LTIME_MAX - 1))], + left_members: vec![99u32], + events: vec![], + }; + let encoded = AnyMessage::::PushPull(pp) + .encode() + .expect("encode must succeed"); + // Must not panic. + e.test_merge_remote_state(encoded, false); + + // The leave is skipped (derived LTIME_MAX rejected); member stays Alive. + let status = e.test_member_status(99u32); + assert_eq!( + status, + Some(MemberStatus::Alive), + "node 99 must stay Alive when derived leave ltime = LTIME_MAX is rejected; got {status:?}" + ); + + // Integrity floor: member clock is not 0, not u64::MAX. + assert_ne!( + e.member_time(), + 0, + "member clock must not be 0; got {}", + e.member_time() + ); + assert_ne!( + e.member_time(), + u64::MAX, + "member clock must not be u64::MAX; got {}", + e.member_time() + ); +} + +/// load_snapshot with all three clocks at LTIME_MAX - 1 followed by +/// user_event, query, force_leave, and resync_local_state: every stored clock +/// and every push-pull emission must satisfy the integrity floor (not 0, not +/// u64::MAX, no wrap). +/// +/// After load_snapshot with last_event_clock = LTIME_MAX - 1, the stored +/// event_clock is LTIME_MAX - 1 (from witness) and event_buffer.min_time is +/// LTIME_MAX (saturating_add(1)). user_event() stamps LTIME_MAX - 1 via +/// next_ltime and the stamp equals the floor, so the event is NOT dropped. +#[test] +fn all_clock_derived_values_satisfy_integrity_floor_after_near_watermark_snapshot() { + use crate::snapshot::ReplayResult; + + let mut e = ep(); + // Seed node 99 as Alive so force_leave has a target. + e.test_seed_member(99u32, MemberStatus::Alive, LamportTime::new(1)); + + let replay = ReplayResult { + alive_nodes: vec![], + last_clock: LamportTime::new(LTIME_MAX - 1), + last_event_clock: LamportTime::new(LTIME_MAX - 1), + last_query_clock: LamportTime::new(LTIME_MAX - 1), + }; + let now = memberlist_proto::Instant::ORIGIN; + e.load_snapshot(replay, now); + + // After witnessing LTIME_MAX - 1, stored clocks = LTIME_MAX - 1. + // Integrity floor: no 0, no u64::MAX. + assert_ne!( + e.member_time(), + 0, + "member_clock must not be 0 after snapshot" + ); + assert_ne!( + e.member_time(), + u64::MAX, + "member_clock must not be u64::MAX after snapshot" + ); + assert_ne!( + e.event_time(), + 0, + "event_clock must not be 0 after snapshot" + ); + assert_ne!( + e.event_time(), + u64::MAX, + "event_clock must not be u64::MAX after snapshot" + ); + assert_ne!( + e.query_time(), + 0, + "query_clock must not be 0 after snapshot" + ); + assert_ne!( + e.query_time(), + u64::MAX, + "query_clock must not be u64::MAX after snapshot" + ); + + // Buffer floors: saturating_add(1) of LTIME_MAX - 1 = LTIME_MAX. The floor + // may equal LTIME_MAX; that is not 0 and not u64::MAX. + assert_ne!( + e.test_event_min_time(), + 0, + "event_buffer.min_time must not be 0" + ); + assert_ne!( + e.test_event_min_time(), + u64::MAX, + "event_buffer.min_time must not be u64::MAX" + ); + assert_ne!( + e.test_query_min_time(), + 0, + "query_buffer.min_time must not be 0" + ); + assert_ne!( + e.test_query_min_time(), + u64::MAX, + "query_buffer.min_time must not be u64::MAX" + ); + + // force_leave stamps clock.saturating_add(1): no UB. + let fl = e.force_leave(99u32, false, now); + assert!( + fl.is_ok(), + "force_leave must succeed after near-watermark snapshot: {fl:?}" + ); + + // resync_local_state must emit push-pull with non-0 / non-u64::MAX clocks. + e.resync_local_state(); + let snap = e.test_inner_local_state_snapshot(); + let pp = e.test_decode_pushpull(&snap); + assert_ne!( + pp.ltime.0, 0, + "push-pull member ltime must not be 0; got {}", + pp.ltime.0 + ); + assert_ne!( + pp.ltime.0, + u64::MAX, + "push-pull member ltime must not be u64::MAX; got {}", + pp.ltime.0 + ); + assert_ne!( + pp.event_ltime.0, 0, + "push-pull event ltime must not be 0; got {}", + pp.event_ltime.0 + ); + assert_ne!( + pp.event_ltime.0, + u64::MAX, + "push-pull event ltime must not be u64::MAX; got {}", + pp.event_ltime.0 + ); + // Final stored member clock: no wrap, no tombstone. + assert_ne!( + e.member_time(), + 0, + "member_clock must not be 0 after all operations" + ); + assert_ne!( + e.member_time(), + u64::MAX, + "member_clock must not be u64::MAX after all operations; got {}", + e.member_time() + ); +} + +// ── Near-watermark status_time ingress: integrity floor ─────────────────────── +// +// A join or leave intent whose ltime is LTIME_MAX - 1 passes the +// ltime_is_acceptable gate (LTIME_MAX - 1 < LTIME_MAX) and is stored directly +// as the member's status_time. The integrity floor: the stored value must not +// be 0 (no wrap) and not u64::MAX (no tombstone). A subsequent leave intent +// at a strictly greater ltime (LTIME_MAX or above) is rejected by +// ltime_is_acceptable, so the stale guard on stored ltime = LTIME_MAX - 1 is +// the effective ceiling for further intents on that member. +// +// The provable assertions after the headroom apparatus is removed: +// 1. stored status_time == the accepted ingress value (no clamping). +// 2. A leave intent strictly below the stored status_time is stale. +// 3. An organic leave intent strictly above the stored status_time applies. +// 4. No wrap (0) and no tombstone (u64::MAX) appear anywhere. + +/// A push-pull with status_ltimes at LTIME_MAX - 1 must store that value +/// directly (it passes the ingress gate). The stored status_time must not be +/// 0 or u64::MAX. A subsequent leave intent at a strictly greater ltime is +/// rejected by the gate; a leave at a ltime below the stored value is stale. +#[test] +fn push_pull_near_watermark_status_time_integrity_floor() { + use crate::typed::PushPullMessage; + + let mut e = ep(); + let now = memberlist_proto::Instant::ORIGIN; + + // LTIME_MAX - 1 passes ltime_is_acceptable (< LTIME_MAX). + let near_max = LTIME_MAX - 1; + assert!( + ltime_is_acceptable(near_max), + "LTIME_MAX-1 must pass the ingress gate" + ); + + // Part A: stored status_time == ingress value. + // Deliver a push-pull join intent for node 99 at LTIME_MAX - 1. + let pp = PushPullMessage:: { + ltime: LamportTime::new(10), + event_ltime: LamportTime::new(10), + query_ltime: LamportTime::new(10), + status_ltimes: vec![(99u32, LamportTime::new(near_max))], + left_members: vec![], + events: vec![], + }; + let encoded = AnyMessage::::PushPull(pp) + .encode() + .expect("encode must succeed"); + e.test_merge_remote_state(encoded, false); + + // The inner NodeJoined event materialises node 99. + e.test_inner_node_joined(99u32, now); + + let stored = e + .test_member_status_time(99u32) + .expect("node 99 must be in the store after NodeJoined"); + // Integrity floor: not 0, not u64::MAX. + assert_ne!( + stored.0, 0, + "stored status_time must not be 0; got {}", + stored.0 + ); + assert_ne!( + stored.0, + u64::MAX, + "stored status_time must not be u64::MAX; got {}", + stored.0 + ); + // No-clamp property: stored == ingress value. + assert_eq!( + stored.0, near_max, + "stored status_time must equal the accepted ingress value; got {}", + stored.0 + ); + + // Part B: the leave-intent path also stores the ingress value as-is. + e.test_seed_member(55u32, MemberStatus::Alive, LamportTime::new(10)); + let _leave_applied = e.test_handle_leave_intent(55u32, LamportTime::new(near_max), now); + let stored_55 = e + .test_member_status_time(55u32) + .expect("node 55 must remain in the store"); + assert_ne!( + stored_55.0, 0, + "leave-intent stored value must not be 0; got {}", + stored_55.0 + ); + assert_ne!( + stored_55.0, + u64::MAX, + "leave-intent stored value must not be u64::MAX; got {}", + stored_55.0 + ); + + // Part C: an organic leave at a low ltime (20) applies on a SEPARATE endpoint + // whose member has status_time = 10 (never witnessed the near-max value). + let mut e2 = ep(); + e2.test_seed_member(55u32, MemberStatus::Alive, LamportTime::new(10)); + let organic_20 = LamportTime::new(20); + let applied = e2.test_handle_leave_intent(55u32, organic_20, now); + assert!( + applied, + "organic leave at ltime 20 must apply to a member with status_time=10" + ); + assert_eq!( + e2.test_member_status(55u32), + Some(MemberStatus::Leaving), + "node 55 must be Leaving after a valid organic leave intent" + ); +} + +/// A direct join intent at LTIME_MAX - 1 on an unknown node must buffer a +/// recent_intent whose ltime equals the accepted ingress value (no clamping). +/// The buffered ltime must not be 0 or u64::MAX. A leave at the same ltime +/// must not supersede the existing join (equal ltime is stale). +#[test] +fn join_intent_near_watermark_buffers_ingress_value_exactly() { + let mut e = ep(); + let now = memberlist_proto::Instant::ORIGIN; + + let near_max = LTIME_MAX - 1; + assert!( + ltime_is_acceptable(near_max), + "LTIME_MAX-1 must pass the ingress gate" + ); + + // Node 77 is unknown: the intent goes into recent_intents. + let buffered = e.test_handle_join_intent(77u32, LamportTime::new(near_max), now); + assert!( + buffered, + "near-watermark join intent for unknown node must be buffered (returned true)" + ); + + // Buffered ltime == ingress value (no clamping). + let intent_ltime = e + .test_intent_ltime(77u32, IntentKind::Join) + .expect("recent_intent for node 77 must exist"); + // Integrity floor: not 0, not u64::MAX. + assert_ne!( + intent_ltime.0, 0, + "buffered intent ltime must not be 0; got {}", + intent_ltime.0 + ); + assert_ne!( + intent_ltime.0, + u64::MAX, + "buffered intent ltime must not be u64::MAX; got {}", + intent_ltime.0 + ); + assert_eq!( + intent_ltime.0, near_max, + "buffered intent ltime must equal the accepted ingress value; got {}", + intent_ltime.0 + ); + + // A leave intent at LTIME_MAX - 1 for an unknown node also buffers the + // ingress value as-is. + let buffered_leave = e.test_handle_leave_intent(88u32, LamportTime::new(near_max), now); + assert!( + buffered_leave, + "near-watermark leave intent for unknown node must be buffered" + ); + let leave_intent_ltime = e + .test_intent_ltime(88u32, IntentKind::Leave) + .expect("recent_intent for node 88 must exist"); + assert_ne!( + leave_intent_ltime.0, 0, + "buffered leave ltime must not be 0; got {}", + leave_intent_ltime.0 + ); + assert_ne!( + leave_intent_ltime.0, + u64::MAX, + "buffered leave ltime must not be u64::MAX; got {}", + leave_intent_ltime.0 + ); + + // A leave at the SAME ltime as the buffered join must not supersede it: + // equal ltime is stale per upsert_intent semantics. + let no_supersede = e.test_handle_leave_intent(77u32, LamportTime::new(near_max), now); + assert!( + !no_supersede, + "leave at ltime == existing join ltime must not supersede \ + (equal ltime is stale per upsert_intent semantics)" + ); +} + +// ── G4 near-watermark event-floor: integrity floor ──────────────────────────── +// +// A push-pull whose event_ltime is LTIME_MAX - 1 passes the +// ltime_is_acceptable gate and is stored directly as event_buffer.min_time. +// The floor is LTIME_MAX - 1 (the ingress value as-is). A local user_event() +// then stamps LTIME_MAX - 1 via next_ltime (which stamps the current +// event_clock, also LTIME_MAX - 1 after witness) — the stamp equals the +// floor, so the event is NOT dropped (handle_user_event's guard is `< +// min_time`, not `<=`). Integrity floor: min_time is not 0, not u64::MAX. + +/// merge_remote_state with event_ltime = LTIME_MAX - 1 must store that value +/// directly as event_buffer.min_time (not 0, not u64::MAX), and a subsequent +/// local user_event() must emit Event::User rather than being silently dropped. +#[test] +fn push_pull_near_watermark_event_floor_integrity_and_delivery() { + use crate::typed::PushPullMessage; + + let mut e = ep(); + // Set event_join_ignore = true so the G4 path fires on is_join = true. + e.set_event_join_ignore(true); + + // LTIME_MAX - 1 passes ltime_is_acceptable. + let near_max = LTIME_MAX - 1; + assert!( + ltime_is_acceptable(near_max), + "LTIME_MAX-1 must pass the ingress gate" + ); + + let pp = PushPullMessage:: { + ltime: LamportTime::new(10), + event_ltime: LamportTime::new(near_max), + query_ltime: LamportTime::new(10), + status_ltimes: vec![], + left_members: vec![], + events: vec![], + }; + let encoded = AnyMessage::::PushPull(pp) + .encode() + .expect("encode must succeed"); + // is_join = true triggers the G4 event_buffer.min_time update path. + e.test_merge_remote_state(encoded, true); + + // Integrity floor: min_time is the ingress value, not 0, not u64::MAX. + assert_ne!( + e.test_event_min_time(), + 0, + "event_buffer.min_time must not be 0 after near-watermark push-pull; got {}", + e.test_event_min_time() + ); + assert_ne!( + e.test_event_min_time(), + u64::MAX, + "event_buffer.min_time must not be u64::MAX; got {}", + e.test_event_min_time() + ); + + // A local user_event must succeed and emit Event::User. + // witness(event_clock, LTIME_MAX-1) → event_clock = LTIME_MAX - 1. + // next_ltime stamps LTIME_MAX - 1; handle_user_event drops if stamp < min_time + // (LTIME_MAX - 1). LTIME_MAX - 1 is NOT < LTIME_MAX - 1, so the event is kept. + e.user_event("post-join", bytes::Bytes::new(), false) + .expect("user_event must succeed after near-watermark push-pull"); + assert!( + matches!(e.poll_event(), Some(Event::User(_))), + "user_event must emit Event::User — must not be dropped by near-watermark event floor" + ); +} diff --git a/serf-proto/src/event/mod.rs b/serf-proto/src/event/mod.rs new file mode 100644 index 00000000..f7a1dbe5 --- /dev/null +++ b/serf-proto/src/event/mod.rs @@ -0,0 +1,499 @@ +//! Machine-output event types for the serf [`crate::endpoint::Endpoint`]. +//! +//! The serf `Endpoint` surfaces cluster changes, user events, queries, and +//! control signals through this enum rather than callbacks, mirroring the +//! quinn-proto / memberlist-proto pull-style event model. + +use std::sync::Arc; + +use bytes::Bytes; +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +use memberlist_proto::SecretKey; +use memberlist_proto::{Instant, Node, StreamId}; +use smol_str::SmolStr; + +use crate::{LamportTime, UserEventMessage, members::Member}; + +// ── MemberEventKind ────────────────────────────────────────────────────────── + +/// The kind of membership change carried by a [`MemberEvent`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, derive_more::IsVariant, derive_more::Display)] +#[display("{}", self.as_str())] +pub enum MemberEventKind { + /// A node joined the cluster. + Join, + /// A node gracefully left the cluster. + Leave, + /// A node was detected as failed (no graceful leave). + Failed, + /// A node's tags or metadata were updated. + Update, + /// A node was reaped from the membership store (tombstone expired). + Reap, +} + +impl MemberEventKind { + /// Returns a `'static` string representation. + pub const fn as_str(self) -> &'static str { + match self { + Self::Join => "join", + Self::Leave => "leave", + Self::Failed => "failed", + Self::Update => "update", + Self::Reap => "reap", + } + } +} + +// ── MemberEvent ─────────────────────────────────────────────────────────────── + +/// Payload for [`Event::Member`]: a batch of membership changes of the same +/// kind. +/// +/// Members are `Arc`-wrapped so the event is cheap to clone and multiple +/// consumers can inspect the same snapshot without copying. +#[derive(Debug, Clone)] +pub struct MemberEvent { + kind: MemberEventKind, + members: Arc>>, +} + +impl MemberEvent { + /// Constructs a new `MemberEvent`. + pub(crate) fn new(kind: MemberEventKind, members: Vec>) -> Self { + Self { + kind, + members: Arc::new(members), + } + } + + /// The kind of membership change. + pub const fn kind(&self) -> MemberEventKind { + self.kind + } + + /// The affected members. + pub fn members(&self) -> &[Member] { + &self.members + } +} + +// ── QueryEvent ──────────────────────────────────────────────────────────────── + +/// Payload for [`Event::Query`]: an incoming query that the application may +/// respond to. +/// +/// The response token (`id` + `from`) must be passed to +/// `Endpoint::respond_to_query` to send a reply before the `deadline`. +#[derive(Debug, Clone)] +pub struct QueryEvent { + /// Opaque query identifier used to route responses back to the originator. + pub(crate) id: u32, + /// Lamport clock time carried by the query message. + pub(crate) ltime: LamportTime, + /// The node that originated the query. + pub(crate) from: Node, + /// The query name (application-defined). + pub(crate) name: SmolStr, + /// The query payload. + pub(crate) payload: Bytes, + /// Number of relay hops requested. + pub(crate) relay_factor: u8, + /// Deadline by which a response is useful. + pub(crate) deadline: Instant, +} + +impl QueryEvent { + /// Opaque query id (the response token). + pub const fn id(&self) -> u32 { + self.id + } + + /// Lamport time of the query. + pub const fn ltime(&self) -> LamportTime { + self.ltime + } + + /// The originating node. + pub const fn from(&self) -> &Node { + &self.from + } + + /// The query name. + pub fn name(&self) -> &str { + &self.name + } + + /// The query payload bytes. + pub const fn payload(&self) -> &Bytes { + &self.payload + } + + /// The relay factor requested. + pub const fn relay_factor(&self) -> u8 { + self.relay_factor + } + + /// The deadline by which a response should be sent. + pub const fn deadline(&self) -> Instant { + self.deadline + } +} + +// ── QueryResponse ───────────────────────────────────────────────────────────── + +/// Payload for [`Event::QueryResponse`]: a response to a query this node +/// originated, folded from an inbound `QueryResponseMessage`. +#[derive(Debug, Clone)] +pub struct QueryResponse { + /// The query this response is for. + pub(crate) id: u32, + /// The node that sent this response. + pub(crate) from: Node, + /// The response payload. + pub(crate) payload: Bytes, +} + +impl QueryResponse { + /// The query id this response belongs to. + pub const fn id(&self) -> u32 { + self.id + } + + /// The node that sent this response. + pub const fn from(&self) -> &Node { + &self.from + } + + /// The response payload. + pub const fn payload(&self) -> &Bytes { + &self.payload + } +} + +// ── QueryAck ────────────────────────────────────────────────────────────────── + +/// Payload for [`Event::QueryAck`]: a delivery acknowledgement for a query this +/// node originated, from a peer that received the query and passed its filters. +/// +/// Emitted only for queries issued with `request_ack`. An ack carries no +/// payload — it confirms receipt, distinct from a [`QueryResponse`] which +/// carries the responder's reply. A single peer may emit both: an ack on +/// receipt and, later, a response if the application calls `respond`. Mirrors +/// the per-query `ack_ch` channel in serf-core `query.rs`. +#[derive(Debug, Clone)] +pub struct QueryAck { + /// The query this ack is for. + pub(crate) id: u32, + /// The node that acknowledged the query. + pub(crate) from: Node, +} + +impl QueryAck { + /// The query id this ack belongs to. + pub const fn id(&self) -> u32 { + self.id + } + + /// The node that acknowledged the query. + pub const fn from(&self) -> &Node { + &self.from + } +} + +// ── DialPassthrough ─────────────────────────────────────────────────────────── + +/// Payload for [`Event::DialRequested`]: the inner memberlist Endpoint asks the +/// driver to open a TCP/QUIC stream to `peer`. +/// +/// The driver calls `inner.dial_succeeded(id, now)` on success or +/// `inner.dial_failed(id, err, now)` on failure. The `StreamId` is opaque to +/// serf and is forwarded verbatim from the inner [`memberlist_proto::DialRequested`]. +/// +/// serf emits this via `start_push_pull(addr, PushPullKind::Join, now)` in the +/// reconnector; the inner machine queues the `Event::DialRequested` and the +/// sieve passes it through as this type. H3: serf does NO dial itself. +#[derive(Debug, Clone)] +pub struct DialPassthrough { + /// Opaque stream id the driver reports back to the inner endpoint. + id: StreamId, + /// The peer address to dial. + peer: A, +} + +impl DialPassthrough { + /// Construct a new passthrough payload. + pub(crate) fn new(id: StreamId, peer: A) -> Self { + Self { id, peer } + } + + /// The stream id (report back via `inner.dial_succeeded` / `dial_failed`). + pub const fn id(&self) -> StreamId { + self.id + } + + /// The peer address to dial. + pub const fn peer(&self) -> &A { + &self.peer + } +} + +// ── RelayDropped ────────────────────────────────────────────────────────────── + +/// Payload for [`Event::RelayDropped`]: a relay forward could not be delivered. +/// +/// A relay is always directed (`send_user_packet`), never re-broadcast. If the +/// directed send fails, the machine emits this event instead of silently +/// discarding the failure. +#[derive(Debug, Clone)] +pub struct RelayDropped { + /// The destination address the relay was directed to. + pub(crate) destination: A, +} + +impl RelayDropped { + /// The destination that could not be reached. + pub const fn destination(&self) -> &A { + &self.destination + } +} + +// ── KeyResponse ─────────────────────────────────────────────────────────────── + +/// Aggregated result of a cluster-wide key-management query. +/// +/// Produced after a `list_keys`, `install_key`, `use_key`, or `remove_key` +/// query's deadline fires. All per-node `KeyResponseMessage`s received before +/// the deadline are folded into this summary. +/// +/// Requires the `aes-gcm` or `chacha20-poly1305` feature. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) +)] +#[derive(Debug, Clone)] +pub struct KeyResponse { + /// Total number of nodes queried. + pub num_nodes: usize, + /// Number that responded before the deadline. + pub num_resp: usize, + /// Number with `result = false` (error) in their response. + pub num_err: usize, + /// Key → count of nodes that have that key installed. + pub keys: std::collections::HashMap, + /// Primary key → count of nodes using it as primary. + pub primary_keys: std::collections::HashMap, + /// Per-node error messages (only for nodes with `result = false`). + pub messages: std::collections::HashMap, +} + +// ── KeyRequestOperation ─────────────────────────────────────────────────────── + +/// The key-management operation requested by a [`KeyRequest`]. +/// +/// Requires the `aes-gcm` or `chacha20-poly1305` feature. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) +)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum KeyRequestOperation { + /// Install a new key into the keyring. + Install, + /// Promote a key to primary. + Use, + /// Remove a key from the keyring. + Remove, + /// List all installed keys. + List, +} + +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +impl KeyRequestOperation { + /// Returns a `'static` string representation of the operation. + pub const fn as_str(self) -> &'static str { + match self { + Self::Install => "install", + Self::Use => "use", + Self::Remove => "remove", + Self::List => "list", + } + } + + /// Returns `true` if the operation carries a key (all except [`List`](Self::List)). + pub const fn has_key(self) -> bool { + !matches!(self, Self::List) + } +} + +// ── KeyRequest ──────────────────────────────────────────────────────────────── + +/// An inbound key-management request that the driver must apply to its keyring +/// and then answer via [`crate::endpoint::Endpoint::respond_key`]. +/// +/// The machine emits this event instead of `Event::Query` for the four internal +/// key query names (`_serf_install_key`, `_serf_use_key`, `_serf_remove_key`, +/// `_serf_list_keys`). The machine never holds or inspects live key material +/// beyond routing the already-typed [`memberlist_proto::SecretKey`] to the +/// driver. +/// +/// Requires the `aes-gcm` or `chacha20-poly1305` feature. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) +)] +#[derive(Debug, Clone)] +pub struct KeyRequest { + /// The key-management operation. + pub(crate) op: KeyRequestOperation, + /// The key, if the operation requires one (`None` for [`KeyRequestOperation::List`]). + /// + /// `SecretKey`'s `Debug` implementation redacts key bytes; the derived `Debug` + /// on `KeyRequest` therefore does NOT leak raw key material. + pub(crate) key: Option, + /// Opaque query identifier (used by `respond_key` to route the response). + pub(crate) id: u32, + /// Lamport clock time of the originating query. + pub(crate) ltime: LamportTime, + /// The node that originated the query. + pub(crate) from: Node, + /// Number of relay hops requested by the originator. + pub(crate) relay_factor: u8, + /// Deadline by which a response is useful. + pub(crate) deadline: Instant, +} + +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +impl KeyRequest { + /// The key-management operation. + pub const fn op(&self) -> KeyRequestOperation { + self.op + } + + /// The key, if the operation carries one. + pub fn key(&self) -> Option<&SecretKey> { + self.key.as_ref() + } + + /// The node that originated the query. + pub const fn from(&self) -> &Node { + &self.from + } + + /// Opaque query id (forwarded to `Endpoint::respond_key`). + pub const fn id(&self) -> u32 { + self.id + } + + /// Deadline by which a response should be sent. + pub const fn deadline(&self) -> Instant { + self.deadline + } +} + +// ── KeyResponseArgs ─────────────────────────────────────────────────────────── + +/// The driver's per-node answer to a [`KeyRequest`]. +/// +/// This is a projection type for the driver to fill in; it is NOT the wire +/// `KeyResponseMessage` (which stays `pub(crate)`). The driver applies the +/// requested operation to its keyring, then calls +/// [`crate::endpoint::Endpoint::respond_key`] with the result. +/// +/// For `install`, `use`, and `remove` operations, `keys` and `primary_key` +/// are typically left empty; for `list`, the driver fills `keys` with all +/// installed keys and `primary_key` with the current primary. +/// +/// Requires the `aes-gcm` or `chacha20-poly1305` feature. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) +)] +#[derive(Debug, Default, Clone)] +pub struct KeyResponseArgs { + /// `true` if the operation succeeded on this node. + pub result: bool, + /// Human-readable result or error description. + pub message: smol_str::SmolStr, + /// Installed keys (used for `list` responses). + pub keys: Vec, + /// The current primary key, if reporting it. + pub primary_key: Option, +} + +// ── Event ───────────────────────────────────────────────────────────────────── + +/// Machine-output events produced by the serf [`crate::endpoint::Endpoint`]. +/// +/// The driver calls `poll_event()` in a loop and dispatches on these variants. +/// The enum is `#[non_exhaustive]` so that adding new variants is a +/// backwards-compatible change (drivers must include a `_ => {}` arm). +#[derive(Debug, Clone, derive_more::IsVariant)] +#[non_exhaustive] +pub enum Event { + /// A batch of membership changes (join, leave, failed, update, or reap). + Member(MemberEvent), + /// An application-level user event was received from the cluster. + User(UserEventMessage), + /// An incoming query that this node should handle (and optionally respond to). + Query(QueryEvent), + /// A response to a query this node originated arrived from a peer. + QueryResponse(QueryResponse), + /// A delivery acknowledgement for a query this node originated (only emitted + /// for queries issued with `request_ack`). + QueryAck(QueryAck), + /// The local node lost an id-conflict vote and must shut down. + /// + /// The machine emits this signal; the driver is responsible for actually + /// stopping the process or reinitialising — the pure machine never exits. + /// Corresponds to Go serf `serf.go` conflict-resolution shutdown path. + Shutdown, + /// A responder-side relay forward could not be delivered to its destination. + RelayDropped(RelayDropped), + /// The graceful leave chain completed; the node has fully left the cluster. + /// + /// Emitted after the inner memberlist `LeftCluster` event is received and + /// the `leave_propagate_delay` has elapsed. + LeftCluster, + /// The inner memberlist Endpoint requests that the driver open a stream to + /// the given peer. + /// + /// Produced when the reconnector calls `inner.start_push_pull()` and the + /// inner queues its own `Event::DialRequested`. The driver must dial `peer` + /// and report back via `inner.dial_succeeded` / `inner.dial_failed`. + /// H3: serf does NO I/O itself — this is machine output, not machine I/O. + DialRequested(DialPassthrough), + /// Aggregated key-management query results. + /// + /// Emitted after an `install_key`, `use_key`, `remove_key`, or `list_keys` + /// query closes (deadline fires). The driver uses this to surface results + /// to the operator. + /// + /// Requires the `aes-gcm` or `chacha20-poly1305` feature. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + KeyResponse(KeyResponse), + /// An inbound key-management request that this node must handle. + /// + /// The driver applies the requested operation to its keyring and then calls + /// `Endpoint::respond_key` with the result. The machine never holds or + /// inspects live key material — it only routes the typed fields. + /// + /// Requires the `aes-gcm` or `chacha20-poly1305` feature. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + KeyRequest(KeyRequest), +} + +#[cfg(test)] +mod tests; diff --git a/serf-proto/src/event/tests.rs b/serf-proto/src/event/tests.rs new file mode 100644 index 00000000..fa13dadd --- /dev/null +++ b/serf-proto/src/event/tests.rs @@ -0,0 +1,33 @@ +use super::*; + +#[test] +fn member_event_kind_as_str_round_trips() { + assert_eq!(MemberEventKind::Join.as_str(), "join"); + assert_eq!(MemberEventKind::Leave.as_str(), "leave"); + assert_eq!(MemberEventKind::Failed.as_str(), "failed"); + assert_eq!(MemberEventKind::Update.as_str(), "update"); + assert_eq!(MemberEventKind::Reap.as_str(), "reap"); +} + +#[test] +fn member_event_kind_display_matches_as_str() { + for kind in [ + MemberEventKind::Join, + MemberEventKind::Leave, + MemberEventKind::Failed, + MemberEventKind::Update, + MemberEventKind::Reap, + ] { + assert_eq!(kind.to_string(), kind.as_str()); + } +} + +#[test] +fn event_is_variant_helpers() { + use std::net::SocketAddr; + + let ev: Event = Event::Shutdown; + assert!(ev.is_shutdown()); + let ev: Event = Event::LeftCluster; + assert!(ev.is_left_cluster()); +} diff --git a/serf-proto/src/framing/mod.rs b/serf-proto/src/framing/mod.rs index a05e2b72..36ed7d29 100644 --- a/serf-proto/src/framing/mod.rs +++ b/serf-proto/src/framing/mod.rs @@ -252,6 +252,43 @@ where Ok(out) } +/// Peek the header of the leading plain frame in `buf` without allocating the body. +/// +/// Returns `(MessageType, total_frame_len)` on success, where `total_frame_len` +/// is the full encoded length of the frame (tag byte + varint + body). The body +/// bytes are NOT extracted — the caller can use this to apply a cheap size gate +/// before doing a full decode. +/// +/// # Errors +/// +/// Returns the same [`FrameError`] variants as [`decode_message`] — `Empty`, +/// `Incomplete`, or `VarintOverflow`. +pub(crate) fn peek_frame_header(buf: &[u8]) -> Result<(MessageType, usize), FrameError> { + if buf.is_empty() { + return Err(FrameError::Empty); + } + + let ty = MessageType::from(buf[0]); + + let (body_len, varint_bytes) = match decode_varint_u32(&buf[1..]) { + Ok(v) => v, + Err(FrameError::Incomplete(_)) => { + return Err(FrameError::Incomplete(IncompleteFrame::new( + buf.len(), + buf.len() + 1, + ))); + } + Err(e) => return Err(e), + }; + + let header_len = 1 + varint_bytes; + let frame_end = header_len + .checked_add(body_len as usize) + .ok_or(FrameError::VarintOverflow)?; + + Ok((ty, frame_end)) +} + /// Decode the leading plain frame from `buf`. /// /// Returns `(MessageType, body_bytes, bytes_consumed)` on success. diff --git a/serf-proto/src/lib.rs b/serf-proto/src/lib.rs index f2fdd302..d99d32c3 100644 --- a/serf-proto/src/lib.rs +++ b/serf-proto/src/lib.rs @@ -1,23 +1,22 @@ -//! The serf wire codec — pure, no-I/O message types shared by the serf driver crates. +//! The serf wire codec and Sans-I/O state machine — pure, no-I/O types shared +//! by the serf driver crates. //! //! Depends on `memberlist-proto` for the `Data`/`DataRef` codec primitives; defines serf's //! own message set and framing on top of them. #![deny(missing_docs)] -pub use any::{AnyMessage, DecodeError, EncodeError}; -pub use bridge::BridgeError; -pub use framing::{FrameError, IncompleteFrame, MessageType}; +pub(crate) use any::{AnyMessage, EncodeError}; +pub(crate) use bridge::BridgeError; +pub(crate) use framing::{FrameError, MessageType}; +pub(crate) use typed::{ + ConflictResponseMessage, JoinMessage, LeaveMessage, PushPullMessage, RelayMessage, +}; pub use typed::{ - ConflictResponseMessage, Coordinate, Filter, JoinMessage, LeaveMessage, PushPullMessage, - QueryFlag, QueryMessage, QueryResponseMessage, RelayMessage, TagFilter, Tags, UserEvent, + Coordinate, Filter, QueryFlag, QueryMessage, QueryResponseMessage, TagFilter, Tags, UserEvent, UserEventMessage, UserEvents, }; #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] -#[cfg_attr( - docsrs, - doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) -)] -pub use typed::{KeyRequestMessage, KeyResponseMessage}; +pub(crate) use typed::{KeyRequestMessage, KeyResponseMessage}; /// A lamport logical clock value — a monotonically increasing counter used to /// order serf events. @@ -53,8 +52,40 @@ impl LamportTime { } } -pub mod any; +pub(crate) mod any; pub(crate) mod bridge; pub(crate) mod framing; pub(crate) mod messages; pub mod typed; + +#[cfg(feature = "coordinates")] +mod coordinate_client; +#[cfg(feature = "coordinates")] +pub use coordinate_client::{ + CoordinateClient, CoordinateClientStats, CoordinateError, CoordinateOptions, +}; + +pub mod endpoint; +pub mod event; +pub mod members; +pub mod options; +pub mod snapshot; + +#[cfg(feature = "coordinates")] +#[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] +pub use snapshot::CoordinateRecord; +pub use snapshot::{ReplayResult, SnapshotError, SnapshotRecord}; + +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) +)] +pub use event::{KeyRequest, KeyRequestOperation, KeyResponseArgs}; + +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) +)] +pub use memberlist_proto::SecretKey; diff --git a/serf-proto/src/members/mod.rs b/serf-proto/src/members/mod.rs new file mode 100644 index 00000000..d16df5a0 --- /dev/null +++ b/serf-proto/src/members/mod.rs @@ -0,0 +1,404 @@ +//! Membership types for the serf state machine. +//! +//! [`MemberStatus`] is the per-node status within a serf cluster. +//! [`SerfState`] is the lifecycle state of the local serf endpoint (distinct +//! from any single node's status). [`MemberState`] pairs a [`Member`] with +//! its lamport clock timestamp and optional leave wall-time. [`Members`] is +//! the in-memory store that the endpoint mutates as intents and inner +//! memberlist events arrive. +//! +//! Design: `MemberState.leave_time` and `NodeIntent.wall_time` carry a +//! `memberlist_proto::Instant` threaded in from the driver — no wall-clock +//! reads occur inside the pure machine. + +use std::collections::HashMap; + +/// Hard cardinality cap on `Members::recent_intents`. +/// +/// Unlike `states`, `left_members`, and `failed_members` (which are bounded by +/// the real cluster membership), `recent_intents` holds intents for nodes whose +/// inner memberlist `NodeJoined`/`NodeLeft` events have not yet arrived — and +/// those nodes may be entirely unknown to the local machine. A flooder can +/// therefore grow `recent_intents` without bound by sending join/leave intents +/// for an unlimited stream of distinct unknown ids. +/// +/// When the cap is reached the entry with the oldest `wall_time` is evicted +/// before inserting the new one. Oldest-first eviction preserves the most +/// recently observed intents, which are the ones most likely to still be +/// relevant when the corresponding inner event fires. Normal clusters will +/// never approach this limit; it is only reachable under adversarial flood. +pub(crate) const MAX_RECENT_INTENTS: usize = 8192; + +use memberlist_proto::Instant; + +use crate::LamportTime; + +// ── MemberStatus ───────────────────────────────────────────────────────────── + +/// The status of a node in the serf cluster. +/// +/// Variants mirror the Go serf `MemberStatus` constants. `None` is the +/// zero/default and indicates the node has never been seen. +#[derive( + Debug, Clone, Copy, PartialEq, Eq, Hash, Default, derive_more::IsVariant, derive_more::Display, +)] +#[display("{}", self.as_str())] +pub enum MemberStatus { + /// No status (zero value). + #[default] + None, + /// Node is alive and participating in the cluster. + Alive, + /// Node has announced that it is leaving. + Leaving, + /// Node has completed a graceful leave. + Left, + /// Node appears to have failed (no graceful leave observed). + Failed, +} + +impl MemberStatus { + /// Returns a `'static` string representation of the status. + /// + /// Consistent with Go serf's `String()` on `MemberStatus`. + pub const fn as_str(self) -> &'static str { + match self { + Self::None => "none", + Self::Alive => "alive", + Self::Leaving => "leaving", + Self::Left => "left", + Self::Failed => "failed", + } + } +} + +// ── SerfState ───────────────────────────────────────────────────────────────── + +/// The lifecycle state of the local serf endpoint. +/// +/// Distinct from any single node's [`MemberStatus`]. Transitions: +/// `Alive → Leaving → Left` (graceful) or `Alive/Leaving → Shutdown` (forced). +/// Modelled after `SerfState` in Go serf `base.go`. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, derive_more::IsVariant, derive_more::Display)] +#[display("{}", self.as_str())] +pub enum SerfState { + /// Endpoint is alive and participating in the cluster. + Alive, + /// A graceful leave has been initiated but not yet completed. + Leaving, + /// The endpoint has fully left the cluster. + Left, + /// The endpoint has been shut down (abnormal or forced). + Shutdown, +} + +impl SerfState { + /// Returns a `'static` string representation of the state. + pub const fn as_str(self) -> &'static str { + match self { + Self::Alive => "alive", + Self::Leaving => "leaving", + Self::Left => "left", + Self::Shutdown => "shutdown", + } + } +} + +// ── Member ──────────────────────────────────────────────────────────────────── + +/// A single member of the serf cluster as seen by the local node. +/// +/// Holds the node identity + address, the latest advertised tags, and the +/// current [`MemberStatus`]. +#[derive(Debug, Clone)] +pub struct Member { + /// The underlying memberlist node (id + address). + node: memberlist_proto::Node, + /// Key/value metadata advertised by this node. + tags: crate::Tags, + /// Current cluster status of this node. + status: MemberStatus, +} + +impl Member { + /// Constructs a new `Member`. + pub fn new(node: memberlist_proto::Node, tags: crate::Tags, status: MemberStatus) -> Self { + Self { node, tags, status } + } + + /// The underlying memberlist node. + pub const fn node(&self) -> &memberlist_proto::Node { + &self.node + } + + /// The advertised tags. + pub const fn tags(&self) -> &crate::Tags { + &self.tags + } + + /// The current status of this node. + pub const fn status(&self) -> MemberStatus { + self.status + } +} + +// ── MemberState ─────────────────────────────────────────────────────────────── + +/// Tracks the full per-node state used by the serf membership FSM. +/// +/// `status_time` is the lamport clock value at which the last status change +/// was witnessed. `leave_time` is the driver-threaded `Instant` at which the +/// node was observed leaving or failing; it is `None` while the node is alive. +#[derive(Debug, Clone)] +pub struct MemberState { + member: Member, + /// Lamport clock time of last received status-change message. + status_time: LamportTime, + /// Wall-clock (driver-threaded `Instant`) at which the leave/failure was + /// observed. `None` while the node is alive. + leave_time: Option, +} + +impl MemberState { + /// Constructs a new `MemberState`. + pub fn new(member: Member, status_time: LamportTime, leave_time: Option) -> Self { + Self { + member, + status_time, + leave_time, + } + } + + /// The underlying `Member`. + pub const fn member(&self) -> &Member { + &self.member + } + + /// Mutable access to the underlying `Member` (used by the FSM to update tags/status). + pub fn member_mut(&mut self) -> &mut Member { + &mut self.member + } + + /// Current status (shortcut accessor). + pub const fn status(&self) -> MemberStatus { + self.member.status + } + + /// Lamport clock time of the last status change. + pub const fn status_time(&self) -> LamportTime { + self.status_time + } + + /// Sets the status_time. + pub fn set_status_time(&mut self, t: LamportTime) { + self.status_time = t; + } + + /// The leave time, if the node is no longer alive. + pub const fn leave_time(&self) -> Option { + self.leave_time + } + + /// Sets the leave time. + pub fn set_leave_time(&mut self, t: Option) { + self.leave_time = t; + } + + /// Sets the status on the inner Member. + pub fn set_status(&mut self, s: MemberStatus) { + self.member.status = s; + } +} + +// ── IntentKind + NodeIntent ─────────────────────────────────────────────────── + +/// Discriminates between join and leave intents buffered in `recent_intents`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IntentKind { + /// A join intent (node announcing it is joining). + Join, + /// A leave intent (node announcing it is leaving). + Leave, +} + +/// A buffered join/leave intent for a node whose inner memberlist event has +/// not yet arrived. +/// +/// Intents arrive over gossip before the underlying memberlist `NodeJoined` / +/// `NodeLeft` events; `recent_intents` holds the newest one per node so the +/// FSM can apply it when the inner event eventually fires. +/// +/// `wall_time` is a driver-threaded `Instant` — no wall-clock reads in the +/// machine (Go serf used `time.Now()` here; the Sans-I/O design threads it in). +/// +/// `sequence` is a monotonically increasing counter assigned by `Members` at +/// insertion/update time (see `Members::recent_intent_seq`). It breaks ties in +/// the cap-eviction comparator when two entries share the same `wall_time` so +/// that eviction order is deterministic regardless of `HashMap` iteration order. +/// Oldest-inserted entry (smallest sequence) is evicted first. +#[derive(Debug, Clone, Copy)] +pub struct NodeIntent { + kind: IntentKind, + ltime: LamportTime, + /// Driver-threaded instant at which this intent was received. + wall_time: Instant, + /// Monotonic insertion counter — breaks `wall_time` ties in eviction. + /// Assigned by `Members::upsert_intent`; NOT drawn from the RNG. + sequence: u64, +} + +impl NodeIntent { + /// Constructs a new `NodeIntent`. + pub fn new(kind: IntentKind, ltime: LamportTime, wall_time: Instant, sequence: u64) -> Self { + Self { + kind, + ltime, + wall_time, + sequence, + } + } + + /// The kind of intent (join or leave). + pub const fn kind(&self) -> IntentKind { + self.kind + } + + /// The lamport clock value carried by this intent. + pub const fn ltime(&self) -> LamportTime { + self.ltime + } + + /// The driver-threaded instant at which this intent was received. + pub const fn wall_time(&self) -> Instant { + self.wall_time + } + + /// The monotonic insertion sequence number (for deterministic tie-breaking in eviction). + pub const fn sequence(&self) -> u64 { + self.sequence + } +} + +// ── Members ─────────────────────────────────────────────────────────────────── + +/// The in-memory membership store for the serf endpoint. +/// +/// Mutated by the FSM handlers (`handle_node_join`, `handle_node_leave`, etc.) +/// and the intent reconciler. All lookups are by node id `I`. +/// +/// `left_members` and `failed_members` are index lists of ids for the reaper; +/// the full state lives in `states`. +pub(crate) struct Members +where + I: Eq + core::hash::Hash, +{ + /// Full state for every known node (alive, leaving, left, or failed). + pub(crate) states: HashMap>, + /// Buffered join/leave intents whose inner memberlist event has not yet + /// arrived. Newest ltime wins (upsert_intent). + pub(crate) recent_intents: HashMap, + /// Ids of nodes in the `Left` state, for tombstone reaping. + pub(crate) left_members: Vec, + /// Ids of nodes in the `Failed` state, for reconnect and reaping. + pub(crate) failed_members: Vec, + /// Monotonically increasing counter incremented on every `recent_intents` + /// insert or update. Assigned to `NodeIntent::sequence` so that cap-eviction + /// breaks `wall_time` ties deterministically (oldest-inserted, smallest + /// sequence, evicted first) regardless of `HashMap` iteration order. + /// This is a plain counter — NOT derived from the RNG. + pub(crate) recent_intent_seq: u64, +} + +impl Default for Members +where + I: Eq + core::hash::Hash, +{ + fn default() -> Self { + Self { + states: HashMap::default(), + recent_intents: HashMap::default(), + left_members: Vec::new(), + failed_members: Vec::new(), + recent_intent_seq: 0, + } + } +} + +impl Members +where + I: Eq + core::hash::Hash + Clone, +{ + /// Returns the most recent intent for a node + kind, if any. + /// + /// Used by `handle_node_join` to check whether a Leave intent arrived before + /// the inner `NodeJoined` event (Go serf `base.go` `getRecentIntent`). + pub(crate) fn recent_intent(&self, id: &I, kind: IntentKind) -> Option { + self + .recent_intents + .get(id) + .filter(|i| i.kind == kind) + .map(|i| i.ltime) + } + + /// Inserts or updates the intent for a node, keeping the newest ltime. + /// + /// Returns `true` if the intent was inserted or updated (i.e. it was newer). + /// Mirrors Go serf `base.go` `upsertIntent`. + /// + /// **Bounded memory**: when the map is at `MAX_RECENT_INTENTS` and a new + /// entry would be inserted, the entry with the oldest `(wall_time, sequence)` + /// is evicted first. The composite key breaks `wall_time` ties by insertion + /// order so eviction is deterministic regardless of `HashMap` iteration order. + /// Oldest-inserted (smallest sequence) is evicted first among equal timestamps. + /// + /// Every insert AND every update (refresh of an existing node's intent) + /// assigns a fresh sequence number so a re-observed intent is treated as + /// "more recent" and won't be the next eviction candidate. + pub(crate) fn upsert_intent( + &mut self, + id: &I, + kind: IntentKind, + ltime: LamportTime, + wall_time: Instant, + ) -> bool { + match self.recent_intents.get_mut(id) { + Some(existing) if ltime <= existing.ltime => false, + Some(existing) => { + existing.kind = kind; + existing.ltime = ltime; + existing.wall_time = wall_time; + existing.sequence = self.recent_intent_seq; + self.recent_intent_seq += 1; + true + } + None => { + // Enforce the cap before inserting a new entry. Evict the entry with + // the smallest (wall_time, sequence) to preserve the most recently + // observed intents — those are most likely to still be relevant when + // their corresponding inner memberlist event arrives. The sequence + // tie-break makes eviction order deterministic regardless of HashMap + // iteration order (two entries with identical wall_time always evict + // the older-inserted one first). + if self.recent_intents.len() >= MAX_RECENT_INTENTS { + if let Some(oldest_id) = self + .recent_intents + .iter() + .min_by_key(|(_, intent)| (intent.wall_time(), intent.sequence())) + .map(|(k, _)| k.clone()) + { + self.recent_intents.remove(&oldest_id); + } + } + let seq = self.recent_intent_seq; + self.recent_intent_seq += 1; + self + .recent_intents + .insert(id.clone(), NodeIntent::new(kind, ltime, wall_time, seq)); + true + } + } + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-proto/src/members/tests.rs b/serf-proto/src/members/tests.rs new file mode 100644 index 00000000..d384a4e4 --- /dev/null +++ b/serf-proto/src/members/tests.rs @@ -0,0 +1,178 @@ +use super::*; + +// ── determinism regression ──────────────────────────────────────────────────── + +/// Flood two identically-initialised `Members` stores with more than +/// `MAX_RECENT_INTENTS` leave intents for distinct unknown node ids, all +/// stamped with the SAME `Instant`. Without a sequence-number tie-break the +/// eviction is decided by `HashMap` iteration order (randomised per process), +/// so the two stores retain *different* subsets. With the fix the composite +/// `(wall_time, sequence)` key makes eviction purely a function of insertion +/// order — identical input sequence → identical retained set on both stores. +/// +/// A subsequent `recent_intent` lookup for a specific node then returns the +/// same answer on both stores, and a downstream `handle_node_join` for that +/// node would produce the same `MemberStatus` on both machines. +#[test] +fn upsert_intent_cap_eviction_is_deterministic() { + use memberlist_proto::Instant; + + type TestMembers = Members; + + // Use cap + a surplus so we trigger exactly one eviction per surplus insert. + let n = MAX_RECENT_INTENTS + 16; + + // Both stores receive the SAME sequence of intents in the SAME order. + // All share the same wall_time to maximise tie collisions. + let shared_now = Instant::ORIGIN; + + let mut m1 = TestMembers::default(); + let mut m2 = TestMembers::default(); + + for node_id in 0u32..(n as u32) { + let ltime = LamportTime::new(node_id as u64 + 1); + m1.upsert_intent(&node_id, IntentKind::Leave, ltime, shared_now); + m2.upsert_intent(&node_id, IntentKind::Leave, ltime, shared_now); + } + + // Both stores must be at exactly MAX_RECENT_INTENTS. + assert_eq!(m1.recent_intents.len(), MAX_RECENT_INTENTS); + assert_eq!(m2.recent_intents.len(), MAX_RECENT_INTENTS); + + // The retained key sets must be identical. + let keys1: std::collections::BTreeSet = m1.recent_intents.keys().cloned().collect(); + let keys2: std::collections::BTreeSet = m2.recent_intents.keys().cloned().collect(); + assert_eq!( + keys1, keys2, + "two identically-seeded stores retained different intent subsets after cap eviction" + ); + + // The first `n - MAX_RECENT_INTENTS` ids (oldest-inserted) must have been + // evicted; the most recently inserted ones must be retained. + let evicted_count = n - MAX_RECENT_INTENTS; + for evicted_id in 0u32..(evicted_count as u32) { + assert!( + !m1.recent_intents.contains_key(&evicted_id), + "node {evicted_id} should have been evicted (oldest-inserted) but was retained" + ); + } + for retained_id in (evicted_count as u32)..(n as u32) { + assert!( + m1.recent_intents.contains_key(&retained_id), + "node {retained_id} should be retained but was evicted" + ); + } + + // Verify that a `recent_intent` lookup for a retained node returns the same + // answer on both stores — confirming downstream FSM decisions are identical. + let probe_retained = evicted_count as u32; + assert_eq!( + m1.recent_intent(&probe_retained, IntentKind::Leave), + m2.recent_intent(&probe_retained, IntentKind::Leave), + "retained node lookup diverged between the two stores" + ); + + // And for an evicted node both stores agree it is absent. + let probe_evicted = 0u32; + assert_eq!( + m1.recent_intent(&probe_evicted, IntentKind::Leave), + m2.recent_intent(&probe_evicted, IntentKind::Leave), + ); + assert_eq!(m1.recent_intent(&probe_evicted, IntentKind::Leave), None); +} + +#[test] +fn member_status_as_str_round_trips() { + assert_eq!(MemberStatus::None.as_str(), "none"); + assert_eq!(MemberStatus::Alive.as_str(), "alive"); + assert_eq!(MemberStatus::Leaving.as_str(), "leaving"); + assert_eq!(MemberStatus::Left.as_str(), "left"); + assert_eq!(MemberStatus::Failed.as_str(), "failed"); + assert!(MemberStatus::default().is_none()); +} + +#[test] +fn serf_state_as_str() { + assert_eq!(SerfState::Alive.as_str(), "alive"); + assert_eq!(SerfState::Shutdown.as_str(), "shutdown"); +} + +#[test] +fn member_status_display_matches_as_str() { + for status in [ + MemberStatus::None, + MemberStatus::Alive, + MemberStatus::Leaving, + MemberStatus::Left, + MemberStatus::Failed, + ] { + assert_eq!(status.to_string(), status.as_str()); + } +} + +#[test] +fn serf_state_display_matches_as_str() { + for state in [ + SerfState::Alive, + SerfState::Leaving, + SerfState::Left, + SerfState::Shutdown, + ] { + assert_eq!(state.to_string(), state.as_str()); + } +} + +#[test] +fn upsert_intent_newest_ltime_wins() { + use memberlist_proto::Instant; + + type TestMembers = Members; + let mut m = TestMembers::default(); + let t0 = Instant::ORIGIN; + + // First insert + assert!(m.upsert_intent(&1u32, IntentKind::Join, LamportTime::new(5), t0)); + assert_eq!( + m.recent_intent(&1u32, IntentKind::Join), + Some(LamportTime::new(5)) + ); + + // Same ltime — not updated + assert!(!m.upsert_intent(&1u32, IntentKind::Join, LamportTime::new(5), t0)); + + // Older ltime — not updated + assert!(!m.upsert_intent(&1u32, IntentKind::Join, LamportTime::new(3), t0)); + assert_eq!( + m.recent_intent(&1u32, IntentKind::Join), + Some(LamportTime::new(5)) + ); + + // Newer ltime — updated + assert!(m.upsert_intent(&1u32, IntentKind::Join, LamportTime::new(7), t0)); + assert_eq!( + m.recent_intent(&1u32, IntentKind::Join), + Some(LamportTime::new(7)) + ); +} + +#[test] +fn recent_intent_kind_mismatch_returns_none() { + use memberlist_proto::Instant; + + type TestMembers = Members; + let mut m = TestMembers::default(); + + m.upsert_intent( + &2u32, + IntentKind::Leave, + LamportTime::new(3), + Instant::ORIGIN, + ); + // Looking for Join when only Leave is stored + assert_eq!(m.recent_intent(&2u32, IntentKind::Join), None); + // Leave matches + assert_eq!( + m.recent_intent(&2u32, IntentKind::Leave), + Some(LamportTime::new(3)) + ); +} diff --git a/serf-proto/src/options/mod.rs b/serf-proto/src/options/mod.rs new file mode 100644 index 00000000..eb028bdc --- /dev/null +++ b/serf-proto/src/options/mod.rs @@ -0,0 +1,438 @@ +//! Configuration knobs for the serf [`crate::endpoint::Endpoint`]. +//! +//! All timer defaults are taken verbatim from Go serf `options.go` / the legacy +//! `serf-core/src/options.rs` port. The struct itself is a plain value type +//! (no atomics, no locks) consistent with the single-threaded Sans-I/O machine. + +use std::time::Duration; + +/// Configuration for the serf `Endpoint`. +/// +/// Construct via [`Options::new`] (which sets all defaults) or build up from +/// `new()` using the `with_*` setter methods. `Default` delegates to `new()`. +/// +/// Timer semantics follow Go serf `options.go`: +/// - Coalescing is disabled by default (all four coalesce periods are `ZERO`); +/// enable by setting a non-zero `coalesce_period` and `quiescent_period`. +/// - `recent_intent_timeout` guards against out-of-order intent delivery; it +/// should be long enough that a re-broadcasted intent does not arrive after +/// the buffer has already expired. +#[derive(Debug, Clone)] +pub struct Options { + // ── timers ──────────────────────────────────────────────────────────────── + /// How often the reaper runs to remove tombstoned left/failed nodes. + reap_interval: Duration, + /// How often we attempt to reconnect to failed nodes. + reconnect_interval: Duration, + /// How long we try to reconnect to a failed node before giving up. + reconnect_timeout: Duration, + /// How long gracefully-left tombstones are kept for anti-entropy syncing. + tombstone_timeout: Duration, + /// How long recent join/leave intents are buffered to handle out-of-order + /// delivery before the inner memberlist `NodeJoined`/`NodeLeft` event. + recent_intent_timeout: Duration, + /// How long to wait for a broadcast (leave, force-remove) to propagate. + broadcast_timeout: Duration, + /// Extra delay after calling inner `leave()` before transitioning to `Left`, + /// giving in-flight probes time to observe the leave intent. + leave_propagate_delay: Duration, + /// How often the broadcast queue depth is checked for pruning/warning. + queue_check_interval: Duration, + + // ── coalescing (disabled when both period fields are ZERO) ──────────────── + /// Duration over which member events are coalesced. Zero = disabled. + coalesce_period: Duration, + /// If no new events arrive within this window, coalescing fires immediately. + quiescent_period: Duration, + /// Same as `coalesce_period` but for user (application-defined) events only. + user_coalesce_period: Duration, + /// Same as `quiescent_period` but for user events only. + user_quiescent_period: Duration, + + // ── buffers ─────────────────────────────────────────────────────────────── + /// Number of user-event slots in the dedup ring buffer. + event_buffer_size: usize, + /// Number of query slots in the dedup ring buffer. + query_buffer_size: usize, + + // ── size limits ─────────────────────────────────────────────────────────── + /// Maximum `name + payload` byte size for a user event. + max_user_event_size: usize, + /// Maximum inbound payload size for a query message. + query_size_limit: usize, + /// Maximum outbound payload size for a query response. + query_response_size_limit: usize, + + // ── queue pruning ───────────────────────────────────────────────────────── + /// Hard cap on the broadcast queue; messages beyond this are dropped. + max_queue_depth: usize, + /// If `> 0`, replaces `max_queue_depth` with `max(min_queue_depth, 2 * cluster_size)`. + min_queue_depth: usize, + /// Emit a warning when the broadcast queue exceeds this depth. + queue_depth_warning: usize, + + // ── behavior ────────────────────────────────────────────────────────────── + /// When `true`, the endpoint runs the conflict-resolution query when two + /// nodes claim the same id. + enable_id_conflict_resolution: bool, + /// Multiplier applied to gossip interval to compute the default query timeout: + /// `timeout = gossip_interval * query_timeout_mult * log(N+1)`. + query_timeout_mult: usize, + /// A node is considered to have "flapped" if it fails and rejoins within + /// this window (used for telemetry). + flap_timeout: Duration, + /// When `true`, the endpoint rejoins the last known cluster on startup even + /// after a graceful leave was recorded in the snapshot. + rejoin_after_leave: bool, + + // ── feature gates ───────────────────────────────────────────────────────── + /// When `true`, the Vivaldi coordinate subsystem is disabled at runtime even + /// if the `coordinates` compile feature is active. + #[cfg(feature = "coordinates")] + disable_coordinates: bool, +} + +impl Options { + /// Returns a new `Options` with all defaults as specified in Go serf + /// `options.go` and the legacy `serf-core/src/options.rs` port. + pub fn new() -> Self { + Self { + reap_interval: Duration::from_secs(15), + reconnect_interval: Duration::from_secs(30), + reconnect_timeout: Duration::from_secs(3600 * 24), + tombstone_timeout: Duration::from_secs(3600 * 24), + recent_intent_timeout: Duration::from_secs(60 * 5), + broadcast_timeout: Duration::from_secs(5), + leave_propagate_delay: Duration::from_secs(1), + queue_check_interval: Duration::from_secs(30), + coalesce_period: Duration::ZERO, + quiescent_period: Duration::ZERO, + user_coalesce_period: Duration::ZERO, + user_quiescent_period: Duration::ZERO, + event_buffer_size: 512, + query_buffer_size: 512, + max_user_event_size: 512, + query_size_limit: 1024, + query_response_size_limit: 1024, + max_queue_depth: 4096, + min_queue_depth: 0, + queue_depth_warning: 128, + enable_id_conflict_resolution: true, + query_timeout_mult: 16, + flap_timeout: Duration::from_secs(60), + rejoin_after_leave: false, + #[cfg(feature = "coordinates")] + disable_coordinates: false, + } + } + + // ── getters ─────────────────────────────────────────────────────────────── + + /// How often the reaper runs. + pub const fn reap_interval(&self) -> Duration { + self.reap_interval + } + + /// How often we attempt to reconnect to failed nodes. + pub const fn reconnect_interval(&self) -> Duration { + self.reconnect_interval + } + + /// How long we try to reconnect before giving up on a failed node. + pub const fn reconnect_timeout(&self) -> Duration { + self.reconnect_timeout + } + + /// How long gracefully-left tombstones are retained. + pub const fn tombstone_timeout(&self) -> Duration { + self.tombstone_timeout + } + + /// How long recent intents are buffered to handle out-of-order delivery. + pub const fn recent_intent_timeout(&self) -> Duration { + self.recent_intent_timeout + } + + /// Broadcast propagation timeout. + pub const fn broadcast_timeout(&self) -> Duration { + self.broadcast_timeout + } + + /// Extra delay before transitioning to `Left` after calling inner `leave()`. + pub const fn leave_propagate_delay(&self) -> Duration { + self.leave_propagate_delay + } + + /// How often the broadcast queue depth is checked. + pub const fn queue_check_interval(&self) -> Duration { + self.queue_check_interval + } + + /// Member-event coalesce window. Zero means coalescing is disabled. + pub const fn coalesce_period(&self) -> Duration { + self.coalesce_period + } + + /// Quiescent window for member-event coalescing. + pub const fn quiescent_period(&self) -> Duration { + self.quiescent_period + } + + /// User-event coalesce window. + pub const fn user_coalesce_period(&self) -> Duration { + self.user_coalesce_period + } + + /// User-event quiescent window. + pub const fn user_quiescent_period(&self) -> Duration { + self.user_quiescent_period + } + + /// Number of slots in the user-event dedup ring buffer. + pub const fn event_buffer_size(&self) -> usize { + self.event_buffer_size + } + + /// Number of slots in the query dedup ring buffer. + pub const fn query_buffer_size(&self) -> usize { + self.query_buffer_size + } + + /// Maximum `name + payload` byte size for a user event. + pub const fn max_user_event_size(&self) -> usize { + self.max_user_event_size + } + + /// Maximum inbound payload size for a query. + pub const fn query_size_limit(&self) -> usize { + self.query_size_limit + } + + /// Maximum outbound payload size for a query response. + pub const fn query_response_size_limit(&self) -> usize { + self.query_response_size_limit + } + + /// Hard cap on the broadcast queue depth. + pub const fn max_queue_depth(&self) -> usize { + self.max_queue_depth + } + + /// Dynamic queue depth floor (`0` = disabled; uses `max_queue_depth` instead). + pub const fn min_queue_depth(&self) -> usize { + self.min_queue_depth + } + + /// Broadcast queue depth at which a warning is emitted. + pub const fn queue_depth_warning(&self) -> usize { + self.queue_depth_warning + } + + /// Whether conflict-resolution queries are enabled. + pub const fn enable_id_conflict_resolution(&self) -> bool { + self.enable_id_conflict_resolution + } + + /// Query timeout multiplier. + pub const fn query_timeout_mult(&self) -> usize { + self.query_timeout_mult + } + + /// Flap-detection window for telemetry. + pub const fn flap_timeout(&self) -> Duration { + self.flap_timeout + } + + /// Whether to rejoin the last cluster after a graceful leave snapshot. + pub const fn rejoin_after_leave(&self) -> bool { + self.rejoin_after_leave + } + + /// Whether the Vivaldi coordinate subsystem is disabled at runtime. + #[cfg(feature = "coordinates")] + #[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] + pub const fn disable_coordinates(&self) -> bool { + self.disable_coordinates + } + + // ── setters (builder pattern) ───────────────────────────────────────────── + + /// Sets `reap_interval`. + /// + /// Clamped to a minimum of 1 ms: a zero interval would cause `handle_timeout` + /// to rearm the deadline to `now + 0`, which is already-due every tick and + /// busy-loops a conforming Sans-I/O driver. Go serf always runs the reaper + /// (no disable path), so zero is not a meaningful disable signal. + pub fn with_reap_interval(mut self, v: Duration) -> Self { + self.reap_interval = v.max(Duration::from_millis(1)); + self + } + + /// Sets `reconnect_interval`. + /// + /// Clamped to a minimum of 1 ms for the same reason as `reap_interval`: a + /// zero interval busy-loops the driver. + pub fn with_reconnect_interval(mut self, v: Duration) -> Self { + self.reconnect_interval = v.max(Duration::from_millis(1)); + self + } + + /// Sets `reconnect_timeout`. + pub fn with_reconnect_timeout(mut self, v: Duration) -> Self { + self.reconnect_timeout = v; + self + } + + /// Sets `tombstone_timeout`. + pub fn with_tombstone_timeout(mut self, v: Duration) -> Self { + self.tombstone_timeout = v; + self + } + + /// Sets `recent_intent_timeout`. + pub fn with_recent_intent_timeout(mut self, v: Duration) -> Self { + self.recent_intent_timeout = v; + self + } + + /// Sets `broadcast_timeout`. + pub fn with_broadcast_timeout(mut self, v: Duration) -> Self { + self.broadcast_timeout = v; + self + } + + /// Sets `leave_propagate_delay`. + pub fn with_leave_propagate_delay(mut self, v: Duration) -> Self { + self.leave_propagate_delay = v; + self + } + + /// Sets `queue_check_interval`. + /// + /// Clamped to a minimum of 1 ms for the same reason as `reap_interval`: a + /// zero interval busy-loops the driver. + pub fn with_queue_check_interval(mut self, v: Duration) -> Self { + self.queue_check_interval = v.max(Duration::from_millis(1)); + self + } + + /// Sets `coalesce_period`. + pub fn with_coalesce_period(mut self, v: Duration) -> Self { + self.coalesce_period = v; + self + } + + /// Sets `quiescent_period`. + pub fn with_quiescent_period(mut self, v: Duration) -> Self { + self.quiescent_period = v; + self + } + + /// Sets `user_coalesce_period`. + pub fn with_user_coalesce_period(mut self, v: Duration) -> Self { + self.user_coalesce_period = v; + self + } + + /// Sets `user_quiescent_period`. + pub fn with_user_quiescent_period(mut self, v: Duration) -> Self { + self.user_quiescent_period = v; + self + } + + /// Sets `event_buffer_size`. + pub fn with_event_buffer_size(mut self, v: usize) -> Self { + self.event_buffer_size = v; + self + } + + /// Sets `query_buffer_size`. + pub fn with_query_buffer_size(mut self, v: usize) -> Self { + self.query_buffer_size = v; + self + } + + /// Sets `max_user_event_size`. + pub fn with_max_user_event_size(mut self, v: usize) -> Self { + self.max_user_event_size = v; + self + } + + /// Sets `query_size_limit`. + pub fn with_query_size_limit(mut self, v: usize) -> Self { + self.query_size_limit = v; + self + } + + /// Sets `query_response_size_limit`. + pub fn with_query_response_size_limit(mut self, v: usize) -> Self { + self.query_response_size_limit = v; + self + } + + /// Sets `max_queue_depth`. + pub fn with_max_queue_depth(mut self, v: usize) -> Self { + self.max_queue_depth = v; + self + } + + /// Sets `min_queue_depth`. + pub fn with_min_queue_depth(mut self, v: usize) -> Self { + self.min_queue_depth = v; + self + } + + /// Sets `queue_depth_warning`. + pub fn with_queue_depth_warning(mut self, v: usize) -> Self { + self.queue_depth_warning = v; + self + } + + /// Sets `enable_id_conflict_resolution`. + pub fn with_enable_id_conflict_resolution(mut self, v: bool) -> Self { + self.enable_id_conflict_resolution = v; + self + } + + /// Sets `query_timeout_mult`. + /// + /// A value of 0 is clamped to 1: the default query timeout is computed as + /// `gossip_interval * query_timeout_mult * log(N+1)`. A zero multiplier + /// produces a zero `Duration`, which causes every query to expire immediately + /// before the first `poll_timeout` tick — a correctness hazard equivalent to + /// a zero-capacity buffer. Mirrors the `.max(1)` guards on buffer sizes. + pub fn with_query_timeout_mult(mut self, v: usize) -> Self { + self.query_timeout_mult = v.max(1); + self + } + + /// Sets `flap_timeout`. + pub fn with_flap_timeout(mut self, v: Duration) -> Self { + self.flap_timeout = v; + self + } + + /// Sets `rejoin_after_leave`. + pub fn with_rejoin_after_leave(mut self, v: bool) -> Self { + self.rejoin_after_leave = v; + self + } + + /// Sets `disable_coordinates`. + #[cfg(feature = "coordinates")] + #[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] + pub fn with_disable_coordinates(mut self, v: bool) -> Self { + self.disable_coordinates = v; + self + } +} + +impl Default for Options { + /// Delegates to [`Options::new`]. Never derives `Default` when `new()` exists. + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-proto/src/options/tests.rs b/serf-proto/src/options/tests.rs new file mode 100644 index 00000000..336441c6 --- /dev/null +++ b/serf-proto/src/options/tests.rs @@ -0,0 +1,117 @@ +use std::time::Duration; + +use super::Options; + +#[test] +fn options_defaults_match_legacy() { + let o = Options::new(); + assert_eq!(o.reap_interval(), Duration::from_secs(15)); + assert_eq!(o.reconnect_timeout(), Duration::from_secs(3600 * 24)); + assert_eq!(o.tombstone_timeout(), Duration::from_secs(3600 * 24)); + assert_eq!(o.max_user_event_size(), 512); + assert_eq!(o.query_size_limit(), 1024); + assert!(o.enable_id_conflict_resolution()); +} + +#[test] +fn options_default_and_new_are_equal() { + // Verify Default delegates to new() — the two instances should have the + // same field values (confirmed via individual assertions rather than PartialEq, + // which is not derived, to avoid coupling tests to unrelated fields). + let a = Options::new(); + let b = Options::default(); + assert_eq!(a.reap_interval(), b.reap_interval()); + assert_eq!(a.reconnect_interval(), b.reconnect_interval()); + assert_eq!(a.reconnect_timeout(), b.reconnect_timeout()); + assert_eq!(a.tombstone_timeout(), b.tombstone_timeout()); + assert_eq!(a.recent_intent_timeout(), b.recent_intent_timeout()); + assert_eq!(a.broadcast_timeout(), b.broadcast_timeout()); + assert_eq!(a.leave_propagate_delay(), b.leave_propagate_delay()); + assert_eq!(a.queue_check_interval(), b.queue_check_interval()); + assert_eq!(a.coalesce_period(), b.coalesce_period()); + assert_eq!(a.quiescent_period(), b.quiescent_period()); + assert_eq!(a.event_buffer_size(), b.event_buffer_size()); + assert_eq!(a.query_buffer_size(), b.query_buffer_size()); + assert_eq!(a.max_user_event_size(), b.max_user_event_size()); + assert_eq!(a.query_size_limit(), b.query_size_limit()); + assert_eq!(a.query_response_size_limit(), b.query_response_size_limit()); + assert_eq!(a.max_queue_depth(), b.max_queue_depth()); + assert_eq!(a.min_queue_depth(), b.min_queue_depth()); + assert_eq!(a.queue_depth_warning(), b.queue_depth_warning()); + assert_eq!( + a.enable_id_conflict_resolution(), + b.enable_id_conflict_resolution() + ); + assert_eq!(a.query_timeout_mult(), b.query_timeout_mult()); + assert_eq!(a.flap_timeout(), b.flap_timeout()); + assert_eq!(a.rejoin_after_leave(), b.rejoin_after_leave()); +} + +#[test] +fn options_all_defaults() { + let o = Options::new(); + // Timers + assert_eq!(o.reap_interval(), Duration::from_secs(15)); + assert_eq!(o.reconnect_interval(), Duration::from_secs(30)); + assert_eq!(o.reconnect_timeout(), Duration::from_secs(3600 * 24)); + assert_eq!(o.tombstone_timeout(), Duration::from_secs(3600 * 24)); + assert_eq!(o.recent_intent_timeout(), Duration::from_secs(60 * 5)); + assert_eq!(o.broadcast_timeout(), Duration::from_secs(5)); + assert_eq!(o.leave_propagate_delay(), Duration::from_secs(1)); + assert_eq!(o.queue_check_interval(), Duration::from_secs(30)); + // Coalescing disabled by default + assert_eq!(o.coalesce_period(), Duration::ZERO); + assert_eq!(o.quiescent_period(), Duration::ZERO); + assert_eq!(o.user_coalesce_period(), Duration::ZERO); + assert_eq!(o.user_quiescent_period(), Duration::ZERO); + // Buffers + assert_eq!(o.event_buffer_size(), 512); + assert_eq!(o.query_buffer_size(), 512); + // Size limits + assert_eq!(o.max_user_event_size(), 512); + assert_eq!(o.query_size_limit(), 1024); + assert_eq!(o.query_response_size_limit(), 1024); + // Queue pruning + assert_eq!(o.max_queue_depth(), 4096); + assert_eq!(o.min_queue_depth(), 0); + assert_eq!(o.queue_depth_warning(), 128); + // Behavior + assert!(o.enable_id_conflict_resolution()); + assert_eq!(o.query_timeout_mult(), 16); + assert_eq!(o.flap_timeout(), Duration::from_secs(60)); + assert!(!o.rejoin_after_leave()); +} + +#[test] +fn options_builder_overrides_defaults() { + let o = Options::new() + .with_reap_interval(Duration::from_secs(30)) + .with_max_queue_depth(8192) + .with_enable_id_conflict_resolution(false) + .with_rejoin_after_leave(true); + + assert_eq!(o.reap_interval(), Duration::from_secs(30)); + assert_eq!(o.max_queue_depth(), 8192); + assert!(!o.enable_id_conflict_resolution()); + assert!(o.rejoin_after_leave()); + // Untouched defaults remain + assert_eq!(o.reconnect_timeout(), Duration::from_secs(3600 * 24)); +} + +// ── FIX 2 sweep: zero-value hazard clamps ──────────────────────────────────── + +/// `Options::with_query_timeout_mult(0)` must be clamped to 1. +/// +/// A zero multiplier causes `gossip_interval * 0 * log(N+1) = Duration::ZERO`, +/// which makes every query expire immediately before the first `poll_timeout` +/// tick — a correctness hazard. Mirrors the `.max(1)` guards on buffer sizes +/// and `CoordinateOptions::with_dimensionality`. +#[test] +fn query_timeout_mult_zero_is_clamped_to_one() { + let o = Options::new().with_query_timeout_mult(0); + assert_eq!( + o.query_timeout_mult(), + 1, + "with_query_timeout_mult(0) must be clamped to 1" + ); +} diff --git a/serf-proto/src/snapshot/mod.rs b/serf-proto/src/snapshot/mod.rs new file mode 100644 index 00000000..a805a5cc --- /dev/null +++ b/serf-proto/src/snapshot/mod.rs @@ -0,0 +1,580 @@ +//! Pure snapshot record format — encode/decode a single `SnapshotRecord` to/from bytes. +//! +//! The snapshot file is a flat append-only stream of variable-length binary records, +//! each beginning with a one-byte discriminant tag followed by a record-type-specific +//! body. The tag values are fixed and must match the legacy `serf-core` layout for +//! forward-compatible snapshots: +//! +//! | Tag | Variant | Body | +//! |-----|----------------|-----------------------------------------------------| +//! | 0 | Alive | u32 LE node-len + node bytes | +//! | 1 | NotAlive | u32 LE node-len + node bytes | +//! | 2 | Clock | u64 LE lamport time | +//! | 3 | EventClock | u64 LE lamport time | +//! | 4 | QueryClock | u64 LE lamport time | +//! | 5 | Coordinate | (coordinates feature) see below | +//! | 6 | Leave | (empty body) | +//! | 7 | Comment | (empty body) | +//! +//! The `Coordinate` record body (tag 5, feature="coordinates"): +//! `[node_len: u32 LE][node bytes][dim: u32 LE][dim × f64 LE][error: f64 LE][adj: f64 LE][height: f64 LE]` +//! +//! The serf-core oracle did not persist any coordinate payload (tag only, silently skipped on +//! replay). This implementation stores the node + Vivaldi coordinate so the driver can warm +//! the `CoordinateClient` on restart without re-probing. +//! +//! The encode/decode API is pure: `&[u8]` slice in, `Bytes` out. +//! No file handles, no `std::fs`, no `Instant::now()`. The driver owns I/O; +//! this module is consumed by the snapshot replay function in the driver. + +use bytes::{BufMut, Bytes, BytesMut}; +use memberlist_proto::{Data, DataRef}; + +use crate::LamportTime; + +#[cfg(feature = "coordinates")] +use crate::Coordinate; + +// ── Tag constants ───────────────────────────────────────────────────────────── + +const TAG_ALIVE: u8 = 0; +const TAG_NOT_ALIVE: u8 = 1; +const TAG_CLOCK: u8 = 2; +const TAG_EVENT_CLOCK: u8 = 3; +const TAG_QUERY_CLOCK: u8 = 4; +const TAG_COORDINATE: u8 = 5; +const TAG_LEAVE: u8 = 6; +const TAG_COMMENT: u8 = 7; + +// ── SnapshotError ───────────────────────────────────────────────────────────── + +/// Errors produced by snapshot record encode/decode. +/// +/// Pure codec errors only — no file-I/O variants here; those belong to the driver. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum SnapshotError { + /// The tag byte identifies an unknown record type. + #[error("unknown snapshot record tag: {0}")] + UnknownTag(u8), + /// The record body is shorter than required for its type. + #[error("snapshot record truncated: need {need} bytes, have {have}")] + Truncated { + /// Bytes needed to complete the record. + need: usize, + /// Bytes actually available. + have: usize, + }, + /// A `memberlist_proto::Data` encode error. + #[error("encode error: {0}")] + Encode(#[from] memberlist_proto::EncodeError), + /// A `memberlist_proto::Data` decode error. + #[error("decode error: {0}")] + Decode(#[from] memberlist_proto::data::DecodeError), +} + +// ── CoordinateRecord ───────────────────────────────────────────────────────── + +/// A node-coordinate pair stored inside a [`SnapshotRecord::Coordinate`] record. +/// +/// Unlike the serf-core oracle (which wrote only the tag byte and skipped the +/// payload on replay), this implementation stores the full `Node`+`Coordinate` +/// so the driver can warm the `CoordinateClient` from disk without re-probing. +#[cfg(feature = "coordinates")] +#[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] +#[derive(Debug, Clone, PartialEq)] +pub struct CoordinateRecord { + /// The node whose coordinate is stored. + node: memberlist_proto::Node, + /// The cached Vivaldi coordinate for that node. + coordinate: Coordinate, +} + +#[cfg(feature = "coordinates")] +impl CoordinateRecord { + /// Constructs a new `CoordinateRecord`. + pub fn new(node: memberlist_proto::Node, coordinate: Coordinate) -> Self { + Self { node, coordinate } + } + + /// The node this coordinate describes. + pub const fn node(&self) -> &memberlist_proto::Node { + &self.node + } + + /// The Vivaldi coordinate cached for this node. + pub const fn coordinate(&self) -> &Coordinate { + &self.coordinate + } +} + +// ── SnapshotRecord ──────────────────────────────────────────────────────────── + +/// A single record in the serf snapshot file. +/// +/// The binary format mirrors the `serf-core` `SnapshotRecord` discriminants so +/// snapshots are forward-compatible. See the module-level table for the per-tag +/// layout. +/// +/// Note: `Eq` is not derived when the `coordinates` feature is enabled because +/// the `Coordinate` variant contains `f64` fields (`f64` does not implement `Eq`). +/// Under the feature, only `PartialEq` is available. +#[derive(Debug, Clone, PartialEq, derive_more::IsVariant)] +#[cfg_attr(not(feature = "coordinates"), derive(Eq))] +#[non_exhaustive] +pub enum SnapshotRecord { + /// A node that was alive at snapshot time. + Alive(memberlist_proto::Node), + /// A node that was no longer alive at snapshot time. + NotAlive(memberlist_proto::Node), + /// The member Lamport clock at snapshot time. + Clock(LamportTime), + /// The event Lamport clock at snapshot time. + EventClock(LamportTime), + /// The query Lamport clock at snapshot time. + QueryClock(LamportTime), + /// A cached Vivaldi coordinate for a node (requires `feature = "coordinates"`). + #[cfg(feature = "coordinates")] + #[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] + Coordinate(CoordinateRecord), + /// A leave marker: the local node cleanly left before this snapshot was written. + Leave, + /// An informational comment record; ignored on replay. + Comment, +} + +impl SnapshotRecord +where + I: Data, + A: Data, +{ + /// Encode this record to a self-framed [`Bytes`] buffer. + /// + /// The returned buffer begins with the discriminant tag byte and is fully + /// self-contained (the decoder does not need external framing). + pub fn encode(&self) -> Result { + match self { + Self::Alive(node) => encode_node(TAG_ALIVE, node), + Self::NotAlive(node) => encode_node(TAG_NOT_ALIVE, node), + Self::Clock(t) => Ok(encode_clock(TAG_CLOCK, *t)), + Self::EventClock(t) => Ok(encode_clock(TAG_EVENT_CLOCK, *t)), + Self::QueryClock(t) => Ok(encode_clock(TAG_QUERY_CLOCK, *t)), + #[cfg(feature = "coordinates")] + Self::Coordinate(rec) => encode_coordinate_record(rec), + Self::Leave => Ok(Bytes::from_static(&[TAG_LEAVE])), + Self::Comment => Ok(Bytes::from_static(&[TAG_COMMENT])), + } + } + + /// Decode one record from the front of `buf`. + /// + /// Returns `(record, bytes_consumed)`. The caller advances its cursor by + /// `bytes_consumed` to read the next record. + /// + /// # Errors + /// - [`SnapshotError::Truncated`] — buffer ends inside the record body. + /// - [`SnapshotError::UnknownTag`] — unrecognised discriminant byte. + /// - [`SnapshotError::Decode`] — node-id or node-addr decode failure. + pub fn decode(buf: &[u8]) -> Result<(Self, usize), SnapshotError> { + let tag = *buf + .first() + .ok_or(SnapshotError::Truncated { need: 1, have: 0 })?; + + match tag { + TAG_ALIVE => { + let (node, n) = decode_node(&buf[1..])?; + Ok((Self::Alive(node), 1 + n)) + } + TAG_NOT_ALIVE => { + let (node, n) = decode_node(&buf[1..])?; + Ok((Self::NotAlive(node), 1 + n)) + } + TAG_CLOCK => { + let t = decode_clock(&buf[1..])?; + Ok((Self::Clock(t), 9)) + } + TAG_EVENT_CLOCK => { + let t = decode_clock(&buf[1..])?; + Ok((Self::EventClock(t), 9)) + } + TAG_QUERY_CLOCK => { + let t = decode_clock(&buf[1..])?; + Ok((Self::QueryClock(t), 9)) + } + TAG_COORDINATE => { + #[cfg(feature = "coordinates")] + { + let (rec, n) = decode_coordinate_record(&buf[1..])?; + Ok((Self::Coordinate(rec), 1 + n)) + } + // When the coordinates feature is off, a coordinate record from a + // coordinates-enabled snapshot cannot be decoded without knowing its + // length. The driver should not feed such snapshots to a + // coordinates-disabled build. + #[cfg(not(feature = "coordinates"))] + Err(SnapshotError::UnknownTag(TAG_COORDINATE)) + } + TAG_LEAVE => Ok((Self::Leave, 1)), + TAG_COMMENT => Ok((Self::Comment, 1)), + other => Err(SnapshotError::UnknownTag(other)), + } + } +} + +// ── Encoding helpers ────────────────────────────────────────────────────────── + +/// `[tag][node_len: u32 LE][node bytes]` +fn encode_node(tag: u8, node: &memberlist_proto::Node) -> Result +where + I: Data, + A: Data, +{ + let node_len = node.encoded_len(); + let total = 1 + 4 + node_len; + let mut buf = BytesMut::with_capacity(total); + buf.put_u8(tag); + buf.put_u32_le(node_len as u32); + let prev = buf.len(); + buf.resize(prev + node_len, 0); + let written = node.encode(&mut buf[prev..])?; + buf.truncate(prev + written); + Ok(buf.freeze()) +} + +/// `[tag][time: u64 LE]` +fn encode_clock(tag: u8, t: LamportTime) -> Bytes { + let mut buf = [0u8; 9]; + buf[0] = tag; + buf[1..9].copy_from_slice(&u64::from(t).to_le_bytes()); + Bytes::copy_from_slice(&buf) +} + +/// `[TAG_COORDINATE][node_len: u32 LE][node bytes][dim: u32 LE][dim × f64 LE][error f64 LE][adj f64 LE][height f64 LE]` +#[cfg(feature = "coordinates")] +fn encode_coordinate_record(rec: &CoordinateRecord) -> Result +where + I: Data, + A: Data, +{ + let node = &rec.node; + let coord = &rec.coordinate; + + let node_len = node.encoded_len(); + let dim = coord.vec.len(); + // coord body: 4 (dim count) + dim*8 (f64s) + 8 (error) + 8 (adj) + 8 (height) + let coord_body_len = 4 + dim * 8 + 8 + 8 + 8; + let total = 1 + 4 + node_len + coord_body_len; + + let mut buf = BytesMut::with_capacity(total); + buf.put_u8(TAG_COORDINATE); + // node + buf.put_u32_le(node_len as u32); + let prev = buf.len(); + buf.resize(prev + node_len, 0); + let written = node.encode(&mut buf[prev..])?; + buf.truncate(prev + written); + // coordinate: dim count + vec elements + error + adjustment + height + buf.put_u32_le(dim as u32); + for &v in &coord.vec { + buf.put_f64_le(v); + } + buf.put_f64_le(coord.error); + buf.put_f64_le(coord.adjustment); + buf.put_f64_le(coord.height); + Ok(buf.freeze()) +} + +// ── Decoding helpers ────────────────────────────────────────────────────────── + +/// Decode `[node_len: u32 LE][node bytes]` from `buf`. +/// +/// Returns `(node, bytes_consumed)` where consumed includes the 4-byte length prefix. +fn decode_node(buf: &[u8]) -> Result<(memberlist_proto::Node, usize), SnapshotError> +where + I: Data, + A: Data, +{ + if buf.len() < 4 { + return Err(SnapshotError::Truncated { + need: 4, + have: buf.len(), + }); + } + let node_len = u32::from_le_bytes([buf[0], buf[1], buf[2], buf[3]]) as usize; + let body_end = 4 + node_len; + if buf.len() < body_end { + return Err(SnapshotError::Truncated { + need: body_end, + have: buf.len(), + }); + } + let node_bytes = &buf[4..body_end]; + type NodeRef<'a, I, A> = memberlist_proto::Node<::Ref<'a>, ::Ref<'a>>; + let (consumed, node_ref) = + as DataRef<'_, memberlist_proto::Node>>::decode(node_bytes) + .map_err(SnapshotError::Decode)?; + if consumed != node_len { + return Err(SnapshotError::Decode( + memberlist_proto::data::DecodeError::custom(format!( + "node decode consumed {consumed} of {node_len} bytes" + )), + )); + } + let node = memberlist_proto::Node::::from_ref(node_ref).map_err(SnapshotError::Decode)?; + Ok((node, 4 + node_len)) +} + +/// Decode a u64 LE lamport time from `buf[0..8]`. +fn decode_clock(buf: &[u8]) -> Result { + if buf.len() < 8 { + return Err(SnapshotError::Truncated { + need: 8, + have: buf.len(), + }); + } + let t = u64::from_le_bytes([ + buf[0], buf[1], buf[2], buf[3], buf[4], buf[5], buf[6], buf[7], + ]); + Ok(LamportTime::new(t)) +} + +/// Decode a coordinate record body (after the tag byte). +/// +/// Layout: `[node_len: u32 LE][node bytes][dim: u32 LE][dim × f64 LE][error f64][adj f64][height f64]` +#[cfg(feature = "coordinates")] +fn decode_coordinate_record( + buf: &[u8], +) -> Result<(CoordinateRecord, usize), SnapshotError> +where + I: Data, + A: Data, +{ + // Decode the node first. + let (node, node_n) = decode_node(buf)?; + let rest = &buf[node_n..]; + + // Read dim count. + if rest.len() < 4 { + return Err(SnapshotError::Truncated { + need: 4, + have: rest.len(), + }); + } + let dim = u32::from_le_bytes([rest[0], rest[1], rest[2], rest[3]]) as usize; + + // 4 (dim) + dim*8 + 8 (error) + 8 (adj) + 8 (height) + let coord_tail = 4 + dim * 8 + 8 + 8 + 8; + if rest.len() < coord_tail { + return Err(SnapshotError::Truncated { + need: coord_tail, + have: rest.len(), + }); + } + + let mut off = 4; // past the dim u32 + let mut vec = Vec::with_capacity(dim); + for _ in 0..dim { + let v = f64::from_le_bytes([ + rest[off], + rest[off + 1], + rest[off + 2], + rest[off + 3], + rest[off + 4], + rest[off + 5], + rest[off + 6], + rest[off + 7], + ]); + vec.push(v); + off += 8; + } + let error = f64::from_le_bytes([ + rest[off], + rest[off + 1], + rest[off + 2], + rest[off + 3], + rest[off + 4], + rest[off + 5], + rest[off + 6], + rest[off + 7], + ]); + off += 8; + let adjustment = f64::from_le_bytes([ + rest[off], + rest[off + 1], + rest[off + 2], + rest[off + 3], + rest[off + 4], + rest[off + 5], + rest[off + 6], + rest[off + 7], + ]); + off += 8; + let height = f64::from_le_bytes([ + rest[off], + rest[off + 1], + rest[off + 2], + rest[off + 3], + rest[off + 4], + rest[off + 5], + rest[off + 6], + rest[off + 7], + ]); + off += 8; + + let coordinate = Coordinate { + vec, + error, + adjustment, + height, + }; + Ok((CoordinateRecord::new(node, coordinate), node_n + off)) +} + +// ── ReplayResult ───────────────────────────────────────────────────────────── + +/// The output of replaying a snapshot record stream. +/// +/// Contains the set of alive nodes recovered from the snapshot, the three +/// Lamport clock high-water marks, and any coordinate records (when the +/// `coordinates` feature is enabled). +/// +/// The driver reads the snapshot file into [`SnapshotRecord`] values (via +/// [`SnapshotRecord::decode`]) and calls [`ReplayResult::replay`]; serf's +/// [`Endpoint::load_snapshot`][crate::endpoint::Endpoint::load_snapshot] then +/// applies the result to the machine state. This type owns no file handles or +/// I/O state — it is a pure data carrier. +/// +/// Mirrors the pure-record-folding logic in Go serf +/// `snapshot.go` `open_and_replay_snapshot` (lines 228-331). +#[derive(Debug, Clone)] +pub struct ReplayResult { + /// Nodes that were alive at the end of the snapshot, after applying all + /// `Alive` / `NotAlive` / `Leave` records. + /// + /// When `rejoin_after_leave` is `false` and the snapshot ends with a + /// `Leave` record, this list is empty (the leave cleared it). + pub alive_nodes: Vec>, + /// The member Lamport clock high-water mark seen in the snapshot. + /// + /// The `Endpoint` will set its member clock to at least this value + 1 + /// on [`load_snapshot`][crate::endpoint::Endpoint::load_snapshot] (G5). + pub last_clock: LamportTime, + /// The event Lamport clock high-water mark seen in the snapshot. + /// + /// The `Endpoint` will set `event_buffer.min_time` to `last_event_clock + 1` + /// so stale pre-snapshot events are not replayed after restart (G5). + pub last_event_clock: LamportTime, + /// The query Lamport clock high-water mark seen in the snapshot. + /// + /// The `Endpoint` will set `query_buffer.min_time` to `last_query_clock + 1` + /// so stale pre-snapshot queries are not replayed after restart (G5). + pub last_query_clock: LamportTime, +} + +impl ReplayResult +where + I: Eq + core::hash::Hash, +{ + /// Replay a flat sequence of [`SnapshotRecord`] values into a [`ReplayResult`]. + /// + /// This is the pure fold that mirrors Go serf `snapshot.go` + /// `open_and_replay_snapshot` (lines 265–331), but without any file I/O. + /// The driver feeds decoded records; this function folds them into the + /// recovered alive-node set and the three clock high-water marks. + /// + /// **G10 — rejoin-after-leave gating:** + /// When a [`SnapshotRecord::Leave`] record is encountered: + /// - If `rejoin_after_leave` is `true`, the record is **ignored** — the + /// alive-node set and clock floors are preserved so the node re-joins with + /// its previous membership intact. + /// - If `rejoin_after_leave` is `false`, the alive-node set is **cleared** + /// and all three clocks are **reset to zero** — the node starts fresh and + /// does NOT automatically re-dial its previous peers. + /// + /// **Record semantics (mirrors oracle):** + /// - `Alive(node)` — insert `node` into the alive set. + /// - `NotAlive(node)` — remove `node` from the alive set. + /// - `Clock(t)` — update `last_clock` to `t` (last wins). + /// - `EventClock(t)` — update `last_event_clock` to `t` (last wins). + /// - `QueryClock(t)` — update `last_query_clock` to `t` (last wins). + /// - `Coordinate(_)` — ignored here; the driver caches coordinates + /// separately so the `CoordinateClient` can be warmed on restart. + /// - `Leave` — apply the G10 gate described above. + /// - `Comment` — ignored. + pub fn replay( + records: impl IntoIterator>, + rejoin_after_leave: bool, + ) -> Self + where + A: Eq + core::hash::Hash + Clone, + I: Clone, + { + // `alive_vec` preserves snapshot record order (first-seen wins for + // insertion position); `alive_set` gives O(1) membership tests and + // drives dedup so each node appears at most once. Using a Vec here + // rather than a HashSet ensures `alive_nodes` is emitted in a stable, + // input-record order — two drivers that replay the same snapshot bytes + // produce an identical `DialRequested` sequence. + let mut alive_vec: Vec> = Vec::new(); + let mut alive_set: std::collections::HashSet> = + std::collections::HashSet::new(); + let mut last_clock = LamportTime::ZERO; + let mut last_event_clock = LamportTime::ZERO; + let mut last_query_clock = LamportTime::ZERO; + + for record in records { + match record { + SnapshotRecord::Alive(node) => { + if alive_set.insert(node.clone()) { + alive_vec.push(node); + } + } + SnapshotRecord::NotAlive(node) => { + if alive_set.remove(&node) { + alive_vec.retain(|n| n != &node); + } + } + SnapshotRecord::Clock(t) => { + last_clock = t; + } + SnapshotRecord::EventClock(t) => { + last_event_clock = t; + } + SnapshotRecord::QueryClock(t) => { + last_query_clock = t; + } + SnapshotRecord::Leave => { + if rejoin_after_leave { + // G10: ignore the leave record — preserve state for rejoin. + // Go serf logs "ignoring previous leave in snapshot" here. + } else { + // G10: clear everything — the node left and should not auto-rejoin. + alive_vec.clear(); + alive_set.clear(); + last_clock = LamportTime::ZERO; + last_event_clock = LamportTime::ZERO; + last_query_clock = LamportTime::ZERO; + } + } + SnapshotRecord::Comment => { + // Informational only; skip. + } + // Coordinate records are driver-side concerns; the machine does not + // consume them here (the `CoordinateClient` is warmed by the driver + // calling `get_coordinate` / `set_coordinate` after replay). + #[cfg(feature = "coordinates")] + SnapshotRecord::Coordinate(_) => {} + } + } + + Self { + alive_nodes: alive_vec, + last_clock, + last_event_clock, + last_query_clock, + } + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-proto/src/snapshot/tests.rs b/serf-proto/src/snapshot/tests.rs new file mode 100644 index 00000000..3887ce22 --- /dev/null +++ b/serf-proto/src/snapshot/tests.rs @@ -0,0 +1,487 @@ +use std::net::SocketAddr; + +use memberlist_proto::Node; + +use super::*; + +fn make_records_fwd() -> Vec> { + vec![ + SnapshotRecord::Alive(test_node(1, 1001)), + SnapshotRecord::Alive(test_node(2, 1002)), + SnapshotRecord::Alive(test_node(3, 1003)), + SnapshotRecord::Clock(LamportTime::new(10)), + SnapshotRecord::EventClock(LamportTime::new(20)), + SnapshotRecord::QueryClock(LamportTime::new(30)), + ] +} + +fn make_records_rev() -> Vec> { + vec![ + SnapshotRecord::Alive(test_node(3, 1003)), + SnapshotRecord::Alive(test_node(2, 1002)), + SnapshotRecord::Alive(test_node(1, 1001)), + SnapshotRecord::Clock(LamportTime::new(10)), + SnapshotRecord::EventClock(LamportTime::new(20)), + SnapshotRecord::QueryClock(LamportTime::new(30)), + ] +} + +// ── Helpers ─────────────────────────────────────────────────────────────────── + +fn test_node(id: u32, port: u16) -> Node { + Node::new(id, format!("127.0.0.1:{port}").parse().unwrap()) +} + +// ── Round-trip tests ────────────────────────────────────────────────────────── + +#[test] +fn clock_record_round_trips() { + let r = SnapshotRecord::::Clock(42.into()); + let bytes = r.encode().unwrap(); + let (back, n) = SnapshotRecord::::decode(&bytes).unwrap(); + assert_eq!(back, r); + assert_eq!(n, bytes.len(), "bytes_consumed == total encoded length"); +} + +#[test] +fn event_clock_record_round_trips() { + let r = SnapshotRecord::::EventClock(LamportTime::new(u64::MAX)); + let bytes = r.encode().unwrap(); + let (back, n) = SnapshotRecord::::decode(&bytes).unwrap(); + assert_eq!(back, r); + assert_eq!(n, bytes.len()); +} + +#[test] +fn query_clock_record_round_trips() { + let r = SnapshotRecord::::QueryClock(0.into()); + let bytes = r.encode().unwrap(); + let (back, n) = SnapshotRecord::::decode(&bytes).unwrap(); + assert_eq!(back, r); + assert_eq!(n, bytes.len()); +} + +#[test] +fn alive_record_round_trips() { + let node = test_node(7, 7000); + let r = SnapshotRecord::Alive(node); + let bytes = r.encode().unwrap(); + let (back, n) = SnapshotRecord::::decode(&bytes).unwrap(); + assert_eq!(n, bytes.len()); + match back { + SnapshotRecord::Alive(ref got) => assert_eq!(*got, node), + other => panic!("expected Alive, got {other:?}"), + } +} + +#[test] +fn not_alive_record_round_trips() { + let node = test_node(99, 9999); + let r = SnapshotRecord::NotAlive(node); + let bytes = r.encode().unwrap(); + let (back, n) = SnapshotRecord::::decode(&bytes).unwrap(); + assert_eq!(n, bytes.len()); + match back { + SnapshotRecord::NotAlive(ref got) => assert_eq!(*got, node), + other => panic!("expected NotAlive, got {other:?}"), + } +} + +#[test] +fn leave_record_is_single_byte() { + let r = SnapshotRecord::::Leave; + let bytes = r.encode().unwrap(); + assert_eq!(bytes.as_ref(), &[6u8]); + let (back, n) = SnapshotRecord::::decode(&bytes).unwrap(); + assert!(matches!(back, SnapshotRecord::Leave)); + assert_eq!(n, 1); +} + +#[test] +fn comment_record_is_single_byte() { + let r = SnapshotRecord::::Comment; + let bytes = r.encode().unwrap(); + assert_eq!(bytes.as_ref(), &[7u8]); + let (back, n) = SnapshotRecord::::decode(&bytes).unwrap(); + assert!(matches!(back, SnapshotRecord::Comment)); + assert_eq!(n, 1); +} + +// ── bytes_consumed accuracy ─────────────────────────────────────────────────── + +/// Multiple records concatenated: bytes_consumed lets the reader advance correctly. +#[test] +fn concatenated_records_advance_cursor_correctly() { + let r1 = SnapshotRecord::::Clock(10.into()); + let r2 = SnapshotRecord::::Alive(test_node(3, 3000)); + let r3 = SnapshotRecord::::Leave; + + let mut stream = Vec::new(); + stream.extend_from_slice(&r1.encode().unwrap()); + stream.extend_from_slice(&r2.encode().unwrap()); + stream.extend_from_slice(&r3.encode().unwrap()); + + let (got1, n1) = SnapshotRecord::::decode(&stream).unwrap(); + assert!(matches!(got1, SnapshotRecord::Clock(t) if t == LamportTime::new(10))); + let (got2, n2) = SnapshotRecord::::decode(&stream[n1..]).unwrap(); + assert!(matches!(got2, SnapshotRecord::Alive(_))); + let (got3, n3) = SnapshotRecord::::decode(&stream[n1 + n2..]).unwrap(); + assert!(matches!(got3, SnapshotRecord::Leave)); + assert_eq!(n1 + n2 + n3, stream.len()); +} + +// ── Error cases ─────────────────────────────────────────────────────────────── + +#[test] +fn empty_buffer_is_truncated() { + let r = SnapshotRecord::::decode(&[]); + assert!(matches!(r, Err(SnapshotError::Truncated { .. }))); +} + +#[test] +fn clock_record_truncated_body_is_error() { + // Only 4 bytes of the 8-byte clock body. + let buf = [TAG_CLOCK, 1, 2, 3, 4]; + let r = SnapshotRecord::::decode(&buf[..]); + assert!( + matches!(r, Err(SnapshotError::Truncated { .. })), + "truncated clock body must be Truncated" + ); +} + +#[test] +fn alive_record_truncated_node_len_is_error() { + // Only 2 bytes of the 4-byte length prefix. + let buf = [TAG_ALIVE, 0, 0]; + let r = SnapshotRecord::::decode(&buf[..]); + assert!(matches!(r, Err(SnapshotError::Truncated { .. }))); +} + +#[test] +fn alive_record_truncated_node_body_is_error() { + // Length prefix says 100 bytes but buffer has none. + let buf = [TAG_ALIVE, 100, 0, 0, 0]; + let r = SnapshotRecord::::decode(&buf[..]); + assert!(matches!(r, Err(SnapshotError::Truncated { .. }))); +} + +#[test] +fn unknown_tag_is_rejected() { + let buf = [0xFFu8]; + let r = SnapshotRecord::::decode(&buf[..]); + assert!(matches!(r, Err(SnapshotError::UnknownTag(0xFF)))); +} + +// ── ReplayResult tests ──────────────────────────────────────────────────────── + +#[test] +fn replay_empty_stream_gives_empty_result() { + let r = ReplayResult::::replay(vec![], false); + assert!(r.alive_nodes.is_empty()); + assert_eq!(u64::from(r.last_clock), 0); + assert_eq!(u64::from(r.last_event_clock), 0); + assert_eq!(u64::from(r.last_query_clock), 0); +} + +#[test] +fn replay_alive_records_accumulate() { + let recs = vec![ + SnapshotRecord::Alive(test_node(1, 1001)), + SnapshotRecord::Alive(test_node(2, 1002)), + ]; + let r = ReplayResult::::replay(recs, false); + assert_eq!(r.alive_nodes.len(), 2); +} + +#[test] +fn replay_not_alive_removes_node() { + let n1 = test_node(1, 1001); + let recs = vec![ + SnapshotRecord::Alive(n1), + SnapshotRecord::Alive(test_node(2, 1002)), + SnapshotRecord::NotAlive(n1), + ]; + let r = ReplayResult::::replay(recs, false); + // Node 1 was removed; only node 2 remains. + assert_eq!(r.alive_nodes.len(), 1, "NotAlive must remove the node"); + assert!( + r.alive_nodes.iter().any(|n| *n.id_ref() == 2u32), + "node 2 must still be alive" + ); +} + +#[test] +fn replay_clocks_take_last_value() { + let recs = vec![ + SnapshotRecord::::Clock(3.into()), + SnapshotRecord::Clock(7.into()), + SnapshotRecord::EventClock(11.into()), + SnapshotRecord::QueryClock(5.into()), + SnapshotRecord::QueryClock(9.into()), + ]; + let r = ReplayResult::::replay(recs, false); + assert_eq!(u64::from(r.last_clock), 7); + assert_eq!(u64::from(r.last_event_clock), 11); + assert_eq!(u64::from(r.last_query_clock), 9); +} + +#[test] +fn replay_leave_without_rejoin_clears_state() { + let recs = vec![ + SnapshotRecord::Alive(test_node(2, 1002)), + SnapshotRecord::Clock(5.into()), + SnapshotRecord::Leave, + ]; + let r = ReplayResult::::replay(recs, false); + assert!( + r.alive_nodes.is_empty(), + "Leave (no rejoin) must clear alive set" + ); + assert_eq!( + u64::from(r.last_clock), + 0, + "Leave (no rejoin) must reset clock" + ); + assert_eq!(u64::from(r.last_event_clock), 0); + assert_eq!(u64::from(r.last_query_clock), 0); +} + +#[test] +fn replay_leave_with_rejoin_keeps_state() { + let recs = vec![ + SnapshotRecord::Alive(test_node(2, 1002)), + SnapshotRecord::Clock(5.into()), + SnapshotRecord::Leave, + ]; + let r = ReplayResult::::replay(recs, true); + assert_eq!( + r.alive_nodes.len(), + 1, + "Leave with rejoin must keep alive set" + ); + assert_eq!( + u64::from(r.last_clock), + 5, + "Leave with rejoin must keep clock" + ); +} + +#[test] +fn replay_leave_then_alive_after_rejoin_accumulates() { + // Leave (ignore with rejoin=true), then more Alive records after. + let recs = vec![ + SnapshotRecord::Alive(test_node(1, 1001)), + SnapshotRecord::Leave, + SnapshotRecord::Alive(test_node(2, 1002)), + ]; + let r = ReplayResult::::replay(recs, true); + // Both nodes are alive: Leave was ignored. + assert_eq!(r.alive_nodes.len(), 2); +} + +#[test] +fn replay_leave_then_alive_without_rejoin_clears_then_accumulates() { + // Leave (clear with rejoin=false), then more Alive records after. + let recs = vec![ + SnapshotRecord::Alive(test_node(1, 1001)), + SnapshotRecord::Leave, + SnapshotRecord::Alive(test_node(2, 1002)), + SnapshotRecord::Clock(8.into()), + ]; + let r = ReplayResult::::replay(recs, false); + // Only node 2 (alive after leave); node 1 was cleared. + assert_eq!(r.alive_nodes.len(), 1); + assert_eq!(r.alive_nodes[0], test_node(2, 1002)); + assert_eq!(u64::from(r.last_clock), 8); +} + +#[test] +fn replay_comment_is_ignored() { + let recs = vec![ + SnapshotRecord::Alive(test_node(1, 1001)), + SnapshotRecord::::Comment, + ]; + let r = ReplayResult::::replay(recs, false); + assert_eq!(r.alive_nodes.len(), 1); +} + +#[cfg(feature = "coordinates")] +#[test] +fn replay_coordinate_record_is_ignored() { + use crate::Coordinate; + let coord = Coordinate { + vec: vec![1.0; 8], + error: 0.5, + adjustment: 0.0, + height: 0.0, + }; + let recs = vec![ + SnapshotRecord::Alive(test_node(1, 1001)), + SnapshotRecord::Coordinate(super::CoordinateRecord::new(test_node(1, 1001), coord)), + SnapshotRecord::::Clock(3.into()), + ]; + let r = ReplayResult::::replay(recs, false); + // Coordinate is ignored; alive set and clock are unaffected. + assert_eq!(r.alive_nodes.len(), 1); + assert_eq!(u64::from(r.last_clock), 3); +} + +// ── Tag constants ───────────────────────────────────────────────────────────── + +#[test] +fn discriminant_bytes_match_oracle() { + // Verify that each encoded record starts with the correct tag as per the + // serf-core snapshot.rs layout (Alive=0, NotAlive=1, Clock=2, EventClock=3, + // QueryClock=4, Leave=6, Comment=7). + let node = test_node(1, 1234); + + let cases: &[(u8, SnapshotRecord)] = &[ + (0, SnapshotRecord::Alive(node)), + (1, SnapshotRecord::NotAlive(node)), + (2, SnapshotRecord::Clock(1.into())), + (3, SnapshotRecord::EventClock(1.into())), + (4, SnapshotRecord::QueryClock(1.into())), + (6, SnapshotRecord::Leave), + (7, SnapshotRecord::Comment), + ]; + for (expected_tag, rec) in cases { + let bytes = rec.encode().unwrap(); + assert_eq!( + bytes[0], *expected_tag, + "record {rec:?} must encode with tag {expected_tag}" + ); + } +} + +// ── Coordinate feature tests ────────────────────────────────────────────────── + +#[cfg(feature = "coordinates")] +mod coordinates { + use super::*; + use crate::Coordinate; + + fn test_coord() -> Coordinate { + Coordinate { + vec: vec![1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0], + error: 0.5, + adjustment: 0.1, + height: 0.01, + } + } + + #[test] + fn coordinate_record_round_trips() { + // Node is Copy so we can use it freely after passing into the record. + let node = test_node(42, 4242); + let coord = test_coord(); + let rec = CoordinateRecord::new(node, coord.clone()); + let r = SnapshotRecord::Coordinate(rec); + let bytes = r.encode().unwrap(); + // First byte must be the coordinate tag (5). + assert_eq!(bytes[0], 5u8, "Coordinate tag must be 5"); + let (back, n) = SnapshotRecord::::decode(&bytes).unwrap(); + assert_eq!(n, bytes.len()); + match back { + SnapshotRecord::Coordinate(got) => { + assert_eq!(*got.node(), node); // node is Copy; still usable + assert_eq!(got.coordinate().vec, coord.vec); + assert_eq!(got.coordinate().error, coord.error); + assert_eq!(got.coordinate().adjustment, coord.adjustment); + assert_eq!(got.coordinate().height, coord.height); + } + other => panic!("expected Coordinate, got {other:?}"), + } + } + + #[test] + fn coordinate_tag_is_five() { + let node = test_node(1, 1000); + let rec = SnapshotRecord::Coordinate(CoordinateRecord::new(node, test_coord())); + let bytes = rec.encode().unwrap(); + assert_eq!(bytes[0], 5u8); + } + + #[test] + fn zero_dimensional_coordinate_round_trips() { + let node = test_node(1, 1000); + let coord = Coordinate { + vec: vec![], + error: 0.0, + adjustment: 0.0, + height: 0.0, + }; + let rec = SnapshotRecord::Coordinate(CoordinateRecord::new(node, coord.clone())); + let bytes = rec.encode().unwrap(); + let (back, n) = SnapshotRecord::::decode(&bytes).unwrap(); + assert_eq!(n, bytes.len()); + match back { + SnapshotRecord::Coordinate(got) => { + assert_eq!(*got.node(), node); + assert!(got.coordinate().vec.is_empty()); + } + other => panic!("expected Coordinate, got {other:?}"), + } + } +} + +// ── Determinism: snapshot replay dial order ──────────────────────────────────── + +#[test] +fn snapshot_replay_dial_order_deterministic() { + // Replaying the same alive nodes in two different record orderings must + // produce identical alive_nodes lists (same nodes, same sequence). + // This confirms that snapshot replay does not depend on HashSet iteration order. + let result_fwd = ReplayResult::::replay(make_records_fwd(), false); + let result_rev = ReplayResult::::replay(make_records_rev(), false); + + // Both replays must yield the same set of alive nodes. + assert_eq!( + result_fwd.alive_nodes.len(), + result_rev.alive_nodes.len(), + "both replays must yield the same node count" + ); + // The fwd replay preserves record order (1, 2, 3); the rev replay preserves + // its record order (3, 2, 1). The key property: each run is self-consistent — + // running the same record stream twice must yield an identical Vec, not a + // HashSet-order-dependent one. + let result_fwd2 = ReplayResult::::replay(make_records_fwd(), false); + let result_rev2 = ReplayResult::::replay(make_records_rev(), false); + assert_eq!( + result_fwd.alive_nodes, result_fwd2.alive_nodes, + "same record stream must produce identical alive_nodes on every call" + ); + assert_eq!( + result_rev.alive_nodes, result_rev2.alive_nodes, + "same record stream (reversed) must produce identical alive_nodes on every call" + ); + // Verify insertion-order is preserved: fwd → [1,2,3], rev → [3,2,1]. + assert_eq!( + result_fwd.alive_nodes, + vec![test_node(1, 1001), test_node(2, 1002), test_node(3, 1003)], + "forward record order must yield nodes in insertion order" + ); + assert_eq!( + result_rev.alive_nodes, + vec![test_node(3, 1003), test_node(2, 1002), test_node(1, 1001)], + "reversed record order must yield nodes in insertion order" + ); +} + +#[test] +fn snapshot_replay_not_alive_dedup_is_stable() { + // Alive → NotAlive → re-Alive: the node must appear at the re-insertion position. + // Also verifies that dedup and removal are both stable. + let records = vec![ + SnapshotRecord::Alive(test_node(1, 1001)), + SnapshotRecord::Alive(test_node(2, 1002)), + SnapshotRecord::NotAlive(test_node(1, 1001)), // removes node 1 + SnapshotRecord::Alive(test_node(1, 1001)), // re-inserts node 1 at end + ]; + let result = ReplayResult::::replay(records, false); + // Node 2 was inserted first and never removed; node 1 was re-inserted after 2. + assert_eq!( + result.alive_nodes, + vec![test_node(2, 1002), test_node(1, 1001)], + "re-inserted node must appear at its re-insertion position" + ); +} diff --git a/serf-proto/src/typed/mod.rs b/serf-proto/src/typed/mod.rs index 9e7cd5b7..306ee1b2 100644 --- a/serf-proto/src/typed/mod.rs +++ b/serf-proto/src/typed/mod.rs @@ -270,27 +270,27 @@ pub struct UserEvents { /// The `status_ltimes` map is transmitted as a `repeated NodeStatusTime` rather /// than a proto3 `map` because proto3 forbids `bytes` map keys. #[derive(Debug, Clone, PartialEq, Eq)] -pub struct PushPullMessage { +pub(crate) struct PushPullMessage { /// The lamport clock value of the sending node at the time of the exchange. - pub ltime: LamportTime, + pub(crate) ltime: LamportTime, /// Maps each known node-id to its last-seen status lamport time. - pub status_ltimes: Vec<(I, LamportTime)>, + pub(crate) status_ltimes: Vec<(I, LamportTime)>, /// Wire list of node-ids that have left the cluster. /// /// The codec does NOT dedup this list; the consuming machine is responsible /// for treating it as a set (duplicate entries are idempotent leave events). - pub left_members: Vec, + pub(crate) left_members: Vec, /// The lamport clock value for the event subsystem. - pub event_ltime: LamportTime, + pub(crate) event_ltime: LamportTime, /// Buffered user-event batches. - pub events: Vec, + pub(crate) events: Vec, /// The lamport clock value for the query subsystem. - pub query_ltime: LamportTime, + pub(crate) query_ltime: LamportTime, } impl PushPullMessage { /// Construct a new `PushPullMessage`. - pub fn new( + pub(crate) fn new( ltime: LamportTime, status_ltimes: Vec<(I, LamportTime)>, left_members: Vec, @@ -318,20 +318,16 @@ impl PushPullMessage { /// /// Requires the `aes-gcm` or `chacha20-poly1305` feature. #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] -#[cfg_attr( - docsrs, - doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) -)] #[derive(Debug, Clone, PartialEq, Eq)] -pub struct KeyRequestMessage { +pub(crate) struct KeyRequestMessage { /// The encryption key, or `None` for a list-keys request. - pub key: Option, + pub(crate) key: Option, } #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] impl KeyRequestMessage { /// Construct a new `KeyRequestMessage`. - pub fn new(key: Option) -> Self { + pub(crate) fn new(key: Option) -> Self { Self { key } } } @@ -345,28 +341,16 @@ impl KeyRequestMessage { /// /// Requires the `aes-gcm` or `chacha20-poly1305` feature. #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] -#[cfg_attr( - docsrs, - doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) -)] #[derive(Debug, Default, Clone, PartialEq, Eq)] -pub struct KeyResponseMessage { +pub(crate) struct KeyResponseMessage { /// `true` if the operation succeeded on this node. - pub result: bool, + pub(crate) result: bool, /// Human-readable result or error description. - pub message: SmolStr, + pub(crate) message: SmolStr, /// Installed keys (used by list-keys responses). - pub keys: Vec, + pub(crate) keys: Vec, /// The current primary key, if reporting it. - pub primary_key: Option, -} - -#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] -impl KeyResponseMessage { - /// Construct a default (failure, no keys) `KeyResponseMessage`. - pub fn new() -> Self { - Self::default() - } + pub(crate) primary_key: Option, } // ── RelayMessage ────────────────────────────────────────────────────────────── @@ -381,16 +365,16 @@ impl KeyResponseMessage { /// bound is enforced at the bridge layer (`relay_to_pb` / `relay_from_pb`) where /// the destination is encoded/decoded as opaque `bytes`. #[derive(Debug, Clone, PartialEq, Eq)] -pub struct RelayMessage { +pub(crate) struct RelayMessage { /// The node to forward the message to. - pub destination: Node, + pub(crate) destination: Node, /// The inner serf framed message, carried unparsed. - pub payload: Bytes, + pub(crate) payload: Bytes, } impl RelayMessage { /// Construct a new `RelayMessage`. - pub fn new(destination: Node, payload: Bytes) -> Self { + pub(crate) fn new(destination: Node, payload: Bytes) -> Self { Self { destination, payload, @@ -406,16 +390,16 @@ impl RelayMessage { /// `memberlist_proto::Data` so it can be encoded as opaque proto `bytes` in /// the bridge layer. #[derive(Debug, Clone, PartialEq, Eq)] -pub struct JoinMessage { +pub(crate) struct JoinMessage { /// The lamport clock value at the time the node joined. - pub ltime: crate::LamportTime, + pub(crate) ltime: crate::LamportTime, /// The joining node's identifier. - pub id: I, + pub(crate) id: I, } impl JoinMessage { /// Construct a new `JoinMessage`. - pub fn new(ltime: crate::LamportTime, id: I) -> Self { + pub(crate) fn new(ltime: crate::LamportTime, id: I) -> Self { Self { ltime, id } } } @@ -426,18 +410,18 @@ impl JoinMessage { /// `memberlist_proto::Data` so it can be encoded as opaque proto `bytes` in /// the bridge layer. #[derive(Debug, Clone, PartialEq, Eq)] -pub struct LeaveMessage { +pub(crate) struct LeaveMessage { /// The lamport clock value at the time the leave was emitted. - pub ltime: crate::LamportTime, + pub(crate) ltime: crate::LamportTime, /// The leaving node's identifier. - pub id: I, + pub(crate) id: I, /// Whether the leave is a prune (permanent removal) rather than a graceful leave. - pub prune: bool, + pub(crate) prune: bool, } impl LeaveMessage { /// Construct a new `LeaveMessage`. - pub fn new(ltime: crate::LamportTime, id: I, prune: bool) -> Self { + pub(crate) fn new(ltime: crate::LamportTime, id: I, prune: bool) -> Self { Self { ltime, id, prune } } } @@ -448,14 +432,14 @@ impl LeaveMessage { /// implement `memberlist_proto::Data` so the embedded `Node` can be /// encoded as opaque proto `bytes` in the bridge layer. #[derive(Debug, Clone, PartialEq, Eq)] -pub struct ConflictResponseMessage { +pub(crate) struct ConflictResponseMessage { /// The winning node in the conflict resolution. - pub member: Node, + pub(crate) member: Node, } impl ConflictResponseMessage { /// Construct a new `ConflictResponseMessage`. - pub fn new(member: Node) -> Self { + pub(crate) fn new(member: Node) -> Self { Self { member } } } From 311f2130989dd90eaeba75c785f4dddaa9d674ee Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 26 Jun 2026 13:51:50 +1200 Subject: [PATCH 010/128] ci: test serf-proto across feature combos and add to coverage Modernize the coverage nightly-toolchain step (drop the deprecated actions-rs/toolchain action) and quote the tarpaulin output-dir path. --- .github/workflows/ci.yml | 41 ++++++++++++++++++++++++++++++++++ .github/workflows/coverage.yml | 10 ++++----- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 17d882b3..8790dd5a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -72,3 +72,44 @@ jobs: ~/.cargo/git target key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-${{ matrix.runtime }}-${{ matrix.stream_layer }} + + serf-proto: + name: serf-proto - ${{ matrix.name }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - { name: "default", flags: "" } + - { name: "coords-aes-gcm", flags: "--features coordinates,aes-gcm" } + - { name: "coords-chacha", flags: "--features coordinates,chacha20-poly1305" } + - { name: "all-features", flags: "--all-features" } + - { name: "no-default", flags: "--no-default-features" } + steps: + - uses: actions/checkout@v6 + + - name: Install Rust + run: | + rustup update stable && rustup default stable + rustup component add clippy + rustup component add rustfmt + + - name: Format check + if: matrix.name == 'default' + run: cargo fmt --check + working-directory: serf-proto + + - name: Clippy + run: cargo clippy -p serf-proto --all-targets ${{ matrix.flags }} -- -D warnings + + - name: Test + run: cargo test -p serf-proto ${{ matrix.flags }} + + - name: Cache Cargo registry + uses: actions/cache@v5 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-serf-proto-${{ hashFiles('**/Cargo.lock') }}-${{ matrix.name }} diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 8aebbca7..ee2d1b2d 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -37,20 +37,20 @@ jobs: - crate: serf features: "test,tokio,tcp,encryption,metrics" name: "serf-tcp-encryption" + - crate: serf-proto + features: "encryption,coordinates,tag-regex" + name: "serf-proto" steps: - uses: actions/checkout@v6 - name: Install latest nightly - uses: actions-rs/toolchain@v1 - with: - toolchain: nightly - override: true + run: rustup toolchain install nightly && rustup default nightly - name: Setup loopback interface run: ci/setup_subnet_ubuntu.sh - name: Install cargo-tarpaulin run: cargo install cargo-tarpaulin - name: Run Coverage and Generate Report run: | - cargo tarpaulin --out xml --no-default-features --features ${{ matrix.features }} --no-fail-fast --output-dir $PWD -j 1 + cargo tarpaulin --out xml --no-default-features --features ${{ matrix.features }} --no-fail-fast --output-dir "$PWD" -j 1 working-directory: ${{ matrix.crate }} continue-on-error: true - name: Upload Coverage Report as Artifact From f6ce74c5c0f40be64f90eb8312292cc493dd4ad2 Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 26 Jun 2026 19:41:12 +1200 Subject: [PATCH 011/128] feat(serf-proto): add the Reliable seam bridging the memberlist coordinators A narrow pub(crate) trait the serf core uses to reach a memberlist reliable coordinator (queue, send, snapshot, push-pull, leave, poll), implemented for the Stream (tcp/tls) and QUIC coordinators. Adds the tcp/tls/quic feature forwards to serf-proto. --- serf-proto/Cargo.toml | 12 + serf-proto/src/endpoint/mod.rs | 2 + serf-proto/src/endpoint/reliable/mod.rs | 377 ++++++++++++++++++++++ serf-proto/src/endpoint/reliable/tests.rs | 85 +++++ 4 files changed, 476 insertions(+) create mode 100644 serf-proto/src/endpoint/reliable/mod.rs create mode 100644 serf-proto/src/endpoint/reliable/tests.rs diff --git a/serf-proto/Cargo.toml b/serf-proto/Cargo.toml index e39a1e30..4e1912fe 100644 --- a/serf-proto/Cargo.toml +++ b/serf-proto/Cargo.toml @@ -30,6 +30,18 @@ coordinates = [] # `tag-regex` is the forward-compatible preparation for a future no_std port, # not current no_std support. tag-regex = ["dep:regex"] +# Plain-TCP reliable coordinator — enables `memberlist_proto::streams::StreamEndpoint` +# (record-layer-shaped, label-prefixed stream transport). Activating this feature +# makes `Reliable` implementable for `memberlist_proto::TcpEndpoint` and friends. +tcp = ["memberlist-proto/tcp"] +# TLS-over-TCP reliable coordinator — enables the TLS record layer on top of the +# plain-TCP stream transport. Implies `tcp`. +tls = ["tcp", "memberlist-proto/tls"] +# QUIC coordinator — enables `memberlist_proto::QuicEndpoint` (unreliable datagram + +# reliable stream over a single QUIC connection per peer). Requires a TLS backend; +# at least one of `quic-rustls-ring` / `quic-rustls-aws-lc-rs` must be selected. +quic = ["memberlist-proto/quic"] +quic-rustls-ring = ["quic", "memberlist-proto/quic-rustls-ring"] [dependencies] memberlist-proto = { git = "https://github.com/al8n/memberlist", branch = "main", default-features = false, features = ["std"] } diff --git a/serf-proto/src/endpoint/mod.rs b/serf-proto/src/endpoint/mod.rs index dc42fea6..3e223913 100644 --- a/serf-proto/src/endpoint/mod.rs +++ b/serf-proto/src/endpoint/mod.rs @@ -5015,6 +5015,8 @@ where } } +pub(crate) mod reliable; + #[cfg(test)] mod serf_parity_tests; #[cfg(test)] diff --git a/serf-proto/src/endpoint/reliable/mod.rs b/serf-proto/src/endpoint/reliable/mod.rs new file mode 100644 index 00000000..4b6cc62b --- /dev/null +++ b/serf-proto/src/endpoint/reliable/mod.rs @@ -0,0 +1,377 @@ +//! Narrow serf-facing interface to a memberlist reliable coordinator. +//! +//! [`Reliable`] is the only serf-to-coordinator boundary: every call serf's +//! `Endpoint` makes into memberlist passes through one of these methods. +//! It hides the concrete coordinator type (`StreamEndpoint` vs `QuicEndpoint`) +//! so that Tasks 3–6 can wire serf logic against a generic `&mut impl +//! Reliable` without knowing which transport is in use. +//! +//! # Method classification +//! +//! **Read-only accessors** (`local_id_ref`, `advertise_ref`, +//! `local_state_snapshot_bytes`, `user_broadcast_queue_len`) reach the inner +//! membership [`memberlist_proto::Endpoint`] directly through +//! [`Reliable::endpoint_ref`]. They are not duplicated as trait methods. +//! +//! **Mutating operations** are trait methods; each has a single, unambiguous +//! entry point whose contract is documented below. +//! +//! **Coordinator-internal surfaces** (`handle_packet`, `handle_stream_event`, +//! `accept_stream`, `handle_timeout`, `poll_timeout`, `poll_transmit`) are +//! NOT part of this trait — the coordinators own the stream / transport +//! lifecycle, and serf should not reach into it. + +use bytes::Bytes; +use memberlist_proto::{Endpoint, Instant, PushPullKind, Rng, StreamId}; + +/// Serf-facing interface to a memberlist reliable coordinator. +/// +/// Implemented by the two concrete coordinators serf composes with: +/// - [`memberlist_proto::streams::StreamEndpoint`] (plain-TCP or TLS record layer) +/// - [`memberlist_proto::QuicEndpoint`] (QUIC) +/// +/// The trait is `pub(crate)` because it is a serf-internal composition +/// boundary, not part of the public API. +// No production call site exists yet: the serf-logic core still reaches its inner +// endpoint directly. This trait is the seam it routes through once composed with a +// reliable coordinator. +#[expect( + dead_code, + reason = "the serf-logic core has not yet been routed through this seam" +)] +pub(crate) trait Reliable +where + I: Eq + core::hash::Hash, +{ + /// The RNG type used by the inner membership [`Endpoint`]. + type Rng: memberlist_proto::Rng; + + /// Borrow the inner membership [`Endpoint`] for read-only access. + /// + /// Callers use this to reach the read-only accessors that are not + /// duplicated as trait methods: `local_id_ref`, `advertise_ref`, + /// `local_state_snapshot_bytes`, `user_broadcast_queue_len`. + fn endpoint_ref(&self) -> &Endpoint; + + /// Drain one pending event from the coordinator's event queue. + /// + /// Serf calls this in a loop (`while let Some(ev) = t.poll_inner_event()`) + /// to intercept every memberlist event (join/leave/suspect/failed/user-data + /// push-pull / stream lifecycle) and fold it into serf's own FSM. + /// + /// Maps to `poll_event` on the coordinator. + fn poll_inner_event(&mut self) -> Option>; + + /// Broadcast `data` on the gossip plane at priority `rank` (`0` = highest). + /// + /// Serf uses rank 0 for leave/join messages and rank 1 for user-event and + /// query messages so that membership churn travels faster than application + /// traffic. + /// + /// # Errors + /// + /// Propagates [`memberlist_proto::Error`] from the inner broadcast queue + /// (e.g. the queue is full or the coordinator is not running). + fn queue_user_broadcast_ranked( + &mut self, + rank: u8, + data: Bytes, + ) -> Result<(), memberlist_proto::Error>; + + /// Enqueue a directed unreliable (gossip-plane) packet to `to`. + /// + /// Used for targeted serf messages that must reach a specific peer rather + /// than being disseminated to the cluster (e.g. conflict-query responses, + /// directed user-event packets). + /// + /// # Errors + /// + /// Propagates [`memberlist_proto::Error`] if the coordinator is not running + /// or the packet would exceed the gossip MTU. + fn send_user_packet(&mut self, to: A, data: Bytes) -> Result<(), memberlist_proto::Error>; + + /// Replace the local push-pull state snapshot. + /// + /// Serf serializes its own membership state (lamport clocks + serf member + /// list) into a `Bytes` blob and stores it here so it is shipped to any + /// peer that initiates a push-pull anti-entropy exchange. The snapshot is + /// re-computed whenever `local_state_dirty` is set (e.g. after a member + /// joins, leaves, or updates tags). + /// + /// # Errors + /// + /// Returns [`memberlist_proto::Error::LocalStateExceedsFrame`] if the + /// snapshot would not fit inside a reliable-stream frame. + fn set_local_state_snapshot(&mut self, bytes: Bytes) -> Result<(), memberlist_proto::Error>; + + /// Initiate an outbound push-pull anti-entropy exchange with `peer`. + /// + /// The coordinator dials `peer`, performs a label handshake, and exchanges + /// the membership state blob in both directions. Serf calls this on join + /// and on periodic anti-entropy ticks. + /// + /// Returns the [`StreamId`] allocated by the inner endpoint for this + /// exchange; serf currently discards it (the exchange outcome arrives via + /// `poll_inner_event`). + fn start_push_pull(&mut self, peer: A, kind: PushPullKind, now: Instant) -> StreamId; + + /// Update the local node's metadata advertised to peers. + /// + /// Serf stores its tag set (and, when relevant, the coordinate bytes) in + /// the memberlist meta field. This call propagates the change through the + /// coordinator so peers learn the new metadata on the next gossip cycle. + /// + /// # Errors + /// + /// Returns [`memberlist_proto::Error::MetaExceedsLimit`] if the serialised + /// metadata exceeds the memberlist wire limit. + fn update_meta( + &mut self, + meta: memberlist_proto::typed::Meta, + ) -> Result<(), memberlist_proto::Error>; + + /// Signal that the local node intends to leave the cluster gracefully. + /// + /// The coordinator disseminates a Leave message, transitions the inner + /// membership state to `Left`, and cancels pending reliable exchanges that + /// have not yet been written to the wire. After this call, `poll_timeout` + /// and `poll_transmit` drain remaining output; no new exchanges may be + /// initiated. + /// + /// # Errors + /// + /// Returns [`memberlist_proto::Error`] if the coordinator is already in a + /// terminal state. + fn leave(&mut self, now: Instant) -> Result<(), memberlist_proto::Error>; + + /// Attach an application payload to outbound probe Ack messages. + /// + /// The payload is included verbatim in the `AckResponse` the coordinator + /// sends when it is probed by another node. Serf uses this slot to carry + /// its per-node coordinate so peers can compute network-distance estimates + /// without a separate round-trip. + /// + /// # Errors + /// + /// Returns [`memberlist_proto::Error::AckPayloadExceedsMtu`] if the framed + /// Ack would not fit the gossip packet budget. + fn set_ack_payload(&mut self, payload: Bytes) -> Result<(), memberlist_proto::Error>; +} + +// ── impl for the raw memberlist_proto::Endpoint ─────────────────────────────── +// +// Covers the transitional period where the serf `Endpoint` still holds a raw +// `memberlist_proto::Endpoint` as its `inner` field. It is removed once the +// concrete coordinator super-machines own the transport. + +impl Reliable for Endpoint +where + I: memberlist_proto::Id, + A: memberlist_proto::CheapClone + memberlist_proto::Data + PartialEq + 'static, + R: Rng, +{ + type Rng = R; + + #[inline] + fn endpoint_ref(&self) -> &Endpoint { + self + } + + #[inline] + fn poll_inner_event(&mut self) -> Option> { + // Fully-qualified call to reach the inherent method; the trait method has + // the same name and would recurse without the explicit path. + Endpoint::poll_event(self) + } + + #[inline] + fn queue_user_broadcast_ranked( + &mut self, + rank: u8, + data: Bytes, + ) -> Result<(), memberlist_proto::Error> { + Endpoint::queue_user_broadcast_ranked(self, rank, data) + } + + #[inline] + fn send_user_packet(&mut self, to: A, data: Bytes) -> Result<(), memberlist_proto::Error> { + Endpoint::send_user_packet(self, to, data) + } + + #[inline] + fn set_local_state_snapshot(&mut self, bytes: Bytes) -> Result<(), memberlist_proto::Error> { + Endpoint::set_local_state_snapshot(self, bytes) + } + + #[inline] + fn start_push_pull(&mut self, peer: A, kind: PushPullKind, now: Instant) -> StreamId { + Endpoint::start_push_pull(self, peer, kind, now) + } + + #[inline] + fn update_meta( + &mut self, + meta: memberlist_proto::typed::Meta, + ) -> Result<(), memberlist_proto::Error> { + Endpoint::update_meta(self, meta) + } + + #[inline] + fn leave(&mut self, now: Instant) -> Result<(), memberlist_proto::Error> { + Endpoint::leave(self, now) + } + + #[inline] + fn set_ack_payload(&mut self, payload: Bytes) -> Result<(), memberlist_proto::Error> { + Endpoint::set_ack_payload(self, payload) + } +} + +// ── impl for memberlist_proto::streams::StreamEndpoint ──────────────────────── +// +// Enabled when the `tcp` or `tls` feature is active (which enables +// `memberlist-proto/tcp` or `memberlist-proto/tls`; both expose +// `streams::StreamEndpoint`). + +#[cfg(any(feature = "tcp", feature = "tls"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "tls"))))] +impl Reliable for memberlist_proto::streams::StreamEndpoint +where + I: memberlist_proto::Id, + A: memberlist_proto::CheapClone + memberlist_proto::Data + PartialEq + 'static, + R: memberlist_proto::streams::StreamTransport, + G: Rng, +{ + type Rng = G; + + #[inline] + fn endpoint_ref(&self) -> &Endpoint { + self.endpoint_ref() + } + + #[inline] + fn poll_inner_event(&mut self) -> Option> { + self.poll_event() + } + + #[inline] + fn queue_user_broadcast_ranked( + &mut self, + rank: u8, + data: Bytes, + ) -> Result<(), memberlist_proto::Error> { + self.queue_user_broadcast_ranked(rank, data) + } + + #[inline] + fn send_user_packet(&mut self, to: A, data: Bytes) -> Result<(), memberlist_proto::Error> { + self.send_user_packet(to, data) + } + + #[inline] + fn set_local_state_snapshot(&mut self, bytes: Bytes) -> Result<(), memberlist_proto::Error> { + self.set_local_state_snapshot(bytes) + } + + #[inline] + fn start_push_pull(&mut self, peer: A, kind: PushPullKind, now: Instant) -> StreamId { + self.start_push_pull(peer, kind, now) + } + + #[inline] + fn update_meta( + &mut self, + meta: memberlist_proto::typed::Meta, + ) -> Result<(), memberlist_proto::Error> { + self.update_meta(meta) + } + + #[inline] + fn leave(&mut self, now: Instant) -> Result<(), memberlist_proto::Error> { + self.leave(now) + } + + #[inline] + fn set_ack_payload(&mut self, payload: Bytes) -> Result<(), memberlist_proto::Error> { + self.set_ack_payload(payload) + } +} + +// ── impl for memberlist_proto::QuicEndpoint ─────────────────────────────────── +// +// Enabled when the `quic` feature is active (which enables +// `memberlist-proto/quic`). The QUIC coordinator pins `A = SocketAddr`. + +#[cfg(feature = "quic")] +#[cfg_attr(docsrs, doc(cfg(feature = "quic")))] +impl Reliable for memberlist_proto::QuicEndpoint +where + I: memberlist_proto::Id, + R: Rng, +{ + type Rng = R; + + #[inline] + fn endpoint_ref(&self) -> &Endpoint { + self.endpoint_ref() + } + + #[inline] + fn poll_inner_event(&mut self) -> Option> { + self.poll_event() + } + + #[inline] + fn queue_user_broadcast_ranked( + &mut self, + rank: u8, + data: Bytes, + ) -> Result<(), memberlist_proto::Error> { + self.queue_user_broadcast_ranked(rank, data) + } + + #[inline] + fn send_user_packet( + &mut self, + to: core::net::SocketAddr, + data: Bytes, + ) -> Result<(), memberlist_proto::Error> { + self.send_user_packet(to, data) + } + + #[inline] + fn set_local_state_snapshot(&mut self, bytes: Bytes) -> Result<(), memberlist_proto::Error> { + self.set_local_state_snapshot(bytes) + } + + #[inline] + fn start_push_pull( + &mut self, + peer: core::net::SocketAddr, + kind: PushPullKind, + now: Instant, + ) -> StreamId { + self.start_push_pull(peer, kind, now) + } + + #[inline] + fn update_meta( + &mut self, + meta: memberlist_proto::typed::Meta, + ) -> Result<(), memberlist_proto::Error> { + self.update_meta(meta) + } + + #[inline] + fn leave(&mut self, now: Instant) -> Result<(), memberlist_proto::Error> { + self.leave(now) + } + + #[inline] + fn set_ack_payload(&mut self, payload: Bytes) -> Result<(), memberlist_proto::Error> { + self.set_ack_payload(payload) + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-proto/src/endpoint/reliable/tests.rs b/serf-proto/src/endpoint/reliable/tests.rs new file mode 100644 index 00000000..1dec0041 --- /dev/null +++ b/serf-proto/src/endpoint/reliable/tests.rs @@ -0,0 +1,85 @@ +use super::Reliable; +use bytes::Bytes; +use memberlist_proto::{EndpointOptions, Instant, PushPullKind, SeedableRng, SmallRng}; + +/// Smoke-test a `Reliable` implementation through a generic function so that +/// the impls are exercised at the trait boundary, not just as concrete calls. +/// +/// Exercises `endpoint_ref` (read-only accessors), `queue_user_broadcast_ranked`, +/// `set_ack_payload`, `set_local_state_snapshot`, `poll_inner_event`, and +/// `start_push_pull` — methods that run on a fresh, un-started endpoint +/// without requiring a live network peer. +fn drive(t: &mut impl Reliable, addr: A) +where + I: Eq + core::hash::Hash + Clone, + A: Clone, +{ + // Read-only accessor via endpoint_ref. + let _ = t.endpoint_ref().local_state_snapshot_bytes(); + let _ = t.endpoint_ref().user_broadcast_queue_len(); + + // Mutating: queue a broadcast at rank 0 (highest priority). + let result = t.queue_user_broadcast_ranked(0, Bytes::from_static(b"hello")); + assert!( + result.is_ok(), + "queue_user_broadcast_ranked failed: {result:?}" + ); + + // Mutating: set ack payload. + let result = t.set_ack_payload(Bytes::from_static(b"coord")); + assert!(result.is_ok(), "set_ack_payload failed: {result:?}"); + + // Mutating: set local state snapshot. + let result = t.set_local_state_snapshot(Bytes::from_static(b"state")); + assert!( + result.is_ok(), + "set_local_state_snapshot failed: {result:?}" + ); + + // poll_inner_event drains without panicking on a fresh endpoint. + while t.poll_inner_event().is_some() {} + + // start_push_pull enqueues an outbound exchange; returns a StreamId we discard. + let _ = t.start_push_pull(addr, PushPullKind::Join, Instant::ORIGIN); +} + +// ── raw memberlist_proto::Endpoint ─────────────────────────────────────────── + +#[test] +fn raw_endpoint_impl_compiles_and_wires_up() { + let addr: core::net::SocketAddr = "127.0.0.1:7946".parse().unwrap(); + let opts = EndpointOptions::new(1u32, addr); + let mut ep = + memberlist_proto::Endpoint::new_at(opts, Instant::ORIGIN, SmallRng::seed_from_u64(0)); + ep.start_scheduling(Instant::ORIGIN); + drive(&mut ep, addr); +} + +// ── memberlist_proto::streams::StreamEndpoint (plain TCP) ──────────────────── + +#[cfg(feature = "tcp")] +#[test] +fn tcp_stream_endpoint_impl_compiles_and_wires_up() { + use memberlist_proto::{ + RawRecords, + streams::{LabelOptions, StreamEndpoint}, + }; + + let addr: core::net::SocketAddr = "127.0.0.1:7947".parse().unwrap(); + let opts = EndpointOptions::new(1u32, addr); + let mut inner = + memberlist_proto::Endpoint::new_at(opts, Instant::ORIGIN, SmallRng::seed_from_u64(1)); + inner.start_scheduling(Instant::ORIGIN); + + // Plain-TCP label options: no cluster label, Passthrough inner transport. + let cfg: LabelOptions<()> = LabelOptions::new_in(None, ()); + let sni_provider: Box Option + Send + Sync> = + Box::new(|_| Some("localhost".to_string())); + let peer_to_socket: Box core::net::SocketAddr + Send + Sync> = + Box::new(|a| *a); + + let mut coord: StreamEndpoint = + StreamEndpoint::new(inner, cfg, sni_provider, peer_to_socket); + + drive(&mut coord, addr); +} From 0ab7ce5aece42d397539469d8917e72b07e79447 Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 26 Jun 2026 20:36:13 +1200 Subject: [PATCH 012/128] refactor(serf-proto): restructure Endpoint into the serf-logic core over the Reliable seam serf_proto::Endpoint drops its inner memberlist Endpoint and reaches the transport through a disjointly-borrowed &mut impl Reliable; a transitional StreamEndpoint { core, transport } runs the composed tick and preserves the test surface. --- serf-proto/src/endpoint/mod.rs | 635 ++++++----- serf-proto/src/endpoint/reliable/mod.rs | 64 +- serf-proto/src/endpoint/reliable/tests.rs | 16 +- serf-proto/src/endpoint/serf_parity_tests.rs | 7 +- serf-proto/src/endpoint/tests.rs | 81 +- serf-proto/src/lib.rs | 4 + serf-proto/src/stream_endpoint/mod.rs | 1014 ++++++++++++++++++ 7 files changed, 1438 insertions(+), 383 deletions(-) create mode 100644 serf-proto/src/stream_endpoint/mod.rs diff --git a/serf-proto/src/endpoint/mod.rs b/serf-proto/src/endpoint/mod.rs index 3e223913..3c40beaf 100644 --- a/serf-proto/src/endpoint/mod.rs +++ b/serf-proto/src/endpoint/mod.rs @@ -1,14 +1,19 @@ -//! The serf `Endpoint` — a pure Sans-I/O super-machine. +//! The serf `Endpoint` — the transport-agnostic serf-logic core of a +//! Sans-I/O super-machine. //! -//! Owns a [`memberlist_proto::Endpoint`] by value and drives it, intercepting -//! its gossip to implement serf's membership FSM, three Lamport clocks, user -//! events, queries/responses/relays, push-pull anti-entropy, and network -//! coordinates. +//! Holds serf's membership FSM, three Lamport clocks, user events, +//! queries/responses/relays, push-pull anti-entropy, and network coordinates, +//! but **no** transport reference. It reaches a memberlist reliable +//! coordinator only through the narrow [`Reliable`](reliable::Reliable) seam: +//! every serf-logic method that touches the membership transport takes a +//! `&mut impl Reliable` (named `t`). The composing super-machine +//! ([`crate::StreamEndpoint`]) owns both the core and the coordinator as +//! separate fields and threads the latter into the former on each call. //! -//! The driver moves opaque `Bytes` in (`handle_packet`) and +//! The composing super-machine moves opaque `Bytes` in (`handle_packet`) and //! `Transmit`/`Bytes` out (`poll_transmit`) and ticks `poll_timeout(now)`; -//! it carries zero serf logic. All wall-clock reads are threaded in as a -//! `now: Instant` parameter — no clock reads occur inside this module. +//! this core carries zero transport I/O. All wall-clock reads are threaded in +//! as a `now: Instant` parameter — no clock reads occur inside this module. //! //! # Threat model //! @@ -55,9 +60,10 @@ use std::collections::VecDeque; use bytes::Bytes; use memberlist_proto::{ - CheapClone, Data, EndpointEvent, Id, Instant, Node, PushPullKind, Rng, SeedableRng, SmallRng, - StreamCommand, Transmit, parse_message, typed::NodeState, + CheapClone, Data, Id, Instant, Node, PushPullKind, Rng, SeedableRng, SmallRng, typed::NodeState, }; + +use self::reliable::Reliable; use rand::RngExt; use smol_str::SmolStr; @@ -622,7 +628,17 @@ fn next_ltime(clock: &mut u64) -> u64 { // ── Endpoint ────────────────────────────────────────────────────────────────── -/// The serf Sans-I/O super-machine. +/// The serf-logic core of the Sans-I/O super-machine. +/// +/// Holds all serf state — the three Lamport clocks, membership store, options, +/// event ring, query bookkeeping, deadlines, and (feature-gated) the coordinate +/// client — and **no** transport reference. It reaches a memberlist reliable +/// coordinator only through the narrow [`Reliable`](reliable::Reliable) seam: +/// every serf-logic method that must touch the membership transport takes a +/// `&mut impl Reliable` (named `t`), borrowed disjointly from the core's +/// own state. The composing super-machine +/// ([`crate::StreamEndpoint`]) owns both the core and the coordinator as +/// separate fields and threads the latter into the former. /// /// `I` is the node-id type; `A` is the (resolved) address type; `R` is the /// random number generator injected at construction time (default: `SmallRng`). @@ -633,8 +649,6 @@ pub struct Endpoint where I: Eq + core::hash::Hash, { - /// The inner memberlist Endpoint (SWIM state machine + gossip transport). - inner: memberlist_proto::Endpoint, /// serf configuration knobs. opts: Options, /// Member (SWIM membership) Lamport clock — plain `u64`, no atomics. @@ -773,16 +787,17 @@ where I: Clone + Eq + core::hash::Hash, R: SeedableRng, { - /// Construct a serf `Endpoint` wrapping `inner`, using `opts` for serf-level - /// knobs, and `rng` as serf's own injected selection entropy. + /// Construct a serf `Endpoint` core using `opts` for serf-level knobs and + /// `rng` as serf's own injected selection entropy. /// - /// The inner `Endpoint` must already have been configured with + /// The core holds no transport; the composing super-machine pairs it with a + /// memberlist coordinator that must already have been configured with /// `EndpointOptions::with_user_broadcast_tiers(NonZeroU8::new(3))` so that /// serf's three broadcast tiers (intent=0, query=1, event=2) are available. /// - /// `rng` is **separate** from the inner Endpoint's `R`. Seed it from the + /// `rng` is **separate** from the coordinator's `R`. Seed it from the /// driver's own entropy source; do not share the same `R` instance. - pub fn new_with_rng(inner: memberlist_proto::Endpoint, opts: Options, rng: R) -> Self { + pub fn new_with_rng(opts: Options, rng: R) -> Self { // Arm the first reap/reconnect/queue-check deadlines relative to the ORIGIN instant. // The driver calls handle_timeout(now) and the deadlines fire when now >= deadline. let first_reap = Instant::ORIGIN + opts.reap_interval(); @@ -802,7 +817,6 @@ where }; Self { - inner, opts, clock: 0, event_clock: 0, @@ -845,8 +859,8 @@ where /// Suitable for tests and environments where determinism or an explicit seed /// is acceptable. Production drivers should use `new_with_rng` and seed from /// a cryptographically-secure source. - pub fn new(inner: memberlist_proto::Endpoint, opts: Options) -> Self { - Self::new_with_rng(inner, opts, R::seed_from_u64(0)) + pub fn new(opts: Options) -> Self { + Self::new_with_rng(opts, R::seed_from_u64(0)) } // ── read accessors ──────────────────────────────────────────────────────── @@ -887,31 +901,25 @@ where { /// Drain one serf event. /// - /// Pumps the inner `Endpoint` to exhaustion, sieving each inner `Event` + /// Pumps the coordinator `t` to exhaustion, sieving each inner `Event` /// into serf state, then returns the next queued serf `Event`. Call in a /// loop until `None` before blocking. - pub fn poll_event(&mut self) -> Option> { - self.drain_inner(); + pub(crate) fn poll_event(&mut self, t: &mut T) -> Option> + where + T: Reliable, + { + self.drain_inner(t); self.pending_events.pop_front() } - /// Drain one outgoing transmit. + /// The earliest serf-level deadline requiring a `handle_timeout` call. /// - /// Passes through to the inner `Endpoint`; serf adds no additional - /// transmit framing at this layer. - pub fn poll_transmit(&mut self) -> Option> { - self.inner.poll_transmit() - } - - /// The earliest deadline requiring a `handle_timeout` call. - /// - /// Returns the minimum of the inner Endpoint's deadline and serf's own - /// periodic deadlines (reap, reconnect, queue-check, leave-broadcast, - /// leave-complete). - pub fn poll_timeout(&self) -> Option { - let inner = self.inner.poll_timeout(); + /// Returns the minimum of serf's own periodic deadlines (reap, reconnect, + /// queue-check, leave-broadcast, leave-complete, and pending-query closes). + /// The composing super-machine folds in the coordinator's own deadline. + pub fn serf_poll_timeout(&self) -> Option { let query_min = self.pending_queries.iter().map(|pq| pq.deadline).min(); - let serf = [ + [ self.next_reap, self.next_reconnect, self.next_queue_check, @@ -921,71 +929,58 @@ where ] .into_iter() .flatten() - .min(); - match (inner, serf) { - (Some(a), Some(b)) => Some(a.min(b)), - (Some(a), None) => Some(a), - (None, Some(b)) => Some(b), - (None, None) => None, - } + .min() } - /// Deliver an inbound datagram or packet to the machine. - /// - /// Decodes the memberlist wire `Message` from `data` and dispatches it - /// to the inner `Endpoint`. SWIM messages (Ping, Alive, Dead, Suspect, …) - /// are handled by the inner machine; when the inner encounters a - /// `Message::UserData` payload it emits `Event::UserPacket`, which the serf - /// sieve (`on_inner_event`) then decodes as a serf `AnyMessage` and - /// routes to the appropriate serf handler (join/leave intent, user event, - /// query, relay). + /// Sieve the coordinator's inner events after a transport ingress. /// - /// Malformed or unrecognised bytes are silently dropped — the machine must - /// not panic on bad input from the network. - pub fn handle_packet(&mut self, from: A, data: Bytes, now: Instant) { - // Malformed frame or unrecognised tag: drop silently. The inner endpoint - // logs its own decode errors; serf takes no serf-level action here. + /// The composing super-machine hands inbound bytes to its coordinator (which + /// runs the SWIM machine and emits inner events: `UserPacket`, + /// `RemoteStateReceived`, `NodeJoined`, …), then calls this to fold those + /// inner events into serf state. Latches `now` so inner events that need a + /// wall-clock reference (e.g. `NodeLeft` leave-time) use the ingress instant. + pub(crate) fn drain_after_ingress(&mut self, t: &mut T, now: Instant) + where + T: Reliable, + { self.drain_now = now; - if let Ok(msg) = parse_message::(data) { - self.inner.handle_packet(from, msg, now); - } - self.drain_inner(); + self.drain_inner(t); } - /// Deliver a stream event to the machine. + /// Pre-inner-timer phase of the composed tick (H6). /// - /// Forwards to the inner `Endpoint`, then drains resulting inner events. - pub fn handle_stream_event( - &mut self, - ev: EndpointEvent, - now: Instant, - ) -> Option> { - self.drain_now = now; - let cmd = self.inner.handle_stream_event(ev, now); - self.drain_inner(); - cmd - } - - /// Accept an inbound stream connection. + /// Latches `now` and, if the local-state snapshot is dirty, resyncs it so the + /// coordinator echoes a fresh serf-clock / member-status snapshot on this + /// tick's anti-entropy exchange rather than a snapshot from a previous tick. + /// The dirty flag is cleared inside `resync_local_state` on success. /// - /// Passes directly to the inner `Endpoint`. - pub fn accept_stream(&mut self, from: A, now: Instant) -> Option> { - self.inner.accept_stream(from, now) + /// The composing super-machine calls this, then drives the coordinator's own + /// `handle_timeout(now)` (the SWIM gossip / probe / push-pull scheduler), then + /// [`Endpoint::after_inner_timeout`]. This three-phase ordering keeps the + /// load-bearing sequence (resync → inner timer → drain → serf deadlines) + /// structural; the trait deliberately excludes the inner timer so the core + /// cannot drive it directly. + pub(crate) fn before_inner_timeout(&mut self, t: &mut T, now: Instant) + where + T: Reliable, + I: Clone, + A: Clone, + { + self.drain_now = now; + if self.local_state_dirty { + self.resync_local_state(t); + } } - /// Advance time and fire any expired serf or inner deadlines. + /// Post-inner-timer phase of the composed tick: drain inner events then fire + /// serf's own deadlines. /// /// Tick order (H1b / decision 5 step 4): /// - /// 1. Resync the push-pull local-state snapshot if dirty (H6) so the inner - /// echoes a fresh serf-clock / member-status snapshot on this tick's - /// anti-entropy exchange — not a snapshot from a previous tick. - /// 2. Run `inner.handle_timeout(now)` first — the inner gossip scheduler - /// produces any pending SWIM transitions and SWIM piggyback gossip. - /// 3. Drain all inner events produced by the inner tick via `drain_inner()`, - /// processing NodeJoined / NodeLeft / UserPacket / etc. through the serf - /// sieve before any serf deadline fires. - /// 4. Then fire serf's own deadlines: reap → reconnect → queue-check → + /// 1. Drain all inner events produced by the coordinator's timer via + /// `drain_inner`, processing NodeJoined / NodeLeft / UserPacket / etc. + /// through the serf sieve **before** any serf deadline fires. + /// 2. Fire serf's own deadlines: reap → reconnect → queue-check → /// query-closes → leave-complete. /// /// **Why this order:** firing serf deadlines after draining the inner's events @@ -993,30 +988,19 @@ where /// the inner machine at the time the reap deadline would otherwise fire. /// Without this ordering, a member that reconnects exactly at the reap /// boundary could be incorrectly pruned. Decision 5 step 4 / pin H1b. - pub fn handle_timeout(&mut self, now: Instant) + pub(crate) fn after_inner_timeout(&mut self, t: &mut T, now: Instant) where + T: Reliable, I: Clone, A: Clone, { self.drain_now = now; - // H6: resync the push-pull snapshot before the inner tick so the next - // anti-entropy exchange carries the current serf state (not a stale one - // from a previous mutation burst). The dirty flag is cleared inside - // resync_local_state on success. - if self.local_state_dirty { - self.resync_local_state(); - } - - // Step 2: drive the inner memberlist machine (SWIM gossip + probe + push-pull). - // This may produce NodeJoined / NodeLeft / UserPacket inner events. - self.inner.handle_timeout(now); - - // Step 3: drain all inner events produced by the tick through the serf sieve. + // Step 1: drain all inner events produced by the tick through the serf sieve. // NodeJoined / NodeLeft / etc. are processed NOW, before any serf deadline fires. - self.drain_inner(); + self.drain_inner(t); - // Step 4: fire serf's own deadlines in deterministic order. + // Step 2: fire serf's own deadlines in deterministic order. // Reaper: remove tombstoned left/failed nodes and stale intents. if let Some(dl) = self.next_reap { @@ -1029,7 +1013,7 @@ where // Reconnector: probabilistically re-dial a random failed peer. if let Some(dl) = self.next_reconnect { if now >= dl { - self.fire_reconnect(now); + self.fire_reconnect(t, now); self.next_reconnect = Some(now + self.opts.reconnect_interval()); } } @@ -1073,18 +1057,21 @@ where // ── inner-event sieve ───────────────────────────────────────────────────── - /// Pump the inner Endpoint to exhaustion, routing each event through the - /// serf sieve. + /// Pump the coordinator `t` to exhaustion, routing each inner event through + /// the serf sieve. /// /// After all inner events are drained, if the local-state snapshot is dirty /// (H6), `resync_local_state` is called so the next push-pull egress ships /// the current serf clock / member-status state, not a stale snapshot. - fn drain_inner(&mut self) { - while let Some(ev) = self.inner.poll_event() { - self.on_inner_event(ev); + fn drain_inner(&mut self, t: &mut T) + where + T: Reliable, + { + while let Some(ev) = t.poll_inner_event() { + self.on_inner_event(t, ev); } if self.local_state_dirty { - self.resync_local_state(); + self.resync_local_state(t); } } @@ -1092,9 +1079,10 @@ where /// handler. /// /// Every variant of the inner `Event` enum is covered (totality / H4). - /// Unimplemented handlers are stubs that will be filled in by later - /// sub-stages. - fn on_inner_event(&mut self, ev: memberlist_proto::Event) { + fn on_inner_event(&mut self, t: &mut T, ev: memberlist_proto::Event) + where + T: Reliable, + { use memberlist_proto::Event as IE; match ev { // ── membership ─────────────────────────────────────────────────────── @@ -1115,7 +1103,7 @@ where IE::NodeConflict(_c) => { if self.opts.enable_id_conflict_resolution() { let now = self.drain_now; - self.resolve_node_conflict(now); + self.resolve_node_conflict(t, now); } } @@ -1126,12 +1114,12 @@ where // fresh `now` for the current call site. let now = self.drain_now; let (from, data, _reliability) = p.into_parts(); - self.handle_user_packet(from, data, now); + self.handle_user_packet(t, from, data, now); } IE::RemoteStateReceived(r) => { let (_peer, user_data, is_join) = r.into_parts(); if !user_data.is_empty() { - self.merge_remote_state(user_data, is_join); + self.merge_remote_state(t, user_data, is_join); } } @@ -1168,7 +1156,7 @@ where let node_id = p.node_ref().id_ref().clone(); let rtt = p.rtt(); let payload = p.payload_ref().clone(); - self.handle_ping_completed(&node_id, rtt, &payload); + self.handle_ping_completed(t, &node_id, rtt, &payload); } // When the feature is disabled, suppress the unused-variable warning. #[cfg(not(feature = "coordinates"))] @@ -1232,8 +1220,9 @@ where /// (a transient alloc failure should not permanently corrupt the snapshot). /// /// Mirrors Go serf `delegate.go` `local_state` (~line 386). - pub fn resync_local_state(&mut self) + pub(crate) fn resync_local_state(&mut self, t: &mut T) where + T: Reliable, I: Clone + Data, A: Data, { @@ -1290,11 +1279,11 @@ where } }; - // Push to the inner Endpoint. On cap-exceeded errors keep dirty for retry; - // the operator must raise max_stream_frame_size if the serf state is too large - // to fit in one push-pull frame. The snapshot is stale but the machine - // continues operating — the next drain attempt will retry. - if self.inner.set_local_state_snapshot(encoded).is_ok() { + // Push to the coordinator's inner Endpoint. On cap-exceeded errors keep + // dirty for retry; the operator must raise max_stream_frame_size if the serf + // state is too large to fit in one push-pull frame. The snapshot is stale + // but the machine continues operating — the next drain attempt will retry. + if t.set_local_state_snapshot(encoded).is_ok() { self.local_state_dirty = false; } } @@ -1346,8 +1335,9 @@ where /// /// Decoding errors in the `user_data` bytes are silently dropped — /// the machine must not panic on bad network input. - fn merge_remote_state(&mut self, user_data: Bytes, is_join: bool) + fn merge_remote_state(&mut self, t: &mut T, user_data: Bytes, is_join: bool) where + T: Reliable, I: Clone + Data, A: Data, { @@ -1429,7 +1419,7 @@ where for node_id in &pp.left_members { if let Some(&status_ltime) = status_map.get(node_id) { let leave_ltime = LamportTime(status_ltime.0.saturating_add(1)); - self.handle_node_leave_intent(leave_ltime, node_id, false, now); + self.handle_node_leave_intent(t, leave_ltime, node_id, false, now); } } @@ -1713,14 +1703,16 @@ where /// Returns `true` if the intent should be rebroadcast. /// /// Mirrors Go serf `base.go` `handleNodeLeaveIntent` (lines 1449-1579). - pub(crate) fn handle_node_leave_intent( + pub(crate) fn handle_node_leave_intent( &mut self, + t: &mut T, ltime: LamportTime, id: &I, prune: bool, now: Instant, ) -> bool where + T: Reliable, I: Clone, { // Whole-message drop gate: reject unacceptable Lamport times before any @@ -1753,12 +1745,12 @@ where // endpoint is Alive, push back with a join broadcast and suppress the rebroadcast. // We check this AFTER the stale guard so stale self-leaves do not trigger refutes. // broadcast_join calls handle_node_join_intent which marks dirty when it buffers. - let is_local = id == self.inner.local_id_ref(); + let is_local = id == t.endpoint_ref().local_id_ref(); if is_local && self.state == SerfState::Alive { // Refute the leave by re-announcing our own Join intent so every peer // clears the spurious Leaving status; the rebroadcast of the leave itself // is suppressed. - self.broadcast_join(LamportTime(self.clock)); + self.broadcast_join(t, LamportTime(self.clock)); return false; } @@ -1979,8 +1971,9 @@ where /// to the driver as `Event::DialRequested(DialPassthrough { .. })`. /// /// H3: serf emits a dial request; it does NO dial itself. - fn fire_reconnect(&mut self, now: Instant) + fn fire_reconnect(&mut self, t: &mut T, now: Instant) where + T: Reliable, A: Clone, { let num_failed = self.members.failed_members.len(); @@ -2013,10 +2006,10 @@ where }; let addr = ms.member().node().addr_ref().clone(); - // Call inner.start_push_pull; the inner queues Event::DialRequested. + // Call start_push_pull; the inner queues Event::DialRequested. // The sieve (on_inner_event) passes it through as Event::DialRequested. - self.inner.start_push_pull(addr, PushPullKind::Join, now); - self.drain_inner(); + t.start_push_pull(addr, PushPullKind::Join, now); + self.drain_inner(t); } // ── join ───────────────────────────────────────────────────────────────── @@ -2032,14 +2025,15 @@ where /// /// State gate: only `Alive` announces a join; any other lifecycle state /// returns [`Error::BadJoinState`]. - pub fn join(&mut self) -> Result<(), Error> + pub(crate) fn join(&mut self, t: &mut T) -> Result<(), Error> where + T: Reliable, I: Clone, { if self.state != SerfState::Alive { return Err(Error::BadJoinState(self.state)); } - self.broadcast_join(LamportTime(self.clock)); + self.broadcast_join(t, LamportTime(self.clock)); Ok(()) } @@ -2050,11 +2044,12 @@ where /// locally via `handle_node_join_intent`, then enqueues the encoded /// `JoinMessage` on the intent tier (rank 0, highest priority). Mirrors /// serf-core `base.rs` `broadcast_join`. - fn broadcast_join(&mut self, ltime: LamportTime) + fn broadcast_join(&mut self, t: &mut T, ltime: LamportTime) where + T: Reliable, I: Clone, { - let local_id = self.inner.local_id_ref().clone(); + let local_id = t.endpoint_ref().local_id_ref().clone(); // Witness the member clock, then apply the intent locally so the local node // is recorded at this ltime (handle_node_join_intent also witnesses, but the @@ -2069,7 +2064,7 @@ where // construction; a dropped intent is re-announced by the next push-pull. let jm = JoinMessage::new(ltime, local_id); if let Ok(encoded) = AnyMessage::::Join(jm).encode() { - let _ = self.inner.queue_user_broadcast_ranked(0, encoded); + let _ = t.queue_user_broadcast_ranked(0, encoded); } } @@ -2100,8 +2095,9 @@ where /// The `Leaving → Left` transition happens later in `handle_timeout` when /// `leave_complete_deadline` (armed on inner `LeftCluster` + `leave_propagate_delay`) /// elapses. `Event::LeftCluster` is emitted at that point. - pub fn leave(&mut self, now: Instant) -> Result<(), Error> + pub(crate) fn leave(&mut self, t: &mut T, now: Instant) -> Result<(), Error> where + T: Reliable, I: Clone, A: Clone, { @@ -2118,26 +2114,26 @@ where // 2. Local leave intent — marks the local node as Leaving in the store // and witnesses the member clock. - let local_id = self.inner.local_id_ref().clone(); + let local_id = t.endpoint_ref().local_id_ref().clone(); // next_ltime stamps the current clock value (clamped to < LTIME_MAX) and // advances the clock in one atomic step, closing the local-emission hole. let ltime = LamportTime(next_ltime(&mut self.clock)); self.mark_local_state_dirty(); // We are setting state = Leaving above so the self-refute guard in // handle_node_leave_intent will NOT fire (it only fires when state == Alive). - self.handle_node_leave_intent(ltime, &local_id, false, now); + self.handle_node_leave_intent(t, ltime, &local_id, false, now); // 4. Broadcast the leave intent on the intent tier (rank 0) so peers learn // the local node is leaving without waiting for anti-entropy. The // driver bounds the flush via the broadcast deadline below. - self.broadcast_leave(ltime, local_id, false); + self.broadcast_leave(t, ltime, local_id, false); // 5. Arm the broadcast-timeout deadline so the driver always has a finite // wait; it can short-circuit by watching `user_broadcast_queue_len()`. self.leave_broadcast_deadline = Some(now + self.opts.broadcast_timeout()); // 5. Call inner leave; this queues the dead-self fan-out packets. - self.inner.leave(now)?; + t.leave(now)?; Ok(()) } @@ -2151,8 +2147,15 @@ where /// /// Does not require the local endpoint to be `Alive` (callers may want to /// clean up failed nodes before leaving themselves), but rejects `Shutdown`. - pub fn force_leave(&mut self, id: I, prune: bool, now: Instant) -> Result<(), Error> + pub(crate) fn force_leave( + &mut self, + t: &mut T, + id: I, + prune: bool, + now: Instant, + ) -> Result<(), Error> where + T: Reliable, I: Clone, A: Clone, { @@ -2166,11 +2169,11 @@ where self.clock = self.clock.saturating_add(1); let ltime = LamportTime(self.clock); self.mark_local_state_dirty(); - self.handle_node_leave_intent(ltime, &id, prune, now); + self.handle_node_leave_intent(t, ltime, &id, prune, now); // Broadcast the leave intent (carrying the prune flag) so peers apply the // same forced removal. - self.broadcast_leave(ltime, id, prune); + self.broadcast_leave(t, ltime, id, prune); // Arm the broadcast deadline so the driver knows how long to wait. self.leave_broadcast_deadline = Some(now + self.opts.broadcast_timeout()); @@ -2190,14 +2193,17 @@ where /// unconditionally — the inner gossip layer only transmits when peers exist, /// so an enqueue against an empty cluster is a harmless no-op rather than a /// special case. - fn broadcast_leave(&mut self, ltime: LamportTime, id: I, prune: bool) { + fn broadcast_leave(&mut self, t: &mut T, ltime: LamportTime, id: I, prune: bool) + where + T: Reliable, + { let lm = LeaveMessage::new(ltime, id, prune); // Ignoring Err: a `Leave` carrying a single id never approaches the gossip // MTU, so the only error path is an encode failure on a degenerate id type // (a construction-time concern the driver surfaces); a dropped intent is // re-announced by the next anti-entropy round. if let Ok(encoded) = AnyMessage::::Leave(lm).encode() { - let _ = self.inner.queue_user_broadcast_ranked(0, encoded); + let _ = t.queue_user_broadcast_ranked(0, encoded); } } @@ -2218,12 +2224,16 @@ where /// Go serf checks `max_user_event_size` in three places; this port /// consolidates to two (pre-name+payload-len, post-encoded-len), matching /// the oracle's intent without the redundant intermediate check. - pub fn user_event( + pub(crate) fn user_event( &mut self, + t: &mut T, name: impl Into, payload: bytes::Bytes, coalesce: bool, - ) -> Result<(), Error> { + ) -> Result<(), Error> + where + T: Reliable, + { let name: smol_str::SmolStr = name.into(); let max_size = self.opts.max_user_event_size(); @@ -2269,9 +2279,7 @@ where // the only remaining path to an error is a frame larger than the inner's // gossip MTU — a configuration mismatch the driver should detect at // startup. We propagate it back rather than silently drop. - self - .inner - .queue_user_broadcast_ranked(2, encoded) + t.queue_user_broadcast_ranked(2, encoded) .map_err(Error::InnerLeave)?; Ok(()) @@ -2351,8 +2359,9 @@ where /// H4: both `Reliability::Reliable` and `Reliability::Unreliable` dispatch /// identically; the reliability value affects delivery guarantees at the /// memberlist layer but not the serf handler logic. - fn handle_user_packet(&mut self, _from: A, data: Bytes, now: Instant) + fn handle_user_packet(&mut self, t: &mut T, _from: A, data: Bytes, now: Instant) where + T: Reliable, I: Clone + Data, A: Clone + Data, { @@ -2398,21 +2407,21 @@ where // decoded `ue` is consumed by handle_user_event for dedup + emission. let is_new = self.handle_user_event(ue); if is_new { - self.rebroadcast(MessageType::UserEvent, data); + self.rebroadcast(t, MessageType::UserEvent, data); } } AnyMessage::Join(join) => { // handle_node_join_intent takes ltime + id reference. let rebroadcast = self.handle_node_join_intent(join.ltime, &join.id.clone(), now); if rebroadcast { - self.rebroadcast(MessageType::Join, data); + self.rebroadcast(t, MessageType::Join, data); } } AnyMessage::Leave(leave) => { let id = leave.id.clone(); - let rebroadcast = self.handle_node_leave_intent(leave.ltime, &id, leave.prune, now); + let rebroadcast = self.handle_node_leave_intent(t, leave.ltime, &id, leave.prune, now); if rebroadcast { - self.rebroadcast(MessageType::Leave, data); + self.rebroadcast(t, MessageType::Leave, data); } } // Sub-stage 3: push-pull is not valid on a UserPacket; drop. @@ -2424,18 +2433,18 @@ where if data.len() > self.opts.query_size_limit() { return; } - let rebroadcast = self.handle_query(q, QueryOrigin::Inbound); + let rebroadcast = self.handle_query(t, q, QueryOrigin::Inbound); if rebroadcast { - self.rebroadcast(MessageType::Query, data); + self.rebroadcast(t, MessageType::Query, data); } } // Fold the query response into the matching PendingQuery. AnyMessage::QueryResponse(resp) => { - self.handle_query_response(resp); + self.handle_query_response(t, resp); } // Relay: forward the inner payload verbatim to the destination (decision 4). AnyMessage::Relay(relay) => { - self.handle_relay(relay); + self.handle_relay(t, relay); } // ConflictResponse and Key* bare packets arrive only as payloads inside // QueryResponseMessage; bare arrivals here are unexpected — drop silently. @@ -2460,7 +2469,10 @@ where /// /// Errors from `queue_user_broadcast_ranked` are silently dropped — the /// inner already applies its own back-pressure and queue-depth limits. - fn rebroadcast(&mut self, ty: MessageType, original: Bytes) { + fn rebroadcast(&mut self, t: &mut T, ty: MessageType, original: Bytes) + where + T: Reliable, + { let rank: u8 = match ty { MessageType::Join | MessageType::Leave => 0, // intent tier MessageType::UserEvent => 2, // event tier @@ -2471,12 +2483,12 @@ where // cap. This prevents unbounded memory growth under a flood of unique first-seen // messages. The gate mirrors Go serf's `getQueueMax` / `checkQueueDepth` // logic applied inline at the rebroadcast site. - if self.inner.user_broadcast_queue_len() >= self.queue_max() { + if t.endpoint_ref().user_broadcast_queue_len() >= self.queue_max() { return; } // Ignoring Err: the inner applies its own MTU back-pressure; a rejected // broadcast is a flow-control decision, not a fatal error. - let _ = self.inner.queue_user_broadcast_ranked(rank, original); + let _ = t.queue_user_broadcast_ranked(rank, original); } /// Compute the effective broadcast queue depth cap (mirrors Go serf `getQueueMax`). @@ -2493,15 +2505,6 @@ where } } - /// Returns the number of unsent items in the user broadcast queue. - /// - /// The driver may poll this during a graceful leave to detect when the - /// leave-intent broadcast has been flushed without waiting the full - /// `broadcast_timeout`. - pub fn user_broadcast_queue_len(&self) -> usize { - self.inner.user_broadcast_queue_len() - } - // ── Query issue + ingress (G8: read-not-increment) ──────────────────────── /// Issue an application query. @@ -2518,14 +2521,16 @@ where /// 7. Encode and enqueue on the **query tier** (rank 1). /// /// Returns the `QueryId` so the caller can correlate responses. - pub fn query( + pub(crate) fn query( &mut self, + t: &mut T, name: impl Into, payload: Bytes, params: QueryParams, now: Instant, ) -> Result where + T: Reliable, I: Clone + Data, A: Clone + Data, { @@ -2580,8 +2585,8 @@ where ltime, id, from: memberlist_proto::Node::new( - self.inner.local_id_ref().clone(), - self.inner.advertise_ref().clone(), + t.endpoint_ref().local_id_ref().clone(), + t.endpoint_ref().advertise_ref().clone(), ), filters: params.filters, flags, @@ -2626,12 +2631,12 @@ where // QueryOrigin::Local bypasses the inbound cap so the initiating node always // self-processes its own query. self.drain_now = now; - self.handle_query(q, QueryOrigin::Local); + self.handle_query(t, q, QueryOrigin::Local); // Enqueue on the query tier (rank 1). // Ignoring Err: the inner applies queue-depth / MTU back-pressure; a rejected // broadcast is a flow-control decision, not a fatal error. - let _ = self.inner.queue_user_broadcast_ranked(1, encoded); + let _ = t.queue_user_broadcast_ranked(1, encoded); Ok(query_id) } @@ -2661,8 +2666,9 @@ where /// /// Returns `true` if the query should be rebroadcast (i.e., first sight AND /// not `NO_BROADCAST`), even when the filter rejects local processing (G6). - fn handle_query(&mut self, msg: QueryMessage, origin: QueryOrigin) -> bool + fn handle_query(&mut self, t: &mut T, msg: QueryMessage, origin: QueryOrigin) -> bool where + T: Reliable, I: Clone + Data, A: Clone, { @@ -2782,7 +2788,7 @@ where } // Filter check (G6): even if the local node is not targeted, still rebroadcast. - if !self.should_process_query(&msg.filters) { + if !self.should_process_query(t, &msg.filters) { return rebroadcast; } @@ -2811,8 +2817,8 @@ where // emitting Event::Query. The ACK carries no payload; it signals receipt. if msg.ack() { let local_node = memberlist_proto::Node::new( - self.inner.local_id_ref().clone(), - self.inner.advertise_ref().clone(), + t.endpoint_ref().local_id_ref().clone(), + t.endpoint_ref().advertise_ref().clone(), ); let ack_resp = QueryResponseMessage { ltime: msg.ltime, @@ -2825,16 +2831,14 @@ where let from_addr = msg.from.addr_ref().clone(); // Ignoring Err: directed ACK sends are best-effort; a missing ACK is // handled by the querier's timeout on its acks set. - let _ = self - .inner - .send_user_packet(from_addr.clone(), ack_encoded.clone()); + let _ = t.send_user_packet(from_addr.clone(), ack_encoded.clone()); #[cfg(test)] { self.last_directed_send = Some((from_addr, ack_encoded.clone())); } // Relay the ACK through relay_factor random intermediary nodes when requested. if msg.relay_factor > 0 { - self.relay_response(msg.from.clone(), ack_encoded, msg.relay_factor); + self.relay_response(t, msg.from.clone(), ack_encoded, msg.relay_factor); } } } @@ -2846,7 +2850,7 @@ where // point when the id was already decoded and exactly consumed. if let Some(conflict_id) = pre_decoded_conflict_id { self.received_queries.remove(&query_id); - self.handle_conflict_query(&msg, conflict_id); + self.handle_conflict_query(t, &msg, conflict_id); return rebroadcast; } @@ -2905,11 +2909,12 @@ where /// Returns `true` if the node should respond locally, `false` if filtered out. /// A `false` return suppresses `Event::Query` but does NOT affect rebroadcast /// (rebroadcast is decided by the caller, not by this function). - fn should_process_query(&self, filters: &[Filter]) -> bool + fn should_process_query(&self, t: &T, filters: &[Filter]) -> bool where + T: Reliable, I: Clone, { - let local_id = self.inner.local_id_ref(); + let local_id = t.endpoint_ref().local_id_ref(); for filter in filters { match filter { Filter::Id(ids) => { @@ -2922,7 +2927,7 @@ where // The local node's tags live in `members.states` under the local id. // If the local node is not yet in the store (before the first join // event has been processed), we conservatively return `false`. - let local_id = self.inner.local_id_ref(); + let local_id = t.endpoint_ref().local_id_ref(); let empty_tags = Tags::new(); let tags = self .members @@ -2978,13 +2983,15 @@ where /// both checks). This port uses three distinct guards in order — size, /// already-responded, deadline — which is strictly more informative to /// callers that need to distinguish the error cases. - pub fn respond( + pub(crate) fn respond( &mut self, + t: &mut T, token: &QueryEvent, payload: Bytes, now: Instant, ) -> Result<(), Error> where + T: Reliable, I: Clone + Data, A: Clone + Data, { @@ -3004,8 +3011,8 @@ where // Build and encode the `QueryResponseMessage`. let local_node = memberlist_proto::Node::new( - self.inner.local_id_ref().clone(), - self.inner.advertise_ref().clone(), + t.endpoint_ref().local_id_ref().clone(), + t.endpoint_ref().advertise_ref().clone(), ); let resp = QueryResponseMessage { ltime: token.ltime(), @@ -3019,6 +3026,7 @@ where .map_err(Error::RespondEncode)?; self.respond_inner( + t, query_id, to, relay_factor, @@ -3040,8 +3048,9 @@ where // All parameters are distinct routing / payload values with no natural sub-grouping; // a wrapper struct would add churn without clarity. #[allow(clippy::too_many_arguments)] - fn respond_inner( + fn respond_inner( &mut self, + t: &mut T, query_id: QueryId, to: A, relay_factor: u8, @@ -3051,6 +3060,7 @@ where now: Instant, ) -> Result<(), Error> where + T: Reliable, I: Clone, A: Clone, { @@ -3066,9 +3076,7 @@ where } // Directed send — never broadcast. - self - .inner - .send_user_packet(to.clone(), encoded.clone()) + t.send_user_packet(to.clone(), encoded.clone()) .map_err(Error::RespondSend)?; #[cfg(test)] @@ -3081,7 +3089,7 @@ where // Relay when requested. if relay_factor > 0 { - self.relay_response(relay_querier, encoded, relay_factor); + self.relay_response(t, relay_querier, encoded, relay_factor); } Ok(()) @@ -3102,13 +3110,15 @@ where docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] - pub fn respond_key( + pub(crate) fn respond_key( &mut self, + t: &mut T, req: &crate::event::KeyRequest, resp: KeyResponseArgs, now: Instant, ) -> Result<(), Error> where + T: Reliable, I: Clone + Data, A: Clone + Data, { @@ -3136,8 +3146,8 @@ where .map_err(Error::RespondEncode)?; let local_node = memberlist_proto::Node::new( - self.inner.local_id_ref().clone(), - self.inner.advertise_ref().clone(), + t.endpoint_ref().local_id_ref().clone(), + t.endpoint_ref().advertise_ref().clone(), ); let qresp = QueryResponseMessage { ltime: req.ltime, @@ -3151,6 +3161,7 @@ where .map_err(Error::RespondEncode)?; self.respond_inner( + t, query_id, to, relay_factor, @@ -3181,8 +3192,9 @@ where /// - `App` → emit `Event::QueryResponse { id, from, payload }`. /// - `Conflict` → tally: decode `ConflictResponseMessage`, compare addr to local advertise. /// - `Key` → tally: decode `KeyResponseMessage`, fold into the `KeyResponseTally`. - fn handle_query_response(&mut self, msg: QueryResponseMessage) + fn handle_query_response(&mut self, t: &mut T, msg: QueryResponseMessage) where + T: Reliable, I: Clone, A: Clone, { @@ -3298,7 +3310,7 @@ where self.pending_events.push_back(Event::QueryResponse(ev)); } QueryPurpose::Conflict => { - self.handle_conflict_response_fold(query_id, msg.payload); + self.handle_conflict_response_fold(t, query_id, msg.payload); } #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] QueryPurpose::Key => { @@ -3336,8 +3348,14 @@ where /// - The destination querier IS reselectable as a relay peer (the relay dedup /// is the query-response dedup at the querier, not here). /// - Only `Alive` non-self members are eligible relay peers. - fn relay_response(&mut self, querier: Node, relay_frame: Bytes, relay_factor: u8) - where + fn relay_response( + &mut self, + t: &mut T, + querier: Node, + relay_frame: Bytes, + relay_factor: u8, + ) where + T: Reliable, I: Clone + Data, A: Clone + Data, { @@ -3384,7 +3402,7 @@ where // establishes a total, stable input order so the shuffle is a deterministic // function of the RNG state (Go serf `random_members` uses a slice with a // stable iteration order for the same reason). - let local_id = self.inner.local_id_ref(); + let local_id = t.endpoint_ref().local_id_ref(); let mut candidates: Vec<(Vec, A)> = self .members .states @@ -3414,9 +3432,7 @@ where // Directed-send to each chosen relay peer. On failure emit RelayDropped. for (_, peer_addr) in candidates { - let result = self - .inner - .send_user_packet(peer_addr.clone(), relay_encoded.clone()); + let result = t.send_user_packet(peer_addr.clone(), relay_encoded.clone()); #[cfg(test)] { // Track the last directed send for test assertions. @@ -3456,13 +3472,14 @@ where /// we omit it as well to keep faithful oracle correspondence — if a relay chain /// were constructed, the recipient would decode an AnyMessage::Relay and call /// handle_relay again, naturally bounding by TTL at the network layer). - fn handle_relay(&mut self, relay: RelayMessage) + fn handle_relay(&mut self, t: &mut T, relay: RelayMessage) where + T: Reliable, I: Clone, A: Clone, { let dest_id = relay.destination.id_ref(); - let local_id = self.inner.local_id_ref(); + let local_id = t.endpoint_ref().local_id_ref(); // Self-destination guard: relay to self is always a no-op failure. if dest_id == local_id { @@ -3477,9 +3494,7 @@ where let dest_addr = relay.destination.addr_ref().clone(); let payload = relay.payload; - let result = self - .inner - .send_user_packet(dest_addr.clone(), payload.clone()); + let result = t.send_user_packet(dest_addr.clone(), payload.clone()); #[cfg(test)] { self.last_directed_send = Some((dest_addr.clone(), payload.clone())); @@ -3549,8 +3564,9 @@ where /// Seed a member with explicit tags into the membership store (test fixture). /// /// Like `test_seed_member` but lets the caller supply a `Tags` map, enabling - /// tag-filter unit tests to place a known value under a known key. - #[cfg(test)] + /// tag-filter unit tests to place a known value under a known key. Only the + /// `tag-regex` test module exercises tag filtering, so this is gated on it. + #[cfg(all(test, feature = "tag-regex"))] pub(crate) fn test_seed_member_with_tags( &mut self, id: I, @@ -3623,11 +3639,18 @@ where /// Invoke `handle_node_leave_intent` with bare parameters (test adapter). #[cfg(test)] - pub(crate) fn test_handle_leave_intent(&mut self, id: I, ltime: LamportTime, now: Instant) -> bool + pub(crate) fn test_handle_leave_intent( + &mut self, + t: &mut T, + id: I, + ltime: LamportTime, + now: Instant, + ) -> bool where + T: Reliable, I: Clone, { - self.handle_node_leave_intent(ltime, &id, false, now) + self.handle_node_leave_intent(t, ltime, &id, false, now) } /// Synthesise an inner `NodeJoined` event for node `id` and drive it @@ -3709,8 +3732,11 @@ where /// finished its dead-self fan-out. Used by leave-chain tests that do not /// have a live inner endpoint to drive. #[cfg(test)] - pub(crate) fn test_inner_left_cluster(&mut self) { - self.on_inner_event(memberlist_proto::Event::LeftCluster); + pub(crate) fn test_inner_left_cluster(&mut self, t: &mut T) + where + T: Reliable, + { + self.on_inner_event(t, memberlist_proto::Event::LeftCluster); } /// Seed a `Failed` member with an explicit address into both `states` and @@ -3732,11 +3758,12 @@ where /// Directly invoke `fire_reconnect` (test adapter). #[cfg(test)] - pub(crate) fn test_fire_reconnect(&mut self, now: Instant) + pub(crate) fn test_fire_reconnect(&mut self, t: &mut T, now: Instant) where + T: Reliable, A: Clone, { - self.fire_reconnect(now); + self.fire_reconnect(t, now); } /// Directly invoke `fire_reap` (test adapter). @@ -3818,18 +3845,18 @@ where /// `Event::UserPacket` carrying the given bytes and `Unreliable` reliability /// (mirrors the gossip-plane delivery path). #[cfg(test)] - pub(crate) fn test_inject_user_packet(&mut self, from: A, data: Bytes, now: Instant) + pub(crate) fn test_inject_user_packet(&mut self, t: &mut T, from: A, data: Bytes, now: Instant) where + T: Reliable, I: Clone + Data, A: Clone + Data, { use memberlist_proto::{Reliability, UserPacket}; self.drain_now = now; - self.on_inner_event(memberlist_proto::Event::UserPacket(UserPacket::new( - from, - data, - Reliability::Unreliable, - ))); + self.on_inner_event( + t, + memberlist_proto::Event::UserPacket(UserPacket::new(from, data, Reliability::Unreliable)), + ); } // ── Clock + intent test helpers ─────────────────────────────────────────── @@ -3864,11 +3891,14 @@ where self.mark_local_state_dirty(); } - /// Read back the bytes currently stored in the inner Endpoint's + /// Read back the bytes currently stored in the coordinator's inner Endpoint /// `local_state_snapshot` (test adapter). #[cfg(test)] - pub(crate) fn test_inner_local_state_snapshot(&self) -> Bytes { - self.inner.local_state_snapshot_bytes() + pub(crate) fn test_inner_local_state_snapshot(&self, t: &T) -> Bytes + where + T: Reliable, + { + t.endpoint_ref().local_state_snapshot_bytes() } /// Decode `bytes` as a `PushPullMessage` for assertion (test adapter). @@ -3919,12 +3949,13 @@ where /// receive a stable `now`. Use `test_set_drain_now` to override the /// timestamp when wall-clock values matter. #[cfg(test)] - pub(crate) fn test_merge_remote_state(&mut self, user_data: Bytes, is_join: bool) + pub(crate) fn test_merge_remote_state(&mut self, t: &mut T, user_data: Bytes, is_join: bool) where + T: Reliable, I: Clone + Data, A: Data, { - self.merge_remote_state(user_data, is_join); + self.merge_remote_state(t, user_data, is_join); } /// Return the `ltime` of the most recently buffered intent for `id` of `kind`, @@ -3943,12 +3974,13 @@ where /// it should rebroadcast (test adapter). Always passes `QueryOrigin::Inbound` /// so the inbound cap and all inbound semantics are exercised. #[cfg(test)] - pub(crate) fn test_handle_query(&mut self, msg: QueryMessage) -> bool + pub(crate) fn test_handle_query(&mut self, t: &mut T, msg: QueryMessage) -> bool where + T: Reliable, I: Clone + Data, A: Clone + Data, { - self.handle_query(msg, QueryOrigin::Inbound) + self.handle_query(t, msg, QueryOrigin::Inbound) } /// Return the `QueryId` of the last pending query entry (test adapter). @@ -4024,12 +4056,13 @@ where /// Call `handle_query_response` directly (test adapter). #[cfg(test)] - pub(crate) fn test_handle_query_response(&mut self, msg: QueryResponseMessage) + pub(crate) fn test_handle_query_response(&mut self, t: &mut T, msg: QueryResponseMessage) where + T: Reliable, I: Clone, A: Clone, { - self.handle_query_response(msg); + self.handle_query_response(t, msg); } /// Return `true` if the received-query entry for `query_id` has been @@ -4069,22 +4102,29 @@ where /// guard can pass when `relay_factor == 1`. The caller is responsible for /// ensuring the membership store is populated to satisfy the guard. #[cfg(test)] - pub(crate) fn test_relay_response(&mut self, querier: Node, frame: Bytes, relay_factor: u8) - where + pub(crate) fn test_relay_response( + &mut self, + t: &mut T, + querier: Node, + frame: Bytes, + relay_factor: u8, + ) where + T: Reliable, I: Clone + Data, A: Clone + Data, { - self.relay_response(querier, frame, relay_factor); + self.relay_response(t, querier, frame, relay_factor); } /// Directly invoke `handle_relay` (test adapter). #[cfg(test)] - pub(crate) fn test_handle_relay(&mut self, relay: RelayMessage) + pub(crate) fn test_handle_relay(&mut self, t: &mut T, relay: RelayMessage) where + T: Reliable, I: Clone, A: Clone, { - self.handle_relay(relay); + self.handle_relay(t, relay); } /// Return the most recent `(address, bytes)` pair sent via a directed @@ -4152,14 +4192,16 @@ where /// uses the oracle's `defaultQueryTimeout` heuristic unconditionally. The /// registered `PendingQuery` will have `kind = purpose` so responses are /// routed to the appropriate fold path. - fn internal_query( + fn internal_query( &mut self, + t: &mut T, name: SmolStr, payload: Bytes, purpose: QueryPurpose, now: Instant, ) -> Result where + T: Reliable, I: Clone + Data, A: Clone + Data, { @@ -4177,8 +4219,8 @@ where ltime, id, from: memberlist_proto::Node::new( - self.inner.local_id_ref().clone(), - self.inner.advertise_ref().clone(), + t.endpoint_ref().local_id_ref().clone(), + t.endpoint_ref().advertise_ref().clone(), ), filters: vec![], flags: QueryFlag::empty(), @@ -4237,10 +4279,10 @@ where // QueryOrigin::Local bypasses the inbound cap so the initiating node always // self-processes its own internal query. self.drain_now = now; - self.handle_query(q, QueryOrigin::Local); + self.handle_query(t, q, QueryOrigin::Local); // Ignoring Err: queue back-pressure is a flow-control decision, not fatal. - let _ = self.inner.queue_user_broadcast_ranked(1, encoded); + let _ = t.queue_user_broadcast_ranked(1, encoded); Ok(query_id) } @@ -4252,12 +4294,13 @@ where /// query carrying the local id; peers respond with their view of that id's /// address. When the deadline fires, `close_conflict_query` tallies votes and /// emits `Event::Shutdown` if the local node lost the majority. - fn resolve_node_conflict(&mut self, now: Instant) + fn resolve_node_conflict(&mut self, t: &mut T, now: Instant) where + T: Reliable, I: Clone + Data, A: Clone + Data, { - let local_id = self.inner.local_id_ref().clone(); + let local_id = t.endpoint_ref().local_id_ref().clone(); let payload = match local_id.encode_to_bytes() { Ok(b) => b, Err(_) => return, @@ -4265,6 +4308,7 @@ where // Ignoring Err: encoding or queue failures are best-effort; if we cannot // broadcast the conflict query the cluster will simply time out the conflict. let _ = self.internal_query( + t, SmolStr::new("_serf_conflict"), payload, QueryPurpose::Conflict, @@ -4281,13 +4325,14 @@ where /// /// `conflict_id` is pre-decoded and exact-consumption-validated by the /// internal-query payload gate in `handle_query`; it is NOT re-decoded here. - fn handle_conflict_query(&mut self, msg: &QueryMessage, conflict_id: I) + fn handle_conflict_query(&mut self, t: &mut T, msg: &QueryMessage, conflict_id: I) where + T: Reliable, I: Clone + Data, A: Clone + Data, { // The originator does not respond to its own conflict query. - if &conflict_id == self.inner.local_id_ref() { + if &conflict_id == t.endpoint_ref().local_id_ref() { return; } @@ -4306,8 +4351,8 @@ where // Build and encode the QueryResponseMessage to send to the originator. let local_node = memberlist_proto::Node::new( - self.inner.local_id_ref().clone(), - self.inner.advertise_ref().clone(), + t.endpoint_ref().local_id_ref().clone(), + t.endpoint_ref().advertise_ref().clone(), ); let qresp = QueryResponseMessage { ltime: msg.ltime, @@ -4325,7 +4370,7 @@ where let dest_addr = msg.from.addr_ref().clone(); // Ignoring Err: directed-send failure on the conflict-response path is // best-effort; the originator will simply count this node as non-responding. - let _ = self.inner.send_user_packet(dest_addr, qresp_encoded); + let _ = t.send_user_packet(dest_addr, qresp_encoded); } /// Fold a conflict-resolution response into the matching `PendingQuery`. @@ -4333,8 +4378,9 @@ where /// The payload is a serf-framed `ConflictResponseMessage`. If the reported /// member's address matches the local advertise address, `conflict_matching` /// is incremented. - fn handle_conflict_response_fold(&mut self, query_id: QueryId, payload: Bytes) + fn handle_conflict_response_fold(&mut self, t: &mut T, query_id: QueryId, payload: Bytes) where + T: Reliable, A: PartialEq, { // Exact-consumption decode: the validation gate in handle_query_response @@ -4346,7 +4392,7 @@ where _ => return, }; - let local_addr = self.inner.advertise_ref().clone(); + let local_addr = t.endpoint_ref().advertise_ref().clone(); let pending = match self .pending_queries .iter_mut() @@ -4503,17 +4549,20 @@ where docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] - pub fn install_key( + pub(crate) fn install_key( &mut self, + t: &mut T, key: memberlist_proto::SecretKey, now: Instant, ) -> Result where + T: Reliable, I: Clone + Data, A: Clone + Data, { let payload = self.encode_key_request(Some(key))?; self.internal_query( + t, SmolStr::new("_serf_install_key"), payload, QueryPurpose::Key, @@ -4529,17 +4578,20 @@ where docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] - pub fn use_key( + pub(crate) fn use_key( &mut self, + t: &mut T, key: memberlist_proto::SecretKey, now: Instant, ) -> Result where + T: Reliable, I: Clone + Data, A: Clone + Data, { let payload = self.encode_key_request(Some(key))?; self.internal_query( + t, SmolStr::new("_serf_use_key"), payload, QueryPurpose::Key, @@ -4555,17 +4607,20 @@ where docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] - pub fn remove_key( + pub(crate) fn remove_key( &mut self, + t: &mut T, key: memberlist_proto::SecretKey, now: Instant, ) -> Result where + T: Reliable, I: Clone + Data, A: Clone + Data, { let payload = self.encode_key_request(Some(key))?; self.internal_query( + t, SmolStr::new("_serf_remove_key"), payload, QueryPurpose::Key, @@ -4581,13 +4636,15 @@ where docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] - pub fn list_keys(&mut self, now: Instant) -> Result + pub(crate) fn list_keys(&mut self, t: &mut T, now: Instant) -> Result where + T: Reliable, I: Clone + Data, A: Clone + Data, { let payload = self.encode_key_request(None)?; self.internal_query( + t, SmolStr::new("_serf_list_keys"), payload, QueryPurpose::Key, @@ -4758,16 +4815,22 @@ where /// Enqueue raw `bytes` on the intent broadcast tier (rank 0, highest priority). #[cfg(test)] - pub(crate) fn test_enqueue_intent_broadcast(&mut self, bytes: Bytes) { + pub(crate) fn test_enqueue_intent_broadcast(&mut self, t: &mut T, bytes: Bytes) + where + T: Reliable, + { // Ignoring Err: test helper; queue back-pressure is not exercised here. - let _ = self.inner.queue_user_broadcast_ranked(0, bytes); + let _ = t.queue_user_broadcast_ranked(0, bytes); } /// Enqueue raw `bytes` on the query broadcast tier (rank 1). #[cfg(test)] - pub(crate) fn test_enqueue_query_broadcast(&mut self, bytes: Bytes) { + pub(crate) fn test_enqueue_query_broadcast(&mut self, t: &mut T, bytes: Bytes) + where + T: Reliable, + { // Ignoring Err: test helper; queue back-pressure is not exercised here. - let _ = self.inner.queue_user_broadcast_ranked(1, bytes); + let _ = t.queue_user_broadcast_ranked(1, bytes); } // ── Snapshot replay → Endpoint load (G5 + G10) ─────────────────────────── @@ -4798,8 +4861,13 @@ where /// /// The local state is marked dirty so the next push-pull egress ships the /// recovered clock state. - pub fn load_snapshot(&mut self, replay: crate::snapshot::ReplayResult, now: Instant) - where + pub(crate) fn load_snapshot( + &mut self, + t: &mut T, + replay: crate::snapshot::ReplayResult, + now: Instant, + ) where + T: Reliable, A: Clone, { // G5: advance the member clock to at least last_clock. @@ -4836,7 +4904,7 @@ where // G10: dial each alive peer (skip self) so the node re-joins the cluster. // The inner emits Event::DialRequested; the sieve passes it through to the // driver. The driver owns the actual network dial. - let local_id = self.inner.local_id_ref().clone(); + let local_id = t.endpoint_ref().local_id_ref().clone(); for node in replay.alive_nodes { if node.id_ref() == &local_id { // Self-skip: the local node is already "alive" by definition. @@ -4846,8 +4914,8 @@ where // Capture for test assertions before calling start_push_pull. #[cfg(test)] self.rejoin_dials.push(addr.clone()); - self.inner.start_push_pull(addr, PushPullKind::Join, now); - self.drain_inner(); + t.start_push_pull(addr, PushPullKind::Join, now); + self.drain_inner(t); } } @@ -4872,7 +4940,15 @@ where /// /// No-op when `coord_client` is `None` (coordinates disabled at construction). #[cfg(feature = "coordinates")] - fn handle_ping_completed(&mut self, node_id: &I, rtt: std::time::Duration, payload: &Bytes) { + fn handle_ping_completed( + &mut self, + t: &mut T, + node_id: &I, + rtt: std::time::Duration, + payload: &Bytes, + ) where + T: Reliable, + { use crate::{bridge::coordinate_from_pb, messages::serf::v1 as pb}; use buffa::Message as _; @@ -4907,7 +4983,7 @@ where let ack_payload = coord_ack_payload(new_local_coord); // Ignoring Err: set_ack_payload only fails when Leaving/Left/Shutdown. // PingCompleted arrives on the probe path, active only while Alive. - let _ = self.inner.set_ack_payload(ack_payload); + let _ = t.set_ack_payload(ack_payload); } // ── Snapshot replay test helpers ───────────────────────────────────────── @@ -4931,14 +5007,17 @@ where /// /// Sets `drain_now = Instant::ORIGIN` before the call. #[cfg(all(feature = "coordinates", test))] - pub(crate) fn test_ping_completed( + pub(crate) fn test_ping_completed( &mut self, + t: &mut T, node_id: I, rtt: std::time::Duration, payload: Bytes, - ) { + ) where + T: Reliable, + { self.drain_now = memberlist_proto::Instant::ORIGIN; - self.handle_ping_completed(&node_id, rtt, &payload); + self.handle_ping_completed(t, &node_id, rtt, &payload); } } diff --git a/serf-proto/src/endpoint/reliable/mod.rs b/serf-proto/src/endpoint/reliable/mod.rs index 4b6cc62b..04d92cb3 100644 --- a/serf-proto/src/endpoint/reliable/mod.rs +++ b/serf-proto/src/endpoint/reliable/mod.rs @@ -3,8 +3,8 @@ //! [`Reliable`] is the only serf-to-coordinator boundary: every call serf's //! `Endpoint` makes into memberlist passes through one of these methods. //! It hides the concrete coordinator type (`StreamEndpoint` vs `QuicEndpoint`) -//! so that Tasks 3–6 can wire serf logic against a generic `&mut impl -//! Reliable` without knowing which transport is in use. +//! so the serf-logic core wires against a generic `&mut impl Reliable` +//! without knowing which transport is in use. //! //! # Method classification //! @@ -32,13 +32,6 @@ use memberlist_proto::{Endpoint, Instant, PushPullKind, Rng, StreamId}; /// /// The trait is `pub(crate)` because it is a serf-internal composition /// boundary, not part of the public API. -// No production call site exists yet: the serf-logic core still reaches its inner -// endpoint directly. This trait is the seam it routes through once composed with a -// reliable coordinator. -#[expect( - dead_code, - reason = "the serf-logic core has not yet been routed through this seam" -)] pub(crate) trait Reliable where I: Eq + core::hash::Hash, @@ -115,21 +108,6 @@ where /// `poll_inner_event`). fn start_push_pull(&mut self, peer: A, kind: PushPullKind, now: Instant) -> StreamId; - /// Update the local node's metadata advertised to peers. - /// - /// Serf stores its tag set (and, when relevant, the coordinate bytes) in - /// the memberlist meta field. This call propagates the change through the - /// coordinator so peers learn the new metadata on the next gossip cycle. - /// - /// # Errors - /// - /// Returns [`memberlist_proto::Error::MetaExceedsLimit`] if the serialised - /// metadata exceeds the memberlist wire limit. - fn update_meta( - &mut self, - meta: memberlist_proto::typed::Meta, - ) -> Result<(), memberlist_proto::Error>; - /// Signal that the local node intends to leave the cluster gracefully. /// /// The coordinator disseminates a Leave message, transitions the inner @@ -151,18 +129,23 @@ where /// its per-node coordinate so peers can compute network-distance estimates /// without a separate round-trip. /// + /// Only the `coordinates` feature drives this seam (serf piggybacks its + /// Vivaldi coordinate on probe acks); it is compiled out otherwise. + /// /// # Errors /// /// Returns [`memberlist_proto::Error::AckPayloadExceedsMtu`] if the framed /// Ack would not fit the gossip packet budget. + #[cfg(feature = "coordinates")] fn set_ack_payload(&mut self, payload: Bytes) -> Result<(), memberlist_proto::Error>; } // ── impl for the raw memberlist_proto::Endpoint ─────────────────────────────── // -// Covers the transitional period where the serf `Endpoint` still holds a raw -// `memberlist_proto::Endpoint` as its `inner` field. It is removed once the -// concrete coordinator super-machines own the transport. +// The transitional `StreamEndpoint` super-machine uses the raw packet +// `memberlist_proto::Endpoint` as its reliable transport, so the serf-logic +// core can drive it through this seam before the full stream/QUIC coordinators +// are wired in as the transport. impl Reliable for Endpoint where @@ -208,19 +191,12 @@ where Endpoint::start_push_pull(self, peer, kind, now) } - #[inline] - fn update_meta( - &mut self, - meta: memberlist_proto::typed::Meta, - ) -> Result<(), memberlist_proto::Error> { - Endpoint::update_meta(self, meta) - } - #[inline] fn leave(&mut self, now: Instant) -> Result<(), memberlist_proto::Error> { Endpoint::leave(self, now) } + #[cfg(feature = "coordinates")] #[inline] fn set_ack_payload(&mut self, payload: Bytes) -> Result<(), memberlist_proto::Error> { Endpoint::set_ack_payload(self, payload) @@ -278,19 +254,12 @@ where self.start_push_pull(peer, kind, now) } - #[inline] - fn update_meta( - &mut self, - meta: memberlist_proto::typed::Meta, - ) -> Result<(), memberlist_proto::Error> { - self.update_meta(meta) - } - #[inline] fn leave(&mut self, now: Instant) -> Result<(), memberlist_proto::Error> { self.leave(now) } + #[cfg(feature = "coordinates")] #[inline] fn set_ack_payload(&mut self, payload: Bytes) -> Result<(), memberlist_proto::Error> { self.set_ack_payload(payload) @@ -354,19 +323,12 @@ where self.start_push_pull(peer, kind, now) } - #[inline] - fn update_meta( - &mut self, - meta: memberlist_proto::typed::Meta, - ) -> Result<(), memberlist_proto::Error> { - self.update_meta(meta) - } - #[inline] fn leave(&mut self, now: Instant) -> Result<(), memberlist_proto::Error> { self.leave(now) } + #[cfg(feature = "coordinates")] #[inline] fn set_ack_payload(&mut self, payload: Bytes) -> Result<(), memberlist_proto::Error> { self.set_ack_payload(payload) diff --git a/serf-proto/src/endpoint/reliable/tests.rs b/serf-proto/src/endpoint/reliable/tests.rs index 1dec0041..e01875d8 100644 --- a/serf-proto/src/endpoint/reliable/tests.rs +++ b/serf-proto/src/endpoint/reliable/tests.rs @@ -6,9 +6,9 @@ use memberlist_proto::{EndpointOptions, Instant, PushPullKind, SeedableRng, Smal /// the impls are exercised at the trait boundary, not just as concrete calls. /// /// Exercises `endpoint_ref` (read-only accessors), `queue_user_broadcast_ranked`, -/// `set_ack_payload`, `set_local_state_snapshot`, `poll_inner_event`, and -/// `start_push_pull` — methods that run on a fresh, un-started endpoint -/// without requiring a live network peer. +/// `set_ack_payload` (coordinates only), `set_local_state_snapshot`, +/// `poll_inner_event`, and `start_push_pull` — methods that run on a fresh, +/// un-started endpoint without requiring a live network peer. fn drive(t: &mut impl Reliable, addr: A) where I: Eq + core::hash::Hash + Clone, @@ -25,9 +25,13 @@ where "queue_user_broadcast_ranked failed: {result:?}" ); - // Mutating: set ack payload. - let result = t.set_ack_payload(Bytes::from_static(b"coord")); - assert!(result.is_ok(), "set_ack_payload failed: {result:?}"); + // Mutating: set ack payload (the coordinates-only seam serf uses to + // piggyback its Vivaldi coordinate on probe acks). + #[cfg(feature = "coordinates")] + { + let result = t.set_ack_payload(Bytes::from_static(b"coord")); + assert!(result.is_ok(), "set_ack_payload failed: {result:?}"); + } // Mutating: set local state snapshot. let result = t.set_local_state_snapshot(Bytes::from_static(b"state")); diff --git a/serf-proto/src/endpoint/serf_parity_tests.rs b/serf-proto/src/endpoint/serf_parity_tests.rs index bf756c44..01838352 100644 --- a/serf-proto/src/endpoint/serf_parity_tests.rs +++ b/serf-proto/src/endpoint/serf_parity_tests.rs @@ -10,20 +10,19 @@ use bytes::Bytes; use memberlist_proto::{EndpointOptions, Instant, SeedableRng, SmallRng}; use crate::{ - AnyMessage, LamportTime, - endpoint::Endpoint, + AnyMessage, LamportTime, StreamEndpoint, event::{Event, MemberEventKind}, members::{IntentKind, MemberStatus}, options::Options, typed::{PushPullMessage, UserEvent, UserEvents}, }; -fn ep() -> Endpoint { +fn ep() -> StreamEndpoint { let inner_opts = EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); let inner = memberlist_proto::Endpoint::new_at(inner_opts, Instant::ORIGIN, SmallRng::seed_from_u64(0)); - Endpoint::new(inner, Options::new()) + StreamEndpoint::new(inner, Options::new()) } // ── base.rs handle_node_join invariants ─────────────────────────────────────── diff --git a/serf-proto/src/endpoint/tests.rs b/serf-proto/src/endpoint/tests.rs index d19106d9..301571db 100644 --- a/serf-proto/src/endpoint/tests.rs +++ b/serf-proto/src/endpoint/tests.rs @@ -1,6 +1,6 @@ use super::*; use crate::{ - AnyMessage, JoinMessage, LamportTime, LeaveMessage, + AnyMessage, JoinMessage, LamportTime, LeaveMessage, StreamEndpoint, event::{Event, MemberEventKind}, members::{MemberStatus, SerfState}, typed::{Filter, QueryFlag, QueryMessage, RelayMessage, UserEventMessage}, @@ -13,7 +13,7 @@ use memberlist_proto::{EndpointOptions, SeedableRng, SmallRng}; /// /// Uses `u32` node ids and `SocketAddr` addresses with a deterministically /// seeded `SmallRng` so tests are reproducible. -fn ep() -> Endpoint { +fn ep() -> StreamEndpoint { let inner_opts = EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); let inner = memberlist_proto::Endpoint::new_at( @@ -21,12 +21,12 @@ fn ep() -> Endpoint { memberlist_proto::Instant::ORIGIN, SmallRng::seed_from_u64(0), ); - Endpoint::new(inner, Options::new()) + StreamEndpoint::new(inner, Options::new()) } /// Build a serf `Endpoint` with coordinates enabled (for coordinate-gated tests). #[cfg(feature = "coordinates")] -fn ep_with_coords() -> Endpoint { +fn ep_with_coords() -> StreamEndpoint { let inner_opts = EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); let inner = memberlist_proto::Endpoint::new_at( @@ -35,7 +35,7 @@ fn ep_with_coords() -> Endpoint { SmallRng::seed_from_u64(0), ); let opts = Options::new().with_disable_coordinates(false); - Endpoint::new(inner, opts) + StreamEndpoint::new(inner, opts) } #[test] @@ -356,7 +356,7 @@ fn leave_from_already_left_is_idempotent() { fn leave_from_shutdown_is_rejected() { let mut e = ep(); // Force state to Shutdown. - e.state = SerfState::Shutdown; + e.core_mut().state = SerfState::Shutdown; let err = e .leave(memberlist_proto::Instant::ORIGIN) .expect_err("leave from Shutdown must fail"); @@ -435,7 +435,7 @@ fn shutdown_prevents_leaving_to_left_transition() { e.leave(memberlist_proto::Instant::ORIGIN).unwrap(); e.test_inner_left_cluster(); // Force Shutdown before the deadline fires. - e.state = SerfState::Shutdown; + e.core_mut().state = SerfState::Shutdown; let after_delay = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(2); e.handle_timeout(after_delay); // Must remain Shutdown, not Left. @@ -465,7 +465,7 @@ fn leave_arms_broadcast_deadline() { #[test] fn force_leave_from_shutdown_is_rejected() { let mut e = ep(); - e.state = SerfState::Shutdown; + e.core_mut().state = SerfState::Shutdown; let err = e .force_leave(2u32, false, memberlist_proto::Instant::ORIGIN) .expect_err("force_leave from Shutdown must fail"); @@ -520,7 +520,7 @@ fn poll_timeout_includes_leave_deadlines_when_armed() { // Helper: seed a failed member with an explicit address so we can assert what // addr is dialled by the reconnector. fn seed_failed( - e: &mut Endpoint, + e: &mut StreamEndpoint, id: u32, addr: std::net::SocketAddr, leave_time: memberlist_proto::Instant, @@ -530,7 +530,7 @@ fn seed_failed( // Helper: seed the endpoint's one alive member (the local node) explicitly so // the probability computation has a stable num_alive value. -fn seed_alive(e: &mut Endpoint, id: u32) { +fn seed_alive(e: &mut StreamEndpoint, id: u32) { e.test_seed_member(id, MemberStatus::Alive, LamportTime::new(0)); } @@ -1106,7 +1106,7 @@ fn push_pull_local_state_bytes_deterministic() { // produce byte-identical push-pull wire output after resync_local_state. // This verifies that HashMap iteration order in `members.states` does NOT // leak into the encoded PushPullMessage. - fn build_ep_asc() -> Endpoint { + fn build_ep_asc() -> StreamEndpoint { let mut e = ep(); e.test_set_clocks(5, 10, 15); // Insert members in ascending id order: 1, 2, 3, 4, 5. @@ -1116,7 +1116,7 @@ fn push_pull_local_state_bytes_deterministic() { e.resync_local_state(); e } - fn build_ep_desc() -> Endpoint { + fn build_ep_desc() -> StreamEndpoint { let mut e = ep(); e.test_set_clocks(5, 10, 15); // Insert members in descending id order: 5, 4, 3, 2, 1. @@ -1412,7 +1412,7 @@ fn invalid_tag_regex_does_not_advance_rng() { memberlist_proto::Instant::ORIGIN, SmallRng::seed_from_u64(0), ); - Endpoint::new_with_rng(inner, Options::new(), SmallRng::seed_from_u64(seed)) + StreamEndpoint::new_with_rng(inner, Options::new(), SmallRng::seed_from_u64(seed)) }; let mut ep_a = make_ep(42); @@ -1817,16 +1817,9 @@ fn relay_response_picks_alive_non_self_member_and_sends() { let mut e = ep(); // Seed Alive members with ports 1010 and 1011. e.test_seed_member(10u32, MemberStatus::Alive, LamportTime::new(1)); - // test_seed_member uses port 0; override by using test_seed_failed_member - // approach for an explicit address: - { - let node = memberlist_proto::Node::new(11u32, addr(1011)); - let member = crate::members::Member::new(node, crate::typed::Tags::new(), MemberStatus::Alive); - e.members.states.insert( - 11u32, - crate::members::MemberState::new(member, LamportTime::new(1), None), - ); - } + // test_seed_member uses port 0; seed id 11 at an explicit address so the + // relay-peer selection has two distinct addresses to choose between. + e.test_seed_member_at(11u32, addr(1011), MemberStatus::Alive, LamportTime::new(1)); let querier = relay_node(2000); let frame = bytes::Bytes::from_static(b"\x06relay-payload"); @@ -1919,10 +1912,10 @@ fn relay_sieve_arm_decodes_relay_message_from_user_packet() { assert_eq!(sent, inner_payload); } -/// Build a serf `Endpoint` with an explicit RNG seed for the +/// Build a serf `StreamEndpoint` with an explicit RNG seed for the /// serf-level RNG (the relay/reconnect draws). The inner Endpoint uses a fixed /// seed 0; the serf-level seed is the caller-supplied `serf_seed`. -fn ep_with_serf_seed(serf_seed: u64) -> Endpoint { +fn ep_with_serf_seed(serf_seed: u64) -> StreamEndpoint { let inner_opts = memberlist_proto::EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); let inner = memberlist_proto::Endpoint::new_at( @@ -1930,7 +1923,7 @@ fn ep_with_serf_seed(serf_seed: u64) -> Endpoint { memberlist_proto::Instant::ORIGIN, SmallRng::seed_from_u64(0), ); - Endpoint::new_with_rng(inner, Options::new(), SmallRng::seed_from_u64(serf_seed)) + StreamEndpoint::new_with_rng(inner, Options::new(), SmallRng::seed_from_u64(serf_seed)) } #[test] @@ -2454,7 +2447,7 @@ fn reap_forgets_coordinate() { // // Use a tiny buffer (size=4) so ltime=1 and ltime=5 map to the same ring index // without needing a high clock that would make ltime=1 "too old". -fn ep_tiny_event_buf() -> Endpoint { +fn ep_tiny_event_buf() -> StreamEndpoint { let inner_opts = memberlist_proto::EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); let inner = memberlist_proto::Endpoint::new_at( @@ -2462,7 +2455,7 @@ fn ep_tiny_event_buf() -> Endpoint { memberlist_proto::Instant::ORIGIN, SmallRng::seed_from_u64(0), ); - Endpoint::new(inner, Options::new().with_event_buffer_size(4)) + StreamEndpoint::new(inner, Options::new().with_event_buffer_size(4)) } #[test] @@ -2539,7 +2532,7 @@ fn ack_query_produces_immediate_ack_directed_send() { } // Bug 3: Oversized query responses silently consumed. -fn ep_small_resp_limit() -> Endpoint { +fn ep_small_resp_limit() -> StreamEndpoint { let inner_opts = memberlist_proto::EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); let inner = memberlist_proto::Endpoint::new_at( @@ -2547,7 +2540,7 @@ fn ep_small_resp_limit() -> Endpoint { memberlist_proto::Instant::ORIGIN, SmallRng::seed_from_u64(0), ); - Endpoint::new(inner, Options::new().with_query_response_size_limit(10)) + StreamEndpoint::new(inner, Options::new().with_query_response_size_limit(10)) } #[test] @@ -2759,7 +2752,7 @@ fn respond_send_failure_returns_err_and_leaves_responded_false() { SmallRng::seed_from_u64(0), ); let opts = Options::new().with_query_response_size_limit(50_000); - let mut e = Endpoint::new(inner, opts); + let mut e = StreamEndpoint::new(inner, opts); let qid = QueryId { ltime: LamportTime::new(1), @@ -3192,7 +3185,7 @@ fn resync_keeps_dirty_when_inner_snapshot_rejects() { memberlist_proto::Instant::ORIGIN, SmallRng::seed_from_u64(0), ); - let mut e = Endpoint::new(inner, Options::new()); + let mut e = StreamEndpoint::new(inner, Options::new()); // Force dirty and call resync. e.test_set_clocks(1, 2, 3); @@ -3978,7 +3971,7 @@ fn zero_event_buffer_size_does_not_panic_on_first_event() { ); // event_buffer_size = 0 should be clamped to 1 internally. let opts = crate::options::Options::new().with_event_buffer_size(0); - let mut e = Endpoint::new(inner, opts); + let mut e = StreamEndpoint::new(inner, opts); // Must NOT panic. e.user_event("test", bytes::Bytes::new(), false) @@ -4000,7 +3993,7 @@ fn zero_event_buffer_size_does_not_panic_on_merge_remote_state() { SmallRng::seed_from_u64(0), ); let opts = crate::options::Options::new().with_event_buffer_size(0); - let mut e = Endpoint::new(inner, opts); + let mut e = StreamEndpoint::new(inner, opts); let pp = PushPullMessage:: { ltime: LamportTime::new(1), @@ -4223,7 +4216,7 @@ fn inbound_query_oversized_is_dropped_before_state_mutation() { memberlist_proto::SmallRng::seed_from_u64(0), ); let opts = crate::options::Options::new().with_query_size_limit(64); - let mut e: Endpoint = Endpoint::new(inner, opts); + let mut e: StreamEndpoint = StreamEndpoint::new(inner, opts); // Encode a QueryMessage whose payload pushes the wire encoding over 64 bytes. let oversized_payload = bytes::Bytes::from(vec![0u8; 100]); @@ -4340,7 +4333,7 @@ fn ack_for_non_ack_query_is_dropped() { ltime: LamportTime::new(1), id: 42, }; - e.pending_queries.push(PendingQuery { + e.core_mut().pending_queries.push(PendingQuery { kind: QueryPurpose::App, deadline, responses: std::collections::HashMap::new(), @@ -4381,7 +4374,7 @@ fn ack_for_ack_query_is_delivered() { ltime: LamportTime::new(1), id: 42, }; - e.pending_queries.push(PendingQuery { + e.core_mut().pending_queries.push(PendingQuery { kind: QueryPurpose::App, deadline, responses: std::collections::HashMap::new(), @@ -4657,7 +4650,7 @@ fn zero_reap_interval_advances_deadline() { SmallRng::seed_from_u64(0), ); let opts = Options::new().with_reap_interval(std::time::Duration::ZERO); - let mut e = Endpoint::new(inner, opts); + let mut e = StreamEndpoint::new(inner, opts); // Advance time past the first reap deadline so handle_timeout fires it. let now = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(1); @@ -4687,7 +4680,7 @@ fn zero_reconnect_interval_advances_deadline() { SmallRng::seed_from_u64(0), ); let opts = Options::new().with_reconnect_interval(std::time::Duration::ZERO); - let mut e = Endpoint::new(inner, opts); + let mut e = StreamEndpoint::new(inner, opts); let now = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(1); e.handle_timeout(now); @@ -4715,7 +4708,7 @@ fn zero_queue_check_interval_advances_deadline() { SmallRng::seed_from_u64(0), ); let opts = Options::new().with_queue_check_interval(std::time::Duration::ZERO); - let mut e = Endpoint::new(inner, opts); + let mut e = StreamEndpoint::new(inner, opts); let now = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(1); e.handle_timeout(now); @@ -4745,7 +4738,7 @@ fn user_event_total_packet_with_junk_exceeding_size_limit_is_dropped() { ); // Set a tight size limit: 32 bytes. let opts = crate::options::Options::new().with_max_user_event_size(32); - let mut e = super::Endpoint::new(inner, opts); + let mut e = StreamEndpoint::new(inner, opts); // A small valid UserEvent that fits within 32 bytes on its own. let valid = AnyMessage::::UserEvent(UserEventMessage { @@ -4791,7 +4784,7 @@ fn pre_decode_fence_drops_oversized_valid_query_frame() { ); // Small limit (64 bytes) so that a query with a 100-byte payload exceeds it. let opts = crate::options::Options::new().with_query_size_limit(64); - let mut e: Endpoint = Endpoint::new(inner, opts); + let mut e: StreamEndpoint = StreamEndpoint::new(inner, opts); // Construct a syntactically valid Query whose encoded frame exceeds 64 bytes. let q = QueryMessage:: { @@ -4856,7 +4849,7 @@ fn pre_decode_fence_drops_oversized_valid_user_event_frame() { ); // Tight limit: 32 bytes. let opts = crate::options::Options::new().with_max_user_event_size(32); - let mut e: Endpoint = Endpoint::new(inner, opts); + let mut e: StreamEndpoint = StreamEndpoint::new(inner, opts); // Construct a syntactically valid UserEvent whose encoded frame exceeds 32 bytes. let frame = AnyMessage::::UserEvent(UserEventMessage { @@ -5371,7 +5364,7 @@ mod tag_filter_regex { /// Seed the local node (id=1) with the given tags and return an endpoint /// ready for tag-filter query tests. - fn ep_with_tags(tags: Tags) -> Endpoint { + fn ep_with_tags(tags: Tags) -> StreamEndpoint { let mut e = ep(); e.test_seed_member_with_tags(1u32, tags, MemberStatus::Alive, LamportTime::new(0)); e diff --git a/serf-proto/src/lib.rs b/serf-proto/src/lib.rs index d99d32c3..b1b44f47 100644 --- a/serf-proto/src/lib.rs +++ b/serf-proto/src/lib.rs @@ -70,6 +70,10 @@ pub mod event; pub mod members; pub mod options; pub mod snapshot; +pub mod stream_endpoint; + +#[doc(inline)] +pub use stream_endpoint::StreamEndpoint; #[cfg(feature = "coordinates")] #[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] diff --git a/serf-proto/src/stream_endpoint/mod.rs b/serf-proto/src/stream_endpoint/mod.rs new file mode 100644 index 00000000..59a883e6 --- /dev/null +++ b/serf-proto/src/stream_endpoint/mod.rs @@ -0,0 +1,1014 @@ +//! The serf `StreamEndpoint` super-machine — serf logic composed with a +//! memberlist reliable coordinator over a stream transport. +//! +//! `StreamEndpoint` owns the serf-logic [`Endpoint`] core and a memberlist +//! coordinator (`transport`) as **two disjoint fields**, and drives the core +//! over `&mut transport` through the [`Reliable`](crate::endpoint::reliable) +//! seam. It exposes the coordinator's transport-facing driver surface +//! (`handle_packet`, `handle_timeout`, `poll_transmit`, `poll_timeout`, …) plus +//! serf's own commands and events (`poll_event`, `join`, `leave`, `user_event`, +//! `query`, key ops, …), forwarding each to the right field. +//! +//! The composed `handle_timeout` is where the load-bearing tick ordering lives: +//! the reliable coordinator is drained, the coordinator's SWIM timer fires, the +//! resulting inner events are sieved into serf, then serf's own deadlines fire — +//! all in one place, so no per-runtime driver has to re-establish the order. + +use bytes::Bytes; +use memberlist_proto::{CheapClone, Data, Id, Instant, Rng, SeedableRng, Transmit, parse_message}; +use smol_str::SmolStr; + +use crate::{ + endpoint::{Endpoint, Error, QueryId, QueryParams}, + event::{Event, QueryEvent}, + members::SerfState, + options::Options, +}; + +#[cfg(all(test, feature = "tag-regex"))] +use crate::typed::Tags; +#[cfg(test)] +use crate::{ + LamportTime, + members::{IntentKind, MemberStatus}, + typed::{QueryMessage, QueryResponseMessage, RelayMessage, UserEventMessage}, +}; +#[cfg(test)] +use memberlist_proto::Node; + +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +use crate::event::KeyResponseArgs; + +/// The serf `StreamEndpoint` super-machine. +/// +/// Composes the serf-logic [`Endpoint`] `core` with a memberlist reliable +/// coordinator (`transport`). The driver pumps **one** machine: it feeds the +/// transport ingress, ticks `handle_timeout`, and drains the serf and transport +/// poll surfaces. +pub struct StreamEndpoint +where + I: Eq + core::hash::Hash, +{ + /// The serf-logic core, holding all serf state and no transport reference. + core: Endpoint, + /// The memberlist reliable coordinator serf drives through the `Reliable` + /// seam. Holds the single membership `Endpoint`. + transport: memberlist_proto::Endpoint, +} + +impl StreamEndpoint +where + I: Clone + Eq + core::hash::Hash, + R: SeedableRng, +{ + /// Construct a `StreamEndpoint` from a memberlist coordinator `transport`, + /// serf `opts`, and serf's own injected `rng`. + /// + /// `rng` is **separate** from the coordinator's `R`; seed it from the + /// driver's own entropy source. + pub fn new_with_rng( + transport: memberlist_proto::Endpoint, + opts: Options, + rng: R, + ) -> Self { + Self { + core: Endpoint::new_with_rng(opts, rng), + transport, + } + } + + /// Convenience constructor that seeds serf's `R` with a zero seed. + /// + /// Suitable for tests and deterministic environments. Production drivers + /// should use `new_with_rng` and seed from a cryptographically-secure source. + pub fn new(transport: memberlist_proto::Endpoint, opts: Options) -> Self { + Self::new_with_rng(transport, opts, R::seed_from_u64(0)) + } +} + +// ── transport-level driver surface ──────────────────────────────────────────── +// +// These reach the coordinator (`transport`) directly — the `Reliable` seam +// deliberately excludes transport ingress / timer / poll operations — then drive +// the serf-logic sieve over the coordinator. + +impl StreamEndpoint +where + I: Id + Clone, + A: CheapClone + Data + PartialEq + Clone + 'static, + R: Rng + SeedableRng, +{ + /// Deliver an inbound datagram or packet to the machine. + /// + /// Decodes the memberlist wire `Message` and hands it to the + /// coordinator's inner endpoint, then sieves the resulting inner events into + /// serf. Malformed or unrecognised bytes are silently dropped — the machine + /// must not panic on bad input from the network. + pub fn handle_packet(&mut self, from: A, data: Bytes, now: Instant) { + // Malformed frame or unrecognised tag: drop silently. The inner endpoint + // logs its own decode errors; serf takes no serf-level action here. + if let Ok(msg) = parse_message::(data) { + self.transport.handle_packet(from, msg, now); + } + self.core.drain_after_ingress(&mut self.transport, now); + } + + /// Deliver a stream event to the coordinator, then sieve resulting inner + /// events into serf. + pub fn handle_stream_event( + &mut self, + ev: memberlist_proto::EndpointEvent, + now: Instant, + ) -> Option> { + let cmd = self.transport.handle_stream_event(ev, now); + self.core.drain_after_ingress(&mut self.transport, now); + cmd + } + + /// Accept an inbound stream connection on the coordinator. + pub fn accept_stream(&mut self, from: A, now: Instant) -> Option> { + self.transport.accept_stream(from, now) + } + + /// Advance time and fire any expired serf or coordinator deadlines. + /// + /// The composed tick order (the structural win): the coordinator's SWIM timer + /// fires between the serf pre-tick snapshot resync and the serf post-tick + /// drain + deadline pass, so the load-bearing + /// `resync → inner timer → drain → serf deadlines` sequence is established + /// here, once, rather than in each runtime driver. + pub fn handle_timeout(&mut self, now: Instant) { + // Pre-inner-timer: latch `now` and resync the push-pull snapshot if dirty, + // so the coordinator ships current serf state on this tick's anti-entropy. + self.core.before_inner_timeout(&mut self.transport, now); + // Inner timer: the coordinator's SWIM gossip / probe / push-pull scheduler. + self.transport.handle_timeout(now); + // Post-inner-timer: sieve the inner events this tick produced, then fire + // serf's own deadlines (reap / reconnect / queue-check / query-close / …). + self.core.after_inner_timeout(&mut self.transport, now); + } + + /// Drain one outgoing transmit from the coordinator. + pub fn poll_transmit(&mut self) -> Option> { + self.transport.poll_transmit() + } + + /// The earliest deadline requiring a `handle_timeout` call. + /// + /// The minimum of the coordinator's own deadline and serf's periodic + /// deadlines. + pub fn poll_timeout(&self) -> Option { + let inner = self.transport.poll_timeout(); + let serf = self.core.serf_poll_timeout(); + match (inner, serf) { + (Some(a), Some(b)) => Some(a.min(b)), + (Some(a), None) => Some(a), + (None, Some(b)) => Some(b), + (None, None) => None, + } + } + + /// Number of unsent items in the coordinator's user broadcast queue. + /// + /// The driver may poll this during a graceful leave to detect when the + /// leave-intent broadcast has been flushed without waiting the full + /// `broadcast_timeout`. + pub fn user_broadcast_queue_len(&self) -> usize { + self.transport.user_broadcast_queue_len() + } + + // ── serf-logic + serf-command forwarders ──────────────────────────────────── + // Each forwards to the matching `Endpoint` method, threading `&mut transport` + // through the ones that reach the coordinator (the `Reliable` methods). + + /// Announce the local node's join intent to the cluster. + /// + /// Forwards to [`Endpoint::join`]. + pub fn join(&mut self) -> Result<(), Error> + where + I: Clone, + { + self.core.join(&mut self.transport) + } + + /// Issue a cluster-wide `use_key` query to promote `key` to primary. + /// + /// Forwards to [`Endpoint::use_key`]. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn use_key( + &mut self, + key: memberlist_proto::SecretKey, + now: Instant, + ) -> Result + where + I: Clone + Data, + A: Clone + Data, + { + self.core.use_key(&mut self.transport, key, now) + } + + /// Issue a cluster-wide `remove_key` query to remove `key` from all nodes. + /// + /// Forwards to [`Endpoint::remove_key`]. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn remove_key( + &mut self, + key: memberlist_proto::SecretKey, + now: Instant, + ) -> Result + where + I: Clone + Data, + A: Clone + Data, + { + self.core.remove_key(&mut self.transport, key, now) + } + + /// Issue a cluster-wide `list_keys` query to enumerate installed keys. + /// + /// Forwards to [`Endpoint::list_keys`]. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn list_keys(&mut self, now: Instant) -> Result + where + I: Clone + Data, + A: Clone + Data, + { + self.core.list_keys(&mut self.transport, now) + } + + /// Forwards to [`Endpoint::state`]. + pub const fn state(&self) -> SerfState { + self.core.state() + } + + /// Forwards to [`Endpoint::member_time`]. + pub const fn member_time(&self) -> u64 { + self.core.member_time() + } + + /// Forwards to [`Endpoint::event_time`]. + pub const fn event_time(&self) -> u64 { + self.core.event_time() + } + + /// Forwards to [`Endpoint::query_time`]. + pub const fn query_time(&self) -> u64 { + self.core.query_time() + } + + /// Forwards to [`Endpoint::num_members`]. + pub fn num_members(&self) -> usize { + self.core.num_members() + } + + /// Forwards to [`Endpoint::poll_event`]. + pub fn poll_event(&mut self) -> Option> { + self.core.poll_event(&mut self.transport) + } + + /// Forwards to [`Endpoint::resync_local_state`]. + pub fn resync_local_state(&mut self) + where + I: Clone + Data, + A: Data, + { + self.core.resync_local_state(&mut self.transport) + } + + /// Forwards to [`Endpoint::set_event_join_ignore`]. + pub fn set_event_join_ignore(&mut self, v: bool) { + self.core.set_event_join_ignore(v) + } + + /// Forwards to [`Endpoint::handle_node_join_intent`]. + #[cfg(test)] + pub(crate) fn handle_node_join_intent(&mut self, ltime: LamportTime, id: &I, now: Instant) -> bool + where + I: Clone, + { + self.core.handle_node_join_intent(ltime, id, now) + } + + /// Forwards to [`Endpoint::handle_node_leave_intent`]. + #[cfg(test)] + pub(crate) fn handle_node_leave_intent( + &mut self, + ltime: LamportTime, + id: &I, + prune: bool, + now: Instant, + ) -> bool + where + I: Clone, + { + self + .core + .handle_node_leave_intent(&mut self.transport, ltime, id, prune, now) + } + + /// Forwards to [`Endpoint::leave`]. + pub fn leave(&mut self, now: Instant) -> Result<(), Error> + where + I: Clone, + A: Clone, + { + self.core.leave(&mut self.transport, now) + } + + /// Forwards to [`Endpoint::force_leave`]. + pub fn force_leave(&mut self, id: I, prune: bool, now: Instant) -> Result<(), Error> + where + I: Clone, + A: Clone, + { + self.core.force_leave(&mut self.transport, id, prune, now) + } + + /// Forwards to [`Endpoint::user_event`]. + pub fn user_event( + &mut self, + name: impl Into, + payload: bytes::Bytes, + coalesce: bool, + ) -> Result<(), Error> { + self + .core + .user_event(&mut self.transport, name, payload, coalesce) + } + + /// Forwards to [`Endpoint::handle_user_event`]. + #[cfg(test)] + pub(crate) fn handle_user_event(&mut self, msg: UserEventMessage) -> bool { + self.core.handle_user_event(msg) + } + + /// Forwards to [`Endpoint::query`]. + pub fn query( + &mut self, + name: impl Into, + payload: Bytes, + params: QueryParams, + now: Instant, + ) -> Result + where + I: Clone + Data, + A: Clone + Data, + { + self + .core + .query(&mut self.transport, name, payload, params, now) + } + + /// Forwards to [`Endpoint::respond`]. + pub fn respond( + &mut self, + token: &QueryEvent, + payload: Bytes, + now: Instant, + ) -> Result<(), Error> + where + I: Clone + Data, + A: Clone + Data, + { + self.core.respond(&mut self.transport, token, payload, now) + } + + /// Forwards to [`Endpoint::respond_key`]. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn respond_key( + &mut self, + req: &crate::event::KeyRequest, + resp: KeyResponseArgs, + now: Instant, + ) -> Result<(), Error> + where + I: Clone + Data, + A: Clone + Data, + { + self.core.respond_key(&mut self.transport, req, resp, now) + } + + /// Forwards to [`Endpoint::leave_broadcast_deadline`]. + pub const fn leave_broadcast_deadline(&self) -> Option { + self.core.leave_broadcast_deadline() + } + + /// Forwards to [`Endpoint::leave_complete_deadline`]. + pub const fn leave_complete_deadline(&self) -> Option { + self.core.leave_complete_deadline() + } + + /// Forwards to [`Endpoint::test_member_status`]. + #[cfg(test)] + pub(crate) fn test_member_status(&self, id: I) -> Option + where + I: Clone, + { + self.core.test_member_status(id) + } + + /// Forwards to [`Endpoint::test_member_status_time`]. + #[cfg(test)] + pub(crate) fn test_member_status_time(&self, id: I) -> Option + where + I: Clone, + { + self.core.test_member_status_time(id) + } + + /// Forwards to [`Endpoint::test_seed_member`]. + #[cfg(test)] + pub(crate) fn test_seed_member(&mut self, id: I, status: MemberStatus, status_time: LamportTime) + where + I: Clone, + A: Clone + From, + { + self.core.test_seed_member(id, status, status_time) + } + + /// Forwards to [`Endpoint::test_seed_member_with_tags`]. + #[cfg(all(test, feature = "tag-regex"))] + pub(crate) fn test_seed_member_with_tags( + &mut self, + id: I, + tags: Tags, + status: MemberStatus, + status_time: LamportTime, + ) where + I: Clone, + A: Clone + From, + { + self + .core + .test_seed_member_with_tags(id, tags, status, status_time) + } + + /// Forwards to [`Endpoint::test_seed_failed_member_by_status`]. + #[cfg(test)] + pub(crate) fn test_seed_failed_member_by_status( + &mut self, + id: I, + status_time: LamportTime, + now: Instant, + ) where + I: Clone, + A: Clone + From, + { + self + .core + .test_seed_failed_member_by_status(id, status_time, now) + } + + /// Forwards to [`Endpoint::test_seed_left_member_by_status`]. + #[cfg(test)] + pub(crate) fn test_seed_left_member_by_status( + &mut self, + id: I, + status_time: LamportTime, + now: Instant, + ) where + I: Clone, + A: Clone + From, + { + self + .core + .test_seed_left_member_by_status(id, status_time, now) + } + + /// Forwards to [`Endpoint::test_handle_join_intent`]. + #[cfg(test)] + pub(crate) fn test_handle_join_intent(&mut self, id: I, ltime: LamportTime, now: Instant) -> bool + where + I: Clone, + { + self.core.test_handle_join_intent(id, ltime, now) + } + + /// Forwards to [`Endpoint::test_handle_leave_intent`]. + #[cfg(test)] + pub(crate) fn test_handle_leave_intent(&mut self, id: I, ltime: LamportTime, now: Instant) -> bool + where + I: Clone, + { + self + .core + .test_handle_leave_intent(&mut self.transport, id, ltime, now) + } + + /// Forwards to [`Endpoint::test_inner_node_joined`]. + #[cfg(test)] + pub(crate) fn test_inner_node_joined(&mut self, id: I, now: Instant) + where + I: Clone, + A: Clone + From, + { + self.core.test_inner_node_joined(id, now) + } + + /// Forwards to [`Endpoint::test_inner_node_left`]. + #[cfg(test)] + pub(crate) fn test_inner_node_left(&mut self, id: I, now: Instant) + where + I: Clone, + A: Clone + From, + { + self.core.test_inner_node_left(id, now) + } + + /// Forwards to [`Endpoint::test_inner_node_updated`]. + #[cfg(test)] + pub(crate) fn test_inner_node_updated(&mut self, id: I, now: Instant) + where + I: Clone, + A: Clone + From, + { + self.core.test_inner_node_updated(id, now) + } + + /// Forwards to [`Endpoint::test_in_failed_members`]. + #[cfg(test)] + pub(crate) fn test_in_failed_members(&self, id: I) -> bool + where + I: PartialEq, + { + self.core.test_in_failed_members(id) + } + + /// Forwards to [`Endpoint::test_in_left_members`]. + #[cfg(test)] + pub(crate) fn test_in_left_members(&self, id: I) -> bool + where + I: PartialEq, + { + self.core.test_in_left_members(id) + } + + /// Forwards to [`Endpoint::test_inner_left_cluster`]. + #[cfg(test)] + pub(crate) fn test_inner_left_cluster(&mut self) { + self.core.test_inner_left_cluster(&mut self.transport) + } + + /// Forwards to [`Endpoint::test_seed_failed_member`]. + #[cfg(test)] + pub(crate) fn test_seed_failed_member(&mut self, id: I, addr: A, now: Instant) + where + I: Clone, + A: Clone, + { + self.core.test_seed_failed_member(id, addr, now) + } + + /// Forwards to [`Endpoint::test_fire_reconnect`]. + #[cfg(test)] + pub(crate) fn test_fire_reconnect(&mut self, now: Instant) + where + A: Clone, + { + self.core.test_fire_reconnect(&mut self.transport, now) + } + + /// Forwards to [`Endpoint::test_fire_reap`]. + #[cfg(test)] + pub(crate) fn test_fire_reap(&mut self, now: Instant) + where + I: Clone, + A: Clone, + { + self.core.test_fire_reap(now) + } + + /// Forwards to [`Endpoint::test_last_dial_addr`]. + #[cfg(test)] + pub(crate) fn test_last_dial_addr(&self) -> Option + where + A: Clone, + { + self.core.test_last_dial_addr() + } + + /// Forwards to [`Endpoint::test_handle_user_event`]. + #[cfg(test)] + pub(crate) fn test_handle_user_event(&mut self, msg: UserEventMessage) -> bool { + self.core.test_handle_user_event(msg) + } + + /// Forwards to [`Endpoint::test_set_event_min_time`]. + #[cfg(test)] + pub(crate) fn test_set_event_min_time(&mut self, t: u64) { + self.core.test_set_event_min_time(t) + } + + /// Forwards to [`Endpoint::test_set_event_clock`]. + #[cfg(test)] + pub(crate) fn test_set_event_clock(&mut self, t: u64) { + self.core.test_set_event_clock(t) + } + + /// Forwards to [`Endpoint::test_event_slot_len`]. + #[cfg(test)] + pub(crate) fn test_event_slot_len(&self, ltime: u64) -> usize { + self.core.test_event_slot_len(ltime) + } + + /// Forwards to [`Endpoint::test_inject_user_packet`]. + #[cfg(test)] + pub(crate) fn test_inject_user_packet(&mut self, from: A, data: Bytes, now: Instant) + where + I: Clone + Data, + A: Clone + Data, + { + self + .core + .test_inject_user_packet(&mut self.transport, from, data, now) + } + + /// Forwards to [`Endpoint::test_set_clocks`]. + #[cfg(test)] + pub(crate) fn test_set_clocks(&mut self, member: u64, event: u64, query: u64) { + self.core.test_set_clocks(member, event, query) + } + + /// Forwards to [`Endpoint::test_seed_left_member`]. + #[cfg(test)] + pub(crate) fn test_seed_left_member(&mut self, id: I, status_time: LamportTime) + where + I: Clone, + A: Clone + From, + { + self.core.test_seed_left_member(id, status_time) + } + + /// Forwards to [`Endpoint::test_inner_local_state_snapshot`]. + #[cfg(test)] + pub(crate) fn test_inner_local_state_snapshot(&self) -> Bytes { + self.core.test_inner_local_state_snapshot(&self.transport) + } + + /// Forwards to [`Endpoint::test_decode_pushpull`]. + #[cfg(test)] + pub(crate) fn test_decode_pushpull(&self, bytes: &Bytes) -> crate::typed::PushPullMessage + where + I: Clone + Data, + A: Data, + { + self.core.test_decode_pushpull(bytes) + } + + /// Forwards to [`Endpoint::test_clear_dirty`]. + #[cfg(test)] + pub(crate) fn test_clear_dirty(&mut self) { + self.core.test_clear_dirty() + } + + /// Forwards to [`Endpoint::test_is_dirty`]. + #[cfg(test)] + pub(crate) fn test_is_dirty(&self) -> bool { + self.core.test_is_dirty() + } + + /// Forwards to [`Endpoint::test_set_event_join_ignore`]. + #[cfg(test)] + pub(crate) fn test_set_event_join_ignore(&mut self, v: bool) { + self.core.test_set_event_join_ignore(v) + } + + /// Forwards to [`Endpoint::test_event_min_time`]. + #[cfg(test)] + pub(crate) fn test_event_min_time(&self) -> u64 { + self.core.test_event_min_time() + } + + /// Forwards to [`Endpoint::test_merge_remote_state`]. + #[cfg(test)] + pub(crate) fn test_merge_remote_state(&mut self, user_data: Bytes, is_join: bool) + where + I: Clone + Data, + A: Data, + { + self + .core + .test_merge_remote_state(&mut self.transport, user_data, is_join) + } + + /// Forwards to [`Endpoint::test_intent_ltime`]. + #[cfg(test)] + pub(crate) fn test_intent_ltime(&self, id: I, kind: IntentKind) -> Option + where + I: Clone, + { + self.core.test_intent_ltime(id, kind) + } + + /// Forwards to [`Endpoint::test_handle_query`]. + #[cfg(test)] + pub(crate) fn test_handle_query(&mut self, msg: QueryMessage) -> bool + where + I: Clone + Data, + A: Clone + Data, + { + self.core.test_handle_query(&mut self.transport, msg) + } + + /// Forwards to [`Endpoint::test_last_query_id`]. + #[cfg(test)] + pub(crate) fn test_last_query_id(&self) -> Option { + self.core.test_last_query_id() + } + + /// Forwards to [`Endpoint::test_pending_query_count`]. + #[cfg(test)] + pub(crate) fn test_pending_query_count(&self) -> usize { + self.core.test_pending_query_count() + } + + /// Forwards to [`Endpoint::test_received_queries_len`]. + #[cfg(test)] + pub(crate) fn test_received_queries_len(&self) -> usize { + self.core.test_received_queries_len() + } + + /// Forwards to [`Endpoint::test_query_min_time`]. + #[cfg(test)] + pub(crate) fn test_query_min_time(&self) -> u64 { + self.core.test_query_min_time() + } + + /// Forwards to [`Endpoint::test_register_received_query`]. + #[cfg(test)] + pub(crate) fn test_register_received_query( + &mut self, + query_id: QueryId, + querier: A, + deadline: Instant, + ) -> QueryEvent + where + I: Default + Clone, + A: Clone, + { + self + .core + .test_register_received_query(query_id, querier, deadline) + } + + /// Forwards to [`Endpoint::test_handle_query_response`]. + #[cfg(test)] + pub(crate) fn test_handle_query_response(&mut self, msg: QueryResponseMessage) + where + I: Clone, + A: Clone, + { + self + .core + .test_handle_query_response(&mut self.transport, msg) + } + + /// Forwards to [`Endpoint::test_is_responded`]. + #[cfg(test)] + pub(crate) fn test_is_responded(&self, query_id: QueryId) -> bool { + self.core.test_is_responded(query_id) + } + + /// Forwards to [`Endpoint::test_recent_intents_len`]. + #[cfg(test)] + pub(crate) fn test_recent_intents_len(&self) -> usize { + self.core.test_recent_intents_len() + } + + /// Forwards to [`Endpoint::test_pending_query_conflict_matching`]. + #[cfg(test)] + pub(crate) fn test_pending_query_conflict_matching(&self, query_id: QueryId) -> Option { + self.core.test_pending_query_conflict_matching(query_id) + } + + /// Forwards to [`Endpoint::test_relay_response`]. + #[cfg(test)] + pub(crate) fn test_relay_response(&mut self, querier: Node, frame: Bytes, relay_factor: u8) + where + I: Clone + Data, + A: Clone + Data, + { + self + .core + .test_relay_response(&mut self.transport, querier, frame, relay_factor) + } + + /// Forwards to [`Endpoint::test_handle_relay`]. + #[cfg(test)] + pub(crate) fn test_handle_relay(&mut self, relay: RelayMessage) + where + I: Clone, + A: Clone, + { + self.core.test_handle_relay(&mut self.transport, relay) + } + + /// Forwards to [`Endpoint::test_last_directed_send`]. + #[cfg(test)] + pub(crate) fn test_last_directed_send(&self) -> Option<(A, Bytes)> + where + A: Clone, + { + self.core.test_last_directed_send() + } + + /// Forwards to [`Endpoint::install_key`]. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn install_key( + &mut self, + key: memberlist_proto::SecretKey, + now: Instant, + ) -> Result + where + I: Clone + Data, + A: Clone + Data, + { + self.core.install_key(&mut self.transport, key, now) + } + + /// Forwards to [`Endpoint::test_register_conflict_query`]. + #[cfg(test)] + pub(crate) fn test_register_conflict_query(&mut self, deadline: Instant) -> QueryId + where + I: Clone, + A: Clone, + { + self.core.test_register_conflict_query(deadline) + } + + /// Forwards to [`Endpoint::test_register_key_query`]. + #[cfg(all(test, any(feature = "aes-gcm", feature = "chacha20-poly1305")))] + pub(crate) fn test_register_key_query(&mut self, deadline: Instant) -> QueryId + where + I: Clone, + A: Clone, + { + self.core.test_register_key_query(deadline) + } + + /// Forwards to [`Endpoint::test_fold_conflict_response`]. + #[cfg(test)] + pub(crate) fn test_fold_conflict_response( + &mut self, + query_id: QueryId, + responder_id: I, + agrees: bool, + ) where + I: Clone, + { + self + .core + .test_fold_conflict_response(query_id, responder_id, agrees) + } + + /// Forwards to [`Endpoint::test_pending_query_response_count`]. + #[cfg(test)] + pub(crate) fn test_pending_query_response_count(&self, query_id: QueryId) -> usize { + self.core.test_pending_query_response_count(query_id) + } + + /// Forwards to [`Endpoint::test_query_slot_len`]. + #[cfg(test)] + pub(crate) fn test_query_slot_len(&self, ltime: u64) -> usize { + self.core.test_query_slot_len(ltime) + } + + /// Forwards to [`Endpoint::test_fire_due_query_closes`]. + #[cfg(test)] + pub(crate) fn test_fire_due_query_closes(&mut self, now: Instant) + where + I: Clone, + A: Clone, + { + self.core.test_fire_due_query_closes(now) + } + + /// Forwards to [`Endpoint::test_inject_inner_joined`]. + #[cfg(test)] + pub(crate) fn test_inject_inner_joined(&mut self, id: I, now: Instant) + where + I: Clone, + A: Clone + From, + { + self.core.test_inject_inner_joined(id, now) + } + + /// Forwards to [`Endpoint::test_enqueue_intent_broadcast`]. + #[cfg(test)] + pub(crate) fn test_enqueue_intent_broadcast(&mut self, bytes: Bytes) { + self + .core + .test_enqueue_intent_broadcast(&mut self.transport, bytes) + } + + /// Forwards to [`Endpoint::test_enqueue_query_broadcast`]. + #[cfg(test)] + pub(crate) fn test_enqueue_query_broadcast(&mut self, bytes: Bytes) { + self + .core + .test_enqueue_query_broadcast(&mut self.transport, bytes) + } + + /// Forwards to [`Endpoint::load_snapshot`]. + pub fn load_snapshot(&mut self, replay: crate::snapshot::ReplayResult, now: Instant) + where + A: Clone, + { + self.core.load_snapshot(&mut self.transport, replay, now) + } + + /// Forwards to [`Endpoint::test_rejoin_dials`]. + #[cfg(test)] + pub(crate) fn test_rejoin_dials(&self) -> Vec + where + A: Clone, + { + self.core.test_rejoin_dials() + } + + /// Forwards to [`Endpoint::test_ping_completed`]. + #[cfg(all(feature = "coordinates", test))] + pub(crate) fn test_ping_completed( + &mut self, + node_id: I, + rtt: std::time::Duration, + payload: Bytes, + ) { + self + .core + .test_ping_completed(&mut self.transport, node_id, rtt, payload) + } + + /// Forwards to [`Endpoint::get_coordinate`]. + #[cfg(feature = "coordinates")] + #[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] + pub fn get_coordinate(&self) -> Option { + self.core.get_coordinate() + } + + /// Forwards to [`Endpoint::cached_coordinate`]. + #[cfg(feature = "coordinates")] + #[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] + pub fn cached_coordinate(&self, node: &I) -> Option { + self.core.cached_coordinate(node) + } + + /// Forwards to [`Endpoint::test_seed_member_at`]. + #[cfg(test)] + pub(crate) fn test_seed_member_at( + &mut self, + id: I, + addr: A, + status: MemberStatus, + status_time: LamportTime, + ) where + I: Clone, + A: Clone, + { + self.core.test_seed_member_at(id, addr, status, status_time) + } + + /// Forwards to [`Endpoint::test_relay_all_directed_sends`]. + #[cfg(test)] + pub(crate) fn test_relay_all_directed_sends(&self) -> &[(A, Bytes)] { + self.core.test_relay_all_directed_sends() + } + + /// Forwards to [`Endpoint::test_last_pending_query_num_nodes`]. + #[cfg(all(test, any(feature = "aes-gcm", feature = "chacha20-poly1305")))] + pub(crate) fn test_last_pending_query_num_nodes(&self) -> Option { + self.core.test_last_pending_query_num_nodes() + } + + /// Forwards to [`Endpoint::test_peek_received_query_deadlines`]. + #[cfg(test)] + pub(crate) fn test_peek_received_query_deadlines(&self) -> Vec { + self.core.test_peek_received_query_deadlines() + } + + /// Mutable access to the serf-logic core, for tests that manipulate its + /// private state directly. + #[cfg(test)] + pub(crate) fn core_mut(&mut self) -> &mut Endpoint { + &mut self.core + } +} From 7043bbf8571851916957aca3a9ad49c8ed40cce5 Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 26 Jun 2026 21:17:28 +1200 Subject: [PATCH 013/128] feat(serf-proto): StreamEndpoint super-machine over the memberlist StreamEndpoint Wraps the real reliable coordinator (tcp/tls) plus the serf core; the coordinator owns the stream lifecycle and sieves DialRequested internally, so serf observes only RemoteStateReceived. The composed handle_timeout keeps the before_inner, transport, after_inner order. --- serf-proto/src/endpoint/mod.rs | 41 +-- serf-proto/src/endpoint/reliable/mod.rs | 9 +- serf-proto/src/endpoint/reliable/tests.rs | 13 +- serf-proto/src/endpoint/serf_parity_tests.rs | 14 +- serf-proto/src/endpoint/tests.rs | 101 ++++--- serf-proto/src/lib.rs | 5 + serf-proto/src/stream_endpoint/mod.rs | 206 +++++++++++---- serf-proto/src/stream_endpoint/tests.rs | 261 +++++++++++++++++++ 8 files changed, 533 insertions(+), 117 deletions(-) create mode 100644 serf-proto/src/stream_endpoint/tests.rs diff --git a/serf-proto/src/endpoint/mod.rs b/serf-proto/src/endpoint/mod.rs index 3c40beaf..3eeab504 100644 --- a/serf-proto/src/endpoint/mod.rs +++ b/serf-proto/src/endpoint/mod.rs @@ -1109,9 +1109,9 @@ where // ── user gossip ─────────────────────────────────────────────────────── IE::UserPacket(p) => { - // `drain_now` was latched by handle_packet / handle_stream_event / - // handle_timeout before drain_inner was called, so it is always a - // fresh `now` for the current call site. + // `drain_now` was latched by the ingress entry point (handle_packet / + // handle_transport_data / handle_timeout) before drain_inner was called, + // so it is always a fresh `now` for the current call site. let now = self.drain_now; let (from, data, _reliability) = p.into_parts(); self.handle_user_packet(t, from, data, now); @@ -1172,18 +1172,15 @@ where // serf takes no action on undecodable inner messages. IE::DecodeError(_) => {} - // ── reconnect / dial passthrough (H3) ──────────────────────────────── - // The inner emits DialRequested when serf calls inner.start_push_pull() - // (from fire_reconnect) or when the inner initiates its own anti-entropy - // push/pull. The driver must dial the given peer and report back via - // dial_succeeded / dial_failed. serf passes this through to the driver - // as Event::DialRequested so the driver's event loop can perform the dial. - // - // H3: the reconnector is machine OUTPUT — serf does NO I/O itself. + // ── reconnect / dial passthrough ───────────────────────────────────── + // The reliable coordinator (`StreamEndpoint`/`QuicEndpoint`) sieves the + // inner `DialRequested` into its own dial queue and dials itself, so this + // event never reaches serf's drain over a real coordinator. The arm + // remains for totality over `memberlist_proto::Event` and to re-emit a + // serf-level passthrough for any transport that does surface the dial to + // serf (the driver then performs the dial and reports back via + // dial_succeeded / dial_failed). IE::DialRequested(d) => { - // Re-emit as a serf-level event so the driver sees it from poll_event. - // DialPassthrough carries the StreamId and peer address; the driver uses - // StreamId to call inner.dial_succeeded / inner.dial_failed. let (stream_id, peer, _deadline) = d.into_parts(); // Track the dialled address for test assertions. #[cfg(test)] @@ -2006,8 +2003,14 @@ where }; let addr = ms.member().node().addr_ref().clone(); - // Call start_push_pull; the inner queues Event::DialRequested. - // The sieve (on_inner_event) passes it through as Event::DialRequested. + // Capture the dialled address for test assertions at the call site. The + // reliable coordinator sieves the inner `DialRequested` into its own dial + // queue (it IS the driver and dials itself), so the event never reaches the + // serf sieve — the call site is the only place serf observes the choice. + #[cfg(test)] + { + self.last_dial_addr = Some(addr.clone()); + } t.start_push_pull(addr, PushPullKind::Join, now); self.drain_inner(t); } @@ -5096,7 +5099,9 @@ where pub(crate) mod reliable; -#[cfg(test)] +// These suites drive the serf logic through `crate::StreamEndpoint`, which +// composes the plain-TCP reliable coordinator and is therefore `tcp`-gated. +#[cfg(all(test, feature = "tcp"))] mod serf_parity_tests; -#[cfg(test)] +#[cfg(all(test, feature = "tcp"))] mod tests; diff --git a/serf-proto/src/endpoint/reliable/mod.rs b/serf-proto/src/endpoint/reliable/mod.rs index 04d92cb3..7be05870 100644 --- a/serf-proto/src/endpoint/reliable/mod.rs +++ b/serf-proto/src/endpoint/reliable/mod.rs @@ -16,10 +16,11 @@ //! **Mutating operations** are trait methods; each has a single, unambiguous //! entry point whose contract is documented below. //! -//! **Coordinator-internal surfaces** (`handle_packet`, `handle_stream_event`, -//! `accept_stream`, `handle_timeout`, `poll_timeout`, `poll_transmit`) are -//! NOT part of this trait — the coordinators own the stream / transport -//! lifecycle, and serf should not reach into it. +//! **Coordinator-internal surfaces** (`handle_packet`, `handle_gossip`, +//! `handle_transport_data`, `accept_connection`, `handle_timeout`, +//! `poll_timeout`, `poll_action`, `poll_transport_transmit`, +//! `poll_memberlist_transmit`) are NOT part of this trait — the coordinators own +//! the stream / transport lifecycle, and serf should not reach into it. use bytes::Bytes; use memberlist_proto::{Endpoint, Instant, PushPullKind, Rng, StreamId}; diff --git a/serf-proto/src/endpoint/reliable/tests.rs b/serf-proto/src/endpoint/reliable/tests.rs index e01875d8..01028228 100644 --- a/serf-proto/src/endpoint/reliable/tests.rs +++ b/serf-proto/src/endpoint/reliable/tests.rs @@ -77,13 +77,12 @@ fn tcp_stream_endpoint_impl_compiles_and_wires_up() { // Plain-TCP label options: no cluster label, Passthrough inner transport. let cfg: LabelOptions<()> = LabelOptions::new_in(None, ()); - let sni_provider: Box Option + Send + Sync> = - Box::new(|_| Some("localhost".to_string())); - let peer_to_socket: Box core::net::SocketAddr + Send + Sync> = - Box::new(|a| *a); - - let mut coord: StreamEndpoint = - StreamEndpoint::new(inner, cfg, sni_provider, peer_to_socket); + let mut coord: StreamEndpoint = StreamEndpoint::new( + inner, + cfg, + Box::new(|_| Some("localhost".to_string())), + Box::new(|a| *a), + ); drive(&mut coord, addr); } diff --git a/serf-proto/src/endpoint/serf_parity_tests.rs b/serf-proto/src/endpoint/serf_parity_tests.rs index 01838352..7b1537d4 100644 --- a/serf-proto/src/endpoint/serf_parity_tests.rs +++ b/serf-proto/src/endpoint/serf_parity_tests.rs @@ -7,7 +7,9 @@ //! Each test is named after the invariant it checks. use bytes::Bytes; -use memberlist_proto::{EndpointOptions, Instant, SeedableRng, SmallRng}; +use memberlist_proto::{ + EndpointOptions, Instant, RawRecords, SeedableRng, SmallRng, streams::LabelOptions, +}; use crate::{ AnyMessage, LamportTime, StreamEndpoint, @@ -17,12 +19,18 @@ use crate::{ typed::{PushPullMessage, UserEvent, UserEvents}, }; -fn ep() -> StreamEndpoint { +fn ep() -> StreamEndpoint { let inner_opts = EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); let inner = memberlist_proto::Endpoint::new_at(inner_opts, Instant::ORIGIN, SmallRng::seed_from_u64(0)); - StreamEndpoint::new(inner, Options::new()) + let coord = memberlist_proto::streams::StreamEndpoint::<_, _, RawRecords>::new( + inner, + LabelOptions::new_in(Some(b"serf-test".to_vec()), ()), + Box::new(|_addr: &std::net::SocketAddr| None), + Box::new(|addr: &std::net::SocketAddr| *addr), + ); + StreamEndpoint::new(coord, Options::new()) } // ── base.rs handle_node_join invariants ─────────────────────────────────────── diff --git a/serf-proto/src/endpoint/tests.rs b/serf-proto/src/endpoint/tests.rs index 301571db..21376bb4 100644 --- a/serf-proto/src/endpoint/tests.rs +++ b/serf-proto/src/endpoint/tests.rs @@ -7,13 +7,34 @@ use crate::{ }; #[cfg(feature = "coordinates")] use bytes::Bytes; -use memberlist_proto::{EndpointOptions, SeedableRng, SmallRng}; +use memberlist_proto::{EndpointOptions, RawRecords, SeedableRng, SmallRng, streams::LabelOptions}; + +/// The plain-TCP record layer the unit-test coordinators run over. +type TestTransport = RawRecords; + +/// Wrap a raw membership [`memberlist_proto::Endpoint`] into the plain-TCP +/// reliable coordinator the serf `StreamEndpoint` composes with. +/// +/// All unit tests root at `A = SocketAddr`, so the peer-to-socket resolver is +/// the identity and the SNI provider is unused (the plain-TCP record layer +/// ignores it). A fixed cluster label keeps the handshake well-formed for the +/// loopback tests that complete a real exchange. +fn coord( + inner: memberlist_proto::Endpoint, +) -> memberlist_proto::streams::StreamEndpoint { + memberlist_proto::streams::StreamEndpoint::new( + inner, + LabelOptions::new_in(Some(b"serf-test".to_vec()), ()), + Box::new(|_addr: &std::net::SocketAddr| None), + Box::new(|addr: &std::net::SocketAddr| *addr), + ) +} /// Build a minimal serf `Endpoint` suitable for unit tests. /// /// Uses `u32` node ids and `SocketAddr` addresses with a deterministically /// seeded `SmallRng` so tests are reproducible. -fn ep() -> StreamEndpoint { +fn ep() -> StreamEndpoint { let inner_opts = EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); let inner = memberlist_proto::Endpoint::new_at( @@ -21,12 +42,12 @@ fn ep() -> StreamEndpoint { memberlist_proto::Instant::ORIGIN, SmallRng::seed_from_u64(0), ); - StreamEndpoint::new(inner, Options::new()) + StreamEndpoint::new(coord(inner), Options::new()) } /// Build a serf `Endpoint` with coordinates enabled (for coordinate-gated tests). #[cfg(feature = "coordinates")] -fn ep_with_coords() -> StreamEndpoint { +fn ep_with_coords() -> StreamEndpoint { let inner_opts = EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); let inner = memberlist_proto::Endpoint::new_at( @@ -35,7 +56,7 @@ fn ep_with_coords() -> StreamEndpoint { SmallRng::seed_from_u64(0), ); let opts = Options::new().with_disable_coordinates(false); - StreamEndpoint::new(inner, opts) + StreamEndpoint::new(coord(inner), opts) } #[test] @@ -69,7 +90,7 @@ fn poll_event_drains_when_inner_empty() { fn poll_timeout_is_none_on_idle_alive_endpoint() { // No serf deadlines armed yet; inner scheduler idle at ORIGIN-relative new. // Either None or Some — assert it does not panic. - let e = ep(); + let mut e = ep(); let _ = e.poll_timeout(); } @@ -86,10 +107,10 @@ fn handle_packet_with_garbage_bytes_is_a_noop() { } #[test] -fn poll_transmit_delegates_to_inner() { +fn poll_memberlist_transmit_delegates_to_coordinator() { let mut e = ep(); // No transmits queued at construction time; must not panic. - assert!(e.poll_transmit().is_none()); + assert!(e.poll_memberlist_transmit().is_none()); } // ── Task 1.4: member-status FSM + intent reconciliation + clock witnessing ──── @@ -520,7 +541,7 @@ fn poll_timeout_includes_leave_deadlines_when_armed() { // Helper: seed a failed member with an explicit address so we can assert what // addr is dialled by the reconnector. fn seed_failed( - e: &mut StreamEndpoint, + e: &mut StreamEndpoint, id: u32, addr: std::net::SocketAddr, leave_time: memberlist_proto::Instant, @@ -530,7 +551,7 @@ fn seed_failed( // Helper: seed the endpoint's one alive member (the local node) explicitly so // the probability computation has a stable num_alive value. -fn seed_alive(e: &mut StreamEndpoint, id: u32) { +fn seed_alive(e: &mut StreamEndpoint, id: u32) { e.test_seed_member(id, MemberStatus::Alive, LamportTime::new(0)); } @@ -1106,7 +1127,7 @@ fn push_pull_local_state_bytes_deterministic() { // produce byte-identical push-pull wire output after resync_local_state. // This verifies that HashMap iteration order in `members.states` does NOT // leak into the encoded PushPullMessage. - fn build_ep_asc() -> StreamEndpoint { + fn build_ep_asc() -> StreamEndpoint { let mut e = ep(); e.test_set_clocks(5, 10, 15); // Insert members in ascending id order: 1, 2, 3, 4, 5. @@ -1116,7 +1137,7 @@ fn push_pull_local_state_bytes_deterministic() { e.resync_local_state(); e } - fn build_ep_desc() -> StreamEndpoint { + fn build_ep_desc() -> StreamEndpoint { let mut e = ep(); e.test_set_clocks(5, 10, 15); // Insert members in descending id order: 5, 4, 3, 2, 1. @@ -1412,7 +1433,7 @@ fn invalid_tag_regex_does_not_advance_rng() { memberlist_proto::Instant::ORIGIN, SmallRng::seed_from_u64(0), ); - StreamEndpoint::new_with_rng(inner, Options::new(), SmallRng::seed_from_u64(seed)) + StreamEndpoint::new_with_rng(coord(inner), Options::new(), SmallRng::seed_from_u64(seed)) }; let mut ep_a = make_ep(42); @@ -1915,7 +1936,7 @@ fn relay_sieve_arm_decodes_relay_message_from_user_packet() { /// Build a serf `StreamEndpoint` with an explicit RNG seed for the /// serf-level RNG (the relay/reconnect draws). The inner Endpoint uses a fixed /// seed 0; the serf-level seed is the caller-supplied `serf_seed`. -fn ep_with_serf_seed(serf_seed: u64) -> StreamEndpoint { +fn ep_with_serf_seed(serf_seed: u64) -> StreamEndpoint { let inner_opts = memberlist_proto::EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); let inner = memberlist_proto::Endpoint::new_at( @@ -1923,7 +1944,11 @@ fn ep_with_serf_seed(serf_seed: u64) -> StreamEndpoint StreamEndpoint { +fn ep_tiny_event_buf() -> StreamEndpoint { let inner_opts = memberlist_proto::EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); let inner = memberlist_proto::Endpoint::new_at( @@ -2455,7 +2480,7 @@ fn ep_tiny_event_buf() -> StreamEndpoint { memberlist_proto::Instant::ORIGIN, SmallRng::seed_from_u64(0), ); - StreamEndpoint::new(inner, Options::new().with_event_buffer_size(4)) + StreamEndpoint::new(coord(inner), Options::new().with_event_buffer_size(4)) } #[test] @@ -2532,7 +2557,7 @@ fn ack_query_produces_immediate_ack_directed_send() { } // Bug 3: Oversized query responses silently consumed. -fn ep_small_resp_limit() -> StreamEndpoint { +fn ep_small_resp_limit() -> StreamEndpoint { let inner_opts = memberlist_proto::EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); let inner = memberlist_proto::Endpoint::new_at( @@ -2540,7 +2565,10 @@ fn ep_small_resp_limit() -> StreamEndpoint { memberlist_proto::Instant::ORIGIN, SmallRng::seed_from_u64(0), ); - StreamEndpoint::new(inner, Options::new().with_query_response_size_limit(10)) + StreamEndpoint::new( + coord(inner), + Options::new().with_query_response_size_limit(10), + ) } #[test] @@ -2752,7 +2780,8 @@ fn respond_send_failure_returns_err_and_leaves_responded_false() { SmallRng::seed_from_u64(0), ); let opts = Options::new().with_query_response_size_limit(50_000); - let mut e = StreamEndpoint::new(inner, opts); + let mut e: StreamEndpoint = + StreamEndpoint::new(coord(inner), opts); let qid = QueryId { ltime: LamportTime::new(1), @@ -3185,7 +3214,8 @@ fn resync_keeps_dirty_when_inner_snapshot_rejects() { memberlist_proto::Instant::ORIGIN, SmallRng::seed_from_u64(0), ); - let mut e = StreamEndpoint::new(inner, Options::new()); + let mut e: StreamEndpoint = + StreamEndpoint::new(coord(inner), Options::new()); // Force dirty and call resync. e.test_set_clocks(1, 2, 3); @@ -3971,7 +4001,8 @@ fn zero_event_buffer_size_does_not_panic_on_first_event() { ); // event_buffer_size = 0 should be clamped to 1 internally. let opts = crate::options::Options::new().with_event_buffer_size(0); - let mut e = StreamEndpoint::new(inner, opts); + let mut e: StreamEndpoint = + StreamEndpoint::new(coord(inner), opts); // Must NOT panic. e.user_event("test", bytes::Bytes::new(), false) @@ -3993,7 +4024,8 @@ fn zero_event_buffer_size_does_not_panic_on_merge_remote_state() { SmallRng::seed_from_u64(0), ); let opts = crate::options::Options::new().with_event_buffer_size(0); - let mut e = StreamEndpoint::new(inner, opts); + let mut e: StreamEndpoint = + StreamEndpoint::new(coord(inner), opts); let pp = PushPullMessage:: { ltime: LamportTime::new(1), @@ -4216,7 +4248,8 @@ fn inbound_query_oversized_is_dropped_before_state_mutation() { memberlist_proto::SmallRng::seed_from_u64(0), ); let opts = crate::options::Options::new().with_query_size_limit(64); - let mut e: StreamEndpoint = StreamEndpoint::new(inner, opts); + let mut e: StreamEndpoint = + StreamEndpoint::new(coord(inner), opts); // Encode a QueryMessage whose payload pushes the wire encoding over 64 bytes. let oversized_payload = bytes::Bytes::from(vec![0u8; 100]); @@ -4650,7 +4683,8 @@ fn zero_reap_interval_advances_deadline() { SmallRng::seed_from_u64(0), ); let opts = Options::new().with_reap_interval(std::time::Duration::ZERO); - let mut e = StreamEndpoint::new(inner, opts); + let mut e: StreamEndpoint = + StreamEndpoint::new(coord(inner), opts); // Advance time past the first reap deadline so handle_timeout fires it. let now = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(1); @@ -4680,7 +4714,8 @@ fn zero_reconnect_interval_advances_deadline() { SmallRng::seed_from_u64(0), ); let opts = Options::new().with_reconnect_interval(std::time::Duration::ZERO); - let mut e = StreamEndpoint::new(inner, opts); + let mut e: StreamEndpoint = + StreamEndpoint::new(coord(inner), opts); let now = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(1); e.handle_timeout(now); @@ -4708,7 +4743,8 @@ fn zero_queue_check_interval_advances_deadline() { SmallRng::seed_from_u64(0), ); let opts = Options::new().with_queue_check_interval(std::time::Duration::ZERO); - let mut e = StreamEndpoint::new(inner, opts); + let mut e: StreamEndpoint = + StreamEndpoint::new(coord(inner), opts); let now = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(1); e.handle_timeout(now); @@ -4738,7 +4774,8 @@ fn user_event_total_packet_with_junk_exceeding_size_limit_is_dropped() { ); // Set a tight size limit: 32 bytes. let opts = crate::options::Options::new().with_max_user_event_size(32); - let mut e = StreamEndpoint::new(inner, opts); + let mut e: StreamEndpoint = + StreamEndpoint::new(coord(inner), opts); // A small valid UserEvent that fits within 32 bytes on its own. let valid = AnyMessage::::UserEvent(UserEventMessage { @@ -4784,7 +4821,8 @@ fn pre_decode_fence_drops_oversized_valid_query_frame() { ); // Small limit (64 bytes) so that a query with a 100-byte payload exceeds it. let opts = crate::options::Options::new().with_query_size_limit(64); - let mut e: StreamEndpoint = StreamEndpoint::new(inner, opts); + let mut e: StreamEndpoint = + StreamEndpoint::new(coord(inner), opts); // Construct a syntactically valid Query whose encoded frame exceeds 64 bytes. let q = QueryMessage:: { @@ -4849,7 +4887,8 @@ fn pre_decode_fence_drops_oversized_valid_user_event_frame() { ); // Tight limit: 32 bytes. let opts = crate::options::Options::new().with_max_user_event_size(32); - let mut e: StreamEndpoint = StreamEndpoint::new(inner, opts); + let mut e: StreamEndpoint = + StreamEndpoint::new(coord(inner), opts); // Construct a syntactically valid UserEvent whose encoded frame exceeds 32 bytes. let frame = AnyMessage::::UserEvent(UserEventMessage { @@ -5364,7 +5403,7 @@ mod tag_filter_regex { /// Seed the local node (id=1) with the given tags and return an endpoint /// ready for tag-filter query tests. - fn ep_with_tags(tags: Tags) -> StreamEndpoint { + fn ep_with_tags(tags: Tags) -> StreamEndpoint { let mut e = ep(); e.test_seed_member_with_tags(1u32, tags, MemberStatus::Alive, LamportTime::new(0)); e diff --git a/serf-proto/src/lib.rs b/serf-proto/src/lib.rs index b1b44f47..f3627518 100644 --- a/serf-proto/src/lib.rs +++ b/serf-proto/src/lib.rs @@ -70,8 +70,13 @@ pub mod event; pub mod members; pub mod options; pub mod snapshot; + +#[cfg(feature = "tcp")] +#[cfg_attr(docsrs, doc(cfg(feature = "tcp")))] pub mod stream_endpoint; +#[cfg(feature = "tcp")] +#[cfg_attr(docsrs, doc(cfg(feature = "tcp")))] #[doc(inline)] pub use stream_endpoint::StreamEndpoint; diff --git a/serf-proto/src/stream_endpoint/mod.rs b/serf-proto/src/stream_endpoint/mod.rs index 59a883e6..9918d5fa 100644 --- a/serf-proto/src/stream_endpoint/mod.rs +++ b/serf-proto/src/stream_endpoint/mod.rs @@ -1,21 +1,34 @@ -//! The serf `StreamEndpoint` super-machine — serf logic composed with a -//! memberlist reliable coordinator over a stream transport. +//! The serf `StreamEndpoint` super-machine — serf logic composed with the +//! memberlist reliable stream coordinator. //! -//! `StreamEndpoint` owns the serf-logic [`Endpoint`] core and a memberlist -//! coordinator (`transport`) as **two disjoint fields**, and drives the core -//! over `&mut transport` through the [`Reliable`](crate::endpoint::reliable) -//! seam. It exposes the coordinator's transport-facing driver surface -//! (`handle_packet`, `handle_timeout`, `poll_transmit`, `poll_timeout`, …) plus -//! serf's own commands and events (`poll_event`, `join`, `leave`, `user_event`, +//! `StreamEndpoint` owns the serf-logic [`Endpoint`] core and the memberlist +//! reliable coordinator ([`memberlist_proto::streams::StreamEndpoint`]) as +//! **two disjoint fields**, and drives the core over `&mut transport` through +//! the [`Reliable`](crate::endpoint::reliable) seam. It exposes the +//! coordinator's transport-facing driver surface (`handle_packet`, +//! `handle_gossip`, `accept_connection`, `handle_transport_data`, +//! `poll_action`, `poll_transport_transmit`, `handle_timeout`, …) plus serf's +//! own commands and events (`poll_event`, `join`, `leave`, `user_event`, //! `query`, key ops, …), forwarding each to the right field. //! //! The composed `handle_timeout` is where the load-bearing tick ordering lives: -//! the reliable coordinator is drained, the coordinator's SWIM timer fires, the -//! resulting inner events are sieved into serf, then serf's own deadlines fire — -//! all in one place, so no per-runtime driver has to re-establish the order. +//! the coordinator's SWIM timer fires between serf's pre-tick snapshot resync +//! and serf's post-tick drain + deadline pass, so no per-runtime driver has to +//! re-establish the order. +//! +//! The coordinator owns the reliable stream lifecycle internally: it dials +//! peers, runs the label / record-layer handshake, and exchanges the membership +//! state blob in both directions, surfacing only its transport I/O intents +//! (`poll_action` → `Connect`, `poll_transport_transmit`) to the driver. Serf +//! observes the merged outcome as [`memberlist_proto::RemoteStateReceived`] on +//! the core's drain over `poll_inner_event`, and never reaches into the stream +//! lifecycle itself. use bytes::Bytes; -use memberlist_proto::{CheapClone, Data, Id, Instant, Rng, SeedableRng, Transmit, parse_message}; +use memberlist_proto::{ + CheapClone, Data, Id, Instant, Rng, SeedableRng, SmallRng, Transmit, parse_message, + streams::{ExchangeId, StreamAction, StreamEndpoint as Coordinator, StreamTransport}, +}; use smol_str::SmolStr; use crate::{ @@ -41,36 +54,44 @@ use crate::event::KeyResponseArgs; /// The serf `StreamEndpoint` super-machine. /// -/// Composes the serf-logic [`Endpoint`] `core` with a memberlist reliable -/// coordinator (`transport`). The driver pumps **one** machine: it feeds the -/// transport ingress, ticks `handle_timeout`, and drains the serf and transport -/// poll surfaces. -pub struct StreamEndpoint +/// Composes the serf-logic [`Endpoint`] `core` with the memberlist reliable +/// stream coordinator ([`memberlist_proto::streams::StreamEndpoint`]) as the +/// `transport`. The driver pumps **one** machine: it feeds the transport +/// ingress, ticks `handle_timeout`, and drains the serf and transport poll +/// surfaces. +/// +/// The serf-logic core carries its **own** injected RNG `R`, distinct from the +/// coordinator's RNG `G`; the two are seeded independently so serf's gossip +/// choices and memberlist's probe choices do not share a stream. `RT` is the +/// record-layer ([`StreamTransport`]) — `RawRecords` for plain-TCP, +/// `Labeled` for TLS. +#[cfg(feature = "tcp")] +#[cfg_attr(docsrs, doc(cfg(feature = "tcp")))] +pub struct StreamEndpoint where I: Eq + core::hash::Hash, + RT: StreamTransport, { /// The serf-logic core, holding all serf state and no transport reference. core: Endpoint, /// The memberlist reliable coordinator serf drives through the `Reliable` /// seam. Holds the single membership `Endpoint`. - transport: memberlist_proto::Endpoint, + transport: Coordinator, } -impl StreamEndpoint +#[cfg(feature = "tcp")] +impl StreamEndpoint where I: Clone + Eq + core::hash::Hash, + RT: StreamTransport, R: SeedableRng, { - /// Construct a `StreamEndpoint` from a memberlist coordinator `transport`, - /// serf `opts`, and serf's own injected `rng`. + /// Construct a `StreamEndpoint` from a memberlist reliable coordinator + /// `transport`, serf `opts`, and serf's own injected `rng`. /// - /// `rng` is **separate** from the coordinator's `R`; seed it from the + /// `rng` is **separate** from the coordinator's RNG `G`; seed it from the /// driver's own entropy source. - pub fn new_with_rng( - transport: memberlist_proto::Endpoint, - opts: Options, - rng: R, - ) -> Self { + pub fn new_with_rng(transport: Coordinator, opts: Options, rng: R) -> Self { Self { core: Endpoint::new_with_rng(opts, rng), transport, @@ -81,7 +102,7 @@ where /// /// Suitable for tests and deterministic environments. Production drivers /// should use `new_with_rng` and seed from a cryptographically-secure source. - pub fn new(transport: memberlist_proto::Endpoint, opts: Options) -> Self { + pub fn new(transport: Coordinator, opts: Options) -> Self { Self::new_with_rng(transport, opts, R::seed_from_u64(0)) } } @@ -92,42 +113,81 @@ where // deliberately excludes transport ingress / timer / poll operations — then drive // the serf-logic sieve over the coordinator. -impl StreamEndpoint +#[cfg(feature = "tcp")] +impl StreamEndpoint where I: Id + Clone, A: CheapClone + Data + PartialEq + Clone + 'static, + RT: StreamTransport, + G: Rng, R: Rng + SeedableRng, { - /// Deliver an inbound datagram or packet to the machine. + /// Feed one decoded unreliable memberlist `Message` into the + /// coordinator, then sieve the resulting inner events into serf. /// - /// Decodes the memberlist wire `Message` and hands it to the - /// coordinator's inner endpoint, then sieves the resulting inner events into - /// serf. Malformed or unrecognised bytes are silently dropped — the machine - /// must not panic on bad input from the network. + /// The composed unit's unreliable ingress is `handle_gossip` → + /// `poll_memberlist_ingress` → (codec decode) → `handle_packet`. This method + /// is the decode-then-feed convenience: it parses the memberlist wire frame + /// and hands the typed message to the coordinator. Malformed or unrecognised + /// bytes are silently dropped — the machine must not panic on bad input from + /// the network. pub fn handle_packet(&mut self, from: A, data: Bytes, now: Instant) { - // Malformed frame or unrecognised tag: drop silently. The inner endpoint - // logs its own decode errors; serf takes no serf-level action here. + // Malformed frame or unrecognised tag: drop silently. The coordinator logs + // its own decode errors; serf takes no serf-level action here. if let Ok(msg) = parse_message::(data) { self.transport.handle_packet(from, msg, now); } self.core.drain_after_ingress(&mut self.transport, now); } - /// Deliver a stream event to the coordinator, then sieve resulting inner - /// events into serf. - pub fn handle_stream_event( - &mut self, - ev: memberlist_proto::EndpointEvent, - now: Instant, - ) -> Option> { - let cmd = self.transport.handle_stream_event(ev, now); + /// Buffer one inbound gossip datagram on the coordinator's ingress queue. + /// + /// The codec-owning driver drains the raw frames via + /// [`Self::poll_memberlist_ingress`], decodes each, and feeds the typed + /// messages back through [`Self::handle_packet`] before ticking + /// [`Self::handle_timeout`]. + pub fn handle_gossip(&mut self, from: A, datagram: &[u8], now: Instant) { + self.transport.handle_gossip(from, datagram, now); + } + + /// Admit an inbound reliable stream connection from `from`. + /// + /// Returns the [`ExchangeId`] the coordinator allocated for the accepted + /// exchange, or `None` if the connection was rejected (e.g. the inbound + /// stream cap is exceeded or the node is leaving). The driver feeds the + /// connection's bytes back through [`Self::handle_transport_data`] under this + /// id. + pub fn accept_connection(&mut self, from: A, now: Instant) -> Option { + self.transport.accept_connection(from, now) + } + + /// Deliver inbound transport bytes for the reliable exchange `id`, then sieve + /// the resulting inner events into serf. + /// + /// `eof` signals the peer half-closed the connection (a transport read of + /// zero). A completed push-pull exchange surfaces as + /// [`memberlist_proto::RemoteStateReceived`] on the core's drain, which serf + /// folds into its membership via `merge_remote_state`. + pub fn handle_transport_data(&mut self, id: ExchangeId, bytes: &[u8], eof: bool, now: Instant) { + self.transport.handle_transport_data(id, bytes, eof, now); + self.core.drain_after_ingress(&mut self.transport, now); + } + + /// Report that the driver's outbound dial for reliable exchange `id` failed, + /// then sieve the resulting inner events into serf. + /// + /// The coordinator retires the exchange (no bridge is opened) and may emit a + /// terminal `ExchangeCompleted`; serf takes no action on it. + pub fn handle_dial_failed(&mut self, id: ExchangeId, now: Instant) { + self.transport.handle_dial_failed(id, now); self.core.drain_after_ingress(&mut self.transport, now); - cmd } - /// Accept an inbound stream connection on the coordinator. - pub fn accept_stream(&mut self, from: A, now: Instant) -> Option> { - self.transport.accept_stream(from, now) + /// Report a transport-level error on reliable exchange `id`, then sieve the + /// resulting inner events into serf. + pub fn handle_transport_error(&mut self, id: ExchangeId, now: Instant) { + self.transport.handle_transport_error(id, now); + self.core.drain_after_ingress(&mut self.transport, now); } /// Advance time and fire any expired serf or coordinator deadlines. @@ -137,6 +197,11 @@ where /// drain + deadline pass, so the load-bearing /// `resync → inner timer → drain → serf deadlines` sequence is established /// here, once, rather than in each runtime driver. + /// + /// The coordinator's own `handle_timeout` services its dial queue and bridge + /// schedule internally; any `DialRequested` the inner endpoint emits is + /// sieved into the coordinator's private dial queue (surfaced to the driver as + /// a `poll_action` → `Connect`), so it never reaches serf's drain. pub fn handle_timeout(&mut self, now: Instant) { // Pre-inner-timer: latch `now` and resync the push-pull snapshot if dirty, // so the coordinator ships current serf state on this tick's anti-entropy. @@ -148,16 +213,38 @@ where self.core.after_inner_timeout(&mut self.transport, now); } - /// Drain one outgoing transmit from the coordinator. - pub fn poll_transmit(&mut self) -> Option> { - self.transport.poll_transmit() + /// Drain one outbound transport directive ([`StreamAction`]) from the + /// coordinator — a `Connect` to dial a peer, or a `Shutdown` / `Close` / + /// `Abort` to tear an exchange's connection down. + pub fn poll_action(&mut self) -> Option { + self.transport.poll_action() + } + + /// Drain one outbound per-exchange transport chunk `(exchange, peer, bytes)` + /// from the coordinator; the driver writes `bytes` on `exchange`'s connection. + pub fn poll_transport_transmit(&mut self) -> Option<(ExchangeId, core::net::SocketAddr, Bytes)> { + self.transport.poll_transport_transmit() + } + + /// Drain one raw inbound gossip datagram `(from, bytes)` the coordinator + /// buffered from [`Self::handle_gossip`]; the codec layer decodes it and feeds + /// the typed messages back through [`Self::handle_packet`]. + pub fn poll_memberlist_ingress(&mut self) -> Option<(A, Bytes)> { + self.transport.poll_memberlist_ingress() + } + + /// Drain one outgoing unreliable (gossip-plane) memberlist [`Transmit`] from + /// the coordinator; the driver encodes and sends it on the UDP socket. + pub fn poll_memberlist_transmit(&mut self) -> Option> { + self.transport.poll_memberlist_transmit() } /// The earliest deadline requiring a `handle_timeout` call. /// /// The minimum of the coordinator's own deadline and serf's periodic - /// deadlines. - pub fn poll_timeout(&self) -> Option { + /// deadlines. Takes `&mut self` because the coordinator folds in + /// immediate-due dial wakes that it tracks mutably. + pub fn poll_timeout(&mut self) -> Option { let inner = self.transport.poll_timeout(); let serf = self.core.serf_poll_timeout(); match (inner, serf) { @@ -174,7 +261,7 @@ where /// leave-intent broadcast has been flushed without waiting the full /// `broadcast_timeout`. pub fn user_broadcast_queue_len(&self) -> usize { - self.transport.user_broadcast_queue_len() + self.transport.endpoint_ref().user_broadcast_queue_len() } // ── serf-logic + serf-command forwarders ──────────────────────────────────── @@ -1011,4 +1098,15 @@ where pub(crate) fn core_mut(&mut self) -> &mut Endpoint { &mut self.core } + + /// Mutable access to the memberlist reliable coordinator, for tests that + /// drive a two-endpoint loopback (relay one side's transport transmits into + /// the other's `handle_transport_data`). + #[cfg(test)] + pub(crate) fn transport_mut(&mut self) -> &mut Coordinator { + &mut self.transport + } } + +#[cfg(all(test, feature = "tcp"))] +mod tests; diff --git a/serf-proto/src/stream_endpoint/tests.rs b/serf-proto/src/stream_endpoint/tests.rs new file mode 100644 index 00000000..5e949c6e --- /dev/null +++ b/serf-proto/src/stream_endpoint/tests.rs @@ -0,0 +1,261 @@ +//! `StreamEndpoint` super-machine tests over the real memberlist reliable +//! coordinator (plain-TCP `RawRecords` record layer). +//! +//! These cover the composition seam the per-runtime drivers depend on: +//! construction, the serf-command surface forwarding to the core, the +//! coordinator-driven dial (`poll_action` → `Connect`), and a two-endpoint +//! loopback that drives serf's `merge_remote_state` through the coordinator's +//! real push-pull path (relay one side's `poll_transport_transmit` into the +//! other's `handle_transport_data`) until the serf membership converges. +//! +//! Packet / FSM-level coverage lives in `endpoint::tests`, which drives the same +//! `StreamEndpoint` through `handle_packet` / `handle_timeout`. + +use bytes::Bytes; +use core::net::SocketAddr; + +use memberlist_proto::{ + EndpointOptions, Instant, PushPullKind, RawRecords, SeedableRng, SmallRng, + streams::{LabelOptions, StreamAction}, +}; + +use crate::{StreamEndpoint, members::MemberStatus, options::Options}; + +/// Loopback cluster label shared by both sides so the record-layer handshake +/// settles. +const CLUSTER: &[u8] = b"serf-loopback"; + +fn sa(port: u16) -> SocketAddr { + format!("127.0.0.1:{port}").parse().unwrap() +} + +/// Build a serf `StreamEndpoint` rooted at `id` / +/// `port`, seeded deterministically. +fn ep(id: u32, port: u16) -> StreamEndpoint { + let inner_opts = EndpointOptions::new(id, sa(port)) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); + let inner = + memberlist_proto::Endpoint::new_at(inner_opts, Instant::ORIGIN, SmallRng::seed_from_u64(0)); + let coord = memberlist_proto::streams::StreamEndpoint::<_, _, RawRecords>::new( + inner, + LabelOptions::new_in(Some(CLUSTER.to_vec()), ()), + Box::new(|_addr: &SocketAddr| None), + Box::new(|addr: &SocketAddr| *addr), + ); + StreamEndpoint::new(coord, Options::new()) +} + +#[test] +fn constructs_alive_with_zero_clocks() { + let e = ep(1, 7946); + assert!(e.state().is_alive()); + assert_eq!(e.member_time(), 0); + assert_eq!(e.event_time(), 0); + assert_eq!(e.query_time(), 0); +} + +#[test] +fn user_event_marks_local_state_dirty() { + let mut e = ep(1, 7946); + e.test_clear_dirty(); + e.user_event("deploy", Bytes::from_static(b"v2"), false) + .expect("user_event on an alive endpoint"); + assert!( + e.test_is_dirty(), + "a user_event must mark the local-state snapshot dirty" + ); +} + +#[test] +fn handle_packet_with_garbage_bytes_is_a_noop() { + let mut e = ep(1, 7946); + e.handle_packet(sa(9999), Bytes::from_static(b"\xff\xff"), Instant::ORIGIN); + assert!( + e.poll_event().is_none(), + "an undecodable frame yields no serf event" + ); +} + +/// A reconnect dial against a failed member surfaces a coordinator +/// `StreamAction::Connect` (the coordinator IS the driver and dials itself), +/// and the chosen address is captured at the `start_push_pull` call site. +#[test] +fn reconnect_dial_surfaces_a_connect_action() { + let mut e = ep(1, 7946); + // Seed the local node alive and one failed peer so the reconnect gate fires + // (prob = num_failed / num_alive = 1/1 = 1.0). + e.test_seed_member(1, MemberStatus::Alive, 1.into()); + e.test_seed_failed_member(2, sa(7000), Instant::ORIGIN); + + e.test_fire_reconnect(Instant::ORIGIN); + + assert_eq!( + e.test_last_dial_addr(), + Some(sa(7000)), + "the reconnect dial targets the failed peer's address" + ); + let action = e.poll_action(); + assert!( + matches!(action, Some(StreamAction::Connect(_))), + "the coordinator surfaces a Connect for the reconnect dial, got {action:?}" + ); +} + +/// Decode a push-pull body fed through the coordinator's merge path and assert +/// serf folds the remote clock state in. Exercises serf's `merge_remote_state` +/// (the serf-side of state exchange) over the real coordinator, without +/// re-implementing the stream wire protocol. +#[test] +fn merge_remote_state_folds_remote_clocks() { + use crate::typed::PushPullMessage; + + let mut e = ep(1, 7946); + e.test_clear_dirty(); + + // A remote push-pull body advancing the member / event / query clocks. + let pp: PushPullMessage = PushPullMessage::new( + 42.into(), + Vec::new(), + Vec::new(), + 7.into(), + Vec::new(), + 9.into(), + ); + let encoded = crate::AnyMessage::::PushPull(pp) + .encode() + .expect("encode push-pull body"); + + e.test_merge_remote_state(encoded, false); + + assert_eq!( + e.member_time(), + 42, + "member clock witnessed the remote ltime" + ); + assert_eq!(e.event_time(), 7, "event clock witnessed the remote ltime"); + assert_eq!(e.query_time(), 9, "query clock witnessed the remote ltime"); +} + +/// Two-endpoint loopback: a dialer initiates a push-pull, the acceptor admits +/// the inbound connection, bytes shuttle both directions through the +/// coordinators' real transport surface, and the acceptor folds the dialer's +/// serf push-pull body — asserting the serf member clock converges across a +/// real exchange. +/// +/// Serf's push-pull body carries the three Lamport clocks + member status +/// ltimes (not the full memberlist roster, which SWIM disseminates), so the +/// observable serf-layer outcome is the acceptor witnessing the dialer's higher +/// member clock (`merge_remote_state` witnesses each clock at `remote - 1`). +#[test] +fn loopback_push_pull_converges_member_clock() { + let now = Instant::ORIGIN; + let mut dialer = ep(1, 7946); + let mut acceptor = ep(2, 7000); + + // Advance the dialer's serf member clock so its push-pull body carries a + // higher ltime than the acceptor's (which starts at 0). The acceptor must + // witness this clock through the merge. + dialer.test_set_clocks(50, 0, 0); + dialer.resync_local_state(); + assert_eq!( + acceptor.member_time(), + 0, + "the acceptor starts at member clock 0" + ); + + // Dialer: start a push-pull and pull the Connect off the action queue. + let dial_exchange = { + dialer + .transport_mut() + .start_push_pull(sa(7000), PushPullKind::Join, now); + match dialer.poll_action() { + Some(StreamAction::Connect(c)) => c.id(), + other => panic!("dialer must surface a Connect, got {other:?}"), + } + }; + while dialer.poll_action().is_some() {} + + // Acceptor: admit the inbound connection, taking its exchange handle. + let accept_exchange = acceptor + .accept_connection(sa(7946), now) + .expect("acceptor admits the inbound connection"); + + // Shuttle bytes + half-close signals both directions until the acceptor has + // witnessed the dialer's clock or both coordinators go idle. A real driver + // maps each coordinator action to a transport operation: bytes from + // `poll_transport_transmit` are written to the peer's connection, and a + // `Shutdown` / `Close` action is a `shutdown(write)` the peer reads as an EOF. + let mut converged = false; + for _ in 0..256 { + let mut moved = false; + + // dialer -> acceptor: bytes, then any half-close as an EOF. + let mut to_acceptor = Vec::new(); + while let Some((id, _peer, bytes)) = dialer.poll_transport_transmit() { + if id == dial_exchange { + to_acceptor.extend_from_slice(&bytes); + } + } + if !to_acceptor.is_empty() { + acceptor.handle_transport_data(accept_exchange, &to_acceptor, false, now); + moved = true; + } + while let Some(action) = dialer.poll_action() { + if let StreamAction::Shutdown(r) | StreamAction::Close(r) | StreamAction::Abort(r) = action { + if r.id() == dial_exchange { + acceptor.handle_transport_data(accept_exchange, &[], true, now); + moved = true; + } + } + } + + // acceptor -> dialer: bytes, then any half-close as an EOF. + let mut to_dialer = Vec::new(); + while let Some((id, _peer, bytes)) = acceptor.poll_transport_transmit() { + if id == accept_exchange { + to_dialer.extend_from_slice(&bytes); + } + } + if !to_dialer.is_empty() { + dialer.handle_transport_data(dial_exchange, &to_dialer, false, now); + moved = true; + } + while let Some(action) = acceptor.poll_action() { + if let StreamAction::Shutdown(r) | StreamAction::Close(r) | StreamAction::Abort(r) = action { + if r.id() == accept_exchange { + dialer.handle_transport_data(dial_exchange, &[], true, now); + moved = true; + } + } + } + + // Tick both so the bridges advance their send/recv halves and reap on + // completion. The merge is sieved into serf inside `handle_transport_data`. + dialer.handle_timeout(now); + acceptor.handle_timeout(now); + while dialer.poll_event().is_some() {} + while acceptor.poll_event().is_some() {} + + // The acceptor witnessed the dialer's serf member clock through the merge. + if acceptor.member_time() >= 49 { + converged = true; + break; + } + if !moved { + break; + } + } + + assert!( + converged, + "the loopback push-pull exchange converged: the acceptor witnessed the dialer's \ + serf member clock (started at 0, advanced to {})", + acceptor.member_time() + ); + // The acceptor also learned the dialer (node 1) as a member through the + // exchange's membership push. + assert!( + acceptor.test_member_status(1).is_some(), + "the acceptor learned the dialer (node 1) as a member over the exchange" + ); +} From 6edaf31d3ace88cfbbf8ba5648f6f2c8b7835345 Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 26 Jun 2026 21:37:45 +1200 Subject: [PATCH 014/128] feat(serf-proto): QuicEndpoint super-machine over the memberlist QuicEndpoint --- serf-proto/Cargo.toml | 9 + serf-proto/src/lib.rs | 9 + serf-proto/src/quic_endpoint/mod.rs | 988 ++++++++++++++++++++++++++ serf-proto/src/quic_endpoint/tests.rs | 323 +++++++++ 4 files changed, 1329 insertions(+) create mode 100644 serf-proto/src/quic_endpoint/mod.rs create mode 100644 serf-proto/src/quic_endpoint/tests.rs diff --git a/serf-proto/Cargo.toml b/serf-proto/Cargo.toml index 4e1912fe..3a335caa 100644 --- a/serf-proto/Cargo.toml +++ b/serf-proto/Cargo.toml @@ -59,3 +59,12 @@ regex = { version = "1", optional = true } buffa-build = "0.7" [dev-dependencies] +# The QUIC super-machine loopback test builds a quinn-proto config bundle with a +# self-signed cert and an accept-any verifier (the same shape memberlist-proto's +# own QUIC coordinator tests use). Only the `quic`-gated test module references +# these; they are unconditional dev-deps so the test crate always resolves. +quinn-proto = { version = "0.11", default-features = false, features = ["rustls-ring"] } +rustls = { version = "0.23", features = ["ring"] } +rustls-pki-types = "1" +rcgen = "0.14" +ring = "0.17" diff --git a/serf-proto/src/lib.rs b/serf-proto/src/lib.rs index f3627518..e70c66fa 100644 --- a/serf-proto/src/lib.rs +++ b/serf-proto/src/lib.rs @@ -80,6 +80,15 @@ pub mod stream_endpoint; #[doc(inline)] pub use stream_endpoint::StreamEndpoint; +#[cfg(feature = "quic")] +#[cfg_attr(docsrs, doc(cfg(feature = "quic")))] +pub mod quic_endpoint; + +#[cfg(feature = "quic")] +#[cfg_attr(docsrs, doc(cfg(feature = "quic")))] +#[doc(inline)] +pub use quic_endpoint::QuicEndpoint; + #[cfg(feature = "coordinates")] #[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] pub use snapshot::CoordinateRecord; diff --git a/serf-proto/src/quic_endpoint/mod.rs b/serf-proto/src/quic_endpoint/mod.rs new file mode 100644 index 00000000..3cd73c18 --- /dev/null +++ b/serf-proto/src/quic_endpoint/mod.rs @@ -0,0 +1,988 @@ +//! The serf `QuicEndpoint` super-machine — serf logic composed with the +//! memberlist QUIC coordinator. +//! +//! `QuicEndpoint` is the QUIC sibling of [`StreamEndpoint`](crate::StreamEndpoint): +//! it owns the same serf-logic [`Endpoint`] core but pairs it with the memberlist +//! QUIC coordinator ([`memberlist_proto::QuicEndpoint`]) instead of the reliable +//! stream coordinator. The two live as **disjoint fields**, and the core is +//! driven over `&mut transport` through the [`Reliable`](crate::endpoint::reliable) +//! seam — identical to the stream super-machine; only the transport surface +//! differs. +//! +//! The QUIC coordinator pins `A = SocketAddr` (quinn dials and accepts wire +//! addresses), so this super-machine pins serf's address type to `SocketAddr` +//! as well. Its driver surface is the QUIC coordinator's: one UDP ingress +//! ([`handle_udp`](Self::handle_udp)), one combined egress +//! ([`poll_transmit`](Self::poll_transmit)), and the memberlist gossip-plane +//! ingress / egress accessors — plus serf's own commands and events. +//! +//! The composed [`handle_timeout`](Self::handle_timeout) is where the +//! load-bearing tick ordering lives: the coordinator's SWIM timer fires between +//! serf's pre-tick snapshot resync and serf's post-tick drain + deadline pass, +//! so no per-runtime driver has to re-establish the order. +//! +//! The coordinator owns the QUIC reliable lifecycle internally: it dials peers, +//! drives the quinn handshake, opens per-peer bidi streams, and exchanges the +//! membership state blob in both directions. Crucially it also sieves its own +//! `DialRequested` events into a private dial queue and dials itself (it *is* +//! the driver), so serf never routes a dial — it observes only the merged +//! outcome as [`memberlist_proto::RemoteStateReceived`] on the core's drain over +//! `poll_inner_event`. + +use bytes::Bytes; +use core::net::SocketAddr; +use memberlist_proto::{ + Data, Id, Instant, QuicEndpoint as Coordinator, Rng, SeedableRng, SmallRng, Transmit, + parse_message, +}; +use smol_str::SmolStr; + +use crate::{ + endpoint::{Endpoint, Error, QueryId, QueryParams}, + event::{Event, QueryEvent}, + members::SerfState, + options::Options, +}; + +#[cfg(all(test, feature = "tag-regex"))] +use crate::typed::Tags; +#[cfg(test)] +use crate::{ + LamportTime, + members::{IntentKind, MemberStatus}, + typed::{QueryMessage, QueryResponseMessage, RelayMessage, UserEventMessage}, +}; +#[cfg(test)] +use memberlist_proto::Node; + +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +use crate::event::KeyResponseArgs; + +/// The serf `QuicEndpoint` super-machine. +/// +/// Composes the serf-logic [`Endpoint`] `core` with the memberlist QUIC +/// coordinator ([`memberlist_proto::QuicEndpoint`]) as the `transport`. The +/// driver pumps **one** machine: it feeds the UDP ingress, ticks +/// `handle_timeout`, and drains the serf and transport poll surfaces. +/// +/// The serf-logic core carries its **own** injected RNG `R`, distinct from the +/// coordinator's RNG `G`; the two are seeded independently so serf's gossip +/// choices and memberlist's probe choices do not share a stream. The QUIC +/// coordinator pins `A = SocketAddr`, so this super-machine pins serf's address +/// type to `SocketAddr` too. +#[cfg(feature = "quic")] +#[cfg_attr(docsrs, doc(cfg(feature = "quic")))] +pub struct QuicEndpoint +where + I: Eq + core::hash::Hash, +{ + /// The serf-logic core, holding all serf state and no transport reference. + core: Endpoint, + /// The memberlist QUIC coordinator serf drives through the `Reliable` seam. + /// Holds the single membership `Endpoint` and the quinn endpoint. + transport: Coordinator, +} + +#[cfg(feature = "quic")] +impl QuicEndpoint +where + I: Clone + Eq + core::hash::Hash, + R: SeedableRng, +{ + /// Construct a `QuicEndpoint` from a memberlist QUIC coordinator `transport`, + /// serf `opts`, and serf's own injected `rng`. + /// + /// `rng` is **separate** from the coordinator's RNG `G`; seed it from the + /// driver's own entropy source. + pub fn new_with_rng(transport: Coordinator, opts: Options, rng: R) -> Self { + Self { + core: Endpoint::new_with_rng(opts, rng), + transport, + } + } + + /// Convenience constructor that seeds serf's `R` with a zero seed. + /// + /// Suitable for tests and deterministic environments. Production drivers + /// should use `new_with_rng` and seed from a cryptographically-secure source. + pub fn new(transport: Coordinator, opts: Options) -> Self { + Self::new_with_rng(transport, opts, R::seed_from_u64(0)) + } +} + +// ── transport-level driver surface + serf commands ───────────────────────────── +// +// The driver-surface methods reach the coordinator (`transport`) directly — the +// `Reliable` seam deliberately excludes transport ingress / timer / poll +// operations — then drive the serf-logic sieve over the coordinator. The serf +// commands forward to the matching `Endpoint` method, threading `&mut transport` +// through the ones that reach the coordinator (the `Reliable` methods). + +#[cfg(feature = "quic")] +impl QuicEndpoint +where + I: Id + Clone, + G: Rng, + R: Rng + SeedableRng, +{ + /// Feed one inbound UDP datagram from `from` into the coordinator. + /// + /// The single conceptual socket carries both QUIC packets and plain-UDP + /// gossip; the coordinator's first-byte demux routes each datagram. A QUIC + /// packet is fully processed in-band (handshake, stream data, datagram); a + /// gossip frame is buffered for the codec-owning driver to drain via + /// [`Self::poll_memberlist_ingress`], decode, and feed back through + /// [`Self::handle_packet`]. After the datagram is handled the resulting inner + /// events are sieved into serf. + pub fn handle_udp(&mut self, from: SocketAddr, datagram: &[u8], now: Instant) { + self.transport.handle_udp(from, datagram, now); + self.core.drain_after_ingress(&mut self.transport, now); + } + + /// Feed one decoded unreliable memberlist `Message` into the + /// coordinator, then sieve the resulting inner events into serf. + /// + /// The composed unit's unreliable ingress is `handle_udp` → + /// `poll_memberlist_ingress` → (codec decode) → `handle_packet`. This method + /// is the decode-then-feed convenience: it parses the memberlist wire frame + /// and hands the typed message to the coordinator. Malformed or unrecognised + /// bytes are silently dropped — the machine must not panic on bad input from + /// the network. + pub fn handle_packet(&mut self, from: SocketAddr, data: Bytes, now: Instant) { + // Malformed frame or unrecognised tag: drop silently. The coordinator logs + // its own decode errors; serf takes no serf-level action here. + if let Ok(msg) = parse_message::(data) { + self.transport.handle_packet(from, msg, now); + } + self.core.drain_after_ingress(&mut self.transport, now); + } + + /// Advance time and fire any expired serf or coordinator deadlines. + /// + /// The composed tick order (the structural win): the coordinator's SWIM timer + /// fires between the serf pre-tick snapshot resync and the serf post-tick + /// drain + deadline pass, so the load-bearing + /// `resync → inner timer → drain → serf deadlines` sequence is established + /// here, once, rather than in each runtime driver. + /// + /// The coordinator's own `handle_timeout` services its dial queue, quinn + /// connections, and bridge schedule internally; any `DialRequested` the inner + /// endpoint emits is sieved into the coordinator's private dial queue (the + /// coordinator dials itself), so it never reaches serf's drain. + pub fn handle_timeout(&mut self, now: Instant) { + // Pre-inner-timer: latch `now` and resync the push-pull snapshot if dirty, + // so the coordinator ships current serf state on this tick's anti-entropy. + self.core.before_inner_timeout(&mut self.transport, now); + // Inner timer: the coordinator's SWIM gossip / probe / push-pull scheduler + // plus its quinn connection + bridge servicing. + self.transport.handle_timeout(now); + // Post-inner-timer: sieve the inner events this tick produced, then fire + // serf's own deadlines (reap / reconnect / queue-check / query-close / …). + self.core.after_inner_timeout(&mut self.transport, now); + } + + /// Drain one outbound UDP datagram `(to, bytes)` from the coordinator (a quinn + /// packet or an encoded memberlist gossip frame); the driver writes `bytes` to + /// `to` on the UDP socket. + pub fn poll_transmit(&mut self) -> Option<(SocketAddr, Bytes)> { + self.transport.poll_transmit() + } + + /// Drain one raw inbound gossip datagram `(from, bytes)` the coordinator + /// buffered from [`Self::handle_udp`]; the codec layer decodes it and feeds + /// the typed messages back through [`Self::handle_packet`]. + pub fn poll_memberlist_ingress(&mut self) -> Option<(SocketAddr, Bytes)> { + self.transport.poll_memberlist_ingress() + } + + /// Drain one outgoing unreliable (gossip-plane) memberlist [`Transmit`] from + /// the coordinator; the driver encodes and sends it on the UDP socket. + pub fn poll_memberlist_transmit(&mut self) -> Option> { + self.transport.poll_memberlist_transmit() + } + + /// The earliest deadline requiring a `handle_timeout` call. + /// + /// The minimum of the coordinator's own deadline and serf's periodic + /// deadlines. Takes `&mut self` because the coordinator folds in + /// immediate-due dial wakes that it tracks mutably. + pub fn poll_timeout(&mut self) -> Option { + let inner = self.transport.poll_timeout(); + let serf = self.core.serf_poll_timeout(); + match (inner, serf) { + (Some(a), Some(b)) => Some(a.min(b)), + (Some(a), None) => Some(a), + (None, Some(b)) => Some(b), + (None, None) => None, + } + } + + /// Number of unsent items in the coordinator's user broadcast queue. + /// + /// The driver may poll this during a graceful leave to detect when the + /// leave-intent broadcast has been flushed without waiting the full + /// `broadcast_timeout`. + pub fn user_broadcast_queue_len(&self) -> usize { + self.transport.endpoint_ref().user_broadcast_queue_len() + } + + // ── serf-logic + serf-command forwarders ──────────────────────────────────── + + /// Announce the local node's join intent to the cluster. + /// + /// Forwards to [`Endpoint::join`]. + pub fn join(&mut self) -> Result<(), Error> + where + I: Clone, + { + self.core.join(&mut self.transport) + } + + /// Issue a cluster-wide `use_key` query to promote `key` to primary. + /// + /// Forwards to [`Endpoint::use_key`]. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn use_key( + &mut self, + key: memberlist_proto::SecretKey, + now: Instant, + ) -> Result + where + I: Clone + Data, + { + self.core.use_key(&mut self.transport, key, now) + } + + /// Issue a cluster-wide `remove_key` query to remove `key` from all nodes. + /// + /// Forwards to [`Endpoint::remove_key`]. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn remove_key( + &mut self, + key: memberlist_proto::SecretKey, + now: Instant, + ) -> Result + where + I: Clone + Data, + { + self.core.remove_key(&mut self.transport, key, now) + } + + /// Issue a cluster-wide `list_keys` query to enumerate installed keys. + /// + /// Forwards to [`Endpoint::list_keys`]. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn list_keys(&mut self, now: Instant) -> Result + where + I: Clone + Data, + { + self.core.list_keys(&mut self.transport, now) + } + + /// Install a new encryption key into the local keyring. + /// + /// Forwards to [`Endpoint::install_key`]. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn install_key( + &mut self, + key: memberlist_proto::SecretKey, + now: Instant, + ) -> Result + where + I: Clone + Data, + { + self.core.install_key(&mut self.transport, key, now) + } + + /// Forwards to [`Endpoint::state`]. + pub const fn state(&self) -> SerfState { + self.core.state() + } + + /// Forwards to [`Endpoint::member_time`]. + pub const fn member_time(&self) -> u64 { + self.core.member_time() + } + + /// Forwards to [`Endpoint::event_time`]. + pub const fn event_time(&self) -> u64 { + self.core.event_time() + } + + /// Forwards to [`Endpoint::query_time`]. + pub const fn query_time(&self) -> u64 { + self.core.query_time() + } + + /// Forwards to [`Endpoint::num_members`]. + pub fn num_members(&self) -> usize { + self.core.num_members() + } + + /// Forwards to [`Endpoint::poll_event`]. + pub fn poll_event(&mut self) -> Option> { + self.core.poll_event(&mut self.transport) + } + + /// Forwards to [`Endpoint::resync_local_state`]. + pub fn resync_local_state(&mut self) + where + I: Clone + Data, + { + self.core.resync_local_state(&mut self.transport) + } + + /// Forwards to [`Endpoint::set_event_join_ignore`]. + pub fn set_event_join_ignore(&mut self, v: bool) { + self.core.set_event_join_ignore(v) + } + + /// Forwards to [`Endpoint::leave`]. + pub fn leave(&mut self, now: Instant) -> Result<(), Error> + where + I: Clone, + { + self.core.leave(&mut self.transport, now) + } + + /// Forwards to [`Endpoint::force_leave`]. + pub fn force_leave(&mut self, id: I, prune: bool, now: Instant) -> Result<(), Error> + where + I: Clone, + { + self.core.force_leave(&mut self.transport, id, prune, now) + } + + /// Forwards to [`Endpoint::user_event`]. + pub fn user_event( + &mut self, + name: impl Into, + payload: bytes::Bytes, + coalesce: bool, + ) -> Result<(), Error> { + self + .core + .user_event(&mut self.transport, name, payload, coalesce) + } + + /// Forwards to [`Endpoint::query`]. + pub fn query( + &mut self, + name: impl Into, + payload: Bytes, + params: QueryParams, + now: Instant, + ) -> Result + where + I: Clone + Data, + { + self + .core + .query(&mut self.transport, name, payload, params, now) + } + + /// Forwards to [`Endpoint::respond`]. + pub fn respond( + &mut self, + token: &QueryEvent, + payload: Bytes, + now: Instant, + ) -> Result<(), Error> + where + I: Clone + Data, + { + self.core.respond(&mut self.transport, token, payload, now) + } + + /// Forwards to [`Endpoint::respond_key`]. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn respond_key( + &mut self, + req: &crate::event::KeyRequest, + resp: KeyResponseArgs, + now: Instant, + ) -> Result<(), Error> + where + I: Clone + Data, + { + self.core.respond_key(&mut self.transport, req, resp, now) + } + + /// Forwards to [`Endpoint::leave_broadcast_deadline`]. + pub const fn leave_broadcast_deadline(&self) -> Option { + self.core.leave_broadcast_deadline() + } + + /// Forwards to [`Endpoint::leave_complete_deadline`]. + pub const fn leave_complete_deadline(&self) -> Option { + self.core.leave_complete_deadline() + } + + /// Forwards to [`Endpoint::load_snapshot`]. + pub fn load_snapshot( + &mut self, + replay: crate::snapshot::ReplayResult, + now: Instant, + ) { + self.core.load_snapshot(&mut self.transport, replay, now) + } + + /// Forwards to [`Endpoint::get_coordinate`]. + #[cfg(feature = "coordinates")] + #[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] + pub fn get_coordinate(&self) -> Option { + self.core.get_coordinate() + } + + /// Forwards to [`Endpoint::cached_coordinate`]. + #[cfg(feature = "coordinates")] + #[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] + pub fn cached_coordinate(&self, node: &I) -> Option { + self.core.cached_coordinate(node) + } +} + +// ── test-only forwarders ────────────────────────────────────────────────────── +// +// These mirror `StreamEndpoint`'s test surface so the migrated endpoint suite +// can drive serf logic through either super-machine. The QUIC unit tests in +// `tests.rs` exercise a subset; the block-level `allow(dead_code)` admits the +// rest until the full suite drives `QuicEndpoint` directly. + +#[cfg(all(test, feature = "quic"))] +#[allow(dead_code)] +impl QuicEndpoint +where + I: Id + Clone, + G: Rng, + R: Rng + SeedableRng, +{ + /// Forwards to [`Endpoint::handle_node_join_intent`]. + pub(crate) fn handle_node_join_intent(&mut self, ltime: LamportTime, id: &I, now: Instant) -> bool + where + I: Clone, + { + self.core.handle_node_join_intent(ltime, id, now) + } + + /// Forwards to [`Endpoint::handle_node_leave_intent`]. + pub(crate) fn handle_node_leave_intent( + &mut self, + ltime: LamportTime, + id: &I, + prune: bool, + now: Instant, + ) -> bool + where + I: Clone, + { + self + .core + .handle_node_leave_intent(&mut self.transport, ltime, id, prune, now) + } + + /// Forwards to [`Endpoint::handle_user_event`]. + pub(crate) fn handle_user_event(&mut self, msg: UserEventMessage) -> bool { + self.core.handle_user_event(msg) + } + + /// Forwards to [`Endpoint::test_member_status`]. + pub(crate) fn test_member_status(&self, id: I) -> Option + where + I: Clone, + { + self.core.test_member_status(id) + } + + /// Forwards to [`Endpoint::test_member_status_time`]. + pub(crate) fn test_member_status_time(&self, id: I) -> Option + where + I: Clone, + { + self.core.test_member_status_time(id) + } + + /// Forwards to [`Endpoint::test_seed_member`]. + pub(crate) fn test_seed_member(&mut self, id: I, status: MemberStatus, status_time: LamportTime) + where + I: Clone, + { + self.core.test_seed_member(id, status, status_time) + } + + /// Forwards to [`Endpoint::test_seed_member_with_tags`]. + #[cfg(feature = "tag-regex")] + pub(crate) fn test_seed_member_with_tags( + &mut self, + id: I, + tags: Tags, + status: MemberStatus, + status_time: LamportTime, + ) where + I: Clone, + { + self + .core + .test_seed_member_with_tags(id, tags, status, status_time) + } + + /// Forwards to [`Endpoint::test_seed_failed_member_by_status`]. + pub(crate) fn test_seed_failed_member_by_status( + &mut self, + id: I, + status_time: LamportTime, + now: Instant, + ) where + I: Clone, + { + self + .core + .test_seed_failed_member_by_status(id, status_time, now) + } + + /// Forwards to [`Endpoint::test_seed_left_member_by_status`]. + pub(crate) fn test_seed_left_member_by_status( + &mut self, + id: I, + status_time: LamportTime, + now: Instant, + ) where + I: Clone, + { + self + .core + .test_seed_left_member_by_status(id, status_time, now) + } + + /// Forwards to [`Endpoint::test_handle_join_intent`]. + pub(crate) fn test_handle_join_intent(&mut self, id: I, ltime: LamportTime, now: Instant) -> bool + where + I: Clone, + { + self.core.test_handle_join_intent(id, ltime, now) + } + + /// Forwards to [`Endpoint::test_handle_leave_intent`]. + pub(crate) fn test_handle_leave_intent(&mut self, id: I, ltime: LamportTime, now: Instant) -> bool + where + I: Clone, + { + self + .core + .test_handle_leave_intent(&mut self.transport, id, ltime, now) + } + + /// Forwards to [`Endpoint::test_inner_node_joined`]. + pub(crate) fn test_inner_node_joined(&mut self, id: I, now: Instant) + where + I: Clone, + { + self.core.test_inner_node_joined(id, now) + } + + /// Forwards to [`Endpoint::test_inner_node_left`]. + pub(crate) fn test_inner_node_left(&mut self, id: I, now: Instant) + where + I: Clone, + { + self.core.test_inner_node_left(id, now) + } + + /// Forwards to [`Endpoint::test_inner_node_updated`]. + pub(crate) fn test_inner_node_updated(&mut self, id: I, now: Instant) + where + I: Clone, + { + self.core.test_inner_node_updated(id, now) + } + + /// Forwards to [`Endpoint::test_in_failed_members`]. + pub(crate) fn test_in_failed_members(&self, id: I) -> bool + where + I: PartialEq, + { + self.core.test_in_failed_members(id) + } + + /// Forwards to [`Endpoint::test_in_left_members`]. + pub(crate) fn test_in_left_members(&self, id: I) -> bool + where + I: PartialEq, + { + self.core.test_in_left_members(id) + } + + /// Forwards to [`Endpoint::test_inner_left_cluster`]. + pub(crate) fn test_inner_left_cluster(&mut self) { + self.core.test_inner_left_cluster(&mut self.transport) + } + + /// Forwards to [`Endpoint::test_seed_failed_member`]. + pub(crate) fn test_seed_failed_member(&mut self, id: I, addr: SocketAddr, now: Instant) + where + I: Clone, + { + self.core.test_seed_failed_member(id, addr, now) + } + + /// Forwards to [`Endpoint::test_fire_reconnect`]. + pub(crate) fn test_fire_reconnect(&mut self, now: Instant) { + self.core.test_fire_reconnect(&mut self.transport, now) + } + + /// Forwards to [`Endpoint::test_fire_reap`]. + pub(crate) fn test_fire_reap(&mut self, now: Instant) + where + I: Clone, + { + self.core.test_fire_reap(now) + } + + /// Forwards to [`Endpoint::test_last_dial_addr`]. + pub(crate) fn test_last_dial_addr(&self) -> Option { + self.core.test_last_dial_addr() + } + + /// Forwards to [`Endpoint::test_handle_user_event`]. + pub(crate) fn test_handle_user_event(&mut self, msg: UserEventMessage) -> bool { + self.core.test_handle_user_event(msg) + } + + /// Forwards to [`Endpoint::test_set_event_min_time`]. + pub(crate) fn test_set_event_min_time(&mut self, t: u64) { + self.core.test_set_event_min_time(t) + } + + /// Forwards to [`Endpoint::test_set_event_clock`]. + pub(crate) fn test_set_event_clock(&mut self, t: u64) { + self.core.test_set_event_clock(t) + } + + /// Forwards to [`Endpoint::test_event_slot_len`]. + pub(crate) fn test_event_slot_len(&self, ltime: u64) -> usize { + self.core.test_event_slot_len(ltime) + } + + /// Forwards to [`Endpoint::test_inject_user_packet`]. + pub(crate) fn test_inject_user_packet(&mut self, from: SocketAddr, data: Bytes, now: Instant) + where + I: Clone + Data, + { + self + .core + .test_inject_user_packet(&mut self.transport, from, data, now) + } + + /// Forwards to [`Endpoint::test_set_clocks`]. + pub(crate) fn test_set_clocks(&mut self, member: u64, event: u64, query: u64) { + self.core.test_set_clocks(member, event, query) + } + + /// Forwards to [`Endpoint::test_seed_left_member`]. + pub(crate) fn test_seed_left_member(&mut self, id: I, status_time: LamportTime) + where + I: Clone, + { + self.core.test_seed_left_member(id, status_time) + } + + /// Forwards to [`Endpoint::test_inner_local_state_snapshot`]. + pub(crate) fn test_inner_local_state_snapshot(&self) -> Bytes { + self.core.test_inner_local_state_snapshot(&self.transport) + } + + /// Forwards to [`Endpoint::test_decode_pushpull`]. + pub(crate) fn test_decode_pushpull(&self, bytes: &Bytes) -> crate::typed::PushPullMessage + where + I: Clone + Data, + { + self.core.test_decode_pushpull(bytes) + } + + /// Forwards to [`Endpoint::test_clear_dirty`]. + pub(crate) fn test_clear_dirty(&mut self) { + self.core.test_clear_dirty() + } + + /// Forwards to [`Endpoint::test_is_dirty`]. + pub(crate) fn test_is_dirty(&self) -> bool { + self.core.test_is_dirty() + } + + /// Forwards to [`Endpoint::test_set_event_join_ignore`]. + pub(crate) fn test_set_event_join_ignore(&mut self, v: bool) { + self.core.test_set_event_join_ignore(v) + } + + /// Forwards to [`Endpoint::test_event_min_time`]. + pub(crate) fn test_event_min_time(&self) -> u64 { + self.core.test_event_min_time() + } + + /// Forwards to [`Endpoint::test_merge_remote_state`]. + pub(crate) fn test_merge_remote_state(&mut self, user_data: Bytes, is_join: bool) + where + I: Clone + Data, + { + self + .core + .test_merge_remote_state(&mut self.transport, user_data, is_join) + } + + /// Forwards to [`Endpoint::test_intent_ltime`]. + pub(crate) fn test_intent_ltime(&self, id: I, kind: IntentKind) -> Option + where + I: Clone, + { + self.core.test_intent_ltime(id, kind) + } + + /// Forwards to [`Endpoint::test_handle_query`]. + pub(crate) fn test_handle_query(&mut self, msg: QueryMessage) -> bool + where + I: Clone + Data, + { + self.core.test_handle_query(&mut self.transport, msg) + } + + /// Forwards to [`Endpoint::test_last_query_id`]. + pub(crate) fn test_last_query_id(&self) -> Option { + self.core.test_last_query_id() + } + + /// Forwards to [`Endpoint::test_pending_query_count`]. + pub(crate) fn test_pending_query_count(&self) -> usize { + self.core.test_pending_query_count() + } + + /// Forwards to [`Endpoint::test_received_queries_len`]. + pub(crate) fn test_received_queries_len(&self) -> usize { + self.core.test_received_queries_len() + } + + /// Forwards to [`Endpoint::test_query_min_time`]. + pub(crate) fn test_query_min_time(&self) -> u64 { + self.core.test_query_min_time() + } + + /// Forwards to [`Endpoint::test_register_received_query`]. + pub(crate) fn test_register_received_query( + &mut self, + query_id: QueryId, + querier: SocketAddr, + deadline: Instant, + ) -> QueryEvent + where + I: Default + Clone, + { + self + .core + .test_register_received_query(query_id, querier, deadline) + } + + /// Forwards to [`Endpoint::test_handle_query_response`]. + pub(crate) fn test_handle_query_response(&mut self, msg: QueryResponseMessage) + where + I: Clone, + { + self + .core + .test_handle_query_response(&mut self.transport, msg) + } + + /// Forwards to [`Endpoint::test_is_responded`]. + pub(crate) fn test_is_responded(&self, query_id: QueryId) -> bool { + self.core.test_is_responded(query_id) + } + + /// Forwards to [`Endpoint::test_recent_intents_len`]. + pub(crate) fn test_recent_intents_len(&self) -> usize { + self.core.test_recent_intents_len() + } + + /// Forwards to [`Endpoint::test_pending_query_conflict_matching`]. + pub(crate) fn test_pending_query_conflict_matching(&self, query_id: QueryId) -> Option { + self.core.test_pending_query_conflict_matching(query_id) + } + + /// Forwards to [`Endpoint::test_relay_response`]. + pub(crate) fn test_relay_response( + &mut self, + querier: Node, + frame: Bytes, + relay_factor: u8, + ) where + I: Clone + Data, + { + self + .core + .test_relay_response(&mut self.transport, querier, frame, relay_factor) + } + + /// Forwards to [`Endpoint::test_handle_relay`]. + pub(crate) fn test_handle_relay(&mut self, relay: RelayMessage) + where + I: Clone, + { + self.core.test_handle_relay(&mut self.transport, relay) + } + + /// Forwards to [`Endpoint::test_last_directed_send`]. + pub(crate) fn test_last_directed_send(&self) -> Option<(SocketAddr, Bytes)> { + self.core.test_last_directed_send() + } + + /// Forwards to [`Endpoint::test_register_conflict_query`]. + pub(crate) fn test_register_conflict_query(&mut self, deadline: Instant) -> QueryId + where + I: Clone, + { + self.core.test_register_conflict_query(deadline) + } + + /// Forwards to [`Endpoint::test_register_key_query`]. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + pub(crate) fn test_register_key_query(&mut self, deadline: Instant) -> QueryId + where + I: Clone, + { + self.core.test_register_key_query(deadline) + } + + /// Forwards to [`Endpoint::test_fold_conflict_response`]. + pub(crate) fn test_fold_conflict_response( + &mut self, + query_id: QueryId, + responder_id: I, + agrees: bool, + ) where + I: Clone, + { + self + .core + .test_fold_conflict_response(query_id, responder_id, agrees) + } + + /// Forwards to [`Endpoint::test_pending_query_response_count`]. + pub(crate) fn test_pending_query_response_count(&self, query_id: QueryId) -> usize { + self.core.test_pending_query_response_count(query_id) + } + + /// Forwards to [`Endpoint::test_query_slot_len`]. + pub(crate) fn test_query_slot_len(&self, ltime: u64) -> usize { + self.core.test_query_slot_len(ltime) + } + + /// Forwards to [`Endpoint::test_fire_due_query_closes`]. + pub(crate) fn test_fire_due_query_closes(&mut self, now: Instant) + where + I: Clone, + { + self.core.test_fire_due_query_closes(now) + } + + /// Forwards to [`Endpoint::test_inject_inner_joined`]. + pub(crate) fn test_inject_inner_joined(&mut self, id: I, now: Instant) + where + I: Clone, + { + self.core.test_inject_inner_joined(id, now) + } + + /// Forwards to [`Endpoint::test_enqueue_intent_broadcast`]. + pub(crate) fn test_enqueue_intent_broadcast(&mut self, bytes: Bytes) { + self + .core + .test_enqueue_intent_broadcast(&mut self.transport, bytes) + } + + /// Forwards to [`Endpoint::test_enqueue_query_broadcast`]. + pub(crate) fn test_enqueue_query_broadcast(&mut self, bytes: Bytes) { + self + .core + .test_enqueue_query_broadcast(&mut self.transport, bytes) + } + + /// Forwards to [`Endpoint::test_rejoin_dials`]. + pub(crate) fn test_rejoin_dials(&self) -> Vec { + self.core.test_rejoin_dials() + } + + /// Forwards to [`Endpoint::test_ping_completed`]. + #[cfg(feature = "coordinates")] + pub(crate) fn test_ping_completed( + &mut self, + node_id: I, + rtt: std::time::Duration, + payload: Bytes, + ) { + self + .core + .test_ping_completed(&mut self.transport, node_id, rtt, payload) + } + + /// Forwards to [`Endpoint::test_seed_member_at`]. + pub(crate) fn test_seed_member_at( + &mut self, + id: I, + addr: SocketAddr, + status: MemberStatus, + status_time: LamportTime, + ) where + I: Clone, + { + self.core.test_seed_member_at(id, addr, status, status_time) + } + + /// Forwards to [`Endpoint::test_relay_all_directed_sends`]. + pub(crate) fn test_relay_all_directed_sends(&self) -> &[(SocketAddr, Bytes)] { + self.core.test_relay_all_directed_sends() + } + + /// Forwards to [`Endpoint::test_last_pending_query_num_nodes`]. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + pub(crate) fn test_last_pending_query_num_nodes(&self) -> Option { + self.core.test_last_pending_query_num_nodes() + } + + /// Forwards to [`Endpoint::test_peek_received_query_deadlines`]. + pub(crate) fn test_peek_received_query_deadlines(&self) -> Vec { + self.core.test_peek_received_query_deadlines() + } + + /// Mutable access to the serf-logic core, for tests that manipulate its + /// private state directly. + pub(crate) fn core_mut(&mut self) -> &mut Endpoint { + &mut self.core + } + + /// Mutable access to the memberlist QUIC coordinator, for tests that drive a + /// two-endpoint loopback (relay one side's `poll_transmit` UDP datagrams into + /// the other's `handle_udp`). + pub(crate) fn transport_mut(&mut self) -> &mut Coordinator { + &mut self.transport + } +} + +#[cfg(all(test, feature = "quic"))] +mod tests; diff --git a/serf-proto/src/quic_endpoint/tests.rs b/serf-proto/src/quic_endpoint/tests.rs new file mode 100644 index 00000000..007c583d --- /dev/null +++ b/serf-proto/src/quic_endpoint/tests.rs @@ -0,0 +1,323 @@ +//! `QuicEndpoint` super-machine tests over the real memberlist QUIC coordinator. +//! +//! These cover the composition seam the per-runtime QUIC driver depends on: +//! construction, the serf-command surface forwarding to the core, and a +//! two-endpoint loopback that drives serf's `merge_remote_state` through the +//! coordinator's real QUIC push-pull path (relay one side's `poll_transmit` UDP +//! datagrams into the other's `handle_udp`) until the serf membership converges. +//! +//! The QUIC coordinator owns the quinn handshake + reliable bidi lifecycle +//! internally and dials itself, so the loopback ferries the single combined UDP +//! egress in both directions (the quinn handshake needs both paths relayed) +//! rather than per-exchange stream chunks. +//! +//! Packet / FSM-level coverage lives in `endpoint::tests`, which drives the same +//! super-machine surface through `handle_packet` / `handle_timeout`. + +use bytes::Bytes; +use core::{net::SocketAddr, time::Duration}; +use std::sync::Arc; + +use memberlist_proto::{ + EndpointOptions, Instant, PushPullKind, QuicOptions, SeedableRng, SmallRng, UnreliableTransport, +}; +use rustls::{ + client::danger::{HandshakeSignatureValid, ServerCertVerified}, + version::TLS13, +}; +use rustls_pki_types::{CertificateDer, PrivateKeyDer}; + +use crate::{QuicEndpoint, members::MemberStatus, options::Options}; + +fn sa(port: u16) -> SocketAddr { + format!("127.0.0.1:{port}").parse().unwrap() +} + +/// A self-signed cert + key for `localhost`, for the test TLS bundle. +fn self_signed() -> (Vec>, PrivateKeyDer<'static>) { + let ck = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let cert = CertificateDer::from(ck.cert.der().to_vec()); + let key = PrivateKeyDer::Pkcs8(ck.signing_key.serialize_der().into()); + (vec![cert], key) +} + +fn test_endpoint_config(reset_key: &[u8]) -> quinn_proto::EndpointConfig { + let hmac = ring::hmac::Key::new(ring::hmac::HMAC_SHA256, reset_key); + quinn_proto::EndpointConfig::new(Arc::new(hmac)) +} + +fn test_server() -> quinn_proto::ServerConfig { + let (chain, key) = self_signed(); + let provider = Arc::new(rustls::crypto::ring::default_provider()); + let rustls_server = rustls::ServerConfig::builder_with_provider(provider) + .with_protocol_versions(&[&TLS13]) + .unwrap() + .with_no_client_auth() + .with_single_cert(chain, key) + .unwrap(); + let qsc = + quinn_proto::crypto::rustls::QuicServerConfig::try_from(Arc::new(rustls_server)).unwrap(); + quinn_proto::ServerConfig::with_crypto(Arc::new(qsc)) +} + +/// Accept-any server-cert verifier — test only. +#[derive(Debug)] +struct AnyServer; + +impl rustls::client::danger::ServerCertVerifier for AnyServer { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer, + _intermediates: &[CertificateDer], + _server_name: &rustls_pki_types::ServerName, + _ocsp_response: &[u8], + _now: rustls_pki_types::UnixTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + _message: &[u8], + _cert: &CertificateDer, + _dss: &rustls::DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + + fn verify_tls13_signature( + &self, + _message: &[u8], + _cert: &CertificateDer, + _dss: &rustls::DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + + fn supported_verify_schemes(&self) -> Vec { + rustls::crypto::ring::default_provider() + .signature_verification_algorithms + .supported_schemes() + } +} + +fn test_client() -> quinn_proto::ClientConfig { + let provider = Arc::new(rustls::crypto::ring::default_provider()); + let cfg = rustls::ClientConfig::builder_with_provider(provider) + .with_protocol_versions(&[&TLS13]) + .unwrap() + .dangerous() + .with_custom_certificate_verifier(Arc::new(AnyServer)) + .with_no_client_auth(); + let qcc = quinn_proto::crypto::rustls::QuicClientConfig::try_from(Arc::new(cfg)).unwrap(); + quinn_proto::ClientConfig::new(Arc::new(qcc)) +} + +/// A QUIC config bundle with a 20s idle timeout (well past a single-instant +/// localhost handshake) and datagram-mode unreliable transport. +fn test_quic_options() -> QuicOptions { + let mut transport = quinn_proto::TransportConfig::default(); + transport.max_idle_timeout(Some( + quinn_proto::IdleTimeout::try_from(Duration::from_secs(20)).unwrap(), + )); + QuicOptions::new( + test_endpoint_config(&[0x5au8; 32]), + test_server(), + test_client(), + transport, + "localhost", + UnreliableTransport::Datagram, + ) +} + +/// Build a serf `QuicEndpoint` rooted at `id` / `port`, seeded +/// deterministically. The quinn rng seed is derived from the port so the two +/// loopback endpoints draw distinct connection IDs. +fn ep(id: u32, port: u16) -> QuicEndpoint { + let inner_opts = EndpointOptions::new(id, sa(port)) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); + let inner = + memberlist_proto::Endpoint::new_at(inner_opts, Instant::ORIGIN, SmallRng::seed_from_u64(0)); + let mut seed = [0u8; 32]; + seed[..2].copy_from_slice(&port.to_le_bytes()); + let coord = memberlist_proto::QuicEndpoint::::with_quinn_rng_seed( + inner, + test_quic_options(), + Some(seed), + ); + QuicEndpoint::new(coord, Options::new()) +} + +#[test] +fn constructs_alive_with_zero_clocks() { + let e = ep(1, 7946); + assert!(e.state().is_alive()); + assert_eq!(e.member_time(), 0); + assert_eq!(e.event_time(), 0); + assert_eq!(e.query_time(), 0); +} + +#[test] +fn user_event_marks_local_state_dirty() { + let mut e = ep(1, 7946); + e.test_clear_dirty(); + e.user_event("deploy", Bytes::from_static(b"v2"), false) + .expect("user_event on an alive endpoint"); + assert!( + e.test_is_dirty(), + "a user_event must mark the local-state snapshot dirty" + ); +} + +#[test] +fn handle_packet_with_garbage_bytes_is_a_noop() { + let mut e = ep(1, 7946); + e.handle_packet(sa(9999), Bytes::from_static(b"\xff\xff"), Instant::ORIGIN); + assert!( + e.poll_event().is_none(), + "an undecodable frame yields no serf event" + ); +} + +/// A reconnect dial against a failed member targets the failed peer's address. +/// On QUIC the coordinator IS the driver and dials itself (sieving its own +/// `DialRequested` into a private queue), so the observable serf-side fact is +/// the chosen address captured at the `start_push_pull` call site. +#[test] +fn reconnect_dial_targets_the_failed_peer() { + let mut e = ep(1, 7946); + // Seed the local node alive and one failed peer so the reconnect gate fires + // (prob = num_failed / num_alive = 1/1 = 1.0). + e.test_seed_member(1, MemberStatus::Alive, 1.into()); + e.test_seed_failed_member(2, sa(7000), Instant::ORIGIN); + + e.test_fire_reconnect(Instant::ORIGIN); + + assert_eq!( + e.test_last_dial_addr(), + Some(sa(7000)), + "the reconnect dial targets the failed peer's address" + ); +} + +/// Decode a push-pull body fed through the coordinator's merge path and assert +/// serf folds the remote clock state in. Exercises serf's `merge_remote_state` +/// (the serf-side of state exchange) over the real QUIC coordinator, without +/// driving the QUIC wire protocol. +#[test] +fn merge_remote_state_folds_remote_clocks() { + use crate::typed::PushPullMessage; + + let mut e = ep(1, 7946); + e.test_clear_dirty(); + + // A remote push-pull body advancing the member / event / query clocks. + let pp: PushPullMessage = PushPullMessage::new( + 42.into(), + Vec::new(), + Vec::new(), + 7.into(), + Vec::new(), + 9.into(), + ); + let encoded = crate::AnyMessage::::PushPull(pp) + .encode() + .expect("encode push-pull body"); + + e.test_merge_remote_state(encoded, false); + + assert_eq!( + e.member_time(), + 42, + "member clock witnessed the remote ltime" + ); + assert_eq!(e.event_time(), 7, "event clock witnessed the remote ltime"); + assert_eq!(e.query_time(), 9, "query clock witnessed the remote ltime"); +} + +/// Two-endpoint QUIC loopback: a dialer initiates a push-pull, the QUIC +/// coordinators run the quinn handshake and the reliable bidi exchange in both +/// directions (their single combined UDP egress relayed into the peer's +/// `handle_udp`), and the acceptor folds the dialer's serf push-pull body — +/// asserting the serf member clock converges across a real QUIC exchange. +/// +/// Serf's push-pull body carries the three Lamport clocks + member status +/// ltimes (not the full memberlist roster, which SWIM disseminates), so the +/// observable serf-layer outcome is the acceptor witnessing the dialer's higher +/// member clock (`merge_remote_state` witnesses each clock at `remote - 1`). +#[test] +fn loopback_push_pull_converges_member_clock() { + let now = Instant::now(); + let dialer_addr = sa(7946); + let acceptor_addr = sa(7000); + let mut dialer = ep(1, 7946); + let mut acceptor = ep(2, 7000); + + // Advance the dialer's serf member clock so its push-pull body carries a + // higher ltime than the acceptor's (which starts at 0). The acceptor must + // witness this clock through the merge. + dialer.test_set_clocks(50, 0, 0); + dialer.resync_local_state(); + assert_eq!( + acceptor.member_time(), + 0, + "the acceptor starts at member clock 0" + ); + + // Dialer: start a push-pull. The QUIC coordinator dials in-band (opens the + // quinn connection itself); the Initial emerges on the next `poll_transmit`. + dialer + .transport_mut() + .start_push_pull(acceptor_addr, PushPullKind::Join, now); + + // Ferry the single combined UDP egress both directions and tick both sides + // until the acceptor has witnessed the dialer's clock, or both coordinators + // go idle. A real driver writes each `poll_transmit` datagram to the named + // peer's socket; here we deliver it straight into the peer's `handle_udp`. + let mut converged = false; + for _ in 0..400 { + let mut moved = false; + + while let Some((to, bytes)) = dialer.poll_transmit() { + if to == acceptor_addr { + acceptor.handle_udp(dialer_addr, &bytes, now); + moved = true; + } + } + while let Some((to, bytes)) = acceptor.poll_transmit() { + if to == dialer_addr { + dialer.handle_udp(acceptor_addr, &bytes, now); + moved = true; + } + } + + // Tick both so quinn drives the handshake, the bridges pump their + // send/recv halves, and the merge is sieved into serf. + dialer.handle_timeout(now); + acceptor.handle_timeout(now); + while dialer.poll_event().is_some() {} + while acceptor.poll_event().is_some() {} + + // The acceptor witnessed the dialer's serf member clock through the merge. + if acceptor.member_time() >= 49 { + converged = true; + break; + } + if !moved { + break; + } + } + + assert!( + converged, + "the loopback QUIC push-pull exchange converged: the acceptor witnessed \ + the dialer's serf member clock (started at 0, advanced to {})", + acceptor.member_time() + ); + // The acceptor also learned the dialer (node 1) as a member through the + // exchange's membership push. + assert!( + acceptor.test_member_status(1).is_some(), + "the acceptor learned the dialer (node 1) as a member over the exchange" + ); +} From 397320f7d3247a81623813f808d6d5ccce3310fa Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 26 Jun 2026 22:10:57 +1200 Subject: [PATCH 015/128] refactor(serf-proto): cut over to the StreamEndpoint/QuicEndpoint super-machines --- serf-proto/src/endpoint/mod.rs | 14 ++--- serf-proto/src/endpoint/reliable/mod.rs | 63 ----------------------- serf-proto/src/endpoint/reliable/tests.rs | 20 +++---- serf-proto/src/lib.rs | 27 ++++++++-- serf-proto/src/members/mod.rs | 7 ++- serf-proto/src/members/tests.rs | 3 ++ serf-proto/src/options/mod.rs | 2 +- serf-proto/src/quic_endpoint/mod.rs | 4 +- serf-proto/src/snapshot/mod.rs | 4 +- serf-proto/src/stream_endpoint/mod.rs | 2 +- serf-proto/src/typed/mod.rs | 31 +++++++++-- 11 files changed, 79 insertions(+), 98 deletions(-) diff --git a/serf-proto/src/endpoint/mod.rs b/serf-proto/src/endpoint/mod.rs index 3eeab504..24eeadce 100644 --- a/serf-proto/src/endpoint/mod.rs +++ b/serf-proto/src/endpoint/mod.rs @@ -4,11 +4,11 @@ //! Holds serf's membership FSM, three Lamport clocks, user events, //! queries/responses/relays, push-pull anti-entropy, and network coordinates, //! but **no** transport reference. It reaches a memberlist reliable -//! coordinator only through the narrow [`Reliable`](reliable::Reliable) seam: +//! coordinator only through the narrow `Reliable` seam: //! every serf-logic method that touches the membership transport takes a //! `&mut impl Reliable` (named `t`). The composing super-machine -//! ([`crate::StreamEndpoint`]) owns both the core and the coordinator as -//! separate fields and threads the latter into the former on each call. +//! (`StreamEndpoint` or `QuicEndpoint`) owns both the core and the coordinator +//! as separate fields and threads the latter into the former on each call. //! //! The composing super-machine moves opaque `Bytes` in (`handle_packet`) and //! `Transmit`/`Bytes` out (`poll_transmit`) and ticks `poll_timeout(now)`; @@ -633,12 +633,12 @@ fn next_ltime(clock: &mut u64) -> u64 { /// Holds all serf state — the three Lamport clocks, membership store, options, /// event ring, query bookkeeping, deadlines, and (feature-gated) the coordinate /// client — and **no** transport reference. It reaches a memberlist reliable -/// coordinator only through the narrow [`Reliable`](reliable::Reliable) seam: +/// coordinator only through the narrow `Reliable` seam: /// every serf-logic method that must touch the membership transport takes a /// `&mut impl Reliable` (named `t`), borrowed disjointly from the core's -/// own state. The composing super-machine -/// ([`crate::StreamEndpoint`]) owns both the core and the coordinator as -/// separate fields and threads the latter into the former. +/// own state. The composing super-machine (`StreamEndpoint` or `QuicEndpoint`) +/// owns both the core and the coordinator as separate fields and threads the +/// latter into the former. /// /// `I` is the node-id type; `A` is the (resolved) address type; `R` is the /// random number generator injected at construction time (default: `SmallRng`). diff --git a/serf-proto/src/endpoint/reliable/mod.rs b/serf-proto/src/endpoint/reliable/mod.rs index 7be05870..a60086b3 100644 --- a/serf-proto/src/endpoint/reliable/mod.rs +++ b/serf-proto/src/endpoint/reliable/mod.rs @@ -141,69 +141,6 @@ where fn set_ack_payload(&mut self, payload: Bytes) -> Result<(), memberlist_proto::Error>; } -// ── impl for the raw memberlist_proto::Endpoint ─────────────────────────────── -// -// The transitional `StreamEndpoint` super-machine uses the raw packet -// `memberlist_proto::Endpoint` as its reliable transport, so the serf-logic -// core can drive it through this seam before the full stream/QUIC coordinators -// are wired in as the transport. - -impl Reliable for Endpoint -where - I: memberlist_proto::Id, - A: memberlist_proto::CheapClone + memberlist_proto::Data + PartialEq + 'static, - R: Rng, -{ - type Rng = R; - - #[inline] - fn endpoint_ref(&self) -> &Endpoint { - self - } - - #[inline] - fn poll_inner_event(&mut self) -> Option> { - // Fully-qualified call to reach the inherent method; the trait method has - // the same name and would recurse without the explicit path. - Endpoint::poll_event(self) - } - - #[inline] - fn queue_user_broadcast_ranked( - &mut self, - rank: u8, - data: Bytes, - ) -> Result<(), memberlist_proto::Error> { - Endpoint::queue_user_broadcast_ranked(self, rank, data) - } - - #[inline] - fn send_user_packet(&mut self, to: A, data: Bytes) -> Result<(), memberlist_proto::Error> { - Endpoint::send_user_packet(self, to, data) - } - - #[inline] - fn set_local_state_snapshot(&mut self, bytes: Bytes) -> Result<(), memberlist_proto::Error> { - Endpoint::set_local_state_snapshot(self, bytes) - } - - #[inline] - fn start_push_pull(&mut self, peer: A, kind: PushPullKind, now: Instant) -> StreamId { - Endpoint::start_push_pull(self, peer, kind, now) - } - - #[inline] - fn leave(&mut self, now: Instant) -> Result<(), memberlist_proto::Error> { - Endpoint::leave(self, now) - } - - #[cfg(feature = "coordinates")] - #[inline] - fn set_ack_payload(&mut self, payload: Bytes) -> Result<(), memberlist_proto::Error> { - Endpoint::set_ack_payload(self, payload) - } -} - // ── impl for memberlist_proto::streams::StreamEndpoint ──────────────────────── // // Enabled when the `tcp` or `tls` feature is active (which enables diff --git a/serf-proto/src/endpoint/reliable/tests.rs b/serf-proto/src/endpoint/reliable/tests.rs index 01028228..fa41051c 100644 --- a/serf-proto/src/endpoint/reliable/tests.rs +++ b/serf-proto/src/endpoint/reliable/tests.rs @@ -1,6 +1,9 @@ +#[cfg(feature = "tcp")] use super::Reliable; +#[cfg(feature = "tcp")] use bytes::Bytes; -use memberlist_proto::{EndpointOptions, Instant, PushPullKind, SeedableRng, SmallRng}; +#[cfg(feature = "tcp")] +use memberlist_proto::{Instant, PushPullKind}; /// Smoke-test a `Reliable` implementation through a generic function so that /// the impls are exercised at the trait boundary, not just as concrete calls. @@ -9,6 +12,7 @@ use memberlist_proto::{EndpointOptions, Instant, PushPullKind, SeedableRng, Smal /// `set_ack_payload` (coordinates only), `set_local_state_snapshot`, /// `poll_inner_event`, and `start_push_pull` — methods that run on a fresh, /// un-started endpoint without requiring a live network peer. +#[cfg(feature = "tcp")] fn drive(t: &mut impl Reliable, addr: A) where I: Eq + core::hash::Hash + Clone, @@ -47,25 +51,13 @@ where let _ = t.start_push_pull(addr, PushPullKind::Join, Instant::ORIGIN); } -// ── raw memberlist_proto::Endpoint ─────────────────────────────────────────── - -#[test] -fn raw_endpoint_impl_compiles_and_wires_up() { - let addr: core::net::SocketAddr = "127.0.0.1:7946".parse().unwrap(); - let opts = EndpointOptions::new(1u32, addr); - let mut ep = - memberlist_proto::Endpoint::new_at(opts, Instant::ORIGIN, SmallRng::seed_from_u64(0)); - ep.start_scheduling(Instant::ORIGIN); - drive(&mut ep, addr); -} - // ── memberlist_proto::streams::StreamEndpoint (plain TCP) ──────────────────── #[cfg(feature = "tcp")] #[test] fn tcp_stream_endpoint_impl_compiles_and_wires_up() { use memberlist_proto::{ - RawRecords, + EndpointOptions, RawRecords, SeedableRng, SmallRng, streams::{LabelOptions, StreamEndpoint}, }; diff --git a/serf-proto/src/lib.rs b/serf-proto/src/lib.rs index e70c66fa..6af0c424 100644 --- a/serf-proto/src/lib.rs +++ b/serf-proto/src/lib.rs @@ -5,9 +5,13 @@ //! own message set and framing on top of them. #![deny(missing_docs)] +#[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) use any::{AnyMessage, EncodeError}; +#[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) use bridge::BridgeError; +#[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) use framing::{FrameError, MessageType}; +#[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) use typed::{ ConflictResponseMessage, JoinMessage, LeaveMessage, PushPullMessage, RelayMessage, }; @@ -15,7 +19,10 @@ pub use typed::{ Coordinate, Filter, QueryFlag, QueryMessage, QueryResponseMessage, TagFilter, Tags, UserEvent, UserEventMessage, UserEvents, }; -#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[cfg(all( + any(feature = "aes-gcm", feature = "chacha20-poly1305"), + any(feature = "tcp", feature = "quic") +))] pub(crate) use typed::{KeyRequestMessage, KeyResponseMessage}; /// A lamport logical clock value — a monotonically increasing counter used to @@ -52,9 +59,13 @@ impl LamportTime { } } +#[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) mod any; +#[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) mod bridge; +#[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) mod framing; +#[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) mod messages; pub mod typed; @@ -65,7 +76,11 @@ pub use coordinate_client::{ CoordinateClient, CoordinateClientStats, CoordinateError, CoordinateOptions, }; +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] pub mod endpoint; +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] pub mod event; pub mod members; pub mod options; @@ -94,10 +109,16 @@ pub use quic_endpoint::QuicEndpoint; pub use snapshot::CoordinateRecord; pub use snapshot::{ReplayResult, SnapshotError, SnapshotRecord}; -#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[cfg(all( + any(feature = "aes-gcm", feature = "chacha20-poly1305"), + any(feature = "tcp", feature = "quic") +))] #[cfg_attr( docsrs, - doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + doc(cfg(all( + any(feature = "aes-gcm", feature = "chacha20-poly1305"), + any(feature = "tcp", feature = "quic") + ))) )] pub use event::{KeyRequest, KeyRequestOperation, KeyResponseArgs}; diff --git a/serf-proto/src/members/mod.rs b/serf-proto/src/members/mod.rs index d16df5a0..b4296526 100644 --- a/serf-proto/src/members/mod.rs +++ b/serf-proto/src/members/mod.rs @@ -3,7 +3,7 @@ //! [`MemberStatus`] is the per-node status within a serf cluster. //! [`SerfState`] is the lifecycle state of the local serf endpoint (distinct //! from any single node's status). [`MemberState`] pairs a [`Member`] with -//! its lamport clock timestamp and optional leave wall-time. [`Members`] is +//! its lamport clock timestamp and optional leave wall-time. `Members` is //! the in-memory store that the endpoint mutates as intents and inner //! memberlist events arrive. //! @@ -11,6 +11,7 @@ //! `memberlist_proto::Instant` threaded in from the driver — no wall-clock //! reads occur inside the pure machine. +#[cfg(any(feature = "tcp", feature = "quic"))] use std::collections::HashMap; /// Hard cardinality cap on `Members::recent_intents`. @@ -27,6 +28,7 @@ use std::collections::HashMap; /// recently observed intents, which are the ones most likely to still be /// relevant when the corresponding inner event fires. Normal clusters will /// never approach this limit; it is only reachable under adversarial flood. +#[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) const MAX_RECENT_INTENTS: usize = 8192; use memberlist_proto::Instant; @@ -288,6 +290,7 @@ impl NodeIntent { /// /// `left_members` and `failed_members` are index lists of ids for the reaper; /// the full state lives in `states`. +#[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) struct Members where I: Eq + core::hash::Hash, @@ -309,6 +312,7 @@ where pub(crate) recent_intent_seq: u64, } +#[cfg(any(feature = "tcp", feature = "quic"))] impl Default for Members where I: Eq + core::hash::Hash, @@ -324,6 +328,7 @@ where } } +#[cfg(any(feature = "tcp", feature = "quic"))] impl Members where I: Eq + core::hash::Hash + Clone, diff --git a/serf-proto/src/members/tests.rs b/serf-proto/src/members/tests.rs index d384a4e4..6ecf9450 100644 --- a/serf-proto/src/members/tests.rs +++ b/serf-proto/src/members/tests.rs @@ -13,6 +13,7 @@ use super::*; /// A subsequent `recent_intent` lookup for a specific node then returns the /// same answer on both stores, and a downstream `handle_node_join` for that /// node would produce the same `MemberStatus` on both machines. +#[cfg(any(feature = "tcp", feature = "quic"))] #[test] fn upsert_intent_cap_eviction_is_deterministic() { use memberlist_proto::Instant; @@ -122,6 +123,7 @@ fn serf_state_display_matches_as_str() { } } +#[cfg(any(feature = "tcp", feature = "quic"))] #[test] fn upsert_intent_newest_ltime_wins() { use memberlist_proto::Instant; @@ -155,6 +157,7 @@ fn upsert_intent_newest_ltime_wins() { ); } +#[cfg(any(feature = "tcp", feature = "quic"))] #[test] fn recent_intent_kind_mismatch_returns_none() { use memberlist_proto::Instant; diff --git a/serf-proto/src/options/mod.rs b/serf-proto/src/options/mod.rs index eb028bdc..8b8245c0 100644 --- a/serf-proto/src/options/mod.rs +++ b/serf-proto/src/options/mod.rs @@ -1,4 +1,4 @@ -//! Configuration knobs for the serf [`crate::endpoint::Endpoint`]. +//! Configuration knobs for the serf `Endpoint`. //! //! All timer defaults are taken verbatim from Go serf `options.go` / the legacy //! `serf-core/src/options.rs` port. The struct itself is a plain value type diff --git a/serf-proto/src/quic_endpoint/mod.rs b/serf-proto/src/quic_endpoint/mod.rs index 3cd73c18..b5a1332d 100644 --- a/serf-proto/src/quic_endpoint/mod.rs +++ b/serf-proto/src/quic_endpoint/mod.rs @@ -1,11 +1,11 @@ //! The serf `QuicEndpoint` super-machine — serf logic composed with the //! memberlist QUIC coordinator. //! -//! `QuicEndpoint` is the QUIC sibling of [`StreamEndpoint`](crate::StreamEndpoint): +//! `QuicEndpoint` is the QUIC sibling of the stream super-machine `StreamEndpoint`: //! it owns the same serf-logic [`Endpoint`] core but pairs it with the memberlist //! QUIC coordinator ([`memberlist_proto::QuicEndpoint`]) instead of the reliable //! stream coordinator. The two live as **disjoint fields**, and the core is -//! driven over `&mut transport` through the [`Reliable`](crate::endpoint::reliable) +//! driven over `&mut transport` through the `Reliable` //! seam — identical to the stream super-machine; only the transport surface //! differs. //! diff --git a/serf-proto/src/snapshot/mod.rs b/serf-proto/src/snapshot/mod.rs index a805a5cc..77270227 100644 --- a/serf-proto/src/snapshot/mod.rs +++ b/serf-proto/src/snapshot/mod.rs @@ -440,7 +440,7 @@ where /// /// The driver reads the snapshot file into [`SnapshotRecord`] values (via /// [`SnapshotRecord::decode`]) and calls [`ReplayResult::replay`]; serf's -/// [`Endpoint::load_snapshot`][crate::endpoint::Endpoint::load_snapshot] then +/// `Endpoint::load_snapshot` then /// applies the result to the machine state. This type owns no file handles or /// I/O state — it is a pure data carrier. /// @@ -457,7 +457,7 @@ pub struct ReplayResult { /// The member Lamport clock high-water mark seen in the snapshot. /// /// The `Endpoint` will set its member clock to at least this value + 1 - /// on [`load_snapshot`][crate::endpoint::Endpoint::load_snapshot] (G5). + /// on `load_snapshot` (G5). pub last_clock: LamportTime, /// The event Lamport clock high-water mark seen in the snapshot. /// diff --git a/serf-proto/src/stream_endpoint/mod.rs b/serf-proto/src/stream_endpoint/mod.rs index 9918d5fa..150bbc31 100644 --- a/serf-proto/src/stream_endpoint/mod.rs +++ b/serf-proto/src/stream_endpoint/mod.rs @@ -4,7 +4,7 @@ //! `StreamEndpoint` owns the serf-logic [`Endpoint`] core and the memberlist //! reliable coordinator ([`memberlist_proto::streams::StreamEndpoint`]) as //! **two disjoint fields**, and drives the core over `&mut transport` through -//! the [`Reliable`](crate::endpoint::reliable) seam. It exposes the +//! the `Reliable` seam. It exposes the //! coordinator's transport-facing driver surface (`handle_packet`, //! `handle_gossip`, `accept_connection`, `handle_transport_data`, //! `poll_action`, `poll_transport_transmit`, `handle_timeout`, …) plus serf's diff --git a/serf-proto/src/typed/mod.rs b/serf-proto/src/typed/mod.rs index 306ee1b2..9b513575 100644 --- a/serf-proto/src/typed/mod.rs +++ b/serf-proto/src/typed/mod.rs @@ -9,8 +9,12 @@ use bytes::Bytes; use smol_str::SmolStr; use crate::LamportTime; +#[cfg(any(feature = "tcp", feature = "quic"))] use memberlist_proto::Node; -#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[cfg(all( + any(feature = "aes-gcm", feature = "chacha20-poly1305"), + any(feature = "tcp", feature = "quic") +))] use memberlist_proto::SecretKey; /// A user-generated event broadcast through the serf cluster. @@ -269,6 +273,7 @@ pub struct UserEvents { /// /// The `status_ltimes` map is transmitted as a `repeated NodeStatusTime` rather /// than a proto3 `map` because proto3 forbids `bytes` map keys. +#[cfg(any(feature = "tcp", feature = "quic"))] #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct PushPullMessage { /// The lamport clock value of the sending node at the time of the exchange. @@ -288,6 +293,7 @@ pub(crate) struct PushPullMessage { pub(crate) query_ltime: LamportTime, } +#[cfg(any(feature = "tcp", feature = "quic"))] impl PushPullMessage { /// Construct a new `PushPullMessage`. pub(crate) fn new( @@ -317,14 +323,20 @@ impl PushPullMessage { /// operations. /// /// Requires the `aes-gcm` or `chacha20-poly1305` feature. -#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[cfg(all( + any(feature = "aes-gcm", feature = "chacha20-poly1305"), + any(feature = "tcp", feature = "quic") +))] #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct KeyRequestMessage { /// The encryption key, or `None` for a list-keys request. pub(crate) key: Option, } -#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[cfg(all( + any(feature = "aes-gcm", feature = "chacha20-poly1305"), + any(feature = "tcp", feature = "quic") +))] impl KeyRequestMessage { /// Construct a new `KeyRequestMessage`. pub(crate) fn new(key: Option) -> Self { @@ -340,7 +352,10 @@ impl KeyRequestMessage { /// the wire codec; only this per-node response message is encoded on the wire. /// /// Requires the `aes-gcm` or `chacha20-poly1305` feature. -#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[cfg(all( + any(feature = "aes-gcm", feature = "chacha20-poly1305"), + any(feature = "tcp", feature = "quic") +))] #[derive(Debug, Default, Clone, PartialEq, Eq)] pub(crate) struct KeyResponseMessage { /// `true` if the operation succeeded on this node. @@ -364,6 +379,7 @@ pub(crate) struct KeyResponseMessage { /// Generic over `I` (node-id) and `A` (node-address). The `memberlist_proto::Data` /// bound is enforced at the bridge layer (`relay_to_pb` / `relay_from_pb`) where /// the destination is encoded/decoded as opaque `bytes`. +#[cfg(any(feature = "tcp", feature = "quic"))] #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct RelayMessage { /// The node to forward the message to. @@ -372,6 +388,7 @@ pub(crate) struct RelayMessage { pub(crate) payload: Bytes, } +#[cfg(any(feature = "tcp", feature = "quic"))] impl RelayMessage { /// Construct a new `RelayMessage`. pub(crate) fn new(destination: Node, payload: Bytes) -> Self { @@ -389,6 +406,7 @@ impl RelayMessage { /// Generic over `I`: the node-id type, which must implement /// `memberlist_proto::Data` so it can be encoded as opaque proto `bytes` in /// the bridge layer. +#[cfg(any(feature = "tcp", feature = "quic"))] #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct JoinMessage { /// The lamport clock value at the time the node joined. @@ -397,6 +415,7 @@ pub(crate) struct JoinMessage { pub(crate) id: I, } +#[cfg(any(feature = "tcp", feature = "quic"))] impl JoinMessage { /// Construct a new `JoinMessage`. pub(crate) fn new(ltime: crate::LamportTime, id: I) -> Self { @@ -409,6 +428,7 @@ impl JoinMessage { /// Generic over `I`: the node-id type, which must implement /// `memberlist_proto::Data` so it can be encoded as opaque proto `bytes` in /// the bridge layer. +#[cfg(any(feature = "tcp", feature = "quic"))] #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct LeaveMessage { /// The lamport clock value at the time the leave was emitted. @@ -419,6 +439,7 @@ pub(crate) struct LeaveMessage { pub(crate) prune: bool, } +#[cfg(any(feature = "tcp", feature = "quic"))] impl LeaveMessage { /// Construct a new `LeaveMessage`. pub(crate) fn new(ltime: crate::LamportTime, id: I, prune: bool) -> Self { @@ -431,12 +452,14 @@ impl LeaveMessage { /// Generic over `I` and `A`: the node-id and address types, which must /// implement `memberlist_proto::Data` so the embedded `Node` can be /// encoded as opaque proto `bytes` in the bridge layer. +#[cfg(any(feature = "tcp", feature = "quic"))] #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct ConflictResponseMessage { /// The winning node in the conflict resolution. pub(crate) member: Node, } +#[cfg(any(feature = "tcp", feature = "quic"))] impl ConflictResponseMessage { /// Construct a new `ConflictResponseMessage`. pub(crate) fn new(member: Node) -> Self { From aee212fff612f9229194d6cd2853265ce86097fb Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 26 Jun 2026 22:34:33 +1200 Subject: [PATCH 016/128] docs(serf-proto): doc(cfg) on the super-machine impl blocks Add the docsrs doc(cfg) attribute to the feature-gated impl blocks so docs.rs renders the gate badges, and tighten the composed-timeout doc comment. --- serf-proto/src/endpoint/mod.rs | 4 ++-- serf-proto/src/quic_endpoint/mod.rs | 2 ++ serf-proto/src/stream_endpoint/mod.rs | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/serf-proto/src/endpoint/mod.rs b/serf-proto/src/endpoint/mod.rs index 24eeadce..06a34696 100644 --- a/serf-proto/src/endpoint/mod.rs +++ b/serf-proto/src/endpoint/mod.rs @@ -975,7 +975,7 @@ where /// Post-inner-timer phase of the composed tick: drain inner events then fire /// serf's own deadlines. /// - /// Tick order (H1b / decision 5 step 4): + /// Tick order: /// /// 1. Drain all inner events produced by the coordinator's timer via /// `drain_inner`, processing NodeJoined / NodeLeft / UserPacket / etc. @@ -987,7 +987,7 @@ where /// prevents tombstoning a member whose `NodeJoined` event is still queued in /// the inner machine at the time the reap deadline would otherwise fire. /// Without this ordering, a member that reconnects exactly at the reap - /// boundary could be incorrectly pruned. Decision 5 step 4 / pin H1b. + /// boundary could be incorrectly pruned. pub(crate) fn after_inner_timeout(&mut self, t: &mut T, now: Instant) where T: Reliable, diff --git a/serf-proto/src/quic_endpoint/mod.rs b/serf-proto/src/quic_endpoint/mod.rs index b5a1332d..f3e15489 100644 --- a/serf-proto/src/quic_endpoint/mod.rs +++ b/serf-proto/src/quic_endpoint/mod.rs @@ -84,6 +84,7 @@ where } #[cfg(feature = "quic")] +#[cfg_attr(docsrs, doc(cfg(feature = "quic")))] impl QuicEndpoint where I: Clone + Eq + core::hash::Hash, @@ -119,6 +120,7 @@ where // through the ones that reach the coordinator (the `Reliable` methods). #[cfg(feature = "quic")] +#[cfg_attr(docsrs, doc(cfg(feature = "quic")))] impl QuicEndpoint where I: Id + Clone, diff --git a/serf-proto/src/stream_endpoint/mod.rs b/serf-proto/src/stream_endpoint/mod.rs index 150bbc31..6ef3d319 100644 --- a/serf-proto/src/stream_endpoint/mod.rs +++ b/serf-proto/src/stream_endpoint/mod.rs @@ -80,6 +80,7 @@ where } #[cfg(feature = "tcp")] +#[cfg_attr(docsrs, doc(cfg(feature = "tcp")))] impl StreamEndpoint where I: Clone + Eq + core::hash::Hash, @@ -114,6 +115,7 @@ where // the serf-logic sieve over the coordinator. #[cfg(feature = "tcp")] +#[cfg_attr(docsrs, doc(cfg(feature = "tcp")))] impl StreamEndpoint where I: Id + Clone, From 1ff24dfcc3dbfb3819fbca8771cf8d13cb4bcb3a Mon Sep 17 00:00:00 2001 From: al8n Date: Sat, 27 Jun 2026 01:01:40 +1200 Subject: [PATCH 017/128] fix(serf-proto): gate the membership FSM module behind a transport The no-transport build exposes only wire/data/config/snapshot types; the whole members module (the membership FSM state) is gated on any(tcp, quic). --- .github/workflows/ci.yml | 2 ++ serf-proto/src/lib.rs | 2 ++ serf-proto/src/members/mod.rs | 5 ----- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8790dd5a..d54f610b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -85,6 +85,8 @@ jobs: - { name: "coords-chacha", flags: "--features coordinates,chacha20-poly1305" } - { name: "all-features", flags: "--all-features" } - { name: "no-default", flags: "--no-default-features" } + - { name: "coords-aes-gcm-tcp", flags: "--features coordinates,aes-gcm,tcp" } + - { name: "quic-rustls-ring", flags: "--no-default-features --features quic-rustls-ring" } steps: - uses: actions/checkout@v6 diff --git a/serf-proto/src/lib.rs b/serf-proto/src/lib.rs index 6af0c424..4cc880ee 100644 --- a/serf-proto/src/lib.rs +++ b/serf-proto/src/lib.rs @@ -82,6 +82,8 @@ pub mod endpoint; #[cfg(any(feature = "tcp", feature = "quic"))] #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] pub mod event; +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] pub mod members; pub mod options; pub mod snapshot; diff --git a/serf-proto/src/members/mod.rs b/serf-proto/src/members/mod.rs index b4296526..affc11ee 100644 --- a/serf-proto/src/members/mod.rs +++ b/serf-proto/src/members/mod.rs @@ -11,7 +11,6 @@ //! `memberlist_proto::Instant` threaded in from the driver — no wall-clock //! reads occur inside the pure machine. -#[cfg(any(feature = "tcp", feature = "quic"))] use std::collections::HashMap; /// Hard cardinality cap on `Members::recent_intents`. @@ -28,7 +27,6 @@ use std::collections::HashMap; /// recently observed intents, which are the ones most likely to still be /// relevant when the corresponding inner event fires. Normal clusters will /// never approach this limit; it is only reachable under adversarial flood. -#[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) const MAX_RECENT_INTENTS: usize = 8192; use memberlist_proto::Instant; @@ -290,7 +288,6 @@ impl NodeIntent { /// /// `left_members` and `failed_members` are index lists of ids for the reaper; /// the full state lives in `states`. -#[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) struct Members where I: Eq + core::hash::Hash, @@ -312,7 +309,6 @@ where pub(crate) recent_intent_seq: u64, } -#[cfg(any(feature = "tcp", feature = "quic"))] impl Default for Members where I: Eq + core::hash::Hash, @@ -328,7 +324,6 @@ where } } -#[cfg(any(feature = "tcp", feature = "quic"))] impl Members where I: Eq + core::hash::Hash + Clone, From 093679757dfe29c6983eb48fcc51726fc6ac954f Mon Sep 17 00:00:00 2001 From: al8n Date: Sat, 27 Jun 2026 01:44:24 +1200 Subject: [PATCH 018/128] feat(serf-proto): add Endpoint::set_tags over the Reliable update_meta seam set_tags encodes the local tag map into the node meta, re-advertises it via the coordinator through the new Reliable::update_meta seam, and refreshes an already-present local member's tags so tag-filtered local queries observe them immediately. Both super-machines forward it. handle_node_update no longer marks local-state dirty for a NodeUpdated, since a member's tags and address are not part of the push-pull snapshot. --- serf-proto/src/bridge/mod.rs | 1 - serf-proto/src/endpoint/mod.rs | 98 ++++++++++++- serf-proto/src/endpoint/reliable/mod.rs | 31 ++++ serf-proto/src/endpoint/tests.rs | 185 ++++++++++++++++++++++++ serf-proto/src/quic_endpoint/mod.rs | 40 ++++- serf-proto/src/stream_endpoint/mod.rs | 41 +++++- 6 files changed, 388 insertions(+), 8 deletions(-) diff --git a/serf-proto/src/bridge/mod.rs b/serf-proto/src/bridge/mod.rs index e6f3d0aa..6e71cfa2 100644 --- a/serf-proto/src/bridge/mod.rs +++ b/serf-proto/src/bridge/mod.rs @@ -108,7 +108,6 @@ pub(crate) fn coordinate_from_pb(b: &pb::Coordinate) -> Coordinate { // ─── Tags ───────────────────────────────────────────────────────────────────── /// Convert typed [`Tags`] → `pb::Tags`. -#[cfg(test)] pub(crate) fn tags_to_pb(t: &Tags) -> pb::Tags { pb::Tags { entries: t diff --git a/serf-proto/src/endpoint/mod.rs b/serf-proto/src/endpoint/mod.rs index 06a34696..90ddf32c 100644 --- a/serf-proto/src/endpoint/mod.rs +++ b/serf-proto/src/endpoint/mod.rs @@ -60,7 +60,8 @@ use std::collections::VecDeque; use bytes::Bytes; use memberlist_proto::{ - CheapClone, Data, Id, Instant, Node, PushPullKind, Rng, SeedableRng, SmallRng, typed::NodeState, + CheapClone, Data, Id, Instant, Node, PushPullKind, Rng, SeedableRng, SmallRng, + typed::{Meta, NodeState}, }; use self::reliable::Reliable; @@ -73,7 +74,7 @@ use crate::KeyRequestMessage; use crate::event::{KeyRequest as KeyRequestEvent, KeyRequestOperation, KeyResponseArgs}; use crate::{ AnyMessage, ConflictResponseMessage, EncodeError, LamportTime, MessageType, - bridge::{tags_from_pb, user_event_to_pb}, + bridge::{tags_from_pb, tags_to_pb, user_event_to_pb}, event::{ DialPassthrough, Event, MemberEvent, MemberEventKind, QueryAck, QueryEvent, QueryResponse as QueryResponseEvent, @@ -556,6 +557,10 @@ pub enum Error { #[cfg_attr(docsrs, doc(cfg(feature = "tag-regex")))] #[error("query filter contains an invalid tag regex")] InvalidQueryFilter, + /// The coordinator's `update_meta()` call from `set_tags()` failed (e.g. the + /// encoded tag map exceeds the metadata cap). + #[error("set_tags update_meta error: {0}")] + SetTagsMeta(memberlist_proto::Error), } // ── clock witness ───────────────────────────────────────────────────────────── @@ -1297,6 +1302,72 @@ where self.event_join_ignore = v; } + /// Update the local node's tags, re-advertise them via the coordinator, and + /// synchronously refresh the local member in the membership store. + /// + /// The coordinator queues an Alive/NodeUpdated broadcast so peers learn the + /// new metadata; the corresponding `NodeUpdated` event arrives later via + /// `poll_event` and is idempotent (it re-applies the same tags via the normal + /// `handle_node_update` path). The synchronous refresh here means + /// tag-filtered local queries (`Filter::Tag` in `should_process_query`) see + /// the new tags immediately, without waiting for that event. + /// + /// If the local node is not yet in `members.states` (the local `NodeJoined` + /// has not been drained yet), the in-place refresh is skipped. Materializing + /// a phantom entry here would cause the queued `NodeUpdated` to emit a + /// `Member(Update)` event before the imminent `Member(Join)`, violating the + /// Join-before-Update ordering consumers rely on. When `NodeJoined` does + /// arrive it creates the member with tags decoded from the `Meta` set by + /// `update_meta`, and with the correct status/ltime from any buffered intent. + /// `handle_node_update` already suppresses `NodeUpdated` for absent members, + /// so no spurious event reaches the driver in this path. + /// + /// Tags are not part of the push-pull snapshot (only Lamport clocks, + /// per-member status times, `left_members`, and the event ring are), so + /// `set_tags` never marks the snapshot dirty. Mirrors Go serf + /// `base.go` `SetTags`. + /// + /// # Errors + /// + /// Returns [`Error::SetTagsMeta`] if the encoded tag map exceeds + /// `Meta::MAX_SIZE` or the coordinator's configured `meta_max_size`. + pub(crate) fn set_tags(&mut self, t: &mut T, tags: Tags) -> Result<(), Error> + where + T: Reliable, + I: Clone, + A: Clone, + { + use buffa::Message as _; + + let pb_tags = tags_to_pb(&tags); + let encoded = pb_tags.encode_to_vec(); + // `Meta::try_from` only fails when the encoded size exceeds `Meta::MAX_SIZE` + // (u16::MAX bytes). Surface this as `SetTagsMeta` rather than panicking so + // the driver can log and retry with fewer or shorter tags. + let n = encoded.len(); + let meta = Meta::try_from(encoded).map_err(|_| { + Error::SetTagsMeta(memberlist_proto::Error::MetaExceedsCap( + memberlist_proto::SizeExceeded::new(n, Meta::MAX_SIZE), + )) + })?; + t.update_meta(meta).map_err(Error::SetTagsMeta)?; + + // Refresh the local member's tags in place if it is already present, so + // tag-filtered local queries see the new tags before the coordinator's + // NodeUpdated event is drained. If the local member is not yet present + // (set_tags before the local NodeJoined), do nothing: materializing it here + // would surface a Member(Update) before the Member(Join), and the join will + // create the member with these tags (decoded from the meta set above) and the + // correct status/ltime from any buffered intent. + if let Some(ms) = self.members.states.get_mut(t.endpoint_ref().local_id_ref()) { + let node = ms.member().node().clone(); + let status = ms.status(); + *ms.member_mut() = Member::new(node, tags, status); + } + + Ok(()) + } + // ── push-pull ingress replay (H8/G2, G3, G4) ──────────────────────────── /// Replay a remote push-pull body received via `RemoteStateReceived`. @@ -1614,10 +1685,13 @@ where let status = ms.status(); *ms.member_mut() = Member::new(n, tags, status); let member = ms.member().clone(); - // Drop the mutable borrow on ms before calling mark_local_state_dirty. - // Membership changed — snapshot is stale. - self.mark_local_state_dirty(); + // No `mark_local_state_dirty` here: a NodeUpdated changes only the member's + // tags and address, and the push-pull snapshot carries neither (only the + // Lamport clocks, per-member status times, `left_members`, and the event + // ring). Status and status time are unchanged, so the snapshot is + // unaffected and a resync would be wasted work — and `set_tags` queues + // exactly this event via `update_meta` on every local tag change. self .pending_events @@ -3920,6 +3994,20 @@ where } } + /// Return the local node's serf-side tags from `members.states` (test + /// adapter for `set_tags` synchronous-observability assertions). + /// + /// Returns `None` when the local node is not yet tracked in the serf + /// membership store. + #[cfg(test)] + pub(crate) fn test_local_tags_in(&self, local_id: &I) -> Option { + self + .members + .states + .get(local_id) + .map(|ms| ms.member().tags().clone()) + } + /// Clear the dirty flag (test adapter for dirty-flag unit assertions). #[cfg(test)] pub(crate) fn test_clear_dirty(&mut self) { diff --git a/serf-proto/src/endpoint/reliable/mod.rs b/serf-proto/src/endpoint/reliable/mod.rs index a60086b3..f0567bc4 100644 --- a/serf-proto/src/endpoint/reliable/mod.rs +++ b/serf-proto/src/endpoint/reliable/mod.rs @@ -98,6 +98,21 @@ where /// snapshot would not fit inside a reliable-stream frame. fn set_local_state_snapshot(&mut self, bytes: Bytes) -> Result<(), memberlist_proto::Error>; + /// Update the local node's advertised metadata. + /// + /// serf encodes its tag map into `meta` and calls this so the coordinator + /// re-advertises it (an Alive/NodeUpdated broadcast) and queues a `NodeUpdated` + /// event observed via [`poll_inner_event`](Reliable::poll_inner_event). + /// + /// # Errors + /// + /// Returns [`memberlist_proto::Error`] if the meta exceeds the wire cap or the + /// coordinator is not running. + fn update_meta( + &mut self, + meta: memberlist_proto::typed::Meta, + ) -> Result<(), memberlist_proto::Error>; + /// Initiate an outbound push-pull anti-entropy exchange with `peer`. /// /// The coordinator dials `peer`, performs a label handshake, and exchanges @@ -187,6 +202,14 @@ where self.set_local_state_snapshot(bytes) } + #[inline] + fn update_meta( + &mut self, + meta: memberlist_proto::typed::Meta, + ) -> Result<(), memberlist_proto::Error> { + self.update_meta(meta) + } + #[inline] fn start_push_pull(&mut self, peer: A, kind: PushPullKind, now: Instant) -> StreamId { self.start_push_pull(peer, kind, now) @@ -251,6 +274,14 @@ where self.set_local_state_snapshot(bytes) } + #[inline] + fn update_meta( + &mut self, + meta: memberlist_proto::typed::Meta, + ) -> Result<(), memberlist_proto::Error> { + self.update_meta(meta) + } + #[inline] fn start_push_pull( &mut self, diff --git a/serf-proto/src/endpoint/tests.rs b/serf-proto/src/endpoint/tests.rs index 21376bb4..e21dd7a0 100644 --- a/serf-proto/src/endpoint/tests.rs +++ b/serf-proto/src/endpoint/tests.rs @@ -313,6 +313,33 @@ fn inner_node_updated_for_unknown_is_a_noop() { assert!(e.poll_event().is_none()); } +#[test] +fn inner_node_updated_does_not_dirty_local_state() { + let mut e = ep(); + e.test_seed_member(2, MemberStatus::Alive, LamportTime::new(3)); + e.resync_local_state(); + assert!( + !e.test_is_dirty(), + "resync_local_state clears the dirty flag" + ); + + // A NodeUpdated refreshes only the member's tags/address — neither is in the + // push-pull snapshot — so it must not dirty local state. set_tags queues + // exactly this event via update_meta on every local tag change, so dirtying + // here would force a wasted resync per tag update. + e.test_inner_node_updated(2, memberlist_proto::Instant::ORIGIN); + + assert!( + !e.test_is_dirty(), + "a tag-only NodeUpdated must not mark local_state_dirty" + ); + // The Update event is still emitted. + assert!(matches!( + e.poll_event(), + Some(Event::Member(ref me)) if me.kind() == MemberEventKind::Update + )); +} + #[test] fn handle_node_join_re_joining_failed_clears_lists() { let mut e = ep(); @@ -6651,3 +6678,161 @@ fn push_pull_near_watermark_event_floor_integrity_and_delivery() { "user_event must emit Event::User — must not be dropped by near-watermark event floor" ); } + +// ── set_tags ────────────────────────────────────────────────────────────────── + +/// Tags written by `set_tags` must round-trip: encoding them into `Meta` and +/// decoding that meta must reproduce the original map. +/// +/// This verifies the `tags_to_pb` → encode → `Meta::try_from` → +/// `update_meta` → coordinator-meta-store path used by `set_tags`. +#[test] +fn set_tags_round_trips_via_local_meta() { + use crate::typed::Tags; + + let mut e = ep(); + + let tags: Tags = [("role", "web"), ("dc", "us-east-1")].into_iter().collect(); + e.set_tags(tags.clone()) + .expect("set_tags must succeed on a live endpoint"); + + let meta = e + .test_local_meta() + .expect("local node must be present in the coordinator's membership store"); + let decoded = decode_tags_from_meta(meta.as_bytes()) + .expect("meta written by set_tags must decode as valid Tags"); + + assert_eq!(decoded.len(), tags.len(), "decoded tag count must match"); + for (k, v) in &tags.0 { + assert_eq!( + decoded.0.get(k), + Some(v), + "tag {k:?} must round-trip correctly" + ); + } +} + +/// `set_tags` must synchronously update `members.states` so that tag-filtered +/// local queries evaluate the new tags without a `poll_event` drain first. +/// +/// The coordinator queues a `NodeUpdated` event for the meta change; this test +/// verifies the synchronous path is independent of that event being drained. +#[test] +fn set_tags_local_member_state_is_observable_without_poll_event() { + use crate::typed::Tags; + + let mut e = ep(); + + // Seed the local node (id=1) into members.states so the existing-member + // refresh arm of set_tags is exercised. + e.test_seed_member(1u32, MemberStatus::Alive, LamportTime::new(0)); + + let tags: Tags = [("role", "db")].into_iter().collect(); + e.set_tags(tags.clone()) + .expect("set_tags must succeed on a live endpoint"); + + let local_tags = e.test_local_tags(); + assert_eq!( + local_tags + .as_ref() + .and_then(|t| t.0.get("role")) + .map(|s| s.as_str()), + Some("db"), + "local member tags in members.states must be updated synchronously by set_tags" + ); +} + +/// When `set_tags` is called before the local `NodeJoined` has been drained, +/// no phantom entry must appear in `members.states`. The new tags are written +/// to the coordinator's meta store via `update_meta` and will be decoded from +/// there when `handle_node_join` materializes the member. +#[test] +fn set_tags_does_not_materialize_absent_local_member() { + use crate::typed::Tags; + + let mut e = ep(); // local member absent from members.states + + let tags: Tags = [("env", "staging")].into_iter().collect(); + e.set_tags(tags) + .expect("set_tags must succeed even before the local node is in members.states"); + + // No phantom entry must be inserted into members.states. + assert!( + e.test_local_tags().is_none(), + "set_tags must not fabricate a local member when absent from members.states" + ); + + // The tags must have been advertised via the coordinator's meta store. + let meta = e + .test_local_meta() + .expect("coordinator must track local node meta"); + let decoded = decode_tags_from_meta(meta.as_bytes()) + .expect("meta written by set_tags must decode as valid Tags"); + assert_eq!( + decoded.0.get("env").map(|s| s.as_str()), + Some("staging"), + "new tags must be reflected in the coordinator meta even when local member is absent" + ); +} + +/// `set_tags` must never mark the push-pull snapshot dirty, regardless of +/// whether the local member is present or absent. Tags are not part of the +/// snapshot (only Lamport clocks, per-member status times, `left_members`, +/// and the event ring are). +#[test] +fn set_tags_does_not_mark_local_state_dirty() { + use crate::typed::Tags; + + let mut e = ep(); + // Clear the construction dirty flag so the assertion below is unambiguous. + e.resync_local_state(); + assert!(!e.test_is_dirty(), "resync must clear the dirty flag"); + + let tags: Tags = [("dc", "us-west-2")].into_iter().collect(); + e.set_tags(tags).expect("set_tags must succeed"); + + assert!( + !e.test_is_dirty(), + "set_tags must not mark local_state_dirty" + ); +} + +/// When `set_tags` is called before the local member exists in `members.states`, +/// the coordinator's queued `NodeUpdated` event must be suppressed (because the +/// member is absent) and must not surface as `Member(Update)` before the +/// `Member(Join)` that a subsequent `NodeJoined` would produce. If no `Join` +/// event appears in the drain, no `Update` event must appear either. +#[test] +fn set_tags_before_join_does_not_emit_update_before_join() { + use crate::typed::Tags; + + let mut e = ep(); // local member absent + + let tags: Tags = [("role", "cache")].into_iter().collect(); + e.set_tags(tags).expect("set_tags must succeed"); + + // Drive the serf tick so inner events (NodeUpdated) are drained. + e.handle_timeout(t_secs(1)); + + // Collect all events produced so far. + let mut evs = Vec::new(); + while let Some(ev) = e.poll_event() { + evs.push(ev); + } + + // If a Join appears, no Update may precede it. + let join_pos = evs + .iter() + .position(|ev| matches!(ev, Event::Member(me) if me.kind() == MemberEventKind::Join)); + let update_pos = evs + .iter() + .position(|ev| matches!(ev, Event::Member(me) if me.kind() == MemberEventKind::Update)); + + match (join_pos, update_pos) { + (None, Some(_)) => panic!("Member(Update) appeared without a preceding Member(Join): {evs:?}"), + (Some(j), Some(u)) if u < j => { + panic!("Member(Update) at index {u} appeared before Member(Join) at index {j}: {evs:?}") + } + _ => {} + } +} diff --git a/serf-proto/src/quic_endpoint/mod.rs b/serf-proto/src/quic_endpoint/mod.rs index f3e15489..3e3ec73a 100644 --- a/serf-proto/src/quic_endpoint/mod.rs +++ b/serf-proto/src/quic_endpoint/mod.rs @@ -44,7 +44,6 @@ use crate::{ options::Options, }; -#[cfg(all(test, feature = "tag-regex"))] use crate::typed::Tags; #[cfg(test)] use crate::{ @@ -355,6 +354,19 @@ where self.core.set_event_join_ignore(v) } + /// Update the local node's tags, re-advertise them via the coordinator, and + /// synchronously refresh the local member in the membership store. + /// + /// # Errors + /// + /// Returns [`Error::SetTagsMeta`] if the encoded tags exceed the metadata cap. + pub fn set_tags(&mut self, tags: Tags) -> Result<(), Error> + where + I: Clone, + { + self.core.set_tags(&mut self.transport, tags) + } + /// Forwards to [`Endpoint::leave`]. pub fn leave(&mut self, now: Instant) -> Result<(), Error> where @@ -731,6 +743,32 @@ where self.core.test_is_dirty() } + /// Return the local node's serf-side tags from `members.states` (test + /// adapter for `set_tags` observability assertions). + /// + /// Returns `None` when the local node is not yet in the serf membership store. + #[cfg(test)] + pub(crate) fn test_local_tags(&self) -> Option { + self + .core + .test_local_tags_in(self.transport.endpoint_ref().local_id_ref()) + } + + /// Return the local node's advertised meta from the coordinator's inner + /// membership store (test adapter for `set_tags` round-trip assertions). + /// + /// Returns `None` when the local node is not tracked by the coordinator + /// (should not happen after construction). + #[cfg(test)] + pub(crate) fn test_local_meta(&self) -> Option { + let local_id = self.transport.endpoint_ref().local_id_ref(); + self + .transport + .endpoint_ref() + .member(local_id) + .map(|ns| ns.meta_ref().clone()) + } + /// Forwards to [`Endpoint::test_set_event_join_ignore`]. pub(crate) fn test_set_event_join_ignore(&mut self, v: bool) { self.core.test_set_event_join_ignore(v) diff --git a/serf-proto/src/stream_endpoint/mod.rs b/serf-proto/src/stream_endpoint/mod.rs index 6ef3d319..fdb94b94 100644 --- a/serf-proto/src/stream_endpoint/mod.rs +++ b/serf-proto/src/stream_endpoint/mod.rs @@ -38,7 +38,6 @@ use crate::{ options::Options, }; -#[cfg(all(test, feature = "tag-regex"))] use crate::typed::Tags; #[cfg(test)] use crate::{ @@ -380,6 +379,20 @@ where self.core.set_event_join_ignore(v) } + /// Update the local node's tags, re-advertise them via the coordinator, and + /// synchronously refresh the local member in the membership store. + /// + /// # Errors + /// + /// Returns [`Error::SetTagsMeta`] if the encoded tags exceed the metadata cap. + pub fn set_tags(&mut self, tags: Tags) -> Result<(), Error> + where + I: Clone, + A: Clone, + { + self.core.set_tags(&mut self.transport, tags) + } + /// Forwards to [`Endpoint::handle_node_join_intent`]. #[cfg(test)] pub(crate) fn handle_node_join_intent(&mut self, ltime: LamportTime, id: &I, now: Instant) -> bool @@ -771,6 +784,32 @@ where self.core.test_is_dirty() } + /// Return the local node's serf-side tags from `members.states` (test + /// adapter for `set_tags` observability assertions). + /// + /// Returns `None` when the local node is not yet in the serf membership store. + #[cfg(test)] + pub(crate) fn test_local_tags(&self) -> Option { + self + .core + .test_local_tags_in(self.transport.endpoint_ref().local_id_ref()) + } + + /// Return the local node's advertised meta from the coordinator's inner + /// membership store (test adapter for `set_tags` round-trip assertions). + /// + /// Returns `None` when the local node is not tracked by the coordinator + /// (should not happen after construction). + #[cfg(test)] + pub(crate) fn test_local_meta(&self) -> Option { + let local_id = self.transport.endpoint_ref().local_id_ref(); + self + .transport + .endpoint_ref() + .member(local_id) + .map(|ns| ns.meta_ref().cheap_clone()) + } + /// Forwards to [`Endpoint::test_set_event_join_ignore`]. #[cfg(test)] pub(crate) fn test_set_event_join_ignore(&mut self, v: bool) { From c140f937799c2c8b400c5f605135f36d0a1f43ae Mon Sep 17 00:00:00 2001 From: al8n Date: Sat, 27 Jun 2026 02:57:11 +1200 Subject: [PATCH 019/128] feat(serf-driver): runtime-agnostic pure-data crate shared by the serf drivers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit serf-driver holds the membership SerfSnapshot view, the common driver error payloads, and the observation byte-weight helper — the runtime-independent pieces both serf-compio and serf-reactor will need. Mirrors memberlist-driver; the run loops, channel/cell substrate, and delegate dispatch live per-runtime. --- Cargo.toml | 1 + serf-driver/Cargo.toml | 28 ++++ serf-driver/src/error/mod.rs | 138 +++++++++++++++++ serf-driver/src/error/tests.rs | 34 +++++ serf-driver/src/lib.rs | 24 +++ serf-driver/src/observation/mod.rs | 53 +++++++ serf-driver/src/observation/tests.rs | 69 +++++++++ serf-driver/src/snapshot/mod.rs | 201 +++++++++++++++++++++++++ serf-driver/src/snapshot/tests.rs | 217 +++++++++++++++++++++++++++ 9 files changed, 765 insertions(+) create mode 100644 serf-driver/Cargo.toml create mode 100644 serf-driver/src/error/mod.rs create mode 100644 serf-driver/src/error/tests.rs create mode 100644 serf-driver/src/lib.rs create mode 100644 serf-driver/src/observation/mod.rs create mode 100644 serf-driver/src/observation/tests.rs create mode 100644 serf-driver/src/snapshot/mod.rs create mode 100644 serf-driver/src/snapshot/tests.rs diff --git a/Cargo.toml b/Cargo.toml index 6954d6c7..2a3d514f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "serf-proto", + "serf-driver", ] exclude = ["legacy"] resolver = "3" diff --git a/serf-driver/Cargo.toml b/serf-driver/Cargo.toml new file mode 100644 index 00000000..d22e6c1f --- /dev/null +++ b/serf-driver/Cargo.toml @@ -0,0 +1,28 @@ +[package] +name = "serf-driver" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +description = "Runtime-agnostic glue shared by the serf async driver crates" + +[features] +default = ["tag-regex"] +tcp = ["serf-proto/tcp"] +tls = ["tcp", "serf-proto/tls"] +quic = ["serf-proto/quic"] +quic-rustls-ring = ["quic", "serf-proto/quic-rustls-ring"] +coordinates = ["serf-proto/coordinates"] +tag-regex = ["serf-proto/tag-regex"] + +[dependencies] +serf-proto = { path = "../serf-proto", default-features = false } + +[dev-dependencies] +bytes = { workspace = true } +memberlist-proto = { git = "https://github.com/al8n/memberlist", branch = "main", features = ["std"] } + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] diff --git a/serf-driver/src/error/mod.rs b/serf-driver/src/error/mod.rs new file mode 100644 index 00000000..41588617 --- /dev/null +++ b/serf-driver/src/error/mod.rs @@ -0,0 +1,138 @@ +//! Common error payload structs shared by the serf async driver crates. + +/// Payload for the gossip-MTU-too-small error: the configured `gossip_mtu` is +/// below the floor needed to carry the mandatory single-datagram control packets +/// the SWIM layer always emits. Carries the configured value and the required +/// minimum. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GossipMtuTooSmall { + configured: usize, + minimum: usize, +} + +impl GossipMtuTooSmall { + /// Build a new payload from the configured `gossip_mtu` and the minimum. + #[inline] + pub const fn new(configured: usize, minimum: usize) -> Self { + Self { + configured, + minimum, + } + } + + /// The configured `gossip_mtu` that was rejected. + #[must_use] + #[inline] + pub const fn configured(&self) -> usize { + self.configured + } + + /// The required minimum `gossip_mtu` — the floor that fits the mandatory + /// single-datagram control packets the protocol always emits. + #[must_use] + #[inline] + pub const fn minimum(&self) -> usize { + self.minimum + } +} + +impl core::fmt::Display for GossipMtuTooSmall { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!( + f, + "gossip_mtu {} is below the minimum of {} bytes required to carry the mandatory \ + single-datagram control packets the SWIM layer always emits; a smaller gossip_mtu \ + would make normal probes exceed the plaintext gossip ceiling, so peers would reject \ + them and falsely suspect this node", + self.configured, self.minimum, + ) + } +} + +impl std::error::Error for GossipMtuTooSmall {} + +/// Payload for the invalid-option error: an operator-set tuning knob was given a +/// value that would DETERMINISTICALLY break the node — an accept-then-silently-fail +/// configuration the constructor rejects rather than honoring. Carries the knob name +/// and a human-readable reason describing why the value is invalid. +#[derive(Debug, Clone)] +pub struct InvalidOption { + option: &'static str, + reason: String, +} + +impl InvalidOption { + /// Build a new payload from the rejected knob name and the reason. + #[inline] + pub fn new(option: &'static str, reason: String) -> Self { + Self { option, reason } + } + + /// The name of the tuning knob whose value was rejected. + #[must_use] + #[inline] + pub const fn option(&self) -> &'static str { + self.option + } + + /// The reason the value would deterministically break the node. + #[must_use] + #[inline] + pub fn reason(&self) -> &str { + &self.reason + } +} + +impl core::fmt::Display for InvalidOption { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!(f, "invalid {} option: {}", self.option, self.reason) + } +} + +impl std::error::Error for InvalidOption {} + +/// Payload for the join-failed error: a join attempt contacted none of the +/// configured seed nodes successfully. Carries the seed count and a human-readable +/// reason describing why no seed was reachable. +#[derive(Debug, Clone)] +pub struct JoinFailed { + seed_count: usize, + reason: String, +} + +impl JoinFailed { + /// Build a new payload from the seed count and the reason no seed was reached. + #[inline] + pub fn new(seed_count: usize, reason: String) -> Self { + Self { seed_count, reason } + } + + /// The number of seeds that were attempted. + #[must_use] + #[inline] + pub const fn seed_count(&self) -> usize { + self.seed_count + } + + /// The reason no seed was successfully contacted. + #[must_use] + #[inline] + pub fn reason(&self) -> &str { + &self.reason + } +} + +impl core::fmt::Display for JoinFailed { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!( + f, + "join contacted none of {} seed(s): {}", + self.seed_count, self.reason, + ) + } +} + +impl std::error::Error for JoinFailed {} + +#[cfg(test)] +mod tests; diff --git a/serf-driver/src/error/tests.rs b/serf-driver/src/error/tests.rs new file mode 100644 index 00000000..6fe7bc59 --- /dev/null +++ b/serf-driver/src/error/tests.rs @@ -0,0 +1,34 @@ +use super::{GossipMtuTooSmall, InvalidOption, JoinFailed}; + +#[test] +fn gossip_mtu_too_small_fields_and_display() { + let mtu = GossipMtuTooSmall::new(64, 128); + assert_eq!(mtu.configured(), 64); + assert_eq!(mtu.minimum(), 128); + let s = format!("{mtu}"); + assert!(s.contains("64"), "missing configured value: {s}"); + assert!(s.contains("128"), "missing minimum value: {s}"); + let _: &dyn std::error::Error = &mtu; +} + +#[test] +fn invalid_option_fields_and_display() { + let opt = InvalidOption::new("gossip_interval", "must be nonzero".to_string()); + assert_eq!(opt.option(), "gossip_interval"); + assert_eq!(opt.reason(), "must be nonzero"); + let s = format!("{opt}"); + assert!(s.contains("gossip_interval"), "missing option name: {s}"); + assert!(s.contains("must be nonzero"), "missing reason: {s}"); + let _: &dyn std::error::Error = &opt; +} + +#[test] +fn join_failed_fields_and_display() { + let jf = JoinFailed::new(3, "all seeds unreachable".to_string()); + assert_eq!(jf.seed_count(), 3); + assert_eq!(jf.reason(), "all seeds unreachable"); + let s = format!("{jf}"); + assert!(s.contains("3"), "missing seed count: {s}"); + assert!(s.contains("all seeds unreachable"), "missing reason: {s}"); + let _: &dyn std::error::Error = &jf; +} diff --git a/serf-driver/src/lib.rs b/serf-driver/src/lib.rs new file mode 100644 index 00000000..8cfe401d --- /dev/null +++ b/serf-driver/src/lib.rs @@ -0,0 +1,24 @@ +//! Runtime-agnostic glue shared by the serf async driver crates. +//! +//! A driver binds `serf-proto`'s Sans-I/O endpoint to a real async runtime. This crate holds +//! the runtime-independent pieces both the compio and reactor drivers need — the observable +//! [`SerfSnapshot`] (when a transport feature is enabled), the common driver error payloads, +//! and small pure helpers — so they live in one place instead of being copied per runtime. +//! The run loops, the channel and cell substrate, and the delegate dispatch stay in each +//! runtime crate. + +#![deny(missing_docs)] +#![cfg_attr(docsrs, feature(doc_cfg))] + +pub mod error; +#[cfg(any(feature = "tcp", feature = "quic"))] +mod observation; +#[cfg(any(feature = "tcp", feature = "quic"))] +mod snapshot; + +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub use observation::observation_payload_bytes; +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub use snapshot::SerfSnapshot; diff --git a/serf-driver/src/observation/mod.rs b/serf-driver/src/observation/mod.rs new file mode 100644 index 00000000..a5323747 --- /dev/null +++ b/serf-driver/src/observation/mod.rs @@ -0,0 +1,53 @@ +//! Observation channel byte-backstop accounting shared by the serf async driver crates. + +use serf_proto::{event::Event, members::Member}; + +/// Byte-backstop weight of an event for the observation channel queue. +/// +/// The byte cap exists to bound the queue against **application flood vectors** — events whose +/// volume and size an application or peer controls — so those are weighted by their full +/// heap-bearing payload: +/// - `User` / `Query`: event name + payload bytes (serf bounds these against a `name + payload` +/// size limit, so a large name with an empty payload still counts). +/// - `QueryResponse`: payload bytes. +/// - `Member`: total tag key+value byte length across the affected members (peer tag maps are +/// decoded from node metadata and a peer can spam meta updates). +/// +/// Every other variant returns `None` — it is bounded by the event-**count** cap alone, not the +/// byte cap. That covers control/acknowledgement signals (`Shutdown`, `LeftCluster`, `QueryAck`, +/// `RelayDropped`, `DialRequested`), the admin-triggered, cluster-size-bounded key-management +/// results (`KeyResponse` / `KeyRequest`), and any future `#[non_exhaustive]` variant. Key +/// management is matched by the wildcard rather than a feature-gated arm on purpose: a gated arm +/// would depend on serf-driver's own encryption features, which can skew from serf-proto's and +/// silently drop the arm. +#[must_use] +pub fn observation_payload_bytes(ev: &Event) -> Option { + match ev { + Event::User(u) => Some((u.name.len() as u64).saturating_add(u.payload.len() as u64)), + Event::Query(q) => Some((q.name().len() as u64).saturating_add(q.payload().len() as u64)), + Event::QueryResponse(r) => Some(r.payload().len() as u64), + Event::Member(me) => Some(member_tag_bytes(me.members())), + // Bounded by the event-count cap (no large application-controlled payload). + Event::QueryAck(_) + | Event::Shutdown + | Event::RelayDropped(_) + | Event::LeftCluster + | Event::DialRequested(_) => None, + // Key-management results/requests (admin-triggered, cluster-bounded) and any future variant. + _ => None, + } +} + +/// Total tag key+value byte length across `members` — the heap weight a `Member` event +/// contributes to the observation byte-backstop. Tag maps are decoded from peer node metadata +/// and can grow up to the wire meta ceiling under large tag sets. +pub(crate) fn member_tag_bytes(members: &[Member]) -> u64 { + members + .iter() + .flat_map(|m| m.tags().0.iter()) + .map(|(k, v)| (k.len() + v.len()) as u64) + .sum() +} + +#[cfg(test)] +mod tests; diff --git a/serf-driver/src/observation/tests.rs b/serf-driver/src/observation/tests.rs new file mode 100644 index 00000000..5c2dc41a --- /dev/null +++ b/serf-driver/src/observation/tests.rs @@ -0,0 +1,69 @@ +use bytes::Bytes; +use serf_proto::{LamportTime, UserEventMessage, event::Event}; + +use super::observation_payload_bytes; + +fn user_event(name: &str, payload: Bytes) -> Event { + Event::User(UserEventMessage { + ltime: LamportTime::new(1), + cc: false, + name: name.into(), + payload, + }) +} + +#[test] +fn user_event_charges_name_and_payload() { + let ev = user_event("evt", Bytes::from_static(b"hello world")); + // name "evt" (3) + payload "hello world" (11). + assert_eq!(observation_payload_bytes(&ev), Some(14)); +} + +#[test] +fn user_event_with_large_name_and_empty_payload_is_charged() { + // The name is part of the size-limited application event, so a large name with an empty + // payload must not report Some(0) and slip past the byte cap. + let name = "a-very-long-user-event-name"; + let ev = user_event(name, Bytes::new()); + assert_eq!(observation_payload_bytes(&ev), Some(name.len() as u64)); +} + +#[test] +fn shutdown_is_none() { + let ev: Event = Event::Shutdown; + assert_eq!(observation_payload_bytes(&ev), None); +} + +#[test] +fn left_cluster_is_none() { + let ev: Event = Event::LeftCluster; + assert_eq!(observation_payload_bytes(&ev), None); +} + +/// `member_tag_bytes` — the weight charged for a `Member` event — sums the tag key+value byte +/// lengths across all members, so large peer tag maps count against the byte-backstop. +#[test] +fn member_tag_bytes_sums_tag_key_value_lengths() { + use memberlist_proto::Node; + use serf_proto::{ + Tags, + members::{Member, MemberStatus}, + }; + + let tags: Tags = [("role", "leader"), ("dc", "us-east-1")] + .into_iter() + .collect(); + let expected: u64 = ("role".len() + "leader".len() + "dc".len() + "us-east-1".len()) as u64; + let node = Node::new( + 1u64, + "127.0.0.1:7946".parse::().unwrap(), + ); + let member = Member::new(node, tags, MemberStatus::Alive); + + assert_eq!( + super::member_tag_bytes(std::slice::from_ref(&member)), + expected + ); + let empty: [Member; 0] = []; + assert_eq!(super::member_tag_bytes(&empty), 0); +} diff --git a/serf-driver/src/snapshot/mod.rs b/serf-driver/src/snapshot/mod.rs new file mode 100644 index 00000000..938008c3 --- /dev/null +++ b/serf-driver/src/snapshot/mod.rs @@ -0,0 +1,201 @@ +//! [`SerfSnapshot`] — the observable membership view both serf driver crates publish. + +use std::sync::Arc; + +use serf_proto::{ + LamportTime, + members::{Member, MemberStatus, SerfState}, +}; + +/// An immutable snapshot of the serf cluster's observable membership at one instant. +/// +/// A driver republishes this after every membership change; a `Serf` handle reads the +/// latest one with no coordination. It carries member identity, tags, status, the local +/// endpoint's lifecycle state, and the three Lamport clocks the serf protocol uses to +/// order events across the cluster. Application payloads (`User` events, `Query` / +/// `QueryResponse`) are delivered through the observation channel, not retained here. +/// +/// Generic over the wire id / address types ``, mirroring the underlying +/// [`Member`]; a handle instantiates it as `SerfSnapshot`. +#[derive(Debug, Clone)] +pub struct SerfSnapshot { + members: Vec>>, + /// Index into `members` of the local node's entry. + /// + /// Storing an index rather than a separate `Arc` guarantees that `local()` / + /// `local_ref()` always return the exact same `Arc` that lives in `members`, + /// so the local handle can never contradict the member view. + local_index: usize, + state: SerfState, + member_clock: LamportTime, + event_clock: LamportTime, + query_clock: LamportTime, + alive_count: usize, + member_count: usize, +} + +impl SerfSnapshot { + /// Builds a snapshot from the membership view. Called by a driver each time it republishes. + /// + /// `local_id` identifies the local node; the index into `members` is derived by + /// finding the first member whose id equals `local_id`. `local()` / `local_ref()` + /// always return the element already in `members` and can never contradict the view. + /// + /// # Panics + /// + /// Panics if no member in `members` has an id equal to `local_id`. The local node + /// is always a member of its own cluster view; a driver that omits it has a logic + /// error, and failing at construction makes the invariant violation visible immediately + /// rather than allowing a silent mismatch to propagate to callers of `local()`. + /// + /// `member_count` and `alive_count` are derived from `members` and can never + /// contradict the view. + #[must_use] + pub fn new( + members: Vec>>, + local_id: &I, + state: SerfState, + member_clock: LamportTime, + event_clock: LamportTime, + query_clock: LamportTime, + ) -> Self + where + I: PartialEq, + { + let local_index = members + .iter() + .position(|m| m.node().id_ref() == local_id) + .expect("local node must be present in members"); + let member_count = members.len(); + let alive_count = members + .iter() + .filter(|m| m.status() == MemberStatus::Alive) + .count(); + Self { + members, + local_index, + state, + member_clock, + event_clock, + query_clock, + alive_count, + member_count, + } + } + + /// All known members (full [`Member`], carrying tags and status) — alive, leaving, left, + /// and failed within the reap window. + #[must_use] + pub fn members(&self) -> &[Arc>] { + self.members.as_slice() + } + + /// All known members. Alias for [`Self::members`]. + #[must_use] + pub fn members_slice(&self) -> &[Arc>] { + self.members.as_slice() + } + + /// The local node's full [`Member`] view (cheap Arc clone). + /// + /// Returns a clone of the same `Arc` that lives at `local_index` in `members()`, + /// so this is always consistent with the member view. + /// For a borrowed view use [`Self::local_ref`]. + #[must_use] + pub fn local(&self) -> Arc> { + Arc::clone(&self.members[self.local_index]) + } + + /// Borrow the local node's full [`Member`] view. + /// + /// Returns a reference to the `Arc` at `local_index` in `members()`. + #[must_use] + pub fn local_ref(&self) -> &Arc> { + &self.members[self.local_index] + } + + /// The lifecycle state of the local serf endpoint at the instant the snapshot was taken. + #[must_use] + pub const fn state(&self) -> SerfState { + self.state + } + + /// The member Lamport clock at the time of the snapshot. + #[must_use] + pub const fn member_clock(&self) -> LamportTime { + self.member_clock + } + + /// The event Lamport clock at the time of the snapshot. + #[must_use] + pub const fn event_clock(&self) -> LamportTime { + self.event_clock + } + + /// The query Lamport clock at the time of the snapshot. + #[must_use] + pub const fn query_clock(&self) -> LamportTime { + self.query_clock + } + + /// The count of members currently in the alive state. + #[must_use] + pub const fn alive_count(&self) -> usize { + self.alive_count + } + + /// The count of all known members (alive + leaving + left + failed). + #[must_use] + pub const fn member_count(&self) -> usize { + self.member_count + } + + /// The count of all known members. Alias for [`Self::member_count`]. + #[must_use] + pub const fn num_members(&self) -> usize { + self.member_count + } + + /// Look up a member by id. Returns `None` if the id is not in the snapshot. + #[must_use] + #[inline] + pub fn by_id(&self, id: &I) -> Option<&Arc>> + where + I: PartialEq, + { + self.members.iter().find(|m| m.node().id_ref() == id) + } + + /// Iterate members currently in the alive state. + #[inline] + pub fn online_members(&self) -> impl Iterator>> { + self + .members + .iter() + .filter(|m| m.status() == MemberStatus::Alive) + } + + /// Iterate members matching `pred`. + #[inline] + pub fn members_by<'a>( + &'a self, + mut pred: impl FnMut(&Member) -> bool + 'a, + ) -> impl Iterator>> { + self.members.iter().filter(move |m| pred(m)) + } + + /// Count members matching `pred`. + #[inline] + pub fn num_members_by(&self, mut pred: impl FnMut(&Member) -> bool) -> usize { + self.members.iter().filter(|m| pred(m)).count() + } + + /// Map-filter members, collecting all `Some` results into a `Vec`. + #[inline] + pub fn members_map_by(&self, mut f: impl FnMut(&Member) -> Option) -> Vec { + self.members.iter().filter_map(|m| f(m)).collect() + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-driver/src/snapshot/tests.rs b/serf-driver/src/snapshot/tests.rs new file mode 100644 index 00000000..212c41be --- /dev/null +++ b/serf-driver/src/snapshot/tests.rs @@ -0,0 +1,217 @@ +use std::{net::SocketAddr, sync::Arc}; + +use memberlist_proto::Node; +use serf_proto::{ + LamportTime, + members::{Member, MemberStatus, SerfState}, +}; + +use super::SerfSnapshot; + +fn make_member(id: u32, addr: &str, status: MemberStatus) -> Arc> { + let node = Node::new(id, addr.parse::().unwrap()); + Arc::new(Member::new(node, Default::default(), status)) +} + +#[test] +fn snapshot_counts() { + let alice = make_member(1, "127.0.0.1:7946", MemberStatus::Alive); + let bob = make_member(2, "127.0.0.1:7947", MemberStatus::Alive); + let carol = make_member(3, "127.0.0.1:7948", MemberStatus::Left); + + let snap = SerfSnapshot::new( + vec![alice, bob, carol], + &1u32, + SerfState::Alive, + LamportTime::new(10), + LamportTime::new(20), + LamportTime::new(30), + ); + + // Counts are derived from the member view, so they always agree with it. + assert_eq!(snap.member_count(), 3); + assert_eq!(snap.num_members(), 3); + assert_eq!(snap.alive_count(), 2); + assert_eq!(snap.alive_count(), snap.online_members().count()); + assert_eq!(snap.member_count(), snap.members().len()); +} + +#[test] +fn snapshot_by_id() { + let alice = make_member(1, "127.0.0.1:7946", MemberStatus::Alive); + let bob = make_member(2, "127.0.0.1:7947", MemberStatus::Alive); + let carol = make_member(3, "127.0.0.1:7948", MemberStatus::Left); + + let snap = SerfSnapshot::new( + vec![alice, bob, carol], + &1u32, + SerfState::Alive, + LamportTime::new(1), + LamportTime::new(2), + LamportTime::new(3), + ); + + assert!(snap.by_id(&1).is_some()); + assert!(snap.by_id(&2).is_some()); + assert!(snap.by_id(&3).is_some()); + assert!(snap.by_id(&99).is_none()); +} + +#[test] +fn snapshot_online_members() { + let alice = make_member(1, "127.0.0.1:7946", MemberStatus::Alive); + let bob = make_member(2, "127.0.0.1:7947", MemberStatus::Alive); + let carol = make_member(3, "127.0.0.1:7948", MemberStatus::Left); + + let snap = SerfSnapshot::new( + vec![alice, bob, carol], + &1u32, + SerfState::Alive, + LamportTime::new(5), + LamportTime::new(6), + LamportTime::new(7), + ); + + let online: Vec<_> = snap.online_members().collect(); + assert_eq!(online.len(), 2); +} + +#[test] +fn snapshot_clock_accessors() { + let alice = make_member(1, "127.0.0.1:7946", MemberStatus::Alive); + + let member_clock = LamportTime::new(100); + let event_clock = LamportTime::new(200); + let query_clock = LamportTime::new(300); + + let snap = SerfSnapshot::new( + vec![alice], + &1u32, + SerfState::Leaving, + member_clock, + event_clock, + query_clock, + ); + + assert_eq!(snap.member_clock(), member_clock); + assert_eq!(snap.event_clock(), event_clock); + assert_eq!(snap.query_clock(), query_clock); + assert_eq!(snap.state(), SerfState::Leaving); +} + +#[test] +fn snapshot_members_by_and_num_members_by() { + let alice = make_member(1, "127.0.0.1:7946", MemberStatus::Alive); + let bob = make_member(2, "127.0.0.1:7947", MemberStatus::Alive); + let carol = make_member(3, "127.0.0.1:7948", MemberStatus::Left); + + let snap = SerfSnapshot::new( + vec![alice, bob, carol], + &1u32, + SerfState::Alive, + LamportTime::ZERO, + LamportTime::ZERO, + LamportTime::ZERO, + ); + + let alive_count = snap.num_members_by(|m| m.status() == MemberStatus::Alive); + assert_eq!(alive_count, 2); + + let alive_members: Vec<_> = snap + .members_by(|m| m.status() == MemberStatus::Alive) + .collect(); + assert_eq!(alive_members.len(), 2); +} + +#[test] +fn snapshot_local_and_members_slice() { + let alice = make_member(1, "127.0.0.1:7946", MemberStatus::Alive); + + let snap = SerfSnapshot::new( + vec![alice], + &1u32, + SerfState::Alive, + LamportTime::ZERO, + LamportTime::ZERO, + LamportTime::ZERO, + ); + + assert_eq!(snap.local().node().id_ref(), &1u32); + assert_eq!(snap.local_ref().node().id_ref(), &1u32); + assert_eq!(snap.members().len(), 1); + assert_eq!(snap.members_slice().len(), 1); +} + +#[test] +fn snapshot_members_map_by() { + let alice = make_member(1, "127.0.0.1:7946", MemberStatus::Alive); + let bob = make_member(2, "127.0.0.1:7947", MemberStatus::Failed); + + let snap = SerfSnapshot::new( + vec![alice, bob], + &1u32, + SerfState::Alive, + LamportTime::ZERO, + LamportTime::ZERO, + LamportTime::ZERO, + ); + + let ids = snap.members_map_by(|m| Some(*m.node().id_ref())); + assert_eq!(ids.len(), 2); +} + +/// `local()` / `local_ref()` must return the exact same `Arc` that lives at +/// `local_index` in `members()` — no independent copy that could diverge. +#[test] +fn local_is_indexed_member() { + let alice = make_member(1, "127.0.0.1:7946", MemberStatus::Alive); + let bob = make_member(2, "127.0.0.1:7947", MemberStatus::Alive); + + // local is alice (id 1), found at index 0. + let snap = SerfSnapshot::new( + vec![alice, bob], + &1u32, + SerfState::Alive, + LamportTime::ZERO, + LamportTime::ZERO, + LamportTime::ZERO, + ); + + // `local()` is a clone of the same Arc, so ptr_eq holds. + assert!(Arc::ptr_eq(&snap.local(), &snap.members()[0])); + assert_eq!(snap.local().node().id_ref(), &1u32); + + // local is bob (id 2), found at index 1. + let snap2 = SerfSnapshot::new( + vec![ + make_member(1, "127.0.0.1:7946", MemberStatus::Alive), + make_member(2, "127.0.0.1:7947", MemberStatus::Alive), + ], + &2u32, + SerfState::Alive, + LamportTime::ZERO, + LamportTime::ZERO, + LamportTime::ZERO, + ); + assert!(Arc::ptr_eq(&snap2.local(), &snap2.members()[1])); + assert_eq!(snap2.local().node().id_ref(), &2u32); +} + +/// `new` must panic at construction time when `local_id` is not present in `members`, +/// making the driver invariant violation visible immediately rather than returning a +/// snapshot that would panic (or silently misbehave) later on `local()`. +#[test] +#[should_panic(expected = "local node must be present in members")] +fn new_panics_when_local_id_absent() { + let alice = make_member(1, "127.0.0.1:7946", MemberStatus::Alive); + let bob = make_member(2, "127.0.0.1:7947", MemberStatus::Alive); + // id 99 is not present — must panic. + let _ = SerfSnapshot::new( + vec![alice, bob], + &99u32, + SerfState::Alive, + LamportTime::ZERO, + LamportTime::ZERO, + LamportTime::ZERO, + ); +} From 9cccf6128c6afc2a5a5cfa86bd46b74dcd807301 Mon Sep 17 00:00:00 2001 From: al8n Date: Sat, 27 Jun 2026 23:52:18 +1200 Subject: [PATCH 020/128] feat(serf-proto): additive endpoint forwarders for the driver layer Expose the Sans-I/O accessors a runtime driver needs to pump the StreamEndpoint and QuicEndpoint super-machines: start_scheduling, start_push_pull, handle_message, gossip_mtu, max_stream_frame_size, local_id, members_snapshot, and encrypt_gossip/decrypt_gossip, plus members_snapshot on Endpoint. FSM and wire behavior are unchanged. --- serf-proto/src/endpoint/mod.rs | 16 ++++ serf-proto/src/quic_endpoint/mod.rs | 122 +++++++++++++++++++++++++- serf-proto/src/stream_endpoint/mod.rs | 118 ++++++++++++++++++++++++- 3 files changed, 251 insertions(+), 5 deletions(-) diff --git a/serf-proto/src/endpoint/mod.rs b/serf-proto/src/endpoint/mod.rs index 90ddf32c..9af46c86 100644 --- a/serf-proto/src/endpoint/mod.rs +++ b/serf-proto/src/endpoint/mod.rs @@ -894,6 +894,22 @@ where pub fn num_members(&self) -> usize { self.members.states.len() } + + /// A snapshot of every tracked member (alive, leaving, left, or failed within + /// the reap window) as owned [`Member`](crate::members::Member) values, for a + /// driver's observable membership view published after each membership change. + pub fn members_snapshot(&self) -> Vec>> + where + I: Clone, + A: Clone, + { + self + .members + .states + .values() + .map(|ms| std::sync::Arc::new(ms.member().clone())) + .collect() + } } // ── poll API (requires full Id + Data bounds for inner delegation) ───────────── diff --git a/serf-proto/src/quic_endpoint/mod.rs b/serf-proto/src/quic_endpoint/mod.rs index 3e3ec73a..8e41a2f2 100644 --- a/serf-proto/src/quic_endpoint/mod.rs +++ b/serf-proto/src/quic_endpoint/mod.rs @@ -31,16 +31,18 @@ use bytes::Bytes; use core::net::SocketAddr; +use std::sync::Arc; + use memberlist_proto::{ - Data, Id, Instant, QuicEndpoint as Coordinator, Rng, SeedableRng, SmallRng, Transmit, - parse_message, + Data, Id, Instant, PushPullKind, QuicEndpoint as Coordinator, Rng, SeedableRng, SmallRng, + Transmit, event::StreamId, parse_message, typed::Message, }; use smol_str::SmolStr; use crate::{ endpoint::{Endpoint, Error, QueryId, QueryParams}, event::{Event, QueryEvent}, - members::SerfState, + members::{Member, SerfState}, options::Options, }; @@ -227,6 +229,120 @@ where self.transport.endpoint_ref().user_broadcast_queue_len() } + // ── driver-owned transport surface (additive forwarders) ──────────────────── + // + // These reach the memberlist QUIC coordinator's already-public driver methods. + // The serf `Reliable` seam deliberately excludes them (scheduling / outbound- + // dial / wire-sizing / membership read), so the per-runtime QUIC driver + // forwards through here rather than naming the coordinator directly. + + /// Arm the coordinator's periodic probe / gossip / push-pull schedulers. + /// + /// The driver calls this once at loop entry; without it the coordinator's + /// `next_probe` / `next_gossip` / `next_pushpull` stay unset and failure + /// detection, dissemination, and anti-entropy never run. + pub fn start_scheduling(&mut self, now: Instant) { + self.transport.start_scheduling(now); + } + + /// Initiate an outbound push-pull dial to `peer`, then sieve the resulting + /// inner events into serf. + /// + /// The driver owns the inner-memberlist join: serf's [`Self::join`] only + /// announces the local join intent, while contacting each seed is a + /// driver-issued push-pull through the coordinator. Returns the coordinator's + /// [`StreamId`] for the dial; the QUIC coordinator services the dial and + /// flushes its outbound queue in-band, so the handshake packets surface on the + /// next [`Self::poll_transmit`]. + pub fn start_push_pull( + &mut self, + peer: SocketAddr, + kind: PushPullKind, + now: Instant, + ) -> StreamId { + let id = self.transport.start_push_pull(peer, kind, now); + self.core.drain_after_ingress(&mut self.transport, now); + id + } + + /// Feed one already-decoded gossip [`Message`] into the coordinator, then + /// sieve the resulting inner events into serf. + /// + /// The compound-aware counterpart to [`Self::handle_packet`]: a codec-owning + /// driver that has split a compound datagram into individual messages feeds + /// each typed message here, skipping the per-call single-message + /// `parse_message` that `handle_packet` performs. + pub fn handle_message(&mut self, from: SocketAddr, msg: Message, now: Instant) { + self.transport.handle_packet(from, msg, now); + self.core.drain_after_ingress(&mut self.transport, now); + } + + /// The coordinator's configured gossip MTU — the driver sizes its UDP recv + /// buffer and bounds inbound transform stripping by this value. + pub fn gossip_mtu(&self) -> usize { + self.transport.gossip_mtu() + } + + /// Encrypt one outbound gossip datagram for the wire, applying the + /// coordinator's configured encryption keyring. + /// + /// Forwards to [`memberlist_proto::QuicEndpoint::encrypt_gossip`]. The + /// codec-owning driver calls this on the label-framed gossip bytes before + /// handing them to the UDP socket; when no keyring is configured the bytes are + /// returned unchanged. Returns `Err` when encryption is configured but the + /// backend rejects the request — the driver MUST drop the datagram rather than + /// emit plaintext on an encrypted-cluster path. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn encrypt_gossip( + &self, + datagram: &[u8], + ) -> Result, memberlist_proto::EncryptionError> { + self.transport.encrypt_gossip(datagram) + } + + /// Decrypt and unwrap one inbound gossip datagram, reversing the wire + /// transform stack the peer applied before decoding. + /// + /// Forwards to [`memberlist_proto::QuicEndpoint::decrypt_gossip`]. The + /// codec-owning driver calls this on the raw bytes from + /// [`Self::poll_memberlist_ingress`] before stripping the cluster label; a + /// datagram with no encryption wrapper is returned unchanged when no keyring + /// is configured, and a frame the keyring cannot decrypt is an `Err` (the + /// driver drops it, gossip being lossy and self-healing). + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn decrypt_gossip(&self, datagram: &[u8]) -> Result, memberlist_proto::FrameError> { + self.transport.decrypt_gossip(datagram) + } + + /// The coordinator's maximum reliable-stream frame size — the driver uses it + /// to bound the observation byte-backstop budget. + pub fn max_stream_frame_size(&self) -> usize { + self.transport.max_stream_frame_size() + } + + /// The local node's id (the coordinator's membership-endpoint local id). + pub fn local_id(&self) -> &I { + self.transport.endpoint_ref().local_id_ref() + } + + /// A snapshot of every serf member currently tracked (alive, leaving, left, + /// or failed within the reap window), for the observable membership view a + /// driver publishes after each membership change. + pub fn members_snapshot(&self) -> Vec>> + where + I: Clone, + { + self.core.members_snapshot() + } + // ── serf-logic + serf-command forwarders ──────────────────────────────────── /// Announce the local node's join intent to the cluster. diff --git a/serf-proto/src/stream_endpoint/mod.rs b/serf-proto/src/stream_endpoint/mod.rs index fdb94b94..1f69b0d4 100644 --- a/serf-proto/src/stream_endpoint/mod.rs +++ b/serf-proto/src/stream_endpoint/mod.rs @@ -24,17 +24,22 @@ //! the core's drain over `poll_inner_event`, and never reaches into the stream //! lifecycle itself. +use std::sync::Arc; + use bytes::Bytes; use memberlist_proto::{ - CheapClone, Data, Id, Instant, Rng, SeedableRng, SmallRng, Transmit, parse_message, + CheapClone, Data, Id, Instant, PushPullKind, Rng, SeedableRng, SmallRng, Transmit, + event::StreamId, + parse_message, streams::{ExchangeId, StreamAction, StreamEndpoint as Coordinator, StreamTransport}, + typed::Message, }; use smol_str::SmolStr; use crate::{ endpoint::{Endpoint, Error, QueryId, QueryParams}, event::{Event, QueryEvent}, - members::SerfState, + members::{Member, SerfState}, options::Options, }; @@ -265,6 +270,115 @@ where self.transport.endpoint_ref().user_broadcast_queue_len() } + // ── driver-owned transport surface (additive forwarders) ──────────────────── + // + // These reach the memberlist coordinator's already-public driver methods. The + // serf `Reliable` seam deliberately excludes them (scheduling / outbound-dial / + // wire-sizing / membership read), so the per-runtime driver forwards through + // here rather than naming the coordinator directly. + + /// Arm the coordinator's periodic probe / gossip / push-pull schedulers. + /// + /// The driver calls this once at loop entry; without it the coordinator's + /// `next_probe` / `next_gossip` / `next_pushpull` stay unset and failure + /// detection, dissemination, and anti-entropy never run. + pub fn start_scheduling(&mut self, now: Instant) { + self.transport.start_scheduling(now); + } + + /// Initiate an outbound push-pull dial to `peer`, then sieve the resulting + /// inner events into serf. + /// + /// The driver owns the inner-memberlist join: serf's [`Self::join`] only + /// announces the local join intent, while contacting each seed is a + /// driver-issued push-pull through the coordinator. Returns the + /// coordinator's [`StreamId`] for the dial; the queued `Connect` surfaces on + /// the next [`Self::poll_action`]. + pub fn start_push_pull(&mut self, peer: A, kind: PushPullKind, now: Instant) -> StreamId { + let id = self.transport.start_push_pull(peer, kind, now); + self.core.drain_after_ingress(&mut self.transport, now); + id + } + + /// Feed one already-decoded gossip [`Message`] into the coordinator, then + /// sieve the resulting inner events into serf. + /// + /// The compound-aware counterpart to [`Self::handle_packet`]: a codec-owning + /// driver that has split a compound datagram into individual messages feeds + /// each typed message here, skipping the per-call single-message + /// `parse_message` that `handle_packet` performs. + pub fn handle_message(&mut self, from: A, msg: Message, now: Instant) { + self.transport.handle_packet(from, msg, now); + self.core.drain_after_ingress(&mut self.transport, now); + } + + /// The coordinator's configured gossip MTU — the driver sizes its UDP recv + /// buffer and bounds inbound transform stripping by this value. + pub fn gossip_mtu(&self) -> usize { + self.transport.gossip_mtu() + } + + /// Encrypt one outbound gossip datagram for the wire, applying the + /// coordinator's configured encryption keyring. + /// + /// Forwards to [`memberlist_proto::streams::StreamEndpoint::encrypt_gossip`]. + /// The codec-owning driver calls this on the label-framed gossip bytes before + /// handing them to the UDP socket; when no keyring is configured the bytes are + /// returned unchanged. Returns `Err` when encryption is configured but the + /// backend rejects the request — the driver MUST drop the datagram rather than + /// emit plaintext on an encrypted-cluster path. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn encrypt_gossip( + &self, + datagram: &[u8], + ) -> Result, memberlist_proto::EncryptionError> { + self.transport.encrypt_gossip(datagram) + } + + /// Decrypt and unwrap one inbound gossip datagram, reversing the wire + /// transform stack the peer applied before decoding. + /// + /// Forwards to [`memberlist_proto::streams::StreamEndpoint::decrypt_gossip`]. + /// The codec-owning driver calls this on the raw bytes from + /// [`Self::poll_memberlist_ingress`] before stripping the cluster label; a + /// datagram with no encryption wrapper is returned unchanged when no keyring + /// is configured, and a frame the keyring cannot decrypt is an `Err` (the + /// driver drops it, gossip being lossy and self-healing). + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn decrypt_gossip(&self, datagram: &[u8]) -> Result, memberlist_proto::FrameError> { + self.transport.decrypt_gossip(datagram) + } + + /// The coordinator's maximum reliable-stream frame size — the driver uses it + /// to bound the observation byte-backstop budget. + pub fn max_stream_frame_size(&self) -> usize { + self.transport.max_stream_frame_size() + } + + /// The local node's id (the coordinator's membership-endpoint local id). + pub fn local_id(&self) -> &I { + self.transport.endpoint_ref().local_id_ref() + } + + /// A snapshot of every serf member currently tracked (alive, leaving, left, + /// or failed within the reap window), for the observable membership view a + /// driver publishes after each membership change. + pub fn members_snapshot(&self) -> Vec>> + where + I: Clone, + A: Clone, + { + self.core.members_snapshot() + } + // ── serf-logic + serf-command forwarders ──────────────────────────────────── // Each forwards to the matching `Endpoint` method, threading `&mut transport` // through the ones that reach the coordinator (the `Reliable` methods). From 606414bcf44d4b3dd0172a68f6633a31717b4a3c Mon Sep 17 00:00:00 2001 From: al8n Date: Sat, 27 Jun 2026 23:52:19 +1200 Subject: [PATCH 021/128] refactor(serf-driver): drop the unused JoinFailed payload JoinFailed was never wired into a SerfError variant and has no consumer under the dispatch-only Join semantics the drivers use: the Join command is dispatched into the machine, not awaited for a seed-contact result. --- serf-driver/src/error/mod.rs | 43 ---------------------------------- serf-driver/src/error/tests.rs | 13 +--------- 2 files changed, 1 insertion(+), 55 deletions(-) diff --git a/serf-driver/src/error/mod.rs b/serf-driver/src/error/mod.rs index 41588617..973a68d5 100644 --- a/serf-driver/src/error/mod.rs +++ b/serf-driver/src/error/mod.rs @@ -91,48 +91,5 @@ impl core::fmt::Display for InvalidOption { impl std::error::Error for InvalidOption {} -/// Payload for the join-failed error: a join attempt contacted none of the -/// configured seed nodes successfully. Carries the seed count and a human-readable -/// reason describing why no seed was reachable. -#[derive(Debug, Clone)] -pub struct JoinFailed { - seed_count: usize, - reason: String, -} - -impl JoinFailed { - /// Build a new payload from the seed count and the reason no seed was reached. - #[inline] - pub fn new(seed_count: usize, reason: String) -> Self { - Self { seed_count, reason } - } - - /// The number of seeds that were attempted. - #[must_use] - #[inline] - pub const fn seed_count(&self) -> usize { - self.seed_count - } - - /// The reason no seed was successfully contacted. - #[must_use] - #[inline] - pub fn reason(&self) -> &str { - &self.reason - } -} - -impl core::fmt::Display for JoinFailed { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!( - f, - "join contacted none of {} seed(s): {}", - self.seed_count, self.reason, - ) - } -} - -impl std::error::Error for JoinFailed {} - #[cfg(test)] mod tests; diff --git a/serf-driver/src/error/tests.rs b/serf-driver/src/error/tests.rs index 6fe7bc59..11121027 100644 --- a/serf-driver/src/error/tests.rs +++ b/serf-driver/src/error/tests.rs @@ -1,4 +1,4 @@ -use super::{GossipMtuTooSmall, InvalidOption, JoinFailed}; +use super::{GossipMtuTooSmall, InvalidOption}; #[test] fn gossip_mtu_too_small_fields_and_display() { @@ -21,14 +21,3 @@ fn invalid_option_fields_and_display() { assert!(s.contains("must be nonzero"), "missing reason: {s}"); let _: &dyn std::error::Error = &opt; } - -#[test] -fn join_failed_fields_and_display() { - let jf = JoinFailed::new(3, "all seeds unreachable".to_string()); - assert_eq!(jf.seed_count(), 3); - assert_eq!(jf.reason(), "all seeds unreachable"); - let s = format!("{jf}"); - assert!(s.contains("3"), "missing seed count: {s}"); - assert!(s.contains("all seeds unreachable"), "missing reason: {s}"); - let _: &dyn std::error::Error = &jf; -} From d3560cc00f7b5668c30a488a5e8d5c1c2ada4f23 Mon Sep 17 00:00:00 2001 From: al8n Date: Sat, 27 Jun 2026 23:52:19 +1200 Subject: [PATCH 022/128] feat(serf-compio): compio async driver for the serf Sans-I/O machine The compio runtime driver (io_uring on Linux, polling/kqueue on macOS/BSD, IOCP on Windows) for serf's Sans-I/O SWIM machine, mirroring memberlist-compio: TCP, TLS-over-TCP, and QUIC reliable coordinators over the shared UDP gossip plane, gossip encryption (AES-GCM and ChaCha20-Poly1305), DNS/getifs/OS address resolvers, serde and clap option layers, and the Serf handle with observable event and observation drop counters. A single-owner select loop per transport drives the endpoint super-machine and republishes a lock-free snapshot. When a suspicion deadline is already past, the drain reaps the proactor with a zero-timeout poll before re-checking the socket, so a buffered Ack resolves its probe before suspicion fires: a backend-portable empty observation on a completion-based runtime where no synchronous check exists. --- Cargo.toml | 18 + serf-compio/Cargo.toml | 98 ++ serf-compio/build.rs | 16 + serf-compio/src/bridge/mod.rs | 500 ++++++ serf-compio/src/command/mod.rs | 322 ++++ serf-compio/src/command/tests.rs | 128 ++ serf-compio/src/delegate/mod.rs | 242 +++ serf-compio/src/delegate/void.rs | 155 ++ serf-compio/src/driver/mod.rs | 12 + serf-compio/src/driver/options/mod.rs | 588 +++++++ serf-compio/src/driver/options/tests.rs | 415 +++++ serf-compio/src/driver/quic/mod.rs | 1233 ++++++++++++++ serf-compio/src/driver/quic/tests.rs | 267 +++ serf-compio/src/driver/shared/mod.rs | 205 +++ serf-compio/src/driver/shared/tests.rs | 155 ++ serf-compio/src/driver/stream/mod.rs | 1820 +++++++++++++++++++++ serf-compio/src/driver/stream/tests.rs | 147 ++ serf-compio/src/error/mod.rs | 186 +++ serf-compio/src/error/tests.rs | 85 + serf-compio/src/events.rs | 66 + serf-compio/src/lib.rs | 170 ++ serf-compio/src/quic/mod.rs | 356 ++++ serf-compio/src/quic/tests.rs | 569 +++++++ serf-compio/src/resolver/advertise/mod.rs | 81 + serf-compio/src/resolver/dns/mod.rs | 258 +++ serf-compio/src/resolver/dns/tests.rs | 412 +++++ serf-compio/src/resolver/getifs/mod.rs | 170 ++ serf-compio/src/resolver/getifs/tests.rs | 129 ++ serf-compio/src/resolver/mod.rs | 63 + serf-compio/src/resolver/os.rs | 32 + serf-compio/src/resolver/socket_addr.rs | 21 + serf-compio/src/serf/mod.rs | 463 ++++++ serf-compio/src/serf/tests.rs | 481 ++++++ serf-compio/src/snapshot.rs | 14 + serf-compio/src/tcp/mod.rs | 382 +++++ serf-compio/src/tcp/tests.rs | 125 ++ serf-compio/src/tls/mod.rs | 469 ++++++ serf-compio/src/tls/tests.rs | 457 ++++++ serf-compio/src/transport/mod.rs | 211 +++ serf-compio/src/transport/runtime.rs | 97 ++ 40 files changed, 11618 insertions(+) create mode 100644 serf-compio/Cargo.toml create mode 100644 serf-compio/build.rs create mode 100644 serf-compio/src/bridge/mod.rs create mode 100644 serf-compio/src/command/mod.rs create mode 100644 serf-compio/src/command/tests.rs create mode 100644 serf-compio/src/delegate/mod.rs create mode 100644 serf-compio/src/delegate/void.rs create mode 100644 serf-compio/src/driver/mod.rs create mode 100644 serf-compio/src/driver/options/mod.rs create mode 100644 serf-compio/src/driver/options/tests.rs create mode 100644 serf-compio/src/driver/quic/mod.rs create mode 100644 serf-compio/src/driver/quic/tests.rs create mode 100644 serf-compio/src/driver/shared/mod.rs create mode 100644 serf-compio/src/driver/shared/tests.rs create mode 100644 serf-compio/src/driver/stream/mod.rs create mode 100644 serf-compio/src/driver/stream/tests.rs create mode 100644 serf-compio/src/error/mod.rs create mode 100644 serf-compio/src/error/tests.rs create mode 100644 serf-compio/src/events.rs create mode 100644 serf-compio/src/lib.rs create mode 100644 serf-compio/src/quic/mod.rs create mode 100644 serf-compio/src/quic/tests.rs create mode 100644 serf-compio/src/resolver/advertise/mod.rs create mode 100644 serf-compio/src/resolver/dns/mod.rs create mode 100644 serf-compio/src/resolver/dns/tests.rs create mode 100644 serf-compio/src/resolver/getifs/mod.rs create mode 100644 serf-compio/src/resolver/getifs/tests.rs create mode 100644 serf-compio/src/resolver/mod.rs create mode 100644 serf-compio/src/resolver/os.rs create mode 100644 serf-compio/src/resolver/socket_addr.rs create mode 100644 serf-compio/src/serf/mod.rs create mode 100644 serf-compio/src/serf/tests.rs create mode 100644 serf-compio/src/snapshot.rs create mode 100644 serf-compio/src/tcp/mod.rs create mode 100644 serf-compio/src/tcp/tests.rs create mode 100644 serf-compio/src/tls/mod.rs create mode 100644 serf-compio/src/tls/tests.rs create mode 100644 serf-compio/src/transport/mod.rs create mode 100644 serf-compio/src/transport/runtime.rs diff --git a/Cargo.toml b/Cargo.toml index 2a3d514f..c87460f2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ members = [ "serf-proto", "serf-driver", + "serf-compio", ] exclude = ["legacy"] resolver = "3" @@ -23,7 +24,10 @@ bytes = "1" byteorder = "1" derive_more = { version = "2", features = ["full"] } futures = { version = "0.3", default-features = false } +futures-channel = { version = "0.3", default-features = false, features = ["std"] } +futures-util = { version = "0.3", default-features = false } serde = { version = "1", features = ["derive"] } +humantime = "2" humantime-serde = "1" indexmap = "2" thiserror = { version = "2", default-features = false } @@ -32,6 +36,18 @@ regex = "1" smol_str = "0.3" smallvec = "1" rand = "0.10" +tracing = "0.1" +clap = { version = "4", default-features = false, features = ["std", "derive", "env"] } + +# Networking / transport +compio = { version = "0.19.0", default-features = false } +flume = { version = "0.12", default-features = false } +hostaddr = { version = "0.2", default-features = false } +lochan = "0.1" +hickory-proto = { version = "0.26", default-features = false } +resolv-conf = { version = "0.7", default-features = false } +getifs = "0.6" +iprfc = "0.2" arbitrary = { version = "1", default-features = false, features = ["derive"] } quickcheck = { version = "1", default-features = false } @@ -42,3 +58,5 @@ memberlist = { version = "0.8.5", default-features = false } serf-core = { path = "serf-core", version = "0.5", default-features = false } serf = { path = "serf", version = "0.5", default-features = false } +serf-proto = { path = "serf-proto", default-features = false } +serf-driver = { path = "serf-driver", default-features = false } diff --git a/serf-compio/Cargo.toml b/serf-compio/Cargo.toml new file mode 100644 index 00000000..7855a5b8 --- /dev/null +++ b/serf-compio/Cargo.toml @@ -0,0 +1,98 @@ +[package] +name = "serf-compio" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +description = "compio-based async driver for the Sans-I/O serf machine" + +[features] +default = ["tcp", "tag-regex"] +# Plain-TCP reliable coordinator. +tcp = ["serf-proto/tcp", "serf-driver/tcp"] +# TLS-over-TCP reliable coordinator (implies tcp). +tls = ["tcp", "serf-proto/tls", "compio/rustls"] +# QUIC coordinator. +quic = ["serf-proto/quic", "serf-driver/quic"] +quic-rustls-ring = ["quic", "serf-proto/quic-rustls-ring", "serf-driver/quic-rustls-ring"] +# Vivaldi network coordinate estimation. +coordinates = ["serf-proto/coordinates", "serf-driver/coordinates"] +# Encryption backends — forward to serf-proto (key-management messages) and to +# memberlist-proto (the coordinator's gossip-encryption keyring + encrypt/decrypt). +aes-gcm = ["serf-proto/aes-gcm", "memberlist-proto/aes-gcm"] +chacha20-poly1305 = ["serf-proto/chacha20-poly1305", "memberlist-proto/chacha20-poly1305"] +# Regex-backed tag-filter matching. +tag-regex = ["serf-proto/tag-regex", "serf-driver/tag-regex"] +# Emit `tracing` spans around the public driver operations. +tracing = ["dep:tracing"] +# Optional config layering: `serde` adds Serialize/Deserialize; `clap` adds CLI flags. +serde = ["dep:serde", "dep:humantime-serde"] +clap = ["dep:clap", "dep:humantime"] +# DNS resolver support. +dns = ["dep:hickory-proto", "dep:resolv-conf"] +# Auto-detect the advertise address from the host's own interfaces. +getifs = ["dep:getifs", "dep:iprfc"] + +[dependencies] +serf-proto = { path = "../serf-proto", default-features = false } +serf-driver = { path = "../serf-driver", default-features = false } +# Pinned to the same git+branch as serf-proto to guarantee a single copy in the +# resolver; serf-compio uses EncryptionError and the SecretKey type under the +# encryption cfg gate. +memberlist-proto = { git = "https://github.com/al8n/memberlist", branch = "main", default-features = false, features = ["std"] } + +rand = { workspace = true } +smol_str.workspace = true +bytes = { workspace = true } +thiserror.workspace = true +futures-channel.workspace = true +futures-util = { workspace = true, default-features = false, features = [ + "std", + "async-await-macro", +] } +flume = { workspace = true, default-features = false, features = ["async"] } + +# Runtime — unused by C1 foundation types but declared so later chunks +# can add transports without a Cargo.toml edit. +compio = { workspace = true, features = [ + "net", + "runtime", + "macros", + "time", + "io-uring", + "polling", +] } +# Local !Send channels for per-bridge hot paths (declared now, used in later chunks). +lochan.workspace = true +# Address handling (declared now, used in later chunks). +hostaddr = { workspace = true, features = ["std", "smol_str"] } + +tracing = { workspace = true, optional = true } +serde = { workspace = true, optional = true } +humantime-serde = { workspace = true, optional = true } +humantime = { workspace = true, optional = true } +clap = { workspace = true, optional = true } +hickory-proto = { workspace = true, optional = true, default-features = false, features = [ + "std", +] } +resolv-conf = { workspace = true, optional = true } +getifs = { workspace = true, optional = true } +iprfc = { workspace = true, optional = true } + +[dev-dependencies] +# The QUIC two-node loopback smoke test builds a quinn-proto config bundle with a +# self-signed cert and an accept-any verifier (the same shape serf-proto's own +# QUIC super-machine tests use). Only the `quic`-gated test module references +# these; they are unconditional dev-deps so the test crate always resolves. +quinn-proto = { version = "0.11", default-features = false, features = ["rustls-ring"] } +rustls = { version = "0.23", features = ["ring"] } +rustls-pki-types = "1" +rcgen = "0.14" +ring = "0.17" +# Used in the serde option-layer tests. +serde_json = "1" + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] diff --git a/serf-compio/build.rs b/serf-compio/build.rs new file mode 100644 index 00000000..a6e6fc56 --- /dev/null +++ b/serf-compio/build.rs @@ -0,0 +1,16 @@ +//! Emits the aggregate `encryption` cfg when any AEAD cipher backend feature is enabled, +//! so serf-compio code gates on `#[cfg(encryption)]` instead of repeating the full +//! backend list. + +fn any_feature(names: &[&str]) -> bool { + names + .iter() + .any(|name| std::env::var_os(format!("CARGO_FEATURE_{name}")).is_some()) +} + +fn main() { + println!("cargo::rustc-check-cfg=cfg(encryption)"); + if any_feature(&["AES_GCM", "CHACHA20_POLY1305"]) { + println!("cargo::rustc-cfg=encryption"); + } +} diff --git a/serf-compio/src/bridge/mod.rs b/serf-compio/src/bridge/mod.rs new file mode 100644 index 00000000..4ec71116 --- /dev/null +++ b/serf-compio/src/bridge/mod.rs @@ -0,0 +1,500 @@ +//! Per-bridge byte-mover task — owns the negotiated stream after handshake +//! completion and drives it until the driver closes the bridge. +//! +//! Uses [`compio::io::Splittable`] so the recv and send halves are +//! independent: for [`compio::net::TcpStream`] the split is native and +//! lock-free (both halves are `TcpStream` clones); for a TLS stream wrapped +//! via [`compio::io::split`] the halves share an intrinsic `BiLock` because +//! the TLS record layer cannot be touched from two tasks concurrently. +//! +//! ## Push/pull half-close lifecycle +//! +//! The reliable exchange is a one-shot request-response. Each side: +//! +//! 1. Writes its half of the exchange (push request OR pull response). +//! 2. Half-closes its write side (driven by [`BridgeOut::ShutdownWrite`]) +//! so the peer's read side sees FIN — the peer learns "I am done +//! sending on this exchange". +//! 3. Continues reading the peer's bytes on its still-open read half +//! until the peer reciprocates with its own FIN (`read == 0`). +//! 4. Signals EOF to the driver so the coordinator can advance the +//! membership FSM through the peer's reply. +//! 5. Stays alive — accepting any late `Bytes` the coordinator surfaces +//! after observing the peer's EOF (the inbound-server side writes +//! its response AFTER the request EOF, not before) — until the +//! driver sends [`BridgeOut::Close`]. +//! +//! A bridge that exited on peer-EOF would drop the inbound-server's +//! response on the floor: the driver pushes the response into `out_tx` +//! AFTER the coordinator processes the request, but the bridge would +//! already be gone, the out-channel would be disconnected, and +//! `drain_transport_transmits` would silently drop the bytes. +//! +//! ## Flush-then-close ordering +//! +//! `out_rx` is a single channel carrying every [`BridgeOut`] variant in +//! FIFO order. The bridge processes one item per iteration, so every +//! byte the driver queued before a `ShutdownWrite` or `Close` is written +//! to the peer before the close signal fires. Folding a hard abort into +//! this same FIFO would defeat the discard requirement — the abort would +//! queue BEHIND the stale bytes and the bridge would flush them first — +//! so the hard abort rides a SEPARATE out-of-band `cancel_rx` channel +//! that the loop selects on with priority (see below). +//! +//! ## Hard abort (`cancel_rx`) +//! +//! Only a FAILED exchange (`StreamAction::Abort`) aborts: the driver +//! sends an explicit `cancel_tx.send(())`, which resolves `cancel_rx` +//! with `Ok(())`. A single cancel future, hoisted above the loop, resolves +//! ONLY on that explicit `Ok(())`; it is wired into the loop two ways: +//! +//! 1. The `select_biased!` boundary lists the cancel arm FIRST, so an +//! abort resolving while the bridge is between writes wins ahead of +//! both `out_rx` and the read half. +//! 2. Each partial write is RACED against the same cancel future (see +//! [`write_cancellable`]), so an abort that arrives while a write is +//! already in flight drops the write future at its next await point. +//! +//! Either way the bridge breaks immediately and drops its write half +//! WITHOUT draining the `Bytes` still queued in `out_rx` and WITHOUT +//! sending an EOF marker. Those queued bytes are stale (belonging to an +//! exchange the coordinator gave up on) and are discarded with the dropped +//! `out_rx`. The mid-write race is what makes the abort PROMPT even when +//! the peer stopped reading and a write would otherwise block +//! forever — leaking the bridge task and its socket. +//! +//! A graceful `StreamAction::Close` is NOT an abort. It queues +//! `BridgeOut::Close` behind the response `Bytes` and then drops the +//! `BridgeHandle` (and `cancel_tx`), so `cancel_rx` resolves with +//! `Err(Canceled)`. That cancellation must not preempt the FIFO — it +//! winning the bias over already-queued `Bytes`/`Close`, or +//! dropping a write mid-flight, would truncate a clean exchange's final +//! response. So the cancel future maps that `Err` to a future that +//! NEVER resolves: the cancel arm can no longer fire on a graceful Close +//! and no write is ever dropped mid-flight (no partial-write / duplication +//! hazard). The graceful drain+teardown is driven SOLELY by `out_rx`'s +//! FIFO (`Bytes` then `Close`/`Err`). +//! +//! ## Graceful-drain backstop (`close_timeout`) +//! +//! A graceful `StreamAction::Close` has NO remaining cancel path: the driver +//! removed the bridge handle, so it can never send an Abort for this exchange. +//! If the peer sent a valid request+FIN and then STOPPED reading, its receive +//! window collapses to zero and the post-Close drain blocks FOREVER — the +//! machine already emitted Close and forgot the exchange, and the bridge task +//! is detached, so sustained churn accumulates wedged streams and tasks (a leak +//! / DoS vector). To bound it, the drain is a chunked write loop and EACH +//! partial write is raced against a `compio::time::sleep(close_timeout)` that is +//! re-armed FRESH on every chunk. Because progress resets the deadline, this is +//! a NO-PROGRESS (idle) timeout, not a cap on total drain duration: a peer that +//! keeps reading — even slowly, so a large frame takes longer than +//! `close_timeout` overall — advances on every chunk and never trips it. It +//! fires only when a single partial write makes NO progress for the full +//! `close_timeout` (the peer is not reading at all); the bridge then tears down, +//! dropping its write half so the OS RSTs the stuck stream. It never preempts a +//! draining write that is still making progress, or a clean exchange. +//! +//! ## Cancellation model +//! +//! The recv future owns the read buffer by value while it is in flight; +//! if the out arm fires before recv completes, the recv future is dropped +//! (compio cancels the in-flight syscall via `Submit::drop`) and a fresh +//! buffer is allocated on the next loop iteration. The buffer allocation +//! on the cancellation path is deliberate: keeping the recv future pinned +//! across iterations with a stashed buffer requires an unnameable-future +//! slot that adds complexity without affecting correctness. +//! +//! No-data-loss depends on the exchanges being strictly HALF-DUPLEX. Dropping a +//! recv that already completed in the kernel (its CQE consumed bytes off the +//! socket) would lose those bytes — but every exchange kind here is request-then- +//! response, so a recv is only in flight while our out channel is silent, and the +//! late out-messages for a live exchange are `Close` (delivered in read-closed +//! mode, where no recv is in flight) or `Abort` (the exchange already failed, so +//! loss is moot). If a FULL-DUPLEX exchange kind is ever added, its recv must +//! move to a persistent (never-dropped) future like the hoisted accept future, +//! or a completed-but-dropped read will silently lose mid-stream bytes. + +use std::{future::Future, io, time::Duration}; + +use memberlist_proto::Instant; + +use compio::{ + buf::{BufResult, IntoInner, IoBuf}, + io::{AsyncRead, AsyncWrite, util::Splittable}, +}; +use futures_util::{FutureExt, future::FusedFuture, pin_mut, select_biased}; +use lochan::mpsc::{Receiver, Sender}; + +use crate::driver::{ + shared::ExchangeId, + stream::{BridgeBytes, BridgeEof, BridgeError, BridgeInbound, BridgeOut}, +}; + +/// Run the per-bridge byte-mover loop until a [`BridgeOut::Close`] arrives, +/// the driver signals `cancel_rx` (hard abort), the driver drops the +/// out-channel sender, or the stream returns an I/O error. +/// +/// Ownership transfer: +/// - `stream`: the post-handshake stream value (TcpStream or +/// `Split`). +/// - `eid`: the coordinator-allocated handle for tagging inbound events. +/// - `out_rx`: the driver's bytes-or-control FIFO into this bridge. +/// - `cancel_rx`: the driver's out-of-band hard-abort signal. An explicit +/// `cancel_tx.send(())` (the `StreamAction::Abort` arm for a FAILED +/// exchange) resolves the hoisted cancel future with PRIORITY over both +/// `out_rx` and the read half, AND preempts a write already in +/// flight, so the bridge breaks immediately and drops its write half +/// WITHOUT draining any `Bytes` still queued in `out_rx` — those stale +/// bytes are discarded, not written. A graceful-Close handle-drop +/// resolves the cancel receiver with `Err(Canceled)` instead; that is NOT +/// an abort and is mapped to a never-resolving future, so it neither +/// fires the cancel arm nor drops a write mid-flight — the queued `Bytes` +/// then `Close` in `out_rx` flush first. +/// - `inbound_tx`: bytes / eof / error events the driver consumes. +/// - `close_timeout`: no-progress (idle) bound on a drain. A graceful Close has +/// no remaining cancel path, so a peer that stopped reading would wedge the +/// drain forever; if a single partial write makes NO progress for this long +/// the drain is abandoned and the bridge tears down (RST). A peer that keeps +/// reading — even slowly enough that the whole frame outlasts `close_timeout` +/// — resets the deadline on every chunk and never trips it. +/// +/// All `Sender::send` failures are ignored with a justification: if the driver +/// dropped the inbound receiver we are already shutting down and the event can +/// be discarded. +pub(crate) async fn bridge_task( + stream: S, + eid: ExchangeId, + mut out_rx: Receiver, + cancel_rx: futures_channel::oneshot::Receiver<()>, + inbound_tx: Sender, + recv_buf_len: usize, + close_timeout: Duration, +) where + S: Splittable, + S::ReadHalf: AsyncRead, + S::WriteHalf: AsyncWrite, +{ + let (mut read_half, mut write_half) = stream.split(); + let mut write_closed = false; + let mut read_closed = false; + + // Hoist a single cancel future ABOVE the loop that resolves ONLY on an + // explicit abort. An explicit `cancel_tx.send(())` — emitted by + // `StreamAction::Abort` for a FAILED exchange — resolves the cancel + // receiver with `Ok(())`, which both wins the select bias at the boundary + // AND preempts a write already in flight (the write arms race against this + // same future). A graceful `StreamAction::Close` instead drops the + // `BridgeHandle` (and thus `cancel_tx`) AFTER queuing `BridgeOut::Close` + // behind the response `Bytes`, which resolves the cancel receiver with + // `Err(Canceled)`; that cancellation is NOT an abort, so it is mapped + // to a future that NEVER resolves. Mapping that `Err` to `pending()` + // means the cancel future can never win a write race on a graceful close + // (no write is dropped mid-flight — no partial-write / duplication + // hazard); the graceful drain+teardown is driven SOLELY by `out_rx`'s + // FIFO (`Bytes` then `Close`/`Err`). Fusing keeps the future + // `is_terminated()`-safe across the many `select_biased!` re-polls. + let cancel_fut = async { + match cancel_rx.await { + Ok(()) => (), + Err(_) => futures_util::future::pending::<()>().await, + } + } + .fuse(); + pin_mut!(cancel_fut); + + loop { + if read_closed { + // Read-closed mode: peer FIN already observed and the matching + // EOF marker already sent to the driver. The bridge stays alive + // to accept any late `Bytes` the coordinator surfaces (the + // server side writes its response AFTER the request EOF). + // Exits on a hard abort (`cancel_rx`), `Close`, sender drop, or + // the next write error. + let out_fut = out_rx.recv(); + pin_mut!(out_fut); + select_biased! { + // Cancel first: ONLY an explicit abort can resolve this future now + // (a graceful-Close disconnect is mapped to `pending()`), so it + // breaks immediately, dropping the write half WITHOUT writing any + // queued response `Bytes`. The graceful drain+teardown is driven + // SOLELY by the `out_rx` arm's `Close`/`Err` path. + () = &mut cancel_fut => break, + out_msg = out_fut => match out_msg { + Some(BridgeOut::Bytes(bytes)) => { + if write_closed { + // Discard bytes after a half-close — the machine guarantees + // it stops queueing for this exchange after `ShutdownWrite`, + // but the gate is defensive against any in-flight chunk + // that raced past the channel boundary. + continue; + } + // Race each partial write against the cancel future AND a fresh + // `close_timeout`. An explicit abort preempts a write already in + // flight; the idle timeout bounds a post-Close drain whose peer + // stopped reading (it has no remaining cancel path and would + // otherwise block forever, leaking the bridge and socket). + // `cancel_fut` resolves only on a real abort, so a graceful Close + // never drops a progressing write mid-flight — and because progress + // resets the deadline, only a peer making NO progress for the full + // `close_timeout` times out. + match write_cancellable(&mut write_half, bytes, &mut cancel_fut, close_timeout).await { + // Aborted mid-write, or the drain made no progress for + // `close_timeout`: discard and tear down (drop the write half → + // RST), no EOF. + WriteStatus::Aborted | WriteStatus::TimedOut => break, + WriteStatus::Wrote(res) => { + if res.is_err() { + // The io::Error is not surfaced — `handle_transport_error` + // keys only on the exchange id. + // Ignoring Err: shutdown race — driver-side receiver may + // be gone if the driver is tearing down. + let _ = inbound_tx + .send(BridgeInbound::Error(BridgeError { + eid, + received_at: Instant::now(), + })) + .await; + break; + } + } + } + } + Some(BridgeOut::ShutdownWrite) => { + if !write_closed { + // Ignoring Err: a shutdown on a socket the peer FIN'd is + // non-actionable. + let _ = write_half.shutdown().await; + write_closed = true; + } + } + Some(BridgeOut::Close) | None => { + // Graceful teardown (explicit `Close` or the handle-drop + // disconnect). No EOF send here — the read-EOF transition in + // the both-halves-live arm already sent one. Just exit. + break; + } + }, + } + continue; + } + + // Both-halves-live mode: select between a hard abort, out_rx, and read. + let buf = vec![0u8; recv_buf_len]; + let recv_fut = read_half.read(buf).fuse(); + let out_fut = out_rx.recv(); + pin_mut!(recv_fut, out_fut); + + select_biased! { + // Cancel first: ONLY an explicit abort (`StreamAction::Abort` for a + // FAILED exchange) can resolve this future now — a graceful-Close + // disconnect is mapped to `pending()`. It breaks immediately, dropping + // both halves WITHOUT draining any `Bytes` still queued in `out_rx` + // and WITHOUT sending an EOF marker — the driver already removed the + // bridge handle and treats the exchange as failed. The graceful + // drain+teardown is driven SOLELY by the `out_rx` arm's `Close`/`Err` + // path. + () = &mut cancel_fut => break, + out_msg = out_fut => match out_msg { + Some(BridgeOut::Bytes(bytes)) => { + if write_closed { + continue; + } + // Race each partial write against the cancel future AND a fresh + // `close_timeout`. An explicit abort preempts a write already in + // flight; the idle timeout bounds a post-Close drain whose peer + // stopped reading (no remaining cancel path, else it blocks forever, + // leaking the bridge and socket). `cancel_fut` resolves only on a + // real abort, so a graceful Close never drops a progressing write + // mid-flight; progress resets the deadline, so only a peer making NO + // progress for the full `close_timeout` times out. + match write_cancellable(&mut write_half, bytes, &mut cancel_fut, close_timeout).await { + WriteStatus::Aborted | WriteStatus::TimedOut => break, + WriteStatus::Wrote(res) => { + if res.is_err() { + // The io::Error is not surfaced — `handle_transport_error` + // keys only on the exchange id. + // Ignoring Err: shutdown-race reasoning. + let _ = inbound_tx + .send(BridgeInbound::Error(BridgeError { + eid, + received_at: Instant::now(), + })) + .await; + break; + } + } + } + } + Some(BridgeOut::ShutdownWrite) => { + if !write_closed { + // Ignoring Err: a shutdown on a socket the peer FIN'd is + // non-actionable; the read half observes the same condition + // on its next read. + let _ = write_half.shutdown().await; + write_closed = true; + } + } + Some(BridgeOut::Close) => { + // Ignoring Err: shutdown-race reasoning. + let _ = inbound_tx + .send(BridgeInbound::Eof(BridgeEof { + eid, + received_at: Instant::now(), + })) + .await; + break; + } + None => { + // Driver dropped the out-channel sender — treat as clean + // close. + // Ignoring Err: shutdown-race reasoning. + let _ = inbound_tx + .send(BridgeInbound::Eof(BridgeEof { + eid, + received_at: Instant::now(), + })) + .await; + break; + } + }, + recv_msg = recv_fut => { + // Sample `received_at` at the moment the read syscall + // completed — this is the authoritative arrival time the FSM's + // deadline gate compares against, so a successful response + // queued before the exchange deadline is not retroactively + // failed by the driver's later post-deadline `Instant::now()`. + let received_at = Instant::now(); + let BufResult(res, buf) = recv_msg; + match res { + Ok(0) => { + // Peer FIN'd its write side — its half of the exchange is + // complete. Surface a single EOF marker so the coordinator + // advances through `handle_transport_data(.., eof=true)`, + // then flip to read-closed mode (keep the write side alive + // for the local response). + // Ignoring Err: shutdown-race reasoning. + let _ = inbound_tx + .send(BridgeInbound::Eof(BridgeEof { eid, received_at })) + .await; + read_closed = true; + } + Ok(n) => { + // Tight copy into a right-sized Vec; the original `recv_buf_len` + // `buf` drops here. + let bytes = buf[..n].to_vec(); + // Ignoring Err: shutdown-race reasoning. + let _ = inbound_tx + .send(BridgeInbound::Bytes(BridgeBytes { + eid, + bytes, + received_at, + })) + .await; + } + Err(_err) => { + // The io::Error is not surfaced — `handle_transport_error` keys + // only on the exchange id. + // Ignoring Err: shutdown-race reasoning. + let _ = inbound_tx + .send(BridgeInbound::Error(BridgeError { eid, received_at })) + .await; + break; + } + } + } + } + } +} + +/// The result of a [`write_cancellable`] race — one of three outcomes. +enum WriteStatus { + /// An explicit abort preempted the write: the bridge must tear down and + /// discard, writing nothing further. + Aborted, + /// The write completed (or errored); the inner result carries the I/O + /// outcome the caller maps to a `BridgeInbound::Error` on failure. A partial + /// write of zero bytes (no progress possible) is reported here as a + /// [`io::ErrorKind::WriteZero`] error. + Wrote(io::Result<()>), + /// A single partial write made NO progress for the full `close_timeout` (a + /// non-reading peer with no remaining cancel path): the bridge must tear + /// down, dropping its write half so the OS RSTs the stuck stream. A peer that + /// keeps reading — even slowly — resets the deadline on every chunk and never + /// reaches this. + TimedOut, +} + +/// Writes `bytes` fully via a chunked loop, racing EACH partial write against +/// TWO backstops, for one of three outcomes: aborted / written / timed-out. +/// +/// 1. The bridge's hoisted `cancel_fut` — resolves ONLY on an explicit +/// `StreamAction::Abort` (a graceful `StreamAction::Close` maps its +/// handle-drop disconnect to a never-resolving future before this is ever +/// called). Listed FIRST in every iteration so it preempts immediately, even +/// a write stalled mid-frame on a peer that stopped reading. On a graceful +/// close the write always wins this arm and the queued bytes flush — no write +/// is dropped mid-flight and there is no partial-write / duplication hazard. +/// 2. A `compio::time::sleep(close_timeout)` re-armed FRESH on every iteration — +/// the backstop for a post-Close drain that has NO remaining cancel path. +/// After a graceful Close the driver removed the handle, so a non-reading +/// peer could otherwise wedge the drain forever. Because the deadline resets +/// on each partial write, it is a NO-PROGRESS (idle) timeout, not a cap on +/// total write duration: a peer that keeps reading — even slowly, so the +/// whole frame takes longer than `close_timeout` to drain — advances on every +/// chunk and never trips it. It fires only when a single partial write makes +/// NO progress for the full `close_timeout` (a genuinely stalled peer), and +/// the caller then tears the bridge down (dropping the write half → RST). +/// +/// The owned write buffer (`bytes`) is sliced per iteration via +/// [`IoBuf::slice`]; each partial `write` consumes the slice and returns it, +/// from which [`IntoInner`] recovers the `Vec` so the next iteration can +/// re-slice the unwritten tail — no per-chunk reallocation. A partial write of +/// zero bytes cannot make progress and is surfaced as a [`io::ErrorKind::WriteZero`] +/// write error. +async fn write_cancellable( + write_half: &mut W, + mut bytes: Vec, + mut cancel_fut: &mut C, + close_timeout: Duration, +) -> WriteStatus +where + W: AsyncWrite, + C: Future + FusedFuture + Unpin, +{ + let total = bytes.len(); + let mut written = 0; + while written < total { + // Re-arm the deadline FRESH each iteration: progress (a non-empty partial + // write) resets the clock, so this is an idle timeout, not a total-duration + // cap. A slow-but-reading peer advances every chunk and never trips it. + let timeout_fut = compio::time::sleep(close_timeout).fuse(); + pin_mut!(timeout_fut); + let BufResult(res, slice) = select_biased! { + // Explicit abort only (a disconnect was mapped to `pending()`). Listed + // first so it preempts immediately, even a write blocked mid-frame on an + // unresponsive peer, ahead of the timeout backstop. + () = &mut cancel_fut => return WriteStatus::Aborted, + // Backstop: no progress on this partial write for the full + // `close_timeout` (a non-reading peer) → abandon and RST on teardown. + () = timeout_fut => return WriteStatus::TimedOut, + // Write the unwritten tail. `slice` consumes the owned buffer and is + // returned so the next iteration recovers it via `into_inner`. + res = write_half.write(bytes.slice(written..)).fuse() => res, + }; + bytes = slice.into_inner(); + match res { + // A zero-byte write makes no progress; surface it as a write error so the + // caller reports it and tears down, rather than spinning forever. + Ok(0) => { + return WriteStatus::Wrote(Err(io::Error::new( + io::ErrorKind::WriteZero, + "write returned zero bytes", + ))); + } + // Progress: advance and loop with a fresh deadline. + Ok(n) => written += n, + Err(e) => return WriteStatus::Wrote(Err(e)), + } + } + WriteStatus::Wrote(Ok(())) +} diff --git a/serf-compio/src/command/mod.rs b/serf-compio/src/command/mod.rs new file mode 100644 index 00000000..996ee8a7 --- /dev/null +++ b/serf-compio/src/command/mod.rs @@ -0,0 +1,322 @@ +//! Internal command channel — user-facing Serf handle sends; +//! driver task receives and dispatches. + +use crate::error::Result; +use bytes::Bytes; +use futures_channel::oneshot::Sender; +use smol_str::SmolStr; + +#[cfg(any(feature = "tcp", feature = "quic"))] +use memberlist_proto::Instant; +#[cfg(encryption)] +use memberlist_proto::SecretKey; +#[cfg(any(feature = "tcp", feature = "quic"))] +use serf_proto::{ + endpoint::{QueryId, QueryParams}, + event::QueryEvent, + typed::Tags, +}; + +/// Payload for [`Command::Join`]. +pub(crate) struct JoinCmd { + /// Pre-resolved socket addresses of the seed peers to contact. + pub(crate) seeds: Vec, + /// One-shot reply channel delivering the count of seeds the driver dispatched + /// a push-pull to. Join is dispatch-only: the reply reports how many exchanges + /// were initiated, not how many seeds were reached — actual joins surface as + /// membership [`Event`](serf_proto::event::Event)s. + pub(crate) reply: Sender>, +} + +/// Payload for [`Command::Leave`]. +pub(crate) struct LeaveCmd { + /// One-shot reply channel delivering `()` once the graceful leave completes. + pub(crate) reply: Sender>, +} + +/// Payload for [`Command::ForceLeave`]. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) struct ForceLeaveCmd { + /// The node id to force-remove. + pub(crate) id: I, + /// When `true`, the node is pruned immediately rather than after the + /// tombstone timeout — mirrors `Endpoint::force_leave(prune = true)`. + pub(crate) prune: bool, + /// Wall-clock instant passed to the machine's `force_leave` call. + pub(crate) now: Instant, + /// One-shot reply channel for the operation result. + pub(crate) reply: Sender>, +} + +/// Payload for [`Command::UserEvent`]. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) struct UserEventCmd { + /// Event name — must not exceed the configured `max_user_event_size`. + name: SmolStr, + /// Arbitrary application payload bytes. + payload: Bytes, + /// When `true`, the machine deduplicates events with the same name (coalesce mode). + pub(crate) coalesce: bool, + /// One-shot reply channel for the enqueue result. + pub(crate) reply: Sender>, +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +impl UserEventCmd { + /// Construct from event fields and a reply channel. + pub(crate) fn new( + name: SmolStr, + payload: Bytes, + coalesce: bool, + reply: Sender>, + ) -> Self { + Self { + name, + payload, + coalesce, + reply, + } + } + + /// The event name. + pub(crate) fn name(&self) -> &SmolStr { + &self.name + } + + /// The event payload bytes. + pub(crate) const fn payload(&self) -> &Bytes { + &self.payload + } +} + +/// Payload for [`Command::Query`]. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) struct QueryCmd { + /// Query name forwarded to responders. + name: SmolStr, + /// Query payload forwarded to responders. + payload: Bytes, + /// Routing and timing parameters (filters, relay factor, ack, timeout). + pub(crate) params: QueryParams, + /// Wall-clock instant passed to the machine's `query` call. + pub(crate) now: Instant, + /// One-shot reply channel delivering the [`QueryId`] identifying this query. + pub(crate) reply: Sender>, +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +impl QueryCmd { + /// Construct from query fields and a reply channel. + pub(crate) fn new( + name: SmolStr, + payload: Bytes, + params: QueryParams, + now: Instant, + reply: Sender>, + ) -> Self { + Self { + name, + payload, + params, + now, + reply, + } + } + + /// The query name. + pub(crate) fn name(&self) -> &SmolStr { + &self.name + } + + /// The query payload bytes. + pub(crate) const fn payload(&self) -> &Bytes { + &self.payload + } +} + +/// Payload for [`Command::Respond`]. +/// +/// The driver receives a [`serf_proto::event::Event::Query`] carrying a +/// [`QueryEvent`]; the application constructs a `RespondCmd` from that token +/// and its reply bytes, then sends it through the command channel. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) struct RespondCmd { + /// The query token received in `Event::Query` — carries the routing fields + /// (`ltime`, `id`, `from`, `relay_factor`, `deadline`) `Endpoint::respond` + /// needs to dispatch the reply. + pub(crate) token: QueryEvent, + /// The application's response payload bytes. + payload: Bytes, + /// Wall-clock instant at which the driver dispatches the respond call. + pub(crate) now: Instant, + /// One-shot reply channel delivering `Ok(())` on success. + pub(crate) reply: Sender>, +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +impl RespondCmd { + /// Construct from a query token, payload, now instant, and reply channel. + pub(crate) fn new( + token: QueryEvent, + payload: Bytes, + now: Instant, + reply: Sender>, + ) -> Self { + Self { + token, + payload, + now, + reply, + } + } + + /// The response payload bytes. + pub(crate) const fn payload(&self) -> &Bytes { + &self.payload + } +} + +/// Payload for [`Command::SetTags`]. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) struct SetTagsCmd { + /// The new tag map to advertise on the local node. + pub(crate) tags: Tags, + /// One-shot reply channel for the update result. + pub(crate) reply: Sender>, +} + +/// Payload for [`Command::SetEventJoinIgnore`]. +pub(crate) struct SetEventJoinIgnoreCmd { + /// When `true`, the machine suppresses `Event::Member(Join)` events. + pub(crate) ignore: bool, + /// One-shot reply channel (always `Ok(())`; the setter never fails). + pub(crate) reply: Sender>, +} + +/// Payload for [`Command::InstallKey`], [`Command::UseKey`], and +/// [`Command::RemoveKey`]. +/// +/// All three carry a single [`SecretKey`] and reply with the [`QueryId`] of the +/// issued cluster-wide key query. +/// +/// Requires the `aes-gcm` or `chacha20-poly1305` feature. +#[cfg(encryption)] +pub(crate) struct KeyCmd { + /// The secret key to install, promote, or remove. + pub(crate) key: SecretKey, + /// Wall-clock instant passed to the machine's key-op call. + #[cfg(any(feature = "tcp", feature = "quic"))] + pub(crate) now: Instant, + /// One-shot reply channel delivering the [`QueryId`] of the issued query. + #[cfg(any(feature = "tcp", feature = "quic"))] + pub(crate) reply: Sender>, +} + +/// Payload for [`Command::ListKeys`]. +/// +/// Requires the `aes-gcm` or `chacha20-poly1305` feature. +#[cfg(encryption)] +pub(crate) struct ListKeysCmd { + /// Wall-clock instant passed to `Endpoint::list_keys`. + #[cfg(any(feature = "tcp", feature = "quic"))] + pub(crate) now: Instant, + /// One-shot reply channel delivering the [`QueryId`] of the issued query. + #[cfg(any(feature = "tcp", feature = "quic"))] + pub(crate) reply: Sender>, +} + +/// Payload for [`Command::Shutdown`]. +pub(crate) struct ShutdownCmd { + /// One-shot reply channel for the shutdown acknowledgement. + pub(crate) reply: Sender>, +} + +/// Commands sent from the public `Serf` handle to the driver task. +/// +/// `I` is the node-id type; `A` is the resolved peer-address type (typically +/// `std::net::SocketAddr`). All variants are newtype-over-payload-struct. +pub(crate) enum Command { + /// Initiate joins to the given seed peers (addresses already resolved). The + /// reply carries the count of seeds the driver dispatched a push-pull to, not + /// the count reached; actual joins surface as membership events. + Join(JoinCmd), + + /// Begin a graceful leave from the cluster. + Leave(LeaveCmd), + + /// Force-remove a node from the cluster membership. + #[cfg(any(feature = "tcp", feature = "quic"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] + ForceLeave(ForceLeaveCmd), + + /// Broadcast a user-defined event cluster-wide via gossip. + #[cfg(any(feature = "tcp", feature = "quic"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] + UserEvent(UserEventCmd), + + /// Issue a cluster-wide query and collect responses; reply carries the + /// [`QueryId`] identifying the in-flight query. + #[cfg(any(feature = "tcp", feature = "quic"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] + Query(QueryCmd), + + /// Send a response to an inbound query received via `Event::Query`. + #[cfg(any(feature = "tcp", feature = "quic"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] + Respond(RespondCmd), + + /// Update the local node's advertised tags. + #[cfg(any(feature = "tcp", feature = "quic"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] + SetTags(SetTagsCmd), + + /// Enable or disable suppression of member-join events in the driver's + /// observation stream. + SetEventJoinIgnore(SetEventJoinIgnoreCmd), + + /// Issue a cluster-wide install-key query. + /// + /// Requires the `aes-gcm` or `chacha20-poly1305` feature. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + InstallKey(KeyCmd), + + /// Issue a cluster-wide use-key query to promote a key to primary. + /// + /// Requires the `aes-gcm` or `chacha20-poly1305` feature. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + UseKey(KeyCmd), + + /// Issue a cluster-wide remove-key query. + /// + /// Requires the `aes-gcm` or `chacha20-poly1305` feature. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + RemoveKey(KeyCmd), + + /// Issue a cluster-wide list-keys query to enumerate installed keys. + /// + /// Requires the `aes-gcm` or `chacha20-poly1305` feature. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + ListKeys(ListKeysCmd), + + /// Signal the driver task to shut down gracefully. + Shutdown(ShutdownCmd), +} + +#[cfg(test)] +mod tests; diff --git a/serf-compio/src/command/tests.rs b/serf-compio/src/command/tests.rs new file mode 100644 index 00000000..f91fde05 --- /dev/null +++ b/serf-compio/src/command/tests.rs @@ -0,0 +1,128 @@ +use super::*; +use smol_str::SmolStr; + +fn addr() -> std::net::SocketAddr { + "127.0.0.1:5000".parse().unwrap() +} + +fn unit_reply() -> Sender> { + futures_channel::oneshot::channel::>().0 +} + +#[test] +fn join_cmd_carries_seeds_and_reply() { + let (tx, _rx) = futures_channel::oneshot::channel::>(); + let cmd = JoinCmd { + seeds: vec![addr()], + reply: tx, + }; + assert_eq!(cmd.seeds.len(), 1); + assert_eq!(cmd.seeds[0], addr()); + + // The enum variant wraps the payload transparently. + let (tx2, _rx2) = futures_channel::oneshot::channel::>(); + let Command::::Join(c) = Command::Join(JoinCmd { + seeds: vec![addr(), addr()], + reply: tx2, + }) else { + panic!("wrong Command variant"); + }; + assert_eq!(c.seeds.len(), 2); +} + +#[test] +fn leave_cmd_constructs() { + let cmd = Command::::Leave(LeaveCmd { + reply: unit_reply(), + }); + assert!(matches!(cmd, Command::Leave(_))); +} + +#[test] +fn set_event_join_ignore_cmd_round_trips_flag() { + for &flag in &[true, false] { + let cmd = Command::::SetEventJoinIgnore(SetEventJoinIgnoreCmd { + ignore: flag, + reply: unit_reply(), + }); + assert!(matches!(cmd, Command::SetEventJoinIgnore(_))); + if let Command::SetEventJoinIgnore(c) = cmd { + assert_eq!(c.ignore, flag); + } + } +} + +#[test] +fn shutdown_cmd_constructs() { + let cmd = Command::::Shutdown(ShutdownCmd { + reply: unit_reply(), + }); + assert!(matches!(cmd, Command::Shutdown(_))); +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +mod transport_gated { + use super::*; + use bytes::Bytes; + use memberlist_proto::Instant; + use serf_proto::{endpoint::QueryParams, typed::Tags}; + + #[test] + fn force_leave_cmd_round_trips_fields() { + let cmd = Command::::ForceLeave(ForceLeaveCmd { + id: SmolStr::new("node-1"), + prune: true, + now: Instant::now(), + reply: unit_reply(), + }); + assert!(matches!(cmd, Command::ForceLeave(_))); + if let Command::ForceLeave(c) = cmd { + assert_eq!(c.id.as_str(), "node-1"); + assert!(c.prune); + } + } + + #[test] + fn user_event_cmd_round_trips_fields() { + let cmd = UserEventCmd::new( + SmolStr::new("deploy"), + Bytes::from_static(b"v1.2.3"), + true, + unit_reply(), + ); + assert_eq!(cmd.name().as_str(), "deploy"); + assert_eq!(cmd.payload(), &Bytes::from_static(b"v1.2.3")); + assert!(cmd.coalesce); + + let wrapped = Command::::UserEvent(UserEventCmd::new( + SmolStr::new("x"), + Bytes::from_static(b"y"), + false, + unit_reply(), + )); + assert!(matches!(wrapped, Command::UserEvent(_))); + } + + #[test] + fn query_cmd_round_trips_name_and_payload() { + let (tx, _rx) = futures_channel::oneshot::channel::>(); + let cmd: QueryCmd = QueryCmd::new( + SmolStr::new("health"), + Bytes::from_static(b"ping"), + QueryParams::default(), + Instant::now(), + tx, + ); + assert_eq!(cmd.name().as_str(), "health"); + assert_eq!(cmd.payload(), &Bytes::from_static(b"ping")); + } + + #[test] + fn set_tags_cmd_constructs() { + let cmd = Command::::SetTags(SetTagsCmd { + tags: Tags::default(), + reply: unit_reply(), + }); + assert!(matches!(cmd, Command::SetTags(_))); + } +} diff --git a/serf-compio/src/delegate/mod.rs b/serf-compio/src/delegate/mod.rs new file mode 100644 index 00000000..603ca9b2 --- /dev/null +++ b/serf-compio/src/delegate/mod.rs @@ -0,0 +1,242 @@ +//! `Delegate` composite — compio's per-driver observation hook surface for serf. +//! +//! Composes four observation sub-traits (`MemberDelegate` / `UserEventDelegate` +//! / `QueryDelegate` / `KeyringDelegate`) and a join-admission veto +//! (`MergeDelegate`). All observation hooks are `!Send`-tolerant (native +//! `async fn`) — the driver loop owns the delegate and fires hooks on the +//! single `!Send` compio thread. +//! +//! `KeyringDelegate` and `MergeDelegate` are separate from the observation +//! `Delegate` composite: `KeyringDelegate` is sync (keyring ops must not +//! block), and `MergeDelegate` is an async admission veto supplied at +//! construction rather than an observation hook. + +mod void; + +pub use void::{NoopMergeDelegate, VoidDelegate}; + +#[cfg(encryption)] +pub use void::VoidKeyringDelegate; + +#[cfg(any(feature = "tcp", feature = "quic"))] +use std::sync::Arc; + +#[cfg(any(feature = "tcp", feature = "quic"))] +use serf_proto::{event::QueryEvent, members::Member, typed::UserEventMessage}; + +#[cfg(encryption)] +use memberlist_proto::SecretKey; +#[cfg(encryption)] +use serf_proto::KeyResponseArgs; + +/// Async observation hooks for serf membership events. +/// +/// Each method corresponds to one [`serf_proto::event::MemberEventKind`]. +/// Default impls are no-ops; override what the application cares about. +/// +/// `!Send`-tolerant: the driver loop owns the delegate and fires these hooks on +/// the compio thread. +/// +/// Requires a stream or QUIC transport feature (`tcp` or `quic`). +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +#[allow(async_fn_in_trait)] +pub trait MemberDelegate: 'static { + /// Node identifier type. + type Id; + /// Address type. + type Address; + + /// Called when a member joins the cluster. + async fn notify_join(&self, member: Arc>) { + let _ = member; // Unused: default no-op; override to handle. + } + + /// Called when a member gracefully leaves or is reaped as dead. + async fn notify_leave(&self, member: Arc>) { + let _ = member; // Unused: default no-op; override to handle. + } + + /// Called when a member is detected as failed (no graceful leave observed). + async fn notify_failed(&self, member: Arc>) { + let _ = member; // Unused: default no-op; override to handle. + } + + /// Called when a member's tags or metadata are updated. + async fn notify_update(&self, member: Arc>) { + let _ = member; // Unused: default no-op; override to handle. + } + + /// Called when a member is reaped from the membership store (tombstone + /// expired). + async fn notify_reap(&self, member: Arc>) { + let _ = member; // Unused: default no-op; override to handle. + } +} + +/// Async observation hook for cluster-wide user-event broadcasts. +/// +/// `!Send`-tolerant: the driver loop owns the delegate. +/// +/// Requires a stream or QUIC transport feature (`tcp` or `quic`). +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +#[allow(async_fn_in_trait)] +pub trait UserEventDelegate: 'static { + /// Called when a user-event broadcast is received from the cluster. + async fn notify_user_event(&self, event: &UserEventMessage) { + let _ = event; // Unused: default no-op; override to handle. + } +} + +/// Async observation hook for inbound queries. +/// +/// The driver calls `notify_query` when it receives `Event::Query`. The +/// application may respond through the `Serf` handle's `respond` method; +/// the delegate itself does not hold the respond path. +/// +/// `!Send`-tolerant: the driver loop owns the delegate. +/// +/// Requires a stream or QUIC transport feature (`tcp` or `quic`). +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +#[allow(async_fn_in_trait)] +pub trait QueryDelegate: 'static { + /// Node identifier type. + type Id; + /// Address type. + type Address; + + /// Called when an inbound query arrives that the application may respond to. + async fn notify_query(&self, event: &QueryEvent) { + let _ = event; // Unused: default no-op; override to handle. + } +} + +/// compio's per-driver observation hook surface for serf. +/// +/// A type that satisfies `Delegate` implements all three observation sub-traits +/// (`MemberDelegate`, `UserEventDelegate`, `QueryDelegate`) with matching +/// associated types. `!Send`-tolerant: the driver loop owns it and fires all +/// hooks on the compio thread. +/// +/// The keyring delegate (`KeyringDelegate`) and join-admission veto +/// (`MergeDelegate`) are NOT part of this composite — they are supplied +/// separately to the driver constructor. +/// +/// Requires a stream or QUIC transport feature (`tcp` or `quic`). +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub trait Delegate: + MemberDelegate::Id, Address = ::Address> + + UserEventDelegate + + QueryDelegate::Id, Address = ::Address> +{ + /// Node identifier type. + type Id; + /// Address type — always `SocketAddr` in compio. + type Address; +} + +/// Synchronous delegate the driver implements to apply key-management operations +/// to its local keyring. +/// +/// The driver's event loop calls the matching method when it receives a +/// [`serf_proto::event::Event::KeyRequest`] event, then forwards the result to +/// `StreamEndpoint::respond_key` / `QuicEndpoint::respond_key` so the response +/// is routed back to the originating node. +/// +/// All methods are **synchronous and non-blocking**: key storage is the +/// driver's concern. If a key store requires async I/O, shadow the result +/// through pre-computed state the synchronous method can read. +/// +/// Requires the `aes-gcm` or `chacha20-poly1305` feature. +#[cfg(encryption)] +#[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) +)] +pub trait KeyringDelegate: 'static { + /// Install a new key into the keyring. Returns the response args passed to + /// `respond_key`. + fn install(&self, key: SecretKey) -> KeyResponseArgs; + + /// Promote a key to the primary encryption key. Returns the response args + /// passed to `respond_key`. + fn use_key(&self, key: SecretKey) -> KeyResponseArgs; + + /// Remove a key from the keyring. Returns the response args passed to + /// `respond_key`. + fn remove(&self, key: SecretKey) -> KeyResponseArgs; + + /// List all installed keys and the current primary. Returns the response args + /// (with `keys` populated) passed to `respond_key`. + fn list(&self) -> KeyResponseArgs; +} + +/// Async veto hook invoked by the driver on the join path before accepting +/// remote member state from a push-pull exchange. +/// +/// `Ok(())` permits the merge; `Err(Self::Error)` cancels it. The driver wraps +/// the concrete error into [`SerfError`](crate::SerfError) before forwarding it +/// to the join caller. +/// +/// The hook is **async and driver-side** deliberately: the application may need +/// to consult an ACL service or other async resource before deciding whether to +/// accept a batch of remote peers. A synchronous (Sans-I/O) filter would +/// preclude that. +/// +/// `!Send`-agnostic: neither the trait object nor the future returned by +/// `notify_merge` carry a `Send` bound, so compio's `!Send` driver can +/// implement it without wrapping. +/// +/// Requires a stream or QUIC transport feature (`tcp` or `quic`). +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +#[allow(async_fn_in_trait)] +pub trait MergeDelegate: 'static { + /// The veto/error type this delegate reports when a merge is cancelled. + type Error; + + /// Called before the driver accepts inbound push-pull peer state. + /// + /// `peers` is the slice of remote [`Member`]s the cluster is about to merge. + /// Return `Ok(())` to proceed, or `Err(e)` to cancel the merge. + /// + /// The default implementation always permits the merge. + async fn notify_merge(&self, peers: &[Arc>]) -> Result<(), Self::Error> { + let _ = peers; // Unused in the default permit-all impl; an overriding delegate inspects it. + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use smol_str::SmolStr; + use std::net::SocketAddr; + + #[cfg(any(feature = "tcp", feature = "quic"))] + #[test] + fn void_delegate_satisfies_observation_composite() { + fn assert_delegate(_d: &D) + where + D: Delegate, + { + } + let v: VoidDelegate = VoidDelegate::default(); + assert_delegate(&v); + } + + /// Verify `NoopMergeDelegate` satisfies `MergeDelegate` with `Error = + /// Infallible` — a type-level check; no I/O needed. + #[cfg(any(feature = "tcp", feature = "quic"))] + #[test] + fn noop_merge_delegate_satisfies_trait() { + fn assert_merge>( + _: &T, + ) { + } + assert_merge(&NoopMergeDelegate); + } +} diff --git a/serf-compio/src/delegate/void.rs b/serf-compio/src/delegate/void.rs new file mode 100644 index 00000000..8faa6510 --- /dev/null +++ b/serf-compio/src/delegate/void.rs @@ -0,0 +1,155 @@ +//! Zero-cost default delegates — no-op impls of all observation and admission +//! hooks so drivers that do not need them can construct a node without +//! boilerplate. + +use core::marker::PhantomData; + +#[cfg(any(feature = "tcp", feature = "quic"))] +use super::{Delegate, MemberDelegate, MergeDelegate, QueryDelegate, UserEventDelegate}; + +#[cfg(encryption)] +use super::KeyringDelegate; +#[cfg(encryption)] +use serf_proto::KeyResponseArgs; + +/// Zero-cost default observation delegate. Every hook is a no-op. +/// +/// Use when the application does not need to observe membership, user-event, +/// or query notifications. `VoidDelegate` satisfies the +/// [`Delegate`](super::Delegate) composite. +/// +/// Requires a stream or QUIC transport feature (`tcp` or `quic`). +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub struct VoidDelegate { + _phantom: PhantomData, +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +impl VoidDelegate { + /// Construct a `VoidDelegate`. + #[inline] + pub const fn new() -> Self { + Self { + _phantom: PhantomData, + } + } +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +impl Default for VoidDelegate { + fn default() -> Self { + Self::new() + } +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +impl MemberDelegate for VoidDelegate +where + I: 'static, + A: 'static, +{ + type Id = I; + type Address = A; +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +impl UserEventDelegate for VoidDelegate +where + I: 'static, + A: 'static, +{ +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +impl QueryDelegate for VoidDelegate +where + I: 'static, + A: 'static, +{ + type Id = I; + type Address = A; +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +impl Delegate for VoidDelegate +where + I: 'static, + A: 'static, +{ + type Id = I; + type Address = A; +} + +/// A merge delegate that always permits merges. +/// +/// The default delegate for drivers that do not need join admission control. +/// Its associated error type is [`core::convert::Infallible`], reflecting that +/// `notify_merge` can never fail. +/// +/// Requires a stream or QUIC transport feature (`tcp` or `quic`). +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub struct NoopMergeDelegate; + +#[cfg(any(feature = "tcp", feature = "quic"))] +impl MergeDelegate for NoopMergeDelegate +where + I: 'static, + A: 'static, +{ + type Error = core::convert::Infallible; +} + +/// A keyring delegate that manages no key material. +/// +/// The default for nodes that do not participate in key management: every +/// operation reports `result = false` with an explanatory message and an empty +/// key list, so a cluster-wide key query records this node as not key-managed +/// rather than silently timing out. A node that DOES manage encryption keys +/// supplies its own [`KeyringDelegate`](super::KeyringDelegate) bound to the +/// keyring it shares with the transport's encryption configuration. +/// +/// Requires the `aes-gcm` or `chacha20-poly1305` feature. +#[cfg(encryption)] +#[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) +)] +pub struct VoidKeyringDelegate; + +#[cfg(encryption)] +impl VoidKeyringDelegate { + /// The response returned by every operation: not key-managed. + fn unmanaged() -> KeyResponseArgs { + KeyResponseArgs { + result: false, + message: "node has no keyring delegate configured".into(), + keys: Vec::new(), + primary_key: None, + } + } +} + +#[cfg(encryption)] +impl KeyringDelegate for VoidKeyringDelegate { + #[inline] + fn install(&self, _key: memberlist_proto::SecretKey) -> KeyResponseArgs { + Self::unmanaged() + } + + #[inline] + fn use_key(&self, _key: memberlist_proto::SecretKey) -> KeyResponseArgs { + Self::unmanaged() + } + + #[inline] + fn remove(&self, _key: memberlist_proto::SecretKey) -> KeyResponseArgs { + Self::unmanaged() + } + + #[inline] + fn list(&self) -> KeyResponseArgs { + Self::unmanaged() + } +} diff --git a/serf-compio/src/driver/mod.rs b/serf-compio/src/driver/mod.rs new file mode 100644 index 00000000..49bb018f --- /dev/null +++ b/serf-compio/src/driver/mod.rs @@ -0,0 +1,12 @@ +//! The driver layer: shared options and observation helpers. +//! +//! [`shared`] holds the observation / event hand-off helpers used by every +//! transport backend; [`options`] holds the generic-free driver tuning knobs. +//! Per-backend driver loops live in the transport module (e.g. `tcp::run`). + +pub(crate) mod options; +#[cfg(feature = "quic")] +pub(crate) mod quic; +pub(crate) mod shared; +#[cfg(feature = "tcp")] +pub(crate) mod stream; diff --git a/serf-compio/src/driver/options/mod.rs b/serf-compio/src/driver/options/mod.rs new file mode 100644 index 00000000..baebd67b --- /dev/null +++ b/serf-compio/src/driver/options/mod.rs @@ -0,0 +1,588 @@ +//! Per-`Serf` runtime tuning knobs. +//! +//! [`RuntimeOptions`] carries the generic-free runtime knobs; per-backend knobs +//! live on each backend's `*TransportOptions` struct. +//! +//! Each `DEFAULT_*` constant is `pub` so callers can derive new values from the +//! default (e.g. `DEFAULT_LEAVE_TIMEOUT * 3`). + +use core::time::Duration; + +#[cfg(any(feature = "tcp", feature = "quic"))] +use crate::error::{InvalidOption, SerfError}; + +#[cfg(feature = "clap")] +use humantime::parse_duration; + +/// Default per-call deadline for [`Serf::leave`](crate::Serf::leave). +pub const DEFAULT_LEAVE_TIMEOUT: Duration = Duration::from_secs(5); + +/// Default fallback driver-loop sleep when the coordinator's `poll_timeout` +/// returns `None`. +pub const DEFAULT_IDLE_WAKE_INTERVAL: Duration = Duration::from_secs(60); + +/// Default per-iteration drain cap for inbound surfaces. +pub const DEFAULT_ITER_DRAIN_CAP: usize = 256; + +/// Default per-iteration cmd-channel fairness budget. +pub const DEFAULT_CMD_FAIRNESS_BUDGET: usize = 4; + +/// Default events-channel capacity. +pub const DEFAULT_EVENT_QUEUE_CAP: usize = 1024; + +/// Default outbound dial budget for the stream-transport driver. +pub const DEFAULT_DIAL_TIMEOUT: Duration = Duration::from_secs(5); + +/// Default bridge-inbound channel capacity for the stream-transport driver. +pub const DEFAULT_BRIDGE_INBOUND_CAP: usize = 1024; + +/// Default per-bridge TCP read buffer size for the stream-transport driver. +pub const DEFAULT_BRIDGE_RECV_BUF_LEN: usize = 16 * 1024; + +/// Default bound on a per-bridge graceful-drain write for the stream-transport +/// driver: 10 seconds. +pub const DEFAULT_CLOSE_TIMEOUT: Duration = Duration::from_secs(10); + +/// How the per-driver delegate **observation channel** is bounded. +/// +/// The driver hands every machine `Event` to a separate observation task that +/// runs the user [`Delegate`](crate::Delegate) hooks, decoupled from the +/// protocol loop so a slow hook never stalls the FSM. +/// +/// - [`Unbounded`](Self::Unbounded): the hand-off never drops, so the delegate +/// observes every event in order — but a delegate that persistently runs +/// slower than inbound traffic grows the channel without limit. +/// - [`Bounded`](Self::Bounded): caps the channel at `n` queued events. When +/// full, the driver drops the newest event rather than blocking and counts +/// the drop in `observation_dropped`. +/// +/// `Bounded(0)` is rejected at construction: a zero-capacity channel is a +/// rendezvous the driver's non-blocking send can never deposit into. +/// +/// As a config value (serde / CLI) it is open-vocabulary: `Unbounded` is the +/// bare string `"unbounded"`, `Bounded(n)` is `{"bounded": n}` under serde and +/// `bounded:n` on the CLI (via [`FromStr`](core::str::FromStr) / +/// [`Display`](core::fmt::Display)). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] +pub enum Channel { + /// Never drop; the channel grows without bound. + Unbounded, + /// Cap the channel at this many queued events; drop-newest and count in + /// `observation_dropped` when full. + Bounded(usize), +} + +impl core::fmt::Display for Channel { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::Unbounded => f.write_str("unbounded"), + Self::Bounded(n) => write!(f, "bounded:{n}"), + } + } +} + +/// Parse a [`Channel`] from its string form — `"unbounded"` or `"bounded:"`. +/// The inverse of [`Channel`]'s `Display`. +impl core::str::FromStr for Channel { + type Err = ParseChannelError; + + fn from_str(s: &str) -> Result { + if s.eq_ignore_ascii_case("unbounded") { + return Ok(Self::Unbounded); + } + let cap = s + .strip_prefix("bounded:") + .or_else(|| s.strip_prefix("bounded=")) + .ok_or(ParseChannelError(()))?; + let n = cap.parse::().map_err(|_| ParseChannelError(()))?; + Ok(Self::Bounded(n)) + } +} + +/// The error from [`Channel::from_str`]: the input was neither `"unbounded"` +/// nor a `"bounded:"` with a valid capacity. +/// +/// Opaque — the private unit field seals construction to this module. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("invalid observation channel (expected `unbounded` or `bounded:`)")] +pub struct ParseChannelError(()); + +/// Default delegate observation channel: [`Channel::Bounded`] at 1024 events. +pub const DEFAULT_OBSERVATION_CHANNEL: Channel = Channel::Bounded(1024); + +/// Per-`Serf` runtime tuning knobs. Generic-free; per-backend knobs live on +/// each backend's `*TransportOptions` struct. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(default, deny_unknown_fields))] +pub struct RuntimeOptions { + #[cfg_attr(feature = "serde", serde(with = "humantime_serde"))] + leave_timeout: Duration, + #[cfg_attr(feature = "serde", serde(with = "humantime_serde"))] + idle_wake_interval: Duration, + iter_drain_cap: usize, + cmd_fairness_budget: usize, + event_queue_cap: usize, + observation_channel: Channel, +} + +// `clap::Args` is NOT derived on `RuntimeOptions`. The derived +// `update_from_arg_matches` treats every `default_value` / `default_value_t` +// arg as present even when the operator did not pass it, so a partial +// `try_update_from` carrying one unrelated flag would reset every other +// defaulted knob to its default. A private mirror carries the `#[arg(...)]` +// attributes and derives `Args`; `RuntimeOptions` delegates its `Args` / +// `FromArgMatches` to the mirror and, on update, applies ONLY the args whose +// value came from the command line or an env var. +#[cfg(feature = "clap")] +const _: () = { + use clap::{ArgMatches, Args, Command, Error, FromArgMatches, parser::ValueSource}; + + #[derive(Args)] + struct RuntimeOptionsCli { + #[arg( + id = "runtime-leave-timeout", + long = "runtime-leave-timeout", + env = "SERF_RUNTIME_LEAVE_TIMEOUT", + value_parser = parse_duration, + // The humantime spelling of DEFAULT_LEAVE_TIMEOUT. + default_value = "5s", + )] + leave_timeout: Duration, + #[arg( + id = "runtime-idle-wake-interval", + long = "runtime-idle-wake-interval", + env = "SERF_RUNTIME_IDLE_WAKE_INTERVAL", + value_parser = parse_duration, + // The humantime spelling of DEFAULT_IDLE_WAKE_INTERVAL. + default_value = "60s", + )] + idle_wake_interval: Duration, + #[arg( + id = "runtime-iter-drain-cap", + long = "runtime-iter-drain-cap", + env = "SERF_RUNTIME_ITER_DRAIN_CAP", + default_value_t = DEFAULT_ITER_DRAIN_CAP, + )] + iter_drain_cap: usize, + #[arg( + id = "runtime-cmd-fairness-budget", + long = "runtime-cmd-fairness-budget", + env = "SERF_RUNTIME_CMD_FAIRNESS_BUDGET", + default_value_t = DEFAULT_CMD_FAIRNESS_BUDGET, + )] + cmd_fairness_budget: usize, + #[arg( + id = "runtime-event-queue-cap", + long = "runtime-event-queue-cap", + env = "SERF_RUNTIME_EVENT_QUEUE_CAP", + default_value_t = DEFAULT_EVENT_QUEUE_CAP, + )] + event_queue_cap: usize, + #[arg( + id = "runtime-observation-channel", + long = "runtime-observation-channel", + env = "SERF_RUNTIME_OBSERVATION_CHANNEL", + default_value_t = DEFAULT_OBSERVATION_CHANNEL, + )] + observation_channel: Channel, + } + + impl From for RuntimeOptions { + fn from(c: RuntimeOptionsCli) -> Self { + Self { + leave_timeout: c.leave_timeout, + idle_wake_interval: c.idle_wake_interval, + iter_drain_cap: c.iter_drain_cap, + cmd_fairness_budget: c.cmd_fairness_budget, + event_queue_cap: c.event_queue_cap, + observation_channel: c.observation_channel, + } + } + } + + impl Args for RuntimeOptions { + fn augment_args(cmd: Command) -> Command { + RuntimeOptionsCli::augment_args(cmd) + } + + fn augment_args_for_update(cmd: Command) -> Command { + RuntimeOptionsCli::augment_args_for_update(cmd) + } + } + + impl FromArgMatches for RuntimeOptions { + fn from_arg_matches(m: &ArgMatches) -> Result { + RuntimeOptionsCli::from_arg_matches(m).map(Into::into) + } + + fn update_from_arg_matches(&mut self, m: &ArgMatches) -> Result<(), Error> { + // Apply ONLY operator-supplied overrides — args whose value came from the + // command line or an env var, not a clap default. A bare derived update + // treats every `default_value` arg as present and would reset unset fields. + macro_rules! take { + ($id:literal, $field:ident, $ty:ty) => { + if matches!( + m.value_source($id), + Some(ValueSource::CommandLine) | Some(ValueSource::EnvVariable) + ) { + if let Some(v) = m.get_one::<$ty>($id) { + self.$field = *v; + } + } + }; + } + take!("runtime-leave-timeout", leave_timeout, Duration); + take!("runtime-idle-wake-interval", idle_wake_interval, Duration); + take!("runtime-iter-drain-cap", iter_drain_cap, usize); + take!("runtime-cmd-fairness-budget", cmd_fairness_budget, usize); + take!("runtime-event-queue-cap", event_queue_cap, usize); + take!("runtime-observation-channel", observation_channel, Channel); + Ok(()) + } + } +}; + +impl RuntimeOptions { + /// Construct from the canonical base defaults. + #[inline] + pub const fn new() -> Self { + Self { + leave_timeout: DEFAULT_LEAVE_TIMEOUT, + idle_wake_interval: DEFAULT_IDLE_WAKE_INTERVAL, + iter_drain_cap: DEFAULT_ITER_DRAIN_CAP, + cmd_fairness_budget: DEFAULT_CMD_FAIRNESS_BUDGET, + event_queue_cap: DEFAULT_EVENT_QUEUE_CAP, + observation_channel: DEFAULT_OBSERVATION_CHANNEL, + } + } + + /// Builder: per-call deadline for [`Serf::leave`](crate::Serf::leave). + #[must_use] + #[inline] + pub const fn with_leave_timeout(mut self, d: Duration) -> Self { + self.leave_timeout = d; + self + } + + /// Builder: fallback driver-loop sleep when the coordinator has no pending + /// deadline. + #[must_use] + #[inline] + pub const fn with_idle_wake_interval(mut self, d: Duration) -> Self { + self.idle_wake_interval = d; + self + } + + /// Builder: per-iteration drain cap. + #[must_use] + #[inline] + pub const fn with_iter_drain_cap(mut self, n: usize) -> Self { + self.iter_drain_cap = n; + self + } + + /// Builder: per-iteration cmd-channel fairness budget. + #[must_use] + #[inline] + pub const fn with_cmd_fairness_budget(mut self, n: usize) -> Self { + self.cmd_fairness_budget = n; + self + } + + /// Builder: events-channel capacity. + #[must_use] + #[inline] + pub const fn with_event_queue_cap(mut self, n: usize) -> Self { + self.event_queue_cap = n; + self + } + + /// Builder: how the delegate observation channel is bounded. + #[must_use] + #[inline] + pub const fn with_observation_channel(mut self, c: Channel) -> Self { + self.observation_channel = c; + self + } + + /// Per-call deadline for [`Serf::leave`](crate::Serf::leave). + #[inline] + pub const fn leave_timeout(&self) -> Duration { + self.leave_timeout + } + + /// Fallback driver-loop sleep. + #[inline] + pub const fn idle_wake_interval(&self) -> Duration { + self.idle_wake_interval + } + + /// Per-iteration drain cap. + #[inline] + pub const fn iter_drain_cap(&self) -> usize { + self.iter_drain_cap + } + + /// Per-iteration cmd-channel fairness budget. + #[inline] + pub const fn cmd_fairness_budget(&self) -> usize { + self.cmd_fairness_budget + } + + /// Events-channel capacity. + #[inline] + pub const fn event_queue_cap(&self) -> usize { + self.event_queue_cap + } + + /// How the delegate observation channel is bounded. + #[inline] + pub const fn observation_channel(&self) -> Channel { + self.observation_channel + } + + /// Validate the generic-free runtime knobs whose value would DETERMINISTICALLY + /// break (not merely degrade) the driver loop, so the misconfiguration + /// surfaces from [`Serf::new`](crate::Serf::new) — before any socket is bound + /// or the detached driver task is spawned — rather than as a panic, a + /// busy-spin, or a silently dead surface inside that task. Each is rejected + /// (not clamped) so the operator learns and fixes the value; every backend + /// (TCP/TLS/QUIC) routes through the one `Serf::new` path, so this single call + /// covers all three. + /// + /// - `idle_wake_interval == 0`: the driver loop's fallback sleep when the + /// coordinator's `poll_timeout` has no nearer deadline + /// (`poll_timeout().unwrap_or(now + idle_wake_interval)`). Zero makes a + /// quiescent endpoint re-arm a zero-duration timer every pass — a busy-spin + /// that pegs a CPU core. + /// - `cmd_fairness_budget == 0`: the iter-top command fairness drain + /// (`while cmd_drained < cmd_fairness_budget`). The main `select_biased!` + /// biases the network arms ahead of the command arm, so under a continuous + /// inbound flood the command arm is never reached; this drain is the ONLY + /// mechanism that keeps commands progressing under that load, so a zero + /// budget disables it and `shutdown` / `leave` / joins can hang indefinitely. + /// - `event_queue_cap == 0`: the observation task forwards every event to the + /// bounded event-stream channel with a non-blocking `try_send`, and a + /// zero-capacity channel is a rendezvous a non-blocking send can never + /// deposit into, so every event would be dropped and the event-observation + /// surface would be non-functional. + /// - `observation_channel == Channel::Bounded(0)`: each driver builds the + /// delegate observation channel with `lochan::mpsc::bounded`, which panics + /// on a zero capacity, so a `Bounded(0)` channel would panic the detached + /// driver task at startup. + /// + /// `iter_drain_cap == 0` and `leave_timeout == 0` are NOT rejected: the former + /// only caps the per-iteration batch drain (the `select` arms and the uncapped + /// timeout drain still make one-per-pass forward progress), and the latter is a + /// loud immediate [`SerfError::LeaveTimeout`] rather than a silent break. + #[cfg(any(feature = "tcp", feature = "quic"))] + pub(crate) fn validate(&self) -> Result<(), SerfError> { + if self.idle_wake_interval.is_zero() { + return Err(SerfError::InvalidOption(InvalidOption::new( + "idle_wake_interval", + "the driver-loop fallback sleep must be nonzero: the loop sleeps until \ + `poll_timeout().unwrap_or(now + idle_wake_interval)`, so a zero idle_wake_interval \ + makes a quiescent endpoint re-arm a zero-duration timer every pass — a busy-spin \ + that pegs a CPU core" + .to_string(), + ))); + } + if self.cmd_fairness_budget == 0 { + return Err(SerfError::InvalidOption(InvalidOption::new( + "cmd_fairness_budget", + "the iter-top command fairness drain must pull at least one command per pass: the main \ + select biases the network arms ahead of commands, so under a continuous inbound \ + flood the command arm is starved indefinitely and shutdown / leave / joins would \ + never be serviced; a zero budget disables the only drain that guarantees command \ + progress" + .to_string(), + ))); + } + if self.event_queue_cap == 0 { + return Err(SerfError::InvalidOption(InvalidOption::new( + "event_queue_cap", + "the event-stream queue capacity must be nonzero: the observation task forwards every \ + event to this bounded channel with a non-blocking `try_send`, and a zero-capacity \ + channel is a rendezvous a non-blocking send can never deposit into, so every event \ + would be dropped and the event-observation surface would be non-functional" + .to_string(), + ))); + } + if matches!(self.observation_channel, Channel::Bounded(0)) { + return Err(SerfError::InvalidOption(InvalidOption::new( + "observation_channel", + "the delegate observation channel must have a nonzero bounded capacity: each driver \ + builds it with `lochan::mpsc::bounded`, which panics on a zero capacity, so a \ + `Bounded(0)` channel would panic the detached driver task at startup instead of \ + running" + .to_string(), + ))); + } + Ok(()) + } +} + +impl Default for RuntimeOptions { + #[inline] + fn default() -> Self { + Self::new() + } +} + +/// Stream-transport-specific tuning knobs. +/// +/// Apply to the stream-backed (TCP) [`Serf`](crate::Serf) driver. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct StreamTransportOptions { + dial_timeout: Duration, + close_timeout: Duration, + bridge_inbound_cap: usize, + bridge_recv_buf_len: usize, +} + +impl StreamTransportOptions { + /// Construct with the canonical stream-transport defaults. + #[inline] + pub const fn new() -> Self { + Self { + dial_timeout: DEFAULT_DIAL_TIMEOUT, + close_timeout: DEFAULT_CLOSE_TIMEOUT, + bridge_inbound_cap: DEFAULT_BRIDGE_INBOUND_CAP, + bridge_recv_buf_len: DEFAULT_BRIDGE_RECV_BUF_LEN, + } + } + + /// Builder: outbound `TcpStream::connect` budget. + #[must_use] + #[inline] + pub const fn with_dial_timeout(mut self, d: Duration) -> Self { + self.dial_timeout = d; + self + } + + /// Builder: bound on a per-bridge graceful-drain write. + #[must_use] + #[inline] + pub const fn with_close_timeout(mut self, d: Duration) -> Self { + self.close_timeout = d; + self + } + + /// Builder: bridge-inbound channel capacity. + #[must_use] + #[inline] + pub const fn with_bridge_inbound_cap(mut self, n: usize) -> Self { + self.bridge_inbound_cap = n; + self + } + + /// Builder: per-bridge TCP read buffer size. + #[must_use] + #[inline] + pub const fn with_bridge_recv_buf_len(mut self, n: usize) -> Self { + self.bridge_recv_buf_len = n; + self + } + + /// Outbound `TcpStream::connect` budget. + #[inline] + pub const fn dial_timeout(&self) -> Duration { + self.dial_timeout + } + + /// Bound on a per-bridge graceful-drain write. + #[inline] + pub const fn close_timeout(&self) -> Duration { + self.close_timeout + } + + /// Bridge-inbound channel capacity. + #[inline] + pub const fn bridge_inbound_cap(&self) -> usize { + self.bridge_inbound_cap + } + + /// Per-bridge TCP read buffer size. + #[inline] + pub const fn bridge_recv_buf_len(&self) -> usize { + self.bridge_recv_buf_len + } + + /// Validate the stream-transport knobs whose value would DETERMINISTICALLY + /// break (not merely degrade) a stream backend, rejected fail-fast at the + /// transport's `new` (before any socket is bound) rather than constructing `Ok` + /// over a silently-broken backend. Called at the top of both stream backends' + /// `Transport::new` (TCP and TLS); QUIC has no bridges and so no counterpart. + /// + /// - `dial_timeout == 0`: every outbound `StreamAction::Connect` races + /// `TcpStream::connect` against `compio::time::sleep(dial_timeout)` in a + /// biased select. On a completion-based backend the connect is pending on + /// first poll, so a zero timeout wins immediately and EVERY outbound dial + /// resolves as a spurious dial-timeout failure. Because a serf `join` is + /// dispatch-only — it returns the seed count, not a per-exchange outcome — + /// the node returns `Ok` from `join` while no reliable push/pull exchange + /// ever completes, so it silently never joins. + /// - `bridge_recv_buf_len == 0`: the per-bridge byte-mover reads into a + /// `vec![0u8; bridge_recv_buf_len]`, and a zero-length read returns + /// `Ok(0)`, which the bridge treats as peer EOF — so every reliable bridge + /// would report EOF instead of reading frames. + /// - `close_timeout == 0`: the post-`Close` graceful drain fires immediately, + /// so every graceful close abandons (RSTs) its queued response bytes. + /// - `bridge_inbound_cap == 0`: the single-threaded local inbound channel has + /// no zero-capacity rendezvous path, so the first bridge read parks forever. + #[cfg(feature = "tcp")] + pub(crate) fn validate(&self) -> Result<(), SerfError> { + if self.dial_timeout.is_zero() { + return Err(SerfError::InvalidOption(InvalidOption::new( + "dial_timeout", + "the outbound dial budget must be nonzero: every Connect races TcpStream::connect \ + against a sleep(dial_timeout) in a biased select where the connect is pending on \ + first poll, so a zero timeout wins immediately and every outbound push/pull dial \ + fails as a spurious timeout — and since serf's join is dispatch-only, the node \ + returns Ok from join while never completing a single reliable exchange" + .to_string(), + ))); + } + if self.bridge_recv_buf_len == 0 { + return Err(SerfError::InvalidOption(InvalidOption::new( + "bridge_recv_buf_len", + "the per-bridge reliable-stream read buffer must be nonzero: a zero-length read \ + returns Ok(0), which the bridge treats as peer EOF, so every reliable \ + push-pull stream exchange would break" + .to_string(), + ))); + } + if self.close_timeout.is_zero() { + return Err(SerfError::InvalidOption(InvalidOption::new( + "close_timeout", + "the reliable graceful-close drain timeout must be nonzero: a zero timeout fires \ + immediately, so a graceful close abandons (RSTs) queued push/pull response bytes \ + instead of draining them, truncating reliable exchanges" + .to_string(), + ))); + } + if self.bridge_inbound_cap == 0 { + return Err(SerfError::InvalidOption(InvalidOption::new( + "bridge_inbound_cap", + "the per-bridge inbound queue capacity must be nonzero: the thread-per-core driver's \ + local inbound channel has no zero-capacity rendezvous path, so a zero capacity \ + parks every bridge read forever and no reliable-stream frame reaches the driver" + .to_string(), + ))); + } + Ok(()) + } +} + +impl Default for StreamTransportOptions { + #[inline] + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-compio/src/driver/options/tests.rs b/serf-compio/src/driver/options/tests.rs new file mode 100644 index 00000000..8e3e3b4a --- /dev/null +++ b/serf-compio/src/driver/options/tests.rs @@ -0,0 +1,415 @@ +use super::*; +#[cfg(any(feature = "tcp", feature = "quic", feature = "serde", feature = "clap"))] +use core::time::Duration; + +#[test] +fn runtime_options_defaults_are_accessible() { + let opts = RuntimeOptions::new(); + assert_eq!(opts.leave_timeout(), DEFAULT_LEAVE_TIMEOUT); + assert_eq!(opts.idle_wake_interval(), DEFAULT_IDLE_WAKE_INTERVAL); + assert_eq!(opts.iter_drain_cap(), DEFAULT_ITER_DRAIN_CAP); + assert_eq!(opts.cmd_fairness_budget(), DEFAULT_CMD_FAIRNESS_BUDGET); + assert_eq!(opts.event_queue_cap(), DEFAULT_EVENT_QUEUE_CAP); + assert_eq!(opts.observation_channel(), DEFAULT_OBSERVATION_CHANNEL); +} + +#[test] +fn stream_transport_options_defaults_are_accessible() { + let opts = StreamTransportOptions::new(); + assert_eq!(opts.dial_timeout(), DEFAULT_DIAL_TIMEOUT); + assert_eq!(opts.close_timeout(), DEFAULT_CLOSE_TIMEOUT); + assert_eq!(opts.bridge_inbound_cap(), DEFAULT_BRIDGE_INBOUND_CAP); + assert_eq!(opts.bridge_recv_buf_len(), DEFAULT_BRIDGE_RECV_BUF_LEN); +} + +#[test] +fn channel_display_and_from_str_round_trip() { + assert_eq!(Channel::Unbounded.to_string(), "unbounded"); + assert_eq!(Channel::Bounded(42).to_string(), "bounded:42"); + assert_eq!("unbounded".parse::().unwrap(), Channel::Unbounded); + assert_eq!( + "bounded:42".parse::().unwrap(), + Channel::Bounded(42) + ); + assert!("nonsense".parse::().is_err()); +} + +#[cfg(feature = "tcp")] +#[test] +fn stream_transport_validate_rejects_zero_recv_buf() { + let opts = StreamTransportOptions::new().with_bridge_recv_buf_len(0); + assert!(opts.validate().is_err()); +} + +#[cfg(feature = "tcp")] +#[test] +fn stream_transport_validate_rejects_zero_close_timeout() { + let opts = StreamTransportOptions::new().with_close_timeout(Duration::ZERO); + assert!(opts.validate().is_err()); +} + +#[cfg(feature = "tcp")] +#[test] +fn stream_transport_validate_rejects_zero_bridge_inbound_cap() { + let opts = StreamTransportOptions::new().with_bridge_inbound_cap(0); + assert!(opts.validate().is_err()); +} + +// A `Bounded(0)` observation channel would panic the driver's `lochan::mpsc` +// rendezvous-free channel; `validate` rejects it at construction instead. +#[cfg(any(feature = "tcp", feature = "quic"))] +#[test] +fn runtime_validate_rejects_zero_observation_channel() { + let opts = RuntimeOptions::new().with_observation_channel(Channel::Bounded(0)); + assert!(matches!( + opts.validate(), + Err(crate::SerfError::InvalidOption(_)) + )); +} + +// A zero `event_queue_cap` makes the `flume::bounded` event channel a rendezvous +// the non-blocking forward can never deposit into; `validate` rejects it. +#[cfg(any(feature = "tcp", feature = "quic"))] +#[test] +fn runtime_validate_rejects_zero_event_queue_cap() { + let opts = RuntimeOptions::new().with_event_queue_cap(0); + assert!(matches!( + opts.validate(), + Err(crate::SerfError::InvalidOption(_)) + )); +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +#[test] +fn runtime_validate_accepts_unbounded_and_nonzero_caps() { + assert!(RuntimeOptions::new().validate().is_ok()); + assert!( + RuntimeOptions::new() + .with_observation_channel(Channel::Unbounded) + .validate() + .is_ok() + ); + assert!( + RuntimeOptions::new() + .with_observation_channel(Channel::Bounded(1)) + .with_event_queue_cap(1) + .validate() + .is_ok() + ); +} + +// A zero `cmd_fairness_budget` disables the only drain that keeps user commands +// progressing under a continuous inbound flood, so shutdown / leave / joins +// would hang; `validate` rejects it at construction. +#[cfg(any(feature = "tcp", feature = "quic"))] +#[test] +fn runtime_validate_rejects_zero_cmd_fairness_budget() { + let opts = RuntimeOptions::new().with_cmd_fairness_budget(0); + assert!(matches!( + opts.validate(), + Err(crate::SerfError::InvalidOption(_)) + )); +} + +// A zero `idle_wake_interval` makes a quiescent endpoint re-arm a zero-duration +// timer every pass — a busy-spin that pegs a CPU core; `validate` rejects it. +#[cfg(any(feature = "tcp", feature = "quic"))] +#[test] +fn runtime_validate_rejects_zero_idle_wake_interval() { + let opts = RuntimeOptions::new().with_idle_wake_interval(Duration::ZERO); + assert!(matches!( + opts.validate(), + Err(crate::SerfError::InvalidOption(_)) + )); +} + +// `iter_drain_cap == 0` (the per-iteration batch cap; the select arms and the +// uncapped timeout drain still make one-per-pass forward progress) and +// `leave_timeout == 0` (a loud immediate `LeaveTimeout`) degrade-but-function, +// so they are accepted rather than rejected. +#[cfg(any(feature = "tcp", feature = "quic"))] +#[test] +fn runtime_validate_accepts_degrade_but_function_knobs() { + assert!( + RuntimeOptions::new() + .with_iter_drain_cap(0) + .validate() + .is_ok() + ); + assert!( + RuntimeOptions::new() + .with_leave_timeout(Duration::ZERO) + .validate() + .is_ok() + ); +} + +// A zero `dial_timeout` makes every outbound dial resolve as an immediate +// biased-select timeout, so the stream transport rejects it at `validate` +// (before any socket bind). +#[cfg(feature = "tcp")] +#[test] +fn stream_transport_validate_rejects_zero_dial_timeout() { + let opts = StreamTransportOptions::new().with_dial_timeout(Duration::ZERO); + assert!(matches!( + opts.validate(), + Err(crate::SerfError::InvalidOption(_)) + )); +} + +// A zero capacity sourced via serde — both the `bounded:0` `FromStr` form and the +// `{"bounded":0}` deserialize form — parses, then fails validation rather than +// reaching a driver-task panic. +#[cfg(all(feature = "serde", any(feature = "tcp", feature = "quic")))] +#[test] +fn runtime_validate_rejects_zero_observation_channel_from_serde() { + let from_str: Channel = "bounded:0".parse().expect("bounded:0 parses"); + assert_eq!(from_str, Channel::Bounded(0)); + let opts: RuntimeOptions = + serde_json::from_str(r#"{"observation_channel":{"bounded":0}}"#).expect("deserialize"); + assert_eq!(opts.observation_channel(), Channel::Bounded(0)); + assert!(matches!( + opts.validate(), + Err(crate::SerfError::InvalidOption(_)) + )); +} + +#[cfg(all(feature = "serde", any(feature = "tcp", feature = "quic")))] +#[test] +fn runtime_validate_rejects_zero_event_queue_cap_from_serde() { + let opts: RuntimeOptions = serde_json::from_str(r#"{"event_queue_cap":0}"#).expect("deserialize"); + assert_eq!(opts.event_queue_cap(), 0); + assert!(matches!( + opts.validate(), + Err(crate::SerfError::InvalidOption(_)) + )); +} + +// The newly-validated runtime knobs are rejected the same way when sourced via +// serde — a numeric `cmd_fairness_budget` and the humantime `idle_wake_interval` +// duration parse, then fail validation. +#[cfg(all(feature = "serde", any(feature = "tcp", feature = "quic")))] +#[test] +fn runtime_validate_rejects_zero_cmd_fairness_budget_from_serde() { + let opts: RuntimeOptions = + serde_json::from_str(r#"{"cmd_fairness_budget":0}"#).expect("deserialize"); + assert_eq!(opts.cmd_fairness_budget(), 0); + assert!(matches!( + opts.validate(), + Err(crate::SerfError::InvalidOption(_)) + )); +} + +#[cfg(all(feature = "serde", any(feature = "tcp", feature = "quic")))] +#[test] +fn runtime_validate_rejects_zero_idle_wake_interval_from_serde() { + let opts: RuntimeOptions = + serde_json::from_str(r#"{"idle_wake_interval":"0s"}"#).expect("deserialize"); + assert_eq!(opts.idle_wake_interval(), Duration::ZERO); + assert!(matches!( + opts.validate(), + Err(crate::SerfError::InvalidOption(_)) + )); +} + +// A zero capacity sourced via a clap-parsed flag is rejected the same way. +#[cfg(all(feature = "clap", any(feature = "tcp", feature = "quic")))] +#[test] +fn runtime_validate_rejects_zero_observation_channel_from_clap() { + use clap::Parser; + + #[derive(Parser)] + struct Cli { + #[command(flatten)] + runtime: RuntimeOptions, + } + + let cli = Cli::try_parse_from(["app", "--runtime-observation-channel", "bounded:0"]) + .expect("clap parses bounded:0"); + assert_eq!(cli.runtime.observation_channel(), Channel::Bounded(0)); + assert!(matches!( + cli.runtime.validate(), + Err(crate::SerfError::InvalidOption(_)) + )); +} + +// The newly-validated runtime knobs are rejected the same way when sourced via a +// clap-parsed flag. +#[cfg(all(feature = "clap", any(feature = "tcp", feature = "quic")))] +#[test] +fn runtime_validate_rejects_zero_cmd_fairness_budget_from_clap() { + use clap::Parser; + + #[derive(Parser)] + struct Cli { + #[command(flatten)] + runtime: RuntimeOptions, + } + + let cli = Cli::try_parse_from(["app", "--runtime-cmd-fairness-budget", "0"]) + .expect("clap parses cmd-fairness-budget 0"); + assert_eq!(cli.runtime.cmd_fairness_budget(), 0); + assert!(matches!( + cli.runtime.validate(), + Err(crate::SerfError::InvalidOption(_)) + )); +} + +#[cfg(all(feature = "clap", any(feature = "tcp", feature = "quic")))] +#[test] +fn runtime_validate_rejects_zero_idle_wake_interval_from_clap() { + use clap::Parser; + + #[derive(Parser)] + struct Cli { + #[command(flatten)] + runtime: RuntimeOptions, + } + + let cli = Cli::try_parse_from(["app", "--runtime-idle-wake-interval", "0s"]) + .expect("clap parses idle-wake-interval 0s"); + assert_eq!(cli.runtime.idle_wake_interval(), Duration::ZERO); + assert!(matches!( + cli.runtime.validate(), + Err(crate::SerfError::InvalidOption(_)) + )); +} + +#[cfg(feature = "serde")] +#[test] +fn runtime_options_serde_round_trip_and_partial() { + // An empty config deserializes to the full default. + assert_eq!( + serde_json::from_str::("{}").unwrap(), + RuntimeOptions::new() + ); + // A full round-trip preserves every knob, including humantime durations and + // the tagged observation channel. + let opts = RuntimeOptions::new() + .with_leave_timeout(Duration::from_secs(33)) + .with_iter_drain_cap(99) + .with_observation_channel(Channel::Unbounded); + let json = serde_json::to_string(&opts).unwrap(); + assert_eq!(serde_json::from_str::(&json).unwrap(), opts); + // A partial config overrides one field and defaults the rest. + let partial: RuntimeOptions = serde_json::from_str(r#"{"iter_drain_cap": 7}"#).unwrap(); + assert_eq!(partial.iter_drain_cap(), 7); + assert_eq!(partial.leave_timeout(), DEFAULT_LEAVE_TIMEOUT); + assert_eq!(partial.observation_channel(), DEFAULT_OBSERVATION_CHANNEL); + // The bounded channel serializes as its snake_case tagged form. + let bounded = RuntimeOptions::new().with_observation_channel(Channel::Bounded(8)); + let bjson = serde_json::to_string(&bounded).unwrap(); + assert!(bjson.contains("bounded"), "json = {bjson}"); + assert_eq!( + serde_json::from_str::(&bjson).unwrap(), + bounded + ); +} + +#[cfg(feature = "serde")] +#[test] +fn runtime_options_serde_rejects_unknown_field() { + // A misspelled field must be rejected, not silently dropped. + assert!(serde_json::from_str::(r#"{"iter_drain_capp": 7}"#).is_err()); +} + +#[cfg(feature = "clap")] +#[test] +fn runtime_options_clap_parses_flags_and_wires_env() { + use clap::{CommandFactory, Parser}; + + #[derive(Parser)] + struct Cli { + #[command(flatten)] + runtime: RuntimeOptions, + } + + // Flags parse: a humantime duration, a usize, and the channel via FromStr. + let cli = Cli::try_parse_from([ + "app", + "--runtime-leave-timeout", + "30s", + "--runtime-iter-drain-cap", + "12", + "--runtime-observation-channel", + "bounded:64", + ]) + .unwrap(); + assert_eq!(cli.runtime.leave_timeout(), Duration::from_secs(30)); + assert_eq!(cli.runtime.iter_drain_cap(), 12); + assert_eq!(cli.runtime.observation_channel(), Channel::Bounded(64)); + // Unspecified flags stay at the defaults. + let dflt = Cli::try_parse_from(["app"]).unwrap(); + assert_eq!(dflt.runtime, RuntimeOptions::new()); + // The env var is wired — assert via command introspection, never `set_var`. + let cmd = Cli::command(); + let arg = cmd + .get_arguments() + .find(|a| a.get_id().as_str() == "runtime-idle-wake-interval") + .expect("runtime-idle-wake-interval arg is registered"); + assert_eq!( + arg.get_env().and_then(|e| e.to_str()), + Some("SERF_RUNTIME_IDLE_WAKE_INTERVAL") + ); +} + +// A partial `try_update_from` carrying one unrelated flag must NOT reset the +// other defaulted knobs. clap's `default_value` / `default_value_t` makes an +// unset arg look "present" in update mode, so the value-source gate in the +// manual `update_from_arg_matches` is what keeps a seeded non-default value +// alive across an update. +#[cfg(feature = "clap")] +#[test] +fn runtime_options_partial_update_preserves_unset_fields() { + use clap::Parser; + + #[derive(Parser)] + struct Cli { + #[command(flatten)] + o: RuntimeOptions, + } + + // Seed NON-default values on several fields, then run a partial update that + // supplies ONE unrelated flag. + let mut cli = Cli { + o: RuntimeOptions::new() + .with_leave_timeout(Duration::from_secs(42)) + .with_event_queue_cap(7) + .with_cmd_fairness_budget(5) + .with_observation_channel(Channel::Unbounded), + }; + + cli + .try_update_from(["app", "--runtime-iter-drain-cap", "13"]) + .expect("partial update parses"); + + // The supplied flag is applied. + assert_eq!(cli.o.iter_drain_cap(), 13); + // Every seeded non-default field SURVIVES the partial update. + assert_eq!(cli.o.leave_timeout(), Duration::from_secs(42)); + assert_eq!(cli.o.event_queue_cap(), 7); + assert_eq!(cli.o.cmd_fairness_budget(), 5); + assert_eq!(cli.o.observation_channel(), Channel::Unbounded); +} + +// An explicit override on update IS applied (the value-source gate lets a +// command-line value through). +#[cfg(feature = "clap")] +#[test] +fn runtime_options_update_applies_explicit_override() { + use clap::Parser; + + #[derive(Parser)] + struct Cli { + #[command(flatten)] + o: RuntimeOptions, + } + + let mut cli = Cli { + o: RuntimeOptions::new().with_leave_timeout(Duration::from_secs(99)), + }; + cli + .try_update_from(["app", "--runtime-leave-timeout", "3s"]) + .expect("explicit override parses"); + assert_eq!(cli.o.leave_timeout(), Duration::from_secs(3)); +} diff --git a/serf-compio/src/driver/quic/mod.rs b/serf-compio/src/driver/quic/mod.rs new file mode 100644 index 00000000..4bdc290e --- /dev/null +++ b/serf-compio/src/driver/quic/mod.rs @@ -0,0 +1,1233 @@ +//! QUIC-plane driver pump — single owner of the serf `QuicEndpoint` and the +//! shared UDP socket. +//! +//! Unlike the stream plane, QUIC carries no per-exchange bridge table: quinn-proto +//! (inside the coordinator) multiplexes the reliable push-pull streams over the +//! single UDP socket, and serf's datagram gossip rides the same socket. The pump +//! therefore owns exactly one socket and runs a `select_biased` loop over three +//! arms in priority order: gossip/QUIC UDP recv, the coordinator-supplied wake +//! timer, and the command channel. After each fired arm it drains every outbound +//! surface (`poll_memberlist_ingress` decode → `handle_message`, +//! `poll_memberlist_transmit` encode → encrypt → UDP send, `poll_transmit` raw +//! QUIC UDP send, `poll_event` accounting + observation hand-off) until no method +//! makes progress, republishes a fresh [`SerfSnapshot`] when state changed, and +//! re-enters the select. The pump owns the `QuicEndpoint` outright; user-facing +//! handles communicate exclusively via the command channel and read state through +//! the lock-free snapshot. The socket drops when the loop exits so the bound port +//! is released before shutdown returns. + +#![cfg(feature = "quic")] + +use std::{cell::Cell, net::SocketAddr, rc::Rc}; + +use core::time::Duration; + +use bytes::Bytes; +use compio::{buf::BufResult, net::UdpSocket}; +use flume::{Receiver, Sender}; +use futures_channel::oneshot; +use futures_util::{FutureExt, pin_mut, select_biased}; +use lochan::mpsc; +use memberlist_proto::{ + Instant, PushPullKind, Rng, SeedableRng, Transmit, + codec::{ + DecodeOptions, EncodeOptions, decode_incoming, encode_outgoing, encode_outgoing_compound, + parse_messages, + }, +}; +use serf_proto::{LamportTime, QuicEndpoint, event::Event, members::SerfState}; + +#[cfg(encryption)] +use crate::command::{KeyCmd, ListKeysCmd}; +#[cfg(encryption)] +use crate::delegate::KeyringDelegate; +use crate::{ + Channel, + command::{ + Command, ForceLeaveCmd, JoinCmd, LeaveCmd, QueryCmd, RespondCmd, SetEventJoinIgnoreCmd, + SetTagsCmd, ShutdownCmd, + }, + delegate::Delegate, + driver::{ + options::RuntimeOptions, + shared::{ + add_obs_payload, dispatch_event_delegate, drain_past_due_udp, observation_payload_bytes, + yield_once, + }, + }, + error::{Result, SerfError}, + snapshot::{SerfSnapshot, SnapshotCell}, +}; +#[cfg(encryption)] +use serf_proto::{KeyRequestOperation, KeyResponseArgs, event::KeyRequest}; + +/// Driver-side state for the single in-flight graceful-leave operation. +/// +/// A [`Command::Leave`] that finds the endpoint `Alive` initiates the machine's +/// `leave()`, which queues the leave intent + direct notices to peers and +/// withholds [`Event::LeftCluster`] until they have drained. The pump parks this +/// here and replies only once that `LeftCluster` arrives (success) or `deadline` +/// elapses ([`SerfError::LeaveTimeout`]) — so a returned `Ok(())` means the leave +/// actually reached the wire, never merely that it was queued. +/// +/// Leave is a SHARED operation: a second `Command::Leave` racing an in-flight one +/// (cloned handles can both call `leave()`) does NOT re-invoke `endpoint.leave()` +/// (a repeated leave once already `Leaving`/`Left` is a terminal no-op that emits +/// no completion event, so a fresh waiter would hang). Instead it joins this +/// in-flight operation by pushing its reply onto `repliers`. Every terminal path — +/// `LeftCluster` success, timeout reap, shutdown — drains EVERY replier. +struct PendingLeave { + /// Reply channels of every `leave()` caller that joined this in-flight leave — + /// the initiator plus any racing clones. Drained together on the single + /// terminal outcome (`Ok` on `LeftCluster`, `LeaveTimeout` on deadline, + /// `Shutdown` on teardown). + repliers: Vec>>, + /// Wall-clock instant past which the pump replies [`SerfError::LeaveTimeout`] + /// to every replier even if `LeftCluster` has not yet fired. + deadline: Instant, +} + +impl PendingLeave { + /// Reply to every joined `leave()` caller with a fresh `Result<()>` from + /// `make_result`. A constructor closure (rather than a single cloned value) + /// sidesteps `SerfError` not being `Clone` — every terminal outcome here + /// (`Ok(())`, `LeaveTimeout`, `Shutdown`) is trivially reconstructible. + async fn resolve_all(self, mut make_result: impl FnMut() -> Result<()>) { + for replier in self.repliers { + // Ignoring Err: a `leave()` caller dropped its reply receiver (its + // user-facing future was cancelled); nothing to surface. + let _ = replier.send(make_result()); + } + } +} + +/// Pump-loop-local state tracking outstanding commands awaiting completion. +struct PendingCommands { + /// Outstanding graceful-leave waiter (at most one at a time). See [`PendingLeave`]. + leave: Option, +} + +/// Hard ceiling on the gossip-plane contribution to the per-recv UDP buffer — +/// UDP's IPv4 wire payload is capped at 65507 bytes once the IP/UDP headers are +/// deducted, so inflating the gossip path past it just wastes an allocation. The +/// raw-QUIC plane is bounded independently by quinn's `max_udp_payload_size` +/// (≤ 65527) and is intentionally not subject to this gossip cap. +const GOSSIP_RECV_BUF_MAX: usize = 65507; + +/// The largest the encrypted wrapper can inflate a gossip datagram, or `0` when +/// no encryption backend is built in. The wrapper carries the algorithm tag, +/// nonce, and AEAD auth tag; sizing the recv buffer to include it keeps an +/// encrypted gossip datagram from being silently truncated by the kernel. +#[cfg(encryption)] +const ENCRYPTED_WRAPPER_OVERHEAD: usize = memberlist_proto::ENCRYPTED_WRAPPER_OVERHEAD; +#[cfg(not(encryption))] +const ENCRYPTED_WRAPPER_OVERHEAD: usize = 0; + +/// Size the per-recv UDP buffer to the larger of the two planes that share this +/// one socket. +/// +/// The gossip plane needs `gossip_mtu` inflated by the encrypted-wrapper overhead +/// (algorithm tag + nonce + AEAD auth tag), clamped at [`GOSSIP_RECV_BUF_MAX`]. +/// The raw-QUIC plane needs whatever max UDP payload the quinn `EndpointConfig` +/// accepts — which a valid caller can set above the gossip MTU (quinn's default +/// 1472 already exceeds the 1400 default `gossip_mtu`, and callers can raise it +/// further). Sizing below either lets the kernel truncate that plane's largest +/// datagram before the coordinator's first-byte demux sees it, corrupting QUIC +/// handshakes/streams while leaving construction silently successful. +/// +/// `quic_max_udp_payload` is quinn-bounded to `[1200, 65527]`, so it always fits +/// `usize`; the conversion fallback is purely defensive. The QUIC plane is NOT +/// clamped at [`GOSSIP_RECV_BUF_MAX`] — quinn already bounds it, and clamping +/// would shrink the buffer below a configured 65508..=65527 ceiling. +fn recv_buf_len_for(gossip_mtu: usize, quic_max_udp_payload: u64) -> usize { + let gossip_path = gossip_mtu + .saturating_add(ENCRYPTED_WRAPPER_OVERHEAD) + .min(GOSSIP_RECV_BUF_MAX); + let quic_path = usize::try_from(quic_max_udp_payload).unwrap_or(GOSSIP_RECV_BUF_MAX); + gossip_path.max(quic_path) +} + +/// Single-owner pump task. +/// +/// Drives the serf `QuicEndpoint` until the command channel closes (all handles +/// dropped) or a [`Command::Shutdown`] is received. All mutations on the endpoint +/// happen here; reads happen via the published [`SerfSnapshot`]. +#[allow(clippy::too_many_arguments)] +pub(crate) async fn quic_driver_loop( + mut endpoint: QuicEndpoint, + gossip_socket: UdpSocket, + // The quinn `EndpointConfig`'s accepted max UDP payload, read off the + // `QuicOptions` in `QuicTransport::run` (the driver cannot reach the quinn + // config through the `QuicEndpoint`). Sizes the recv buffer's QUIC plane. + quic_max_udp_payload: u64, + commands: Receiver>, + events_tx: Sender>, + events_dropped: Rc>, + observation_dropped: Rc>, + snapshot: SnapshotCell, + shutdown_flag: Rc>, + driver_opts: RuntimeOptions, + delegate: D, + // Cluster label applied to both gossip encode and decode. `None` accepts + // datagrams from any cluster. + label: Option, + // The driver's keyring delegate: applies inbound key-management ops and + // produces the `respond_key` answer. Present only under an encryption backend. + #[cfg(encryption)] keyring: Rc, +) where + D: Delegate, + I: memberlist_proto::Id + Clone, + G: rand::Rng + Unpin, + R: rand::Rng + SeedableRng, +{ + // Spawn the per-driver observation task — it owns the user `Delegate` and the + // `EventStream` sender and runs OFF this pump task: the pump `try_send`s every + // surfaced event onto `obs_tx`, the task dispatches the matching observation + // hook then forwards to subscribers. Decoupling keeps a slow `notify_*` / + // `merge_remote_state` from stalling protocol advancement — and therefore a + // parked leave reply that depends on a follow-up input the pump must service. + let (obs_tx, obs_rx) = match driver_opts.observation_channel() { + Channel::Unbounded => mpsc::unbounded::>(), + Channel::Bounded(n) => mpsc::bounded::>(n), + }; + // `obs_payload_bytes` tracks the bytes of payload-bearing events (`User`) + // currently queued in `obs_tx`: the pump adds on enqueue, the obs task + // subtracts on dequeue. The byte backstop bounds the memory a large payload + // occupies while a delegate falls behind. + let obs_payload_bytes = Rc::new(Cell::new(0u64)); + compio::runtime::spawn(observation_task::( + obs_rx, + delegate, + events_tx, + events_dropped.clone(), + obs_payload_bytes.clone(), + )) + .detach(); + + // Stash for the [`Command::Shutdown`] reply — acked AFTER the post-loop cleanup + // closes the gossip socket so the bound port is free when the caller resumes + // from `shutdown.await`. + let mut shutdown_reply: Option>> = None; + let mut pending = PendingCommands { leave: None }; + + // Per-pump UDP recv buffer size, derived once at entry as the larger of the + // gossip plane (`gossip_mtu` + AEAD wrapper) and the raw-QUIC plane (quinn's + // accepted max UDP payload) — both ride this one socket. Fixed for the + // endpoint lifetime. + let recv_buf_len = recv_buf_len_for(endpoint.gossip_mtu(), quic_max_udp_payload); + + // Observation-channel payload byte backstop. `Bounded(n)`'s count cap bounds + // the NUMBER of queued events, but one `User` event can own up to + // `max_stream_frame_size` bytes; cap the queued payload bytes at four frames' + // worth. `Unbounded` opts out of dropping, so it opts out of the byte backstop. + let obs_payload_budget: Option = match driver_opts.observation_channel() { + Channel::Bounded(_) => Some((endpoint.max_stream_frame_size() as u64).saturating_mul(4)), + Channel::Unbounded => None, + }; + + // Arm the periodic probe / gossip / push-pull schedulers. Without this the + // coordinator's schedulers stay unset, so failure detection, dissemination, + // and anti-entropy never run. + endpoint.start_scheduling(Instant::now()); + refresh_snapshot::(&endpoint, &snapshot); + + loop { + let mut dirty = false; + let mut exit = false; + + // Iter-top command fairness drain so a network flood does not starve user + // commands (the `cmd` select arm sits below the network arms). + let mut cmd_drained = 0; + while cmd_drained < driver_opts.cmd_fairness_budget() { + match commands.try_recv() { + Ok(c) => { + let now = Instant::now(); + let is_shutdown = matches!(c, Command::Shutdown(_)); + if is_shutdown { + exit = true; + } + dispatch_command::( + &mut endpoint, + &mut shutdown_reply, + &mut pending, + driver_opts.leave_timeout(), + c, + now, + ) + .await; + cmd_drained += 1; + dirty = true; + if is_shutdown { + break; + } + } + // All `Serf` handles dropped: tear down exactly as a `Command::Shutdown` + // would. Under a continuous UDP flood the main select's command arm is + // starved by the higher-priority recv arm, so this iter-top drain is the + // only path that observes the disconnect; collapsing it into `Empty` + // would spin the pump forever and leak the bound socket. + Err(flume::TryRecvError::Disconnected) => { + exit = true; + break; + } + // No command queued right now — end the fairness drain. + Err(flume::TryRecvError::Empty) => break, + } + } + + // Honor `exit` from the iter-top cmd drain before the select so a quiet + // shutdown lands promptly. Flush (a preceding `leave()` directly enqueues + // `Dead`-self notices that must reach the wire before the socket drops), + // reap, publish, break. + if exit { + drain_outputs::( + &mut endpoint, + &gossip_socket, + &label, + &obs_tx, + &observation_dropped, + &obs_payload_bytes, + obs_payload_budget, + &mut pending, + #[cfg(encryption)] + &*keyring, + ) + .await; + reap_pending_leave(&mut pending.leave, Instant::now()).await; + refresh_snapshot::(&endpoint, &snapshot); + break; + } + + // Re-poll the deadline AFTER applying drained commands (which may have + // advanced or cleared it). Fold in the earliest pending-leave deadline so + // the timer arm fires by a graceful leave's timeout even when the + // coordinator has no nearer deadline. + let setup_now = Instant::now(); + let endpoint_deadline = endpoint + .poll_timeout() + .unwrap_or(setup_now + driver_opts.idle_wake_interval()); + let timeout_deadline = [ + Some(endpoint_deadline), + min_pending_leave_deadline(&pending.leave), + ] + .into_iter() + .flatten() + .min() + .unwrap_or(endpoint_deadline); + + // BOUNDED past-due preemption. Under a continuous UDP-recv flood the main + // select's recv arm always wins over the timer, so `handle_timeout` would + // never fire and stale connections / overdue probes would never be reaped. + // Route through the `fire_quic_timeout` chokepoint: it drains every + // immediately-ready datagram (decoding each) before firing `handle_timeout` + // only if the deadline is still past, so a buffered Ack that sits BEHIND an + // unrelated datagram in the shared socket's queue still resolves its probe + // ahead of the suspicion sweep. This is a safe site to build the drain's + // `recv_from` SQEs — the main loop's `recv_fut` is not yet in flight, so + // there is no second-SQE race. + if setup_now >= timeout_deadline { + if fire_quic_timeout::( + &mut endpoint, + &gossip_socket, + recv_buf_len, + driver_opts, + &label, + ) + .await + { + dirty = true; + } + + let terminal = drain_outputs::( + &mut endpoint, + &gossip_socket, + &label, + &obs_tx, + &observation_dropped, + &obs_payload_bytes, + obs_payload_budget, + &mut pending, + #[cfg(encryption)] + &*keyring, + ) + .await; + reap_pending_leave(&mut pending.leave, Instant::now()).await; + if dirty { + refresh_snapshot::(&endpoint, &snapshot); + } + // A conflict `Event::Shutdown` observed during the drain is terminal: break + // into teardown rather than re-entering the loop. + if terminal { + break; + } + continue; + } + + // Drained commands may have advanced state without past-due timer pressure — + // flush their outputs before entering the select so a snapshot observer sees + // the post-input state promptly. + if dirty { + let terminal = drain_outputs::( + &mut endpoint, + &gossip_socket, + &label, + &obs_tx, + &observation_dropped, + &obs_payload_bytes, + obs_payload_budget, + &mut pending, + #[cfg(encryption)] + &*keyring, + ) + .await; + reap_pending_leave(&mut pending.leave, Instant::now()).await; + refresh_snapshot::(&endpoint, &snapshot); + dirty = false; + // A conflict `Event::Shutdown` observed during the flush is terminal. + if terminal { + break; + } + } + + let mut timer_fired = false; + { + let recv_buf = vec![0u8; recv_buf_len]; + let recv_fut = gossip_socket.recv_from(recv_buf).fuse(); + let cmd_fut = commands.recv_async().fuse(); + let timer_fut = compio::time::sleep_until(timeout_deadline.into_std()).fuse(); + pin_mut!(recv_fut, cmd_fut, timer_fut); + + // Arm priority (top → bottom): + // 1. recv — kernel-buffered UDP (QUIC handshake/stream data + gossip). An + // Ack resolves a probe deadline before handle_timeout marks the + // peer suspect. + // 2. timer — past-due deadline (quinn's connection timers + serf's periodic + // reapers). + // 3. cmd — user commands (demoted below the network arms so a cloned- + // handle command flood cannot starve them). + select_biased! { + gossip = recv_fut => { + let BufResult(res, buf) = gossip; + if let Ok((n, src)) = res { + let now = Instant::now(); + // The single socket carries both QUIC packets and plain-UDP gossip; the + // coordinator's first-byte demux routes each datagram. Raw gossip is + // buffered for the drain below to decode; QUIC packets are processed + // in-band. + endpoint.handle_udp(src, &buf[..n], now); + dirty = true; + } + // Ignoring Err: a transient recv error is non-fatal — the next iteration + // re-arms recv with a fresh buffer. + } + _ = timer_fut => { + // Defer the timeout to the `fire_quic_timeout` chokepoint AFTER this + // scope drops the in-flight `recv_fut`: a freshly-submitted recv can be + // pending on its first poll on a completion backend, so the timer arm + // winning does NOT prove a would-block — a near-deadline Ack may be + // queued. The chokepoint drains the socket before deciding on + // `handle_timeout`, and dropping `recv_fut` first avoids a second + // concurrent `recv_from` SQE on the shared socket. + timer_fired = true; + } + cmd = cmd_fut => { + match cmd { + Ok(c) => { + exit = matches!(c, Command::Shutdown(_)); + let now = Instant::now(); + dispatch_command::( + &mut endpoint, + &mut shutdown_reply, + &mut pending, + driver_opts.leave_timeout(), + c, + now, + ).await; + dirty = true; + } + // All handles dropped → channel closed. Treat as shutdown. + Err(_) => exit = true, + } + } + } + } + + // Past-due deadline, deferred from the timer arm so the in-flight `recv_fut` + // SQE is dropped before the chokepoint builds its own recv SQEs. Routes + // through the SAME `fire_quic_timeout` as the past-due branch so a + // near-deadline queued Ack is consumed before any suspicion. + if timer_fired + && fire_quic_timeout::( + &mut endpoint, + &gossip_socket, + recv_buf_len, + driver_opts, + &label, + ) + .await + { + dirty = true; + } + + // Post-select drain (sends + sync accounting only — no second recv SQE). A + // conflict `Event::Shutdown` drained here is terminal — fold it into `exit`. + if drain_outputs::( + &mut endpoint, + &gossip_socket, + &label, + &obs_tx, + &observation_dropped, + &obs_payload_bytes, + obs_payload_budget, + &mut pending, + #[cfg(encryption)] + &*keyring, + ) + .await + { + exit = true; + } + reap_pending_leave(&mut pending.leave, Instant::now()).await; + + if dirty { + refresh_snapshot::(&endpoint, &snapshot); + } + + if exit { + break; + } + } + + // Cleanup. Order: flip the shutdown flag so a racing clone observes it on + // entry, drain queued commands with Err(Shutdown), drop the command receiver + // so a late send fails fast, resolve any parked leave, close the bound socket + // (awaited so its port is released), then ack the observed shutdown caller. + shutdown_flag.set(true); + while let Ok(c) = commands.try_recv() { + reply_shutdown(c); + } + drop(commands); + if let Some(pl) = pending.leave.take() { + pl.resolve_all(|| Err(SerfError::Shutdown)).await; + } + // Ignoring Err: socket close on shutdown — the runtime tears down fds anyway. + // Awaiting the close (rather than a plain drop) drains it to completion so the + // kernel slot is released before the stashed reply fires (Windows IOCP closes + // asynchronously, so a plain drop could race a same-port rebind into AddrInUse). + let _ = gossip_socket.close().await; + + if let Some(reply) = shutdown_reply { + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(Ok(())); + } +} + +/// Reply `Err(Shutdown)` to a command drained during teardown. +fn reply_shutdown(c: Command) { + // Ignoring Err on each send: caller dropped the reply receiver. + match c { + Command::Join(JoinCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + Command::Leave(LeaveCmd { reply }) | Command::Shutdown(ShutdownCmd { reply }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + Command::SetEventJoinIgnore(SetEventJoinIgnoreCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + Command::ForceLeave(ForceLeaveCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + Command::UserEvent(cmd) => { + let _ = cmd.reply.send(Err(SerfError::Shutdown)); + } + Command::Query(QueryCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + Command::Respond(RespondCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + Command::SetTags(SetTagsCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + #[cfg(encryption)] + Command::InstallKey(KeyCmd { reply, .. }) + | Command::UseKey(KeyCmd { reply, .. }) + | Command::RemoveKey(KeyCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + #[cfg(encryption)] + Command::ListKeys(ListKeysCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + } +} + +/// Dispatch one [`Command`] onto the serf endpoint. Replies are best-effort via +/// the per-command reply channel — a dropped reply receiver means the caller +/// gave up. +/// +/// The [`Command::Shutdown`] reply is NOT acked inline; it is stashed into +/// `shutdown_reply` so the pump acks the caller only AFTER the socket drops in +/// the post-loop cleanup. +async fn dispatch_command( + endpoint: &mut QuicEndpoint, + shutdown_reply: &mut Option>>, + pending: &mut PendingCommands, + leave_timeout: Duration, + cmd: Command, + now: Instant, +) where + I: memberlist_proto::Id + Clone, + G: Rng, + R: Rng + SeedableRng, +{ + let running = endpoint.state() == SerfState::Alive; + match cmd { + Command::Join(JoinCmd { seeds, reply }) => { + // Gate on a running node: `leave()` is terminal (it stops the periodic + // schedulers), so a join after leave would leave the node non-participating. + if !running { + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(Err(SerfError::NotRunning)); + return; + } + // Announce the serf-level join intent so peers learn the local join ltime + // without waiting for the next anti-entropy round. + if let Err(e) = endpoint.join() { + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(Err(SerfError::from(e))); + return; + } + // Dial every seed via a coordinator push-pull (the driver owns the inner- + // memberlist join). The QUIC coordinator services the dial + flushes the + // outbound queue in-band; the handshake packets surface on the next + // `poll_transmit` drain. + let count = seeds.len(); + for seed in seeds { + let _sid = endpoint.start_push_pull(seed, PushPullKind::Join, now); + } + // serf surfaces a push-pull's outcome as the internal `RemoteStateReceived` + // sieve, not an `ExchangeCompleted` event, so the pump reports the count of + // seeds dispatched rather than parking for per-exchange contact accounting. + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(Ok(count)); + } + Command::Leave(LeaveCmd { reply }) => { + // Leave is a SHARED in-flight operation. If one is in flight, JOIN it (do + // not re-invoke `leave()`, which once `Leaving`/`Left` is a terminal no-op + // emitting no second `LeftCluster`). Otherwise INITIATE: snapshot `Alive` + // before the call (it decides whether a `LeftCluster` will fire), then park + // (was Alive) or reply immediately (idempotent no-op / error). + if let Some(pl) = pending.leave.as_mut() { + pl.repliers.push(reply); + } else { + let was_alive = running; + let res: Result<()> = endpoint.leave(now).map_err(SerfError::from); + match res { + Ok(()) if was_alive => { + pending.leave = Some(PendingLeave { + repliers: vec![reply], + deadline: now + leave_timeout, + }); + } + // Idempotent no-op (not Alive) ⇒ no `LeftCluster` will fire, or the + // call errored. Reply immediately; parking would hang. + other => { + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(other); + } + } + } + } + Command::ForceLeave(ForceLeaveCmd { + id, + prune, + now: at, + reply, + }) => { + let res = if running { + endpoint.force_leave(id, prune, at).map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(res); + } + Command::UserEvent(cmd) => { + let res = if running { + let name = cmd.name().clone(); + let payload = cmd.payload().clone(); + endpoint + .user_event(name, payload, cmd.coalesce) + .map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = cmd.reply.send(res); + } + Command::Query(cmd) => { + let res = if running { + let name = cmd.name().clone(); + let payload = cmd.payload().clone(); + let QueryCmd { + params, now: at, .. + } = &cmd; + endpoint + .query(name, payload, params.clone(), *at) + .map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = cmd.reply.send(res); + } + Command::Respond(cmd) => { + let res = if running { + let payload = cmd.payload().clone(); + endpoint + .respond(&cmd.token, payload, cmd.now) + .map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = cmd.reply.send(res); + } + Command::SetTags(SetTagsCmd { tags, reply }) => { + let res = if running { + endpoint.set_tags(tags).map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(res); + } + Command::SetEventJoinIgnore(SetEventJoinIgnoreCmd { ignore, reply }) => { + endpoint.set_event_join_ignore(ignore); + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(Ok(())); + } + #[cfg(encryption)] + Command::InstallKey(KeyCmd { + key, + now: at, + reply, + }) => { + let res = if running { + endpoint.install_key(key, at).map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(res); + } + #[cfg(encryption)] + Command::UseKey(KeyCmd { + key, + now: at, + reply, + }) => { + let res = if running { + endpoint.use_key(key, at).map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(res); + } + #[cfg(encryption)] + Command::RemoveKey(KeyCmd { + key, + now: at, + reply, + }) => { + let res = if running { + endpoint.remove_key(key, at).map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(res); + } + #[cfg(encryption)] + Command::ListKeys(ListKeysCmd { now: at, reply }) => { + let res = if running { + endpoint.list_keys(at).map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(res); + } + Command::Shutdown(ShutdownCmd { reply }) => { + // Do NOT ack the caller here — the socket is still bound; stash the reply + // and let the post-loop cleanup ack AFTER it drops. + *shutdown_reply = Some(reply); + } + } +} + +/// Drain-first timeout chokepoint for the QUIC pump — the single site that calls +/// `handle_timeout` on this plane. +/// +/// Both the past-due preemption branch and the main select's timer arm route +/// through here. A freshly-submitted `recv` can be pending on its first poll on a +/// completion backend (io_uring), so a biased select's timer arm winning does NOT +/// prove a genuine would-block: a near-deadline Ack may be queued or immediately +/// readable when the timer fires. The bounded `drain_past_due_udp` reads every +/// immediately-ready datagram (decoding each via `drain_ingress` so a drained Ack +/// resolves its probe deadline ahead of the suspicion sweep), then `handle_timeout` +/// fires ONLY if the coordinator's deadline is still past after the drain. Returns +/// `true` iff any work was applied. +/// +/// The caller MUST ensure no other `recv_from` SQE on `gossip_socket` is in flight +/// (the main loop drops its `recv_fut` before invoking this), so the bounded drain +/// is the sole builder of recv SQEs here. +async fn fire_quic_timeout( + endpoint: &mut QuicEndpoint, + gossip_socket: &UdpSocket, + recv_buf_len: usize, + driver_opts: RuntimeOptions, + label: &Option, +) -> bool +where + I: memberlist_proto::Id + Clone, + G: Rng, + R: Rng + SeedableRng, +{ + let mut dirty = false; + // The budget reuses the per-iteration inbound drain cap (`.max(1)` so recv + // always gets at least one shot even at a zero cap). Emptiness is reaped via + // `poll_with(ZERO)` inside the drain, not a time window. + let drained = drain_past_due_udp( + gossip_socket, + recv_buf_len, + driver_opts.iter_drain_cap().max(1), + |src, datagram| { + let now = Instant::now(); + endpoint.handle_udp(src, datagram, now); + // On QUIC `handle_udp` only BUFFERS the raw gossip frame; it is decoded and + // fed back through `handle_message` by `drain_ingress` (the stream pump + // decodes inline in `dispatch_gossip`, QUIC does not). Decode each drained + // frame NOW — before the deadline recheck below decides whether to fire + // `handle_timeout` — so a drained Ack resolves its probe deadline ahead of + // the suspicion sweep. The remaining outbound surfaces are flushed by the + // caller's `drain_outputs` after the timeout decision. + drain_ingress::(endpoint, label); + // Report whether the coordinator deadline is STILL past: a drained Ack that + // resolved it ends the drain so the main select regains fairness. + now + >= endpoint + .poll_timeout() + .unwrap_or(now + driver_opts.idle_wake_interval()) + }, + ) + .await; + if drained { + dirty = true; + } + + // Re-poll the deadline after the drain — a consumed Ack may have resolved it. + // Fire `handle_timeout` iff the deadline is still past: when the socket drained + // empty a buffered Ack was already decoded above (so the deadline is no longer + // past), and when a flood capped the drain, firing for liveness is at worst a + // transient SWIM-refutable false Suspect rather than a frozen-timer stall. + let now = Instant::now(); + let after_drain = endpoint + .poll_timeout() + .unwrap_or(now + driver_opts.idle_wake_interval()); + if now >= after_drain { + endpoint.handle_timeout(now); + dirty = true; + } + dirty +} + +/// Decode the coordinator's buffered inbound gossip frames and feed each decoded +/// message back through `handle_message`. Returns `true` iff any frame was drained. +/// +/// The codec hop is `decrypt → label-strip → decode`: with an encryption backend +/// built in, the encryption wrapper is stripped (and authenticated) before the +/// cluster label is verified; with none built in the serf gossip plane carries no +/// wire transforms, so the raw bytes ARE the label frame. A compound datagram is +/// split into its ordered messages by `parse_messages`, each fed as a typed message. +fn drain_ingress(endpoint: &mut QuicEndpoint, label: &Option) -> bool +where + I: memberlist_proto::Id + Clone, + G: Rng, + R: Rng + SeedableRng, +{ + let now = Instant::now(); + let decode_opts = DecodeOptions::new(label.clone()); + let mut progress = false; + while let Some((from, raw)) = endpoint.poll_memberlist_ingress() { + progress = true; + // Reverse the wire transform stack the peer applied before decoding. With an + // encryption backend built in, `decrypt_gossip` strips (and authenticates) + // the encryption wrapper — returning the frame unchanged when no keyring is + // configured, and dropping a frame the keyring cannot decrypt. With none + // built in the serf gossip plane carries no transforms, so the raw bytes are + // the plain label frame. A dropped datagram is recovered on the next gossip + // round (gossip is lossy and self-healing). + #[cfg(encryption)] + let plain = match endpoint.decrypt_gossip(&raw) { + Ok(p) => Bytes::from(p), + Err(_) => continue, + }; + #[cfg(not(encryption))] + let plain = raw; + // Strip the optional cluster label and verify it matches; a mismatched or + // absent label on a labeled cluster (or vice versa) is dropped here. + let inner = match decode_incoming(plain, &decode_opts) { + Ok(b) => b, + Err(_) => continue, + }; + // Demux plain vs compound and feed each decoded message to the coordinator. + let msgs = match parse_messages::(inner) { + Ok(m) => m, + Err(_) => continue, + }; + for msg in msgs { + endpoint.handle_message(from, msg, now); + } + } + progress +} + +/// Drain every queued unreliable (gossip-plane) [`Transmit`] and send it on the +/// shared UDP socket. Outbound gossip is label-stamped (`encode_outgoing` / +/// `encode_outgoing_compound`), then — with an encryption backend built in — +/// wrapped in the encryption layer (`encrypt_gossip`) before it hits the wire. +async fn drain_transmits( + endpoint: &mut QuicEndpoint, + gossip_socket: &UdpSocket, + label: Option, +) -> bool +where + I: memberlist_proto::Id + Clone, + G: Rng, + R: Rng + SeedableRng, +{ + let encode_opts = EncodeOptions::new(label); + let mut progress = false; + while let Some(transmit) = endpoint.poll_memberlist_transmit() { + progress = true; + let (peer, plain): (SocketAddr, Bytes) = match transmit { + Transmit::Packet(pkt) => { + let (to, msg) = pkt.into_parts(); + match encode_outgoing(&msg, &encode_opts) { + Ok(b) => (to, b), + // A locally-built message that fails to encode is dropped so one bad + // codec invocation cannot wedge the pump. + Err(_) => continue, + } + } + Transmit::Compound(cmp) => { + let (to, msgs) = cmp.into_parts(); + match encode_outgoing_compound(&msgs, &encode_opts) { + Ok(b) => (to, b), + Err(_) => continue, + } + } + }; + // Wrap the label frame in the encryption layer when an encryption backend is + // built in: `encrypt_gossip` is identity when no keyring is configured, and + // drops the datagram if a configured backend rejects it rather than emitting + // plaintext on an encrypted-cluster path. With none built in the frame goes + // out as-is. + #[allow(unused_mut)] + let mut on_wire: Vec = plain.to_vec(); + #[cfg(encryption)] + { + on_wire = match endpoint.encrypt_gossip(&on_wire) { + Ok(bytes) => bytes, + Err(_) => continue, + }; + } + let BufResult(res, _buf) = gossip_socket.send_to(on_wire, peer).await; + // Ignoring Err: a transient send error is non-fatal — gossip is lossy and + // the next probe/gossip round recovers. + let _ = res; + } + progress +} + +/// Drain every raw outbound QUIC datagram (handshake, acks, reliable stream data) +/// the coordinator queued, and send it on the shared UDP socket. These are +/// already framed by quinn-proto, so no codec wrap is applied. +async fn drain_quic_transmits( + endpoint: &mut QuicEndpoint, + gossip_socket: &UdpSocket, +) -> bool +where + I: memberlist_proto::Id + Clone, + G: Rng, + R: Rng + SeedableRng, +{ + let mut progress = false; + while let Some((dest, bytes)) = endpoint.poll_transmit() { + progress = true; + let BufResult(res, _buf) = gossip_socket.send_to(bytes, dest).await; + // Ignoring Err: a transient send error is non-fatal — QUIC retransmits + // unacked frames on its own timer. + let _ = res; + } + progress +} + +/// Drain every queued serf [`Event`]: synchronous protocol accounting (leave +/// completion, conflict-shutdown, key requests), then hand off to the observation +/// task. NO `.await` on user delegate code. Returns `true` iff any event was +/// drained. +/// +/// Sets `*terminal` to `true` if a terminal [`Event::Shutdown`] was observed — +/// the local node lost an id-conflict vote and the pump MUST tear down. The event +/// is still delivered to subscribers before the main loop breaks. +#[allow(clippy::too_many_arguments)] +async fn drain_events( + endpoint: &mut QuicEndpoint, + obs_tx: &mpsc::Sender>, + observation_dropped: &Cell, + obs_payload_bytes: &Cell, + obs_payload_budget: Option, + pending: &mut PendingCommands, + terminal: &mut bool, + #[cfg(encryption)] keyring: &dyn KeyringDelegate, +) -> bool +where + I: memberlist_proto::Id + Clone, + G: Rng, + R: Rng + SeedableRng, +{ + let mut drained = false; + while let Some(ev) = endpoint.poll_event() { + drained = true; + // Leave-completion resolution. `LeftCluster` fires once the leave notices + // have drained to the wire; resolving the parked waiter here — on this pump + // task, ahead of the observation task's `notify_leave` — is what makes + // `leave()` return promptly once the flush is done. + if matches!(ev, Event::LeftCluster) + && let Some(pl) = pending.leave.take() + { + pl.resolve_all(|| Ok(())).await; + } + // Conflict-shutdown enforcement. `Event::Shutdown` means the local node lost + // an id-conflict vote and MUST stop, exactly as for a `Command::Shutdown`. + // Flag it for the main loop (which breaks into teardown after this drain) + // while still delivering the event to subscribers below. + if matches!(ev, Event::Shutdown) { + *terminal = true; + } + // Key-management request enforcement. `Event::KeyRequest` requires the driver + // to apply the install/use/remove/list op to its keyring and answer the + // originator; without this the inbound key op times out and local key state + // never changes. Applied here on the pump (it mutates the endpoint through + // `respond_key`) ahead of the observation hand-off below. + #[cfg(encryption)] + if let Event::KeyRequest(req) = &ev { + let resp = apply_key_request(keyring, req); + // Ignoring Err: `respond_key` fails only when the response cannot be routed + // (originator gone / relay dropped); the key op has already applied locally. + let _ = endpoint.respond_key(req, resp, Instant::now()); + } + + let payload_bytes = observation_payload_bytes(&ev); + + // Byte backstop (bounded channels only): the count cap does not bound memory + // when an event carries a large user payload. If enqueueing would push the + // queued payload bytes over budget, yield once so the obs task can drain, + // re-check, and drop + count if still over. + if let (Some(budget), Some(bytes)) = (obs_payload_budget, payload_bytes) { + if obs_payload_bytes.get().saturating_add(bytes) > budget { + yield_once().await; + } + if obs_payload_bytes.get().saturating_add(bytes) > budget { + observation_dropped.set(observation_dropped.get() + 1); + continue; + } + } + + // Hand off to the observation task (delegate dispatch + EventStream forward, + // off this pump task), non-blocking. `Full` yields once then retries; drop + + // count only if still full. + match obs_tx.try_send(ev) { + Ok(()) => add_obs_payload(obs_payload_bytes, payload_bytes), + Err(mpsc::TrySendError::Closed(_)) => {} + Err(mpsc::TrySendError::Full(ev)) => { + yield_once().await; + match obs_tx.try_send(ev) { + Ok(()) => add_obs_payload(obs_payload_bytes, payload_bytes), + Err(_) => observation_dropped.set(observation_dropped.get() + 1), + } + } + } + } + drained +} + +/// Drain every outbound surface to quiescence in the documented order: ingress +/// decode, gossip transmits, raw QUIC transmits, events; repeat until no method +/// makes progress (a fed inbound message can enqueue an outbound transmit and an +/// event the next pass surfaces). +/// +/// Returns `true` iff a terminal [`Event::Shutdown`] was observed while draining — +/// the local node lost an id-conflict vote and the main loop MUST break into +/// teardown. +#[allow(clippy::too_many_arguments)] +async fn drain_outputs( + endpoint: &mut QuicEndpoint, + gossip_socket: &UdpSocket, + label: &Option, + obs_tx: &mpsc::Sender>, + observation_dropped: &Cell, + obs_payload_bytes: &Cell, + obs_payload_budget: Option, + pending: &mut PendingCommands, + #[cfg(encryption)] keyring: &dyn KeyringDelegate, +) -> bool +where + I: memberlist_proto::Id + Clone, + G: Rng, + R: Rng + SeedableRng, +{ + let mut terminal = false; + loop { + let did_ingress = drain_ingress::(endpoint, label); + let did_transmits = drain_transmits::(endpoint, gossip_socket, label.clone()).await; + let did_quic = drain_quic_transmits::(endpoint, gossip_socket).await; + let did_events = drain_events::( + endpoint, + obs_tx, + observation_dropped, + obs_payload_bytes, + obs_payload_budget, + pending, + &mut terminal, + #[cfg(encryption)] + keyring, + ) + .await; + if !(did_ingress || did_transmits || did_quic || did_events) { + break; + } + } + terminal +} + +/// Apply one inbound [`KeyRequest`] to the driver's keyring delegate, producing +/// the [`KeyResponseArgs`] the pump forwards to `respond_key`. +/// +/// `Install` / `Use` / `Remove` carry a key (the machine enforces op-shape, so a +/// missing key is reported as a failed response rather than panicking); `List` +/// carries no key and enumerates the keyring. +#[cfg(encryption)] +fn apply_key_request( + keyring: &dyn KeyringDelegate, + req: &KeyRequest, +) -> KeyResponseArgs { + match (req.op(), req.key()) { + (KeyRequestOperation::Install, Some(key)) => keyring.install(*key), + (KeyRequestOperation::Use, Some(key)) => keyring.use_key(*key), + (KeyRequestOperation::Remove, Some(key)) => keyring.remove(*key), + (KeyRequestOperation::List, _) => keyring.list(), + (_, None) => KeyResponseArgs { + result: false, + message: "key-management request missing its required key".into(), + keys: Vec::new(), + primary_key: None, + }, + } +} + +/// Per-driver observation task: dispatch each event's [`Delegate`] hook, then fan +/// the event out to the `EventStream`, OFF the pump loop. +/// +/// Every serf [`Event`] (member transitions, user events, queries, responses) is +/// the application's observation surface, so all events are forwarded to +/// subscribers. The forward is best-effort: a full queue (slow subscriber) drops +/// the event and counts it into `events_dropped`, never blocking. The task exits +/// when `obs_rx` closes (pump dropped `obs_tx`). +async fn observation_task( + mut obs_rx: mpsc::Receiver>, + delegate: D, + events_tx: Sender>, + events_dropped: Rc>, + obs_payload_bytes: Rc>, +) where + D: Delegate, + I: Clone, +{ + while let Some(ev) = obs_rx.recv().await { + // Free the byte-backstop budget this event occupied as soon as it leaves the + // channel — before the (possibly slow) delegate hook — so the pump's enqueue + // side sees the reclaimed budget promptly. + let payload = observation_payload_bytes(&ev); + if let Some(b) = payload { + obs_payload_bytes.set(obs_payload_bytes.get().saturating_sub(b)); + } + // Contain a panicking delegate hook so the task SURVIVES and keeps releasing + // the byte-backstop reservations of still-queued events. Ignoring the unwind + // result: the panic is contained and this event is simply dropped. + let _ = std::panic::AssertUnwindSafe(dispatch_event_delegate(&delegate, &ev)) + .catch_unwind() + .await; + if events_tx + .try_send(ev) + .is_err_and(|e| matches!(e, flume::TrySendError::Full(_))) + { + events_dropped.set(events_dropped.get() + 1); + } + } +} + +/// Reap a deadline-expired graceful-leave waiter. If `pending_leave`'s deadline +/// has elapsed without `Event::LeftCluster` having resolved it, reply +/// [`SerfError::LeaveTimeout`] to every joined replier and clear the slot. +async fn reap_pending_leave(pending_leave: &mut Option, now: Instant) { + if let Some(pl) = pending_leave.as_ref() + && now >= pl.deadline + { + let pl = pending_leave.take().expect("checked Some above"); + pl.resolve_all(|| Err(SerfError::LeaveTimeout)).await; + } +} + +/// Earliest pending-leave deadline, if any — folded into the per-iteration +/// `timeout_deadline` so the timer fires even under a continuous network flood. +fn min_pending_leave_deadline(pending_leave: &Option) -> Option { + pending_leave.as_ref().map(|pl| pl.deadline) +} + +/// Publish a fresh [`SerfSnapshot`] of the endpoint's observable membership to +/// the snapshot cell. +/// +/// Skips the publish when the local node is not yet present in the serf +/// membership store (the local `NodeJoined` sieve has not fired): the prior +/// snapshot — seeded at construction — stays current, and `SerfSnapshot::new` +/// (which requires the local node) is never called with it absent. +fn refresh_snapshot(endpoint: &QuicEndpoint, snapshot: &SnapshotCell) +where + I: memberlist_proto::Id + Clone, + G: Rng, + R: Rng + SeedableRng, +{ + let members = endpoint.members_snapshot(); + let local_id = endpoint.local_id(); + if !members.iter().any(|m| m.node().id_ref() == local_id) { + return; + } + let snap = SerfSnapshot::new( + members, + local_id, + endpoint.state(), + LamportTime::from(endpoint.member_time()), + LamportTime::from(endpoint.event_time()), + LamportTime::from(endpoint.query_time()), + ); + *snapshot.borrow_mut() = Rc::new(snap); +} + +#[cfg(test)] +mod tests; diff --git a/serf-compio/src/driver/quic/tests.rs b/serf-compio/src/driver/quic/tests.rs new file mode 100644 index 00000000..5bb7fe16 --- /dev/null +++ b/serf-compio/src/driver/quic/tests.rs @@ -0,0 +1,267 @@ +//! Unit tests for the QUIC driver's recv-buffer sizing and the past-due drain's +//! pre-timeout ingress drain. + +use super::*; + +use core::num::NonZeroU8; +use std::sync::Arc; + +use rand::rngs::StdRng; +use rustls::{ + client::danger::{HandshakeSignatureValid, ServerCertVerified}, + version::TLS13, +}; +use rustls_pki_types::{CertificateDer, PrivateKeyDer}; +use serf_proto::options::Options as SerfOptions; +use smol_str::SmolStr; + +use crate::QuicOptions; +use memberlist_proto::UnreliableTransport; + +/// A self-signed `localhost` cert + key for the test quinn `ServerConfig`. +fn self_signed() -> (Vec>, PrivateKeyDer<'static>) { + let ck = rcgen::generate_simple_self_signed(vec!["localhost".into()]).expect("rcgen"); + let cert = CertificateDer::from(ck.cert.der().to_vec()); + let key = PrivateKeyDer::Pkcs8(ck.signing_key.serialize_der().into()); + (vec![cert], key) +} + +/// Accept-any server-cert verifier — test only; the endpoint built here never +/// completes a handshake (the test feeds only a gossip-class datagram). +#[derive(Debug)] +struct AnyServer; + +impl rustls::client::danger::ServerCertVerifier for AnyServer { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer, + _intermediates: &[CertificateDer], + _server_name: &rustls_pki_types::ServerName, + _ocsp_response: &[u8], + _now: rustls_pki_types::UnixTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } + fn verify_tls12_signature( + &self, + _m: &[u8], + _c: &CertificateDer, + _d: &rustls::DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + fn verify_tls13_signature( + &self, + _m: &[u8], + _c: &CertificateDer, + _d: &rustls::DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + fn supported_verify_schemes(&self) -> Vec { + rustls::crypto::ring::default_provider() + .signature_verification_algorithms + .supported_schemes() + } +} + +/// A minimal datagram-mode [`QuicOptions`] bundle, mirroring the transport-level +/// smoke-test config: a real (if unused) cert/verifier pair so the quinn +/// endpoint builds. +fn test_quic_options() -> QuicOptions { + let endpoint_cfg = quinn_proto::EndpointConfig::new(Arc::new(ring::hmac::Key::new( + ring::hmac::HMAC_SHA256, + &[0x5au8; 32], + ))); + + let (chain, key) = self_signed(); + let provider = Arc::new(rustls::crypto::ring::default_provider()); + let rustls_server = rustls::ServerConfig::builder_with_provider(provider.clone()) + .with_protocol_versions(&[&TLS13]) + .expect("TLS 1.3") + .with_no_client_auth() + .with_single_cert(chain, key) + .expect("valid self-signed cert"); + let server = quinn_proto::ServerConfig::with_crypto(Arc::new( + quinn_proto::crypto::rustls::QuicServerConfig::try_from(Arc::new(rustls_server)).expect("qsc"), + )); + + let rustls_client = rustls::ClientConfig::builder_with_provider(provider) + .with_protocol_versions(&[&TLS13]) + .expect("TLS 1.3") + .dangerous() + .with_custom_certificate_verifier(Arc::new(AnyServer)) + .with_no_client_auth(); + let client = quinn_proto::ClientConfig::new(Arc::new( + quinn_proto::crypto::rustls::QuicClientConfig::try_from(Arc::new(rustls_client)).expect("qcc"), + )); + + QuicOptions::new( + endpoint_cfg, + server, + client, + quinn_proto::TransportConfig::default(), + "localhost", + UnreliableTransport::Datagram, + ) +} + +/// Build a standalone serf `QuicEndpoint` over a memberlist QUIC coordinator — +/// no bound socket, no driver loop; just the composed machine, for driving the +/// ingress surface directly. +fn build_endpoint() -> QuicEndpoint { + let local: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let inner_opts = memberlist_proto::EndpointOptions::new(SmolStr::new("node"), local) + .with_user_broadcast_tiers(NonZeroU8::new(3).expect("3 is nonzero")); + let inner = memberlist_proto::Endpoint::new(inner_opts, StdRng::seed_from_u64(1)); + let coord = memberlist_proto::QuicEndpoint::new(inner, test_quic_options()); + QuicEndpoint::new_with_rng(coord, SerfOptions::new(), StdRng::seed_from_u64(2)) +} + +/// The QUIC past-due drain must DECODE every drained gossip datagram before +/// `handle_timeout` fires, or a buffered Ack would still be undecoded when the +/// suspicion sweep runs and the live peer it acked would be falsely suspected. +/// +/// On QUIC, `handle_udp` only BUFFERS the raw gossip frame (unlike the stream +/// pump, which decodes inline in `dispatch_gossip`); the decode happens in +/// `drain_ingress` (`poll_memberlist_ingress` → decode → `handle_message`). The +/// fix's per-datagram step calls `drain_ingress` for each drained frame BEFORE +/// re-polling the deadline and firing `handle_timeout`. Two frames are buffered +/// here — a non-Ack preceding a (would-be) Ack — to prove `drain_ingress` +/// consumes BOTH, not just the first; the socket-level bound that reads both +/// datagrams off the queue is proven in the driver's `shared::tests`. A full +/// timing-driven false-suspicion exercise is not deterministic as a unit test, +/// so this asserts the fallback the past-due path guarantees: the pre-timeout +/// drain leaves nothing undecoded for `handle_timeout` to race. +#[test] +fn past_due_drain_decodes_buffered_gossip_before_timeout() { + let mut endpoint = build_endpoint(); + let peer: SocketAddr = "127.0.0.1:65000".parse().expect("peer addr"); + let now = Instant::now(); + + // Two `handle_udp` calls, as two drained datagrams would: a first byte of 1 + // (the `Compound` message tag) is demuxed to the gossip plane and BUFFERED raw + // — not processed as a quinn packet, and not decoded inline. The non-Ack + // precedes the would-be Ack in the buffer. + endpoint.handle_udp(peer, &[1u8, 0, 0, 0], now); + endpoint.handle_udp(peer, &[1u8, 0, 0, 0], now); + + // The past-due path's per-datagram drain pops the buffered frames and feeds + // each back through `handle_message`: a drained Ack resolves its probe here, + // ahead of the suspicion sweep. Asserting `true` proves the frames were + // buffered (not decoded inline) and that the drain consumed them. + assert!( + drain_ingress::(&mut endpoint, &None), + "the past-due pre-timeout ingress drain must consume the buffered gossip frames so no \ + buffered Ack is left undecoded when handle_timeout fires" + ); + + // BOTH frames are consumed — nothing remains buffered for a subsequent + // `handle_timeout` to race. + assert!( + endpoint.poll_memberlist_ingress().is_none(), + "drain_ingress must leave the memberlist ingress queue empty before handle_timeout" + ); +} + +/// The recv buffer must hold the larger of the two planes that share the QUIC +/// socket. When a caller's quinn `EndpointConfig` accepts a max UDP payload above +/// the gossip MTU — quinn's own default 1472 already exceeds the 1400 default +/// `gossip_mtu` — the buffer is sized from the QUIC plane, not silently left at +/// the gossip size that would truncate a full-size QUIC packet. +#[test] +fn recv_buf_sizes_to_the_larger_plane() { + // QUIC plane far above the gossip plane (jumbo `max_udp_payload_size`): the + // buffer follows the QUIC plane regardless of the small AEAD overhead. + assert_eq!(recv_buf_len_for(1400, 9000), 9000); + + // quinn's default max UDP payload (1472) exceeds the default gossip MTU (1400), + // so the buffer is sized to 1472, not 1400 — a full-size QUIC packet is not + // truncated. Holds with or without an AEAD backend since + // 1472 > 1400 + ENCRYPTED_WRAPPER_OVERHEAD. + assert_eq!(recv_buf_len_for(1400, 1472), 1472); +} + +/// When the gossip plane is the larger of the two, the QUIC ceiling never shrinks +/// it below the AEAD-inflated gossip requirement. +#[test] +fn recv_buf_keeps_the_gossip_requirement() { + // Large configured gossip MTU, default-ish QUIC payload: the gossip plane wins + // and keeps its encrypted-wrapper headroom. + assert_eq!( + recv_buf_len_for(16_000, 1472), + 16_000 + ENCRYPTED_WRAPPER_OVERHEAD + ); + + // The gossip plane is capped at the IPv4 UDP maximum; a small QUIC payload + // cannot shrink it below that cap. + assert_eq!(recv_buf_len_for(70_000, 1472), GOSSIP_RECV_BUF_MAX); +} + +/// quinn permits a `max_udp_payload_size` up to 65527, just above the gossip +/// plane's 65507 cap; the QUIC plane is NOT clamped to that cap, so such a packet +/// is buffered in full. +#[test] +fn recv_buf_does_not_clamp_quic_below_quinn_max() { + assert_eq!(recv_buf_len_for(70_000, 65_527), 65_527); +} + +/// The drain-first timeout chokepoint must read the shared UDP socket BEFORE +/// deciding on `handle_timeout`: on a completion backend a freshly-submitted recv +/// is pending on first poll, so the main select's timer arm winning is NOT proof +/// of a would-block — a near-deadline Ack can be queued. `fire_quic_timeout` is +/// the single `handle_timeout` site, reached from both the past-due branch and +/// the main timer arm; this proves it actually invokes the drain on the real +/// socket. The socket-level proof that the drain reads EVERY queued datagram is +/// in the driver's `shared::tests`. +#[compio::test] +async fn fire_quic_timeout_drains_socket_before_handle_timeout() { + let mut endpoint = build_endpoint(); + let any: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let driver = UdpSocket::bind(any).await.expect("bind driver socket"); + let driver_addr = driver.local_addr().expect("driver local_addr"); + let peer = UdpSocket::bind(any).await.expect("bind peer socket"); + + // A single gossip compound-tagged datagram queued in the driver's socket — the + // near-deadline "Ack" the chokepoint must consume before any suspicion. + peer + .send_to(vec![1u8, 0, 0, 0], driver_addr) + .await + .0 + .expect("queue a datagram"); + // Let loopback delivery settle so the drain's eager recv reads the datagram on + // its first poll (production's past-due Ack is already buffered). + compio::time::sleep(Duration::from_millis(100)).await; + + let opts = RuntimeOptions::new(); + let dirty = + fire_quic_timeout::(&mut endpoint, &driver, 64, opts, &None).await; + assert!( + dirty, + "the chokepoint consumed the queued datagram (and/or fired handle_timeout)" + ); + + // The datagram was drained off the socket before handle_timeout: a fresh + // bounded recv now BLOCKS (the timer wins) rather than returning the + // still-queued datagram immediately — proving the chokepoint read the socket. + // Scope the recv future so it drops (releasing its borrow of `driver`) before + // the socket is closed below. + let socket_drained = { + let buf = vec![0u8; 64]; + let recv = driver.recv_from(buf).fuse(); + let timer = compio::time::sleep(Duration::from_millis(200)).fuse(); + pin_mut!(recv, timer); + select_biased! { + _ = recv => false, + _ = timer => true, + } + }; + assert!( + socket_drained, + "fire_quic_timeout must drain the queued datagram off the socket before handle_timeout" + ); + + // Ignoring Err: test cleanup of the probe sockets. + let _ = driver.close().await; + let _ = peer.close().await; +} diff --git a/serf-compio/src/driver/shared/mod.rs b/serf-compio/src/driver/shared/mod.rs new file mode 100644 index 00000000..b1d018f4 --- /dev/null +++ b/serf-compio/src/driver/shared/mod.rs @@ -0,0 +1,205 @@ +//! Driver helpers shared by every transport backend's driver loop. +//! +//! These are the observation / event hand-off helpers independent of the +//! reliable plane: the [`Delegate`] hook dispatcher, the cooperative yield +//! that drains a bounded observation channel, and the observation byte-backstop +//! accounting. They live here so every backend reuses them without duplicating +//! the logic. +#[cfg(any(feature = "tcp", feature = "quic"))] +use core::task::Poll; + +/// Yield to the runtime exactly once. +/// +/// The event drain is synchronous — no `.await` fires for membership events — +/// so on a single-threaded runtime the observation task is not scheduled +/// mid-drain. A bounded `obs_tx` would overflow on a single large-but-valid +/// burst (e.g. a join push-pull carrying many members) before the task drains +/// a single event. Yielding hands the scheduler to the already-woken +/// observation task so it can drain `obs_rx` before the drain continues. +/// +/// Runtime-agnostic: re-arms the waker and returns `Pending` once, so the +/// executor runs other ready tasks before re-polling this one. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) async fn yield_once() { + let mut yielded = false; + core::future::poll_fn(move |cx| { + if yielded { + Poll::Ready(()) + } else { + yielded = true; + cx.waker().wake_by_ref(); + Poll::Pending + } + }) + .await +} + +/// Coordinator-allocated handle for one in-flight reliable exchange. +/// +/// Shared by the TCP driver and the per-bridge task so they agree on the +/// same opaque id without the rest of the crate naming the machine's +/// streams module. +#[cfg(feature = "tcp")] +pub(crate) type ExchangeId = memberlist_proto::event::ExchangeId; + +/// Dispatch the matching [`Delegate`] hook for one drained serf [`Event`]. +/// +/// Member hooks run once per affected member in the batch; user-event and +/// query hooks run once per event. The observation delegate observes +/// transitions the FSM has already applied — it is NOT an admission gate. +/// +/// Requires a stream or QUIC transport feature. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) async fn dispatch_event_delegate( + delegate: &D, + ev: &serf_proto::event::Event, +) where + D: crate::delegate::Delegate, + I: Clone, + A: Clone, +{ + use serf_proto::event::{Event, MemberEventKind}; + use std::sync::Arc; + + match ev { + Event::Member(me) => { + for m in me.members() { + let arc = Arc::new(m.clone()); + match me.kind() { + MemberEventKind::Join => delegate.notify_join(arc).await, + MemberEventKind::Leave => delegate.notify_leave(arc).await, + MemberEventKind::Failed => delegate.notify_failed(arc).await, + MemberEventKind::Update => delegate.notify_update(arc).await, + MemberEventKind::Reap => delegate.notify_reap(arc).await, + } + } + } + Event::User(msg) => delegate.notify_user_event(msg).await, + Event::Query(ev) => delegate.notify_query(ev).await, + // Other variants (QueryResponse, QueryAck, Shutdown, LeftCluster, + // RelayDropped, DialRequested, KeyResponse, KeyRequest) carry no + // observation hook — the driver surfaces them through the EventStream. + _ => {} + } +} + +/// Byte-backstop weight accounting: add a just-enqueued event's payload +/// weight (if any) to the counter. Paired with the subtract in each +/// driver's observation task on dequeue. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) fn add_obs_payload(counter: &std::cell::Cell, bytes: Option) { + if let Some(b) = bytes { + counter.set(counter.get().saturating_add(b)); + } +} + +/// Byte-backstop weight of a serf event. Delegates to +/// [`serf_driver::observation_payload_bytes`]. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) use serf_driver::observation_payload_bytes; + +/// Bounded past-due UDP drain shared by both driver pumps. +/// +/// When the coordinator's wake deadline is already past, the driver must not +/// fire `handle_timeout` while a probe Ack still sits unread in the kernel +/// socket queue: the shared UDP socket interleaves gossip Acks with other +/// datagrams (raw QUIC packets on the QUIC plane, unrelated gossip on both), so +/// reading only the FIRST buffered datagram can leave a live peer's Ack unread +/// and the peer falsely suspected. Drain every immediately-ready datagram — +/// bounded by `budget` — so `on_datagram` decodes each before the caller +/// re-checks the deadline. +/// +/// # Emptiness via a completion reap, not a time window +/// +/// Each iteration polls one one-shot `recv_from`. If it is `Pending`, force a +/// synchronous, non-blocking proactor reap — [`Runtime::poll_with`] with a +/// [`Duration::ZERO`](core::time::Duration::ZERO) timeout — then re-poll: +/// `Ready` means a queued datagram was reaped, still-`Pending` means the socket +/// is genuinely empty. That is a CQE/readiness-grounded emptiness signal, not a +/// `sleep`-length guess at how long a kernel-buffered datagram needs to surface. +/// +/// The race this guards against is **io_uring-specific**: there a +/// freshly-submitted `recv` is always `Pending` on its first poll (the SQE has +/// not been reaped yet), so any timer raced against it could fire the timeout +/// while a buffered Ack went unread. `poll_with(ZERO)` submits and reaps that SQE +/// inline, which is the io_uring fix. On kqueue and IOCP the recv runs an eager +/// syscall at submit time and is already `Ready` on its first poll for buffered +/// data, so there the eager syscall is itself the emptiness primitive and the +/// reap is a harmless no-op. +/// +/// `on_datagram` processes each datagram and returns whether the deadline is +/// STILL past — returning `false` (a drained Ack resolved the probe) ends the +/// drain early so the main select regains fairness over any remaining datagrams. +/// `budget` caps the loop so a continuous flood cannot starve the timer (callers +/// pass a nonzero cap so recv always gets at least one shot). +/// +/// Returns `true` iff at least one datagram was consumed (endpoint state may have +/// changed). There is no emptiness gate on firing: the caller fires +/// `handle_timeout` purely on its post-drain deadline re-check. When the socket +/// drained empty the Ack was already reaped and decoded, so the deadline is no +/// longer past; when a budget cap or a flood cut the drain short, firing for +/// liveness yields at worst a transient, SWIM-refutable false Suspect rather than +/// the frozen-timer stall. +/// +/// The residual exact-instant arrival races — an io_uring recv cancelled on drop +/// losing its datagram (pre-existing), and the IOCP window where a completion is +/// posted just after the reap — are the irreducible "the Ack arrives the same +/// instant the timer fires" case, delegated to SWIM suspect-refute with no +/// machine-side compensation. +/// +/// [`Runtime::poll_with`]: compio::runtime::Runtime::poll_with +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) async fn drain_past_due_udp( + socket: &compio::net::UdpSocket, + recv_buf_len: usize, + budget: usize, + mut on_datagram: F, +) -> bool +where + F: FnMut(std::net::SocketAddr, &[u8]) -> bool, +{ + use compio::buf::BufResult; + use core::{future::Future, task::Poll}; + use futures_util::pin_mut; + + let mut drained = false; + for _ in 0..budget { + let buf = vec![0u8; recv_buf_len]; + let recv = socket.recv_from(buf); + pin_mut!(recv); + let outcome = core::future::poll_fn(|cx| match recv.as_mut().poll(cx) { + Poll::Ready(br) => Poll::Ready(Some(br)), + Poll::Pending => { + // Ignoring Err: try_with_current returns Err only with no current runtime, + // which is unreachable inside the driver loop's `block_on`. + let _ = compio::runtime::Runtime::try_with_current(|rt| { + rt.poll_with(Some(core::time::Duration::ZERO)) + }); + match recv.as_mut().poll(cx) { + Poll::Ready(br) => Poll::Ready(Some(br)), + // Still Pending after a real reap — the socket is genuinely empty. + Poll::Pending => Poll::Ready(None), + } + } + }) + .await; + match outcome { + Some(BufResult(Ok((n, src)), buf)) => { + drained = true; + // A drained Ack cleared the deadline — stop so the main select regains + // fairness over any remaining datagrams. + if !on_datagram(src, &buf[..n]) { + break; + } + } + // A transient recv error ends the drain like an empty socket. + Some(BufResult(Err(_), _)) => break, + // Still Pending after a real reap → the socket is empty. + None => break, + } + } + drained +} + +#[cfg(all(test, any(feature = "tcp", feature = "quic")))] +mod tests; diff --git a/serf-compio/src/driver/shared/tests.rs b/serf-compio/src/driver/shared/tests.rs new file mode 100644 index 00000000..a017f851 --- /dev/null +++ b/serf-compio/src/driver/shared/tests.rs @@ -0,0 +1,155 @@ +//! Unit tests for the shared bounded past-due UDP drain. +//! +//! Both driver pumps (QUIC and stream) reach `handle_timeout` through the same +//! [`drain_past_due_udp`] helper, so these socket-level tests are the structural +//! proof for BOTH pumps. They assert the drain consumes EVERY queued datagram +//! (not just the first), honors the early stop when `on_datagram` resolves the +//! deadline, and is bounded by `budget`. On the local kqueue backend the eager +//! recv syscall makes a buffered datagram `Ready` on its first poll, so these are +//! deterministic. + +use super::*; + +use std::{cell::Cell, net::SocketAddr}; + +use compio::net::UdpSocket; + +/// Bind a loopback driver/peer UDP socket pair, returning `(driver, peer, +/// driver_addr)` so the peer can buffer datagrams into the driver's queue. +async fn socket_pair() -> (UdpSocket, UdpSocket, SocketAddr) { + let any: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let driver = UdpSocket::bind(any).await.expect("bind driver socket"); + let driver_addr = driver.local_addr().expect("driver local_addr"); + let peer = UdpSocket::bind(any).await.expect("bind peer socket"); + (driver, peer, driver_addr) +} + +/// Give loopback UDP time to deliver the just-sent datagrams into the driver's +/// receive buffer before the drain runs, so the eager recv reads them on its +/// first poll. This mirrors production — the past-due Ack has been buffered since +/// well before the deadline — and is not a drain-internal time window. +async fn settle() { + compio::time::sleep(core::time::Duration::from_millis(100)).await; +} + +/// The core regression: two datagrams sit buffered (a non-Ack precedes the +/// would-be Ack). Reading only the first could fire `handle_timeout` before the +/// second was read; the bounded drain must consume BOTH — the reap-then-recheck +/// surfaces the second — and report that it drained. +#[compio::test] +async fn drains_all_ready_datagrams_not_just_one() { + let (driver, peer, dst) = socket_pair().await; + peer.send_to(vec![0xA1u8], dst).await.0.expect("send first"); + peer + .send_to(vec![0xA2u8], dst) + .await + .0 + .expect("send second"); + settle().await; + + let count = Cell::new(0usize); + let drained = drain_past_due_udp(&driver, 64, 8, |_src, _bytes| { + count.set(count.get() + 1); + // The deadline is still past after each non-Ack datagram, so keep draining. + true + }) + .await; + + assert!( + drained, + "the drain reports it consumed at least one datagram" + ); + assert_eq!( + count.get(), + 2, + "the bounded drain must consume BOTH queued datagrams, not just the first" + ); + + // Ignoring Err: test cleanup of the probe sockets. + let _ = driver.close().await; + let _ = peer.close().await; +} + +/// Once a consumed datagram resolves the probe deadline (`on_datagram` returns +/// `false`) the drain stops immediately, leaving the remaining datagrams for the +/// main select. +#[compio::test] +async fn stops_when_deadline_resolved() { + let (driver, peer, dst) = socket_pair().await; + peer.send_to(vec![0xC1u8], dst).await.0.expect("send first"); + peer + .send_to(vec![0xC2u8], dst) + .await + .0 + .expect("send second"); + settle().await; + + let count = Cell::new(0usize); + let drained = drain_past_due_udp(&driver, 64, 8, |_src, _bytes| { + count.set(count.get() + 1); + // The first datagram resolved the deadline (a buffered Ack): stop draining. + false + }) + .await; + + assert!(drained); + assert_eq!( + count.get(), + 1, + "a resolved deadline stops the drain after the resolving datagram" + ); + + // Ignoring Err: test cleanup of the probe sockets. + let _ = driver.close().await; + let _ = peer.close().await; +} + +/// More datagrams are queued than the budget, so the drain stops at the cap: the +/// loop is bounded by `budget` no matter how many datagrams remain queued. +#[compio::test] +async fn drain_is_bounded_by_budget() { + let (driver, peer, dst) = socket_pair().await; + for b in [0xB1u8, 0xB2, 0xB3] { + peer.send_to(vec![b], dst).await.0.expect("send"); + } + settle().await; + + let count = Cell::new(0usize); + let drained = drain_past_due_udp(&driver, 64, 2, |_src, _bytes| { + count.set(count.get() + 1); + true + }) + .await; + + assert!(drained); + assert_eq!( + count.get(), + 2, + "the drain stops at the budget cap even when more datagrams are queued" + ); + + // Ignoring Err: test cleanup of the probe sockets. + let _ = driver.close().await; + let _ = peer.close().await; +} + +/// An empty socket queue drains nothing: the recv is `Pending`, the forced reap +/// finds no completion, the re-poll is still `Pending`, and the drain reports it +/// consumed nothing. +#[compio::test] +async fn empty_socket_drains_nothing() { + let (driver, _peer, _dst) = socket_pair().await; + + let count = Cell::new(0usize); + let drained = drain_past_due_udp(&driver, 64, 8, |_src, _bytes| { + count.set(count.get() + 1); + true + }) + .await; + + assert!(!drained, "an empty socket queue drains nothing"); + assert_eq!(count.get(), 0); + + // Ignoring Err: test cleanup of the probe socket. + let _ = driver.close().await; +} diff --git a/serf-compio/src/driver/stream/mod.rs b/serf-compio/src/driver/stream/mod.rs new file mode 100644 index 00000000..6bebb887 --- /dev/null +++ b/serf-compio/src/driver/stream/mod.rs @@ -0,0 +1,1820 @@ +//! Stream-plane driver pump — single owner of the serf `StreamEndpoint`, the +//! UDP gossip socket, the TCP reliable listener, and the per-bridge handle +//! table. +//! +//! Runs a `select_biased` loop over its arms in priority order: gossip UDP +//! recv, the coordinator-supplied wake timer, listener accept, the command +//! channel, outbound-dial completion, and per-bridge inbound bytes. After each +//! fired arm the pump drains every outbound surface (`poll_action`, +//! `poll_transport_transmit`, `poll_memberlist_transmit`, `poll_event`) until +//! no method makes progress, republishes a fresh [`SerfSnapshot`] when state +//! changed, and re-enters the select. The pump owns the `StreamEndpoint` +//! outright; user-facing handles communicate exclusively via the command +//! channel and read state through the lock-free snapshot. The listener and +//! gossip socket are explicitly closed (awaited) when the loop exits so the +//! bound ports are released before shutdown returns. + +use std::{cell::Cell, collections::HashMap, io, net::SocketAddr, rc::Rc}; + +use core::{ + task::{Context, Poll, Waker}, + time::Duration, +}; + +use bytes::Bytes; +use compio::{ + buf::BufResult, + net::{TcpListener, TcpStream, UdpSocket}, +}; +use flume::{Receiver, Sender}; +use futures_util::{FutureExt, future::FusedFuture, pin_mut, select_biased}; +use lochan::mpsc; +use memberlist_proto::{ + Instant, PushPullKind, SeedableRng, Transmit, + codec::{ + DecodeOptions, EncodeOptions, decode_incoming, encode_outgoing, encode_outgoing_compound, + parse_messages, + }, + streams::{StreamAction, StreamTransport}, +}; +use serf_proto::{LamportTime, StreamEndpoint, event::Event, members::SerfState}; + +#[cfg(encryption)] +use crate::command::{KeyCmd, ListKeysCmd}; +#[cfg(encryption)] +use crate::delegate::KeyringDelegate; +use crate::{ + Channel, + command::{ + Command, ForceLeaveCmd, JoinCmd, LeaveCmd, QueryCmd, RespondCmd, SetEventJoinIgnoreCmd, + SetTagsCmd, ShutdownCmd, + }, + delegate::Delegate, + driver::{ + options::{RuntimeOptions, StreamTransportOptions}, + shared::{ + ExchangeId, add_obs_payload, dispatch_event_delegate, drain_past_due_udp, + observation_payload_bytes, yield_once, + }, + }, + error::{Result, SerfError}, + snapshot::{SerfSnapshot, SnapshotCell}, +}; +#[cfg(encryption)] +use serf_proto::{KeyRequestOperation, KeyResponseArgs, event::KeyRequest}; + +/// Driver-side state for the single in-flight graceful-leave operation. +/// +/// A [`Command::Leave`] that finds the endpoint `Alive` initiates the machine's +/// `leave()`, which queues the leave intent + direct notices to peers and +/// withholds [`Event::LeftCluster`] until they have drained. The pump parks this +/// here and replies only once that `LeftCluster` arrives (success) or `deadline` +/// elapses ([`SerfError::LeaveTimeout`]) — so a returned `Ok(())` means the +/// leave actually reached the wire, never merely that it was queued. +/// +/// Leave is a SHARED operation: a second `Command::Leave` racing an in-flight +/// one (cloned handles can both call `leave()`) does NOT re-invoke +/// `endpoint.leave()` (a repeated leave once already `Leaving`/`Left` is a +/// terminal no-op that emits no completion event, so a fresh waiter would hang). +/// Instead it joins this in-flight operation by pushing its reply onto +/// `repliers`. Every terminal path — `LeftCluster` success, timeout reap, +/// shutdown — drains EVERY replier. +struct PendingLeave { + /// Reply channels of every `leave()` caller that joined this in-flight leave + /// — the initiator plus any racing clones. Drained together on the single + /// terminal outcome (`Ok` on `LeftCluster`, `LeaveTimeout` on deadline, + /// `Shutdown` on teardown). + repliers: Vec>>, + /// Wall-clock instant past which the pump replies [`SerfError::LeaveTimeout`] + /// to every replier even if `LeftCluster` has not yet fired. + deadline: Instant, +} + +impl PendingLeave { + /// Reply to every joined `leave()` caller with a fresh `Result<()>` from + /// `make_result`. A constructor closure (rather than a single cloned value) + /// sidesteps `SerfError` not being `Clone` — every terminal outcome here + /// (`Ok(())`, `LeaveTimeout`, `Shutdown`) is trivially reconstructible. + async fn resolve_all(self, mut make_result: impl FnMut() -> Result<()>) { + for replier in self.repliers { + // Ignoring Err: a `leave()` caller dropped its reply receiver (its + // user-facing future was cancelled); nothing to surface. + let _ = replier.send(make_result()); + } + } +} + +/// Pump-loop-local state tracking outstanding commands awaiting completion. +struct PendingCommands { + /// Outstanding graceful-leave waiter (at most one at a time). See [`PendingLeave`]. + leave: Option, +} + +/// Payload for [`BridgeInbound::Bytes`]: a slice of plaintext bytes the +/// per-bridge task read from its stream half, addressed to one exchange. +pub(crate) struct BridgeBytes { + /// Exchange the bytes belong to. + pub(crate) eid: ExchangeId, + /// Owned heap copy of the bytes read off the socket. + pub(crate) bytes: Vec, + /// Wall-clock instant at which the bridge observed these bytes on the + /// socket. Passed to `handle_transport_data` as the observation time so a + /// response that arrived BEFORE the exchange deadline is not retroactively + /// timed out by the pump's later `Instant::now()` sample. + pub(crate) received_at: Instant, +} + +/// Payload for [`BridgeInbound::Eof`]: the per-bridge task observed an orderly +/// close (read returned `Ok(0)` or the close-signal arm fired). +pub(crate) struct BridgeEof { + /// Exchange that hit EOF. + pub(crate) eid: ExchangeId, + /// Wall-clock instant at which the bridge observed the peer's FIN. See + /// [`BridgeBytes::received_at`] for the deadline-gate rationale. + pub(crate) received_at: Instant, +} + +/// Payload for [`BridgeInbound::Error`]: the per-bridge task hit an I/O error +/// on either the read or write half. +/// +/// The underlying `io::Error` is not carried: the coordinator's +/// `handle_transport_error` keys only on the exchange id, so the error value is +/// not needed downstream. +pub(crate) struct BridgeError { + /// Exchange that failed. + pub(crate) eid: ExchangeId, + /// Wall-clock instant at which the bridge observed the error. See + /// [`BridgeBytes::received_at`] for the deadline-gate rationale. + pub(crate) received_at: Instant, +} + +/// Payload for [`BridgeReady::OutboundOk`]: an outbound dial task successfully +/// connected to the peer. +/// +/// Carries the `out_rx` allocated at Connect time alongside the stream so the +/// bridge spawned at receipt sees every byte the pump queued via +/// `drain_transport_transmits` between Connect and dial completion (the machine +/// surfaces the first push/pull request on the same tick the Connect lands). +pub(crate) struct OutboundOkReady { + /// The exchange the connection belongs to. + pub(crate) eid: ExchangeId, + /// The connected stream. + pub(crate) stream: TcpStream, + /// The receive half of the bridge's pre-allocated out-channel. + pub(crate) out_rx: mpsc::Receiver, + /// The receive half of the bridge's pre-allocated cancel channel. + pub(crate) cancel_rx: futures_channel::oneshot::Receiver<()>, +} + +/// Payload for [`BridgeReady::OutboundFail`]: an outbound dial task hit a +/// connect error or dial timeout. +/// +/// The underlying `io::Error` is not carried: `handle_dial_failed` keys only on +/// the exchange id. +pub(crate) struct OutboundFailReady { + /// The exchange whose dial failed. + pub(crate) eid: ExchangeId, + /// Wall-clock instant at which the dial task observed the failure. See + /// [`BridgeBytes::received_at`] for the deadline-gate rationale — a + /// pre-deadline dial failure is observed as a clean terminalization rather + /// than rejected as a timeout. + pub(crate) received_at: Instant, +} + +/// Messages an outbound dial task sends back to the pump. +pub(crate) enum BridgeReady { + /// An outbound dial completed successfully. + OutboundOk(OutboundOkReady), + /// An outbound dial failed. + OutboundFail(OutboundFailReady), +} + +/// Messages a per-bridge task sends back to the pump. +pub(crate) enum BridgeInbound { + /// Plaintext bytes read from the stream. + Bytes(BridgeBytes), + /// Orderly close. + Eof(BridgeEof), + /// Unrecoverable I/O error. + Error(BridgeError), +} + +/// Messages the pump sends to a per-bridge byte-mover task. +/// +/// All variants share one channel so the bridge processes them in FIFO order: +/// every byte queued before a `ShutdownWrite` / `Close` is written to the peer +/// before the close signal fires. +pub(crate) enum BridgeOut { + /// Outbound bytes the coordinator surfaced via `poll_transport_transmit`. + Bytes(Vec), + /// Half-close the write side of the bridge's stream — the FIN-on-send-half + /// anchor of the push/pull half-close. Driven by [`StreamAction::Shutdown`]. + ShutdownWrite, + /// Full close — the bridge sends `BridgeInbound::Eof` and exits. Driven by + /// [`StreamAction::Close`]. + Close, +} + +/// Per-bridge handle the pump owns to communicate with the byte-mover. +/// +/// Two channels: the FIFO `out_tx` for bytes + graceful control, and an +/// out-of-band `cancel_tx` priority signal for a hard abort. A graceful +/// [`StreamAction::Close`] drives a flush-then-exit teardown via `out_tx`; a +/// failed [`StreamAction::Abort`] drives a hard cancel via `cancel_tx` that the +/// bridge selects on with priority, discarding any still-queued stale bytes. +struct BridgeHandle { + /// Bytes-or-graceful-control FIFO into the bridge task. + out_tx: mpsc::Sender, + /// Out-of-band hard-abort signal. + cancel_tx: futures_channel::oneshot::Sender<()>, +} + +/// Hard ceiling on the per-recv UDP buffer — UDP's wire payload is capped at +/// 65507 bytes once the IP/UDP headers are deducted, so a larger buffer just +/// wastes an allocation per iteration. +const GOSSIP_RECV_BUF_MAX: usize = 65507; + +/// The largest the encrypted wrapper can inflate a gossip datagram, or `0` when +/// no encryption backend is built in. The wrapper carries the algorithm tag, +/// nonce, and AEAD auth tag; sizing the recv buffer to include it keeps an +/// encrypted datagram from being silently truncated by the kernel. +#[cfg(encryption)] +const ENCRYPTED_WRAPPER_OVERHEAD: usize = memberlist_proto::ENCRYPTED_WRAPPER_OVERHEAD; +#[cfg(not(encryption))] +const ENCRYPTED_WRAPPER_OVERHEAD: usize = 0; + +/// Compute the per-recv UDP buffer size from the coordinator's `gossip_mtu`, +/// plus the encrypted-wrapper overhead when an encryption backend is built in, +/// clamped at [`GOSSIP_RECV_BUF_MAX`]. Sizing to the inflated value keeps a +/// configured `gossip_mtu` close to the historical default from being truncated +/// once the encryption tag/nonce are added on the wire. +fn gossip_recv_buf_len(endpoint: &StreamEndpoint) -> usize +where + I: memberlist_proto::Id + Clone, + RT: StreamTransport, + G: rand::Rng, + R: rand::Rng + SeedableRng, +{ + endpoint + .gossip_mtu() + .saturating_add(ENCRYPTED_WRAPPER_OVERHEAD) + .min(GOSSIP_RECV_BUF_MAX) +} + +/// Single-owner pump task. +/// +/// Drives the serf `StreamEndpoint` until the command channel closes (all +/// handles dropped) or a [`Command::Shutdown`] is received. All mutations on the +/// endpoint happen here; reads happen via the published [`SerfSnapshot`]. +#[allow(clippy::too_many_arguments)] +pub(crate) async fn stream_driver_loop( + mut endpoint: StreamEndpoint, + gossip_socket: UdpSocket, + listener: TcpListener, + commands: Receiver>, + events_tx: Sender>, + events_dropped: Rc>, + observation_dropped: Rc>, + snapshot: SnapshotCell, + shutdown_flag: Rc>, + driver_opts: RuntimeOptions, + stream_opts: StreamTransportOptions, + delegate: D, + // Cluster label applied to both gossip encode and decode. `None` accepts + // datagrams from any cluster. + label: Option, + // The driver's keyring delegate: applies inbound key-management ops and + // produces the `respond_key` answer. Present only under an encryption backend. + #[cfg(encryption)] keyring: Rc, +) where + D: Delegate, + I: memberlist_proto::Id + Clone, + RT: StreamTransport, + G: rand::Rng + Unpin, + R: rand::Rng + SeedableRng, +{ + let mut bridges: HashMap = HashMap::new(); + let (bridge_inbound_tx, mut bridge_inbound_rx) = + mpsc::bounded::(stream_opts.bridge_inbound_cap()); + let (bridge_ready_tx, bridge_ready_rx) = flume::unbounded::(); + + // Spawn the per-driver observation task — it owns the user `Delegate` and the + // `EventStream` sender and runs OFF this pump task: the pump `try_send`s every + // surfaced event onto `obs_tx`, the task dispatches the matching observation + // hook then forwards to subscribers. Decoupling keeps a slow `notify_*` / + // `merge_remote_state` from stalling protocol advancement — and therefore a + // parked leave reply that depends on a follow-up input the pump must service. + let (obs_tx, obs_rx) = match driver_opts.observation_channel() { + Channel::Unbounded => mpsc::unbounded::>(), + Channel::Bounded(n) => mpsc::bounded::>(n), + }; + // `obs_payload_bytes` tracks the bytes of payload-bearing events (`User`) + // currently queued in `obs_tx`: the pump adds on enqueue, the obs task + // subtracts on dequeue. The byte backstop bounds the memory a large payload + // occupies while a delegate falls behind. + let obs_payload_bytes = Rc::new(Cell::new(0u64)); + compio::runtime::spawn(observation_task::( + obs_rx, + delegate, + events_tx, + events_dropped.clone(), + obs_payload_bytes.clone(), + )) + .detach(); + + // Stash for the [`Command::Shutdown`] reply — acked AFTER the post-loop + // cleanup closes the listener and gossip socket so the bound ports are free + // when the caller resumes from `shutdown.await`. + let mut shutdown_reply: Option>> = None; + let mut pending = PendingCommands { leave: None }; + + // Per-pump UDP recv buffer size, derived once at entry from the coordinator's + // `gossip_mtu` (fixed for the endpoint lifetime). + let recv_buf_len = gossip_recv_buf_len::(&endpoint); + + // Observation-channel payload byte backstop. `Bounded(n)`'s count cap bounds + // the NUMBER of queued events, but one `User` event can own up to + // `max_stream_frame_size` bytes; cap the queued payload bytes at four frames' + // worth. `Unbounded` opts out of dropping, so it opts out of the byte backstop. + let obs_payload_budget: Option = match driver_opts.observation_channel() { + Channel::Bounded(_) => Some((endpoint.max_stream_frame_size() as u64).saturating_mul(4)), + Channel::Unbounded => None, + }; + + // Arm the periodic probe / gossip / push-pull schedulers. Without this the + // coordinator's schedulers stay unset, so failure detection, dissemination, + // and anti-entropy never run. + endpoint.start_scheduling(Instant::now()); + refresh_snapshot::(&endpoint, &snapshot); + + // Hoist the listener-accept future ACROSS loop iterations. On a + // completion-based backend (io_uring) `accept()` is an in-flight SQE; + // recreating it each iteration would drop (cancel) an already-accepted + // connection. Persisting the future means it is only recreated when it + // RESOLVES, never dropped mid-flight. + let mut accept_fut = Box::pin(listener.accept().fuse()); + + loop { + let mut dirty = false; + let mut exit = false; + + // Service any already-ready accept off the select's borrow, with bounded + // fairness, so a busy recv/timer socket cannot hold a kernel-accepted + // connection unbridged until the peer's handshake deadline. + if accept_fut.is_terminated() { + accept_fut.set(listener.accept().fuse()); + } + { + let mut accept_cx = Context::from_waker(Waker::noop()); + let mut accepted_n = 0; + while accepted_n < driver_opts.iter_drain_cap().max(1) { + match accept_fut.as_mut().poll(&mut accept_cx) { + Poll::Ready(accepted) => { + if handle_accepted::( + accepted, + &mut endpoint, + &mut bridges, + &bridge_inbound_tx, + stream_opts, + ) { + dirty = true; + } + accept_fut.set(listener.accept().fuse()); + accepted_n += 1; + } + Poll::Pending => break, + } + } + } + + // Iter-top command fairness drain so a network flood does not starve user + // commands (the `cmd` select arm sits below the network arms). + let mut cmd_drained = 0; + while cmd_drained < driver_opts.cmd_fairness_budget() { + match commands.try_recv() { + Ok(c) => { + let now = Instant::now(); + let is_shutdown = matches!(c, Command::Shutdown(_)); + if is_shutdown { + exit = true; + } + dispatch_command::( + &mut endpoint, + &mut bridges, + &bridge_ready_tx, + stream_opts, + &mut shutdown_reply, + &mut pending, + driver_opts.leave_timeout(), + c, + now, + ) + .await; + cmd_drained += 1; + dirty = true; + if is_shutdown { + break; + } + } + // All `Serf` handles dropped: tear down exactly as a `Command::Shutdown` + // would. Under a continuous UDP flood the main select's command arm is + // starved by the higher-priority recv arm, so this iter-top drain is the + // only path that observes the disconnect; collapsing it into `Empty` + // would spin the pump forever and leak the bound sockets. + Err(flume::TryRecvError::Disconnected) => { + exit = true; + break; + } + // No command queued right now — end the fairness drain. + Err(flume::TryRecvError::Empty) => break, + } + } + + // Drain every already-arrived bridge input BEFORE the timeout decision: a + // peer's push/pull response (or a bridge's EOF / error), AND an + // outbound-dial completion, MUST be applied to the coordinator before any + // `handle_timeout` call — otherwise the timeout sweep would wrongly mark + // exchanges overdue whose terminating bytes are already in the channel. + let mut drained = 0; + while drained < driver_opts.iter_drain_cap() { + match bridge_inbound_rx.try_recv() { + Ok(inbound) => { + dispatch_bridge_inbound::(&mut endpoint, inbound); + drained += 1; + dirty = true; + } + Err(_) => break, + } + } + drained = 0; + while drained < driver_opts.iter_drain_cap() { + match bridge_ready_rx.try_recv() { + Ok(ready) => { + handle_bridge_ready::( + &mut endpoint, + &mut bridges, + &bridge_inbound_tx, + ready, + stream_opts.bridge_recv_buf_len(), + stream_opts.close_timeout(), + ); + drained += 1; + dirty = true; + } + Err(_) => break, + } + } + + // Honor `exit` from the iter-top cmd drain before the select so a quiet + // shutdown lands promptly. Flush, reap, publish, break. + if exit { + drain_outputs::( + &mut endpoint, + &mut bridges, + &bridge_ready_tx, + stream_opts, + &gossip_socket, + &label, + &obs_tx, + &observation_dropped, + &obs_payload_bytes, + obs_payload_budget, + &mut pending, + #[cfg(encryption)] + &*keyring, + ) + .await; + reap_pending_leave(&mut pending.leave, Instant::now()).await; + refresh_snapshot::(&endpoint, &snapshot); + break; + } + + // Re-poll the deadline AFTER applying drained inputs (which may have + // advanced or cleared it). Fold in the earliest pending-leave deadline so + // the timer arm fires by a graceful leave's timeout even when the + // coordinator has no nearer deadline. + let setup_now = Instant::now(); + let endpoint_deadline = endpoint + .poll_timeout() + .unwrap_or(setup_now + driver_opts.idle_wake_interval()); + let timeout_deadline = [ + Some(endpoint_deadline), + min_pending_leave_deadline(&pending.leave), + ] + .into_iter() + .flatten() + .min() + .unwrap_or(endpoint_deadline); + + // BOUNDED past-due preemption. Under a continuous UDP-recv flood the main + // select's recv arm always wins over the timer, so `handle_timeout` would + // never fire and stale bridges / overdue probes would never be reaped. Route + // through the `fire_timeout_with_drain` chokepoint: it drains every + // immediately-ready gossip datagram (decoding each inline) AND every queued + // bridge completion before firing `handle_timeout` only if the deadline is + // still past, so a buffered Ack that sits BEHIND an unrelated datagram in the + // socket's queue still resolves its probe ahead of the suspicion sweep. This + // is a safe site to build the drain's `recv_from` SQEs — the main loop's + // `recv_fut` is not yet in flight, so there is no second-SQE race. + if setup_now >= timeout_deadline { + if fire_timeout_with_drain::( + &mut endpoint, + &mut bridges, + &bridge_inbound_tx, + &mut bridge_inbound_rx, + &bridge_ready_rx, + &gossip_socket, + recv_buf_len, + &label, + driver_opts, + stream_opts, + ) + .await + { + dirty = true; + } + + let terminal = drain_outputs::( + &mut endpoint, + &mut bridges, + &bridge_ready_tx, + stream_opts, + &gossip_socket, + &label, + &obs_tx, + &observation_dropped, + &obs_payload_bytes, + obs_payload_budget, + &mut pending, + #[cfg(encryption)] + &*keyring, + ) + .await; + reap_pending_leave(&mut pending.leave, Instant::now()).await; + if dirty { + refresh_snapshot::(&endpoint, &snapshot); + } + // A conflict `Event::Shutdown` observed during the drain is terminal: break + // into teardown rather than re-entering the loop. + if terminal { + break; + } + continue; + } + + // Drained inputs may have advanced state without past-due timer pressure — + // flush their outputs before entering the select so a snapshot observer + // sees the post-input state promptly. + if dirty { + let terminal = drain_outputs::( + &mut endpoint, + &mut bridges, + &bridge_ready_tx, + stream_opts, + &gossip_socket, + &label, + &obs_tx, + &observation_dropped, + &obs_payload_bytes, + obs_payload_budget, + &mut pending, + #[cfg(encryption)] + &*keyring, + ) + .await; + reap_pending_leave(&mut pending.leave, Instant::now()).await; + refresh_snapshot::(&endpoint, &snapshot); + dirty = false; + // A conflict `Event::Shutdown` observed during the flush is terminal. + if terminal { + break; + } + } + + let mut timer_fired = false; + { + let recv_buf = vec![0u8; recv_buf_len]; + let recv_fut = gossip_socket.recv_from(recv_buf).fuse(); + let cmd_fut = commands.recv_async().fuse(); + let ready_fut = bridge_ready_rx.recv_async().fuse(); + let timer_fut = compio::time::sleep_until(timeout_deadline.into_std()).fuse(); + pin_mut!(recv_fut, cmd_fut, ready_fut, timer_fut); + + // Arm priority (top → bottom): + // 1. recv — kernel-buffered UDP gossip (an Ack resolves a probe + // deadline before handle_timeout marks the peer suspect). + // 2. timer — past-due deadline (ahead of accept so a saturated + // listener cannot starve the suspicion / probe reapers). + // 3. accept — inbound TCP connections (front door for new exchanges). + // 4. cmd — user commands (demoted below the network arms so a + // cloned-handle command flood cannot starve them). + // 5. ready — outbound-dial completions. + // 6. bridge_in — per-bridge byte messages (lowest priority). + select_biased! { + gossip = recv_fut => { + let BufResult(res, buf) = gossip; + if let Ok((n, src)) = res { + let now = Instant::now(); + dispatch_gossip::(&mut endpoint, src, &buf[..n], now, label.clone()); + dirty = true; + } + // Ignoring Err: a transient recv error is non-fatal — the next + // iteration re-arms recv with a fresh buffer. + } + _ = timer_fut => { + // Defer the drain + `handle_timeout` to the `fire_timeout_with_drain` + // chokepoint AFTER this scope drops the in-flight `recv_fut` and the + // bridge-receiver borrows: a freshly-submitted recv can be pending on + // its first poll on a completion backend, so the timer winning does NOT + // prove a would-block. The chokepoint drains the gossip socket and the + // bridge channels before deciding on `handle_timeout`, and dropping + // `recv_fut` first avoids a second concurrent `recv_from` SQE. + timer_fired = true; + } + accepted = accept_fut.as_mut() => { + if handle_accepted::( + accepted, + &mut endpoint, + &mut bridges, + &bridge_inbound_tx, + stream_opts, + ) { + dirty = true; + } + } + cmd = cmd_fut => { + match cmd { + Ok(c) => { + exit = matches!(c, Command::Shutdown(_)); + let now = Instant::now(); + dispatch_command::( + &mut endpoint, + &mut bridges, + &bridge_ready_tx, + stream_opts, + &mut shutdown_reply, + &mut pending, + driver_opts.leave_timeout(), + c, + now, + ).await; + dirty = true; + } + // All handles dropped → channel closed. Treat as shutdown. + Err(_) => exit = true, + } + } + ready = ready_fut => { + if let Ok(ready) = ready { + handle_bridge_ready::( + &mut endpoint, + &mut bridges, + &bridge_inbound_tx, + ready, + stream_opts.bridge_recv_buf_len(), + stream_opts.close_timeout(), + ); + dirty = true; + } + // Ignoring Err: the pump holds its own `bridge_ready_tx`, so the + // channel cannot disconnect while the loop is alive. + } + bi = bridge_inbound_rx.recv() => { + if let Some(inbound) = bi { + dispatch_bridge_inbound::(&mut endpoint, inbound); + dirty = true; + } + // Ignoring None: every bridge dropped its sender; later iterations + // wake on other arms or on a freshly-spawned bridge. + } + } + } + + // Past-due deadline, deferred from the timer arm so the in-flight `recv_fut` + // SQE and the bridge-receiver borrows are dropped before the chokepoint + // drains the gossip socket and `&mut`-drains the bridge receivers. Routes + // through the SAME `fire_timeout_with_drain` as the past-due branch so a + // near-deadline queued Ack is consumed before any suspicion. + if timer_fired + && fire_timeout_with_drain::( + &mut endpoint, + &mut bridges, + &bridge_inbound_tx, + &mut bridge_inbound_rx, + &bridge_ready_rx, + &gossip_socket, + recv_buf_len, + &label, + driver_opts, + stream_opts, + ) + .await + { + dirty = true; + } + + // A conflict `Event::Shutdown` drained here is terminal — fold it into + // `exit` so the loop breaks into teardown after delivering the event. + if drain_outputs::( + &mut endpoint, + &mut bridges, + &bridge_ready_tx, + stream_opts, + &gossip_socket, + &label, + &obs_tx, + &observation_dropped, + &obs_payload_bytes, + obs_payload_budget, + &mut pending, + #[cfg(encryption)] + &*keyring, + ) + .await + { + exit = true; + } + reap_pending_leave(&mut pending.leave, Instant::now()).await; + + if dirty { + refresh_snapshot::(&endpoint, &snapshot); + } + + if exit { + break; + } + } + + // Cleanup. Order: flip the shutdown flag so a racing clone observes it on + // entry, drain queued commands with Err(Shutdown), drop the command receiver + // so a late send fails fast, signal every live bridge to close, close the + // bound sockets (awaited so their ports are released), then ack the observed + // shutdown caller. + shutdown_flag.set(true); + while let Ok(c) = commands.try_recv() { + reply_shutdown(c); + } + drop(commands); + if let Some(pl) = pending.leave.take() { + pl.resolve_all(|| Err(SerfError::Shutdown)).await; + } + for (_eid, handle) in bridges.drain() { + // Ignoring Err: the bridge may have exited already; close is best-effort. + let _ = handle.out_tx.try_send(BridgeOut::Close); + } + // Drop the persistent accept future first: it holds an in-flight accept + // borrowing `listener`, so the listener cannot be moved into `close()` while it + // is alive. Cancelling a pending accept during shutdown is correct — a + // connection arriving as the driver tears down has nothing to be served. + drop(accept_fut); + // Await the listener close rather than a plain drop so the bound TCP port is + // released before the reply fires: a dropped compio listener is not guaranteed + // to close its fd synchronously (Windows IOCP closes asynchronously), so a + // plain drop could race a same-address rebind into AddrInUse. Mirrors the + // awaited close the TCP/TLS construction path uses to discard a retry listener. + // Ignoring Err: a close error during teardown is unactionable. + let _ = listener.close().await; + // Ignoring Err: socket close on shutdown — the runtime tears down fds anyway. + // Awaiting the close drains the UDP gossip socket so its kernel slot is + // released before the stashed reply fires. + let _ = gossip_socket.close().await; + + if let Some(reply) = shutdown_reply { + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(Ok(())); + } +} + +/// Reply `Err(Shutdown)` to a command drained during teardown. +fn reply_shutdown(c: Command) { + // Ignoring Err on each send: caller dropped the reply receiver. + match c { + Command::Join(JoinCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + Command::Leave(LeaveCmd { reply }) | Command::Shutdown(ShutdownCmd { reply }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + Command::SetEventJoinIgnore(SetEventJoinIgnoreCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + Command::ForceLeave(ForceLeaveCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + Command::UserEvent(cmd) => { + let _ = cmd.reply.send(Err(SerfError::Shutdown)); + } + Command::Query(QueryCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + Command::Respond(RespondCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + Command::SetTags(SetTagsCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + #[cfg(encryption)] + Command::InstallKey(KeyCmd { reply, .. }) + | Command::UseKey(KeyCmd { reply, .. }) + | Command::RemoveKey(KeyCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + #[cfg(encryption)] + Command::ListKeys(ListKeysCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + } +} + +/// Dispatch one [`Command`] onto the serf endpoint. Replies are best-effort via +/// the per-command reply channel — a dropped reply receiver means the caller +/// gave up. +/// +/// The [`Command::Shutdown`] reply is NOT acked inline; it is stashed into +/// `shutdown_reply` so the pump acks the caller only AFTER the sockets drop in +/// the post-loop cleanup. +#[allow(clippy::too_many_arguments)] +async fn dispatch_command( + endpoint: &mut StreamEndpoint, + bridges: &mut HashMap, + bridge_ready_tx: &Sender, + stream_opts: StreamTransportOptions, + shutdown_reply: &mut Option>>, + pending: &mut PendingCommands, + leave_timeout: Duration, + cmd: Command, + now: Instant, +) where + I: memberlist_proto::Id + Clone, + RT: StreamTransport, + G: rand::Rng, + R: rand::Rng + SeedableRng, +{ + let running = endpoint.state() == SerfState::Alive; + match cmd { + Command::Join(JoinCmd { seeds, reply }) => { + // Gate on a running node: `leave()` is terminal (it stops the periodic + // schedulers), so a join after leave would leave the node non-participating. + if !running { + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(Err(SerfError::NotRunning)); + return; + } + // Announce the serf-level join intent so peers learn the local join ltime + // without waiting for the next anti-entropy round. + if let Err(e) = endpoint.join() { + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(Err(SerfError::from(e))); + return; + } + // Dial every seed via a coordinator push-pull (the driver owns the + // inner-memberlist join). Each `start_push_pull` queues a `Connect` the + // inline drain routes to its bridge before the next seed dials. + let count = seeds.len(); + for seed in seeds { + let _sid = endpoint.start_push_pull(seed, PushPullKind::Join, now); + while let Some(action) = endpoint.poll_action() { + process_one_action(action, bridges, bridge_ready_tx, stream_opts); + } + } + // serf surfaces a push-pull's outcome as the internal `RemoteStateReceived` + // sieve, not an `ExchangeCompleted` event, so the pump reports the count of + // seeds dispatched rather than parking for per-exchange contact accounting. + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(Ok(count)); + } + Command::Leave(LeaveCmd { reply }) => { + // Leave is a SHARED in-flight operation. If one is in flight, JOIN it (do + // not re-invoke `leave()`, which once `Leaving`/`Left` is a terminal no-op + // emitting no second `LeftCluster`). Otherwise INITIATE: snapshot `Alive` + // before the call (it decides whether a `LeftCluster` will fire), then park + // (was Alive) or reply immediately (idempotent no-op / error). + if let Some(pl) = pending.leave.as_mut() { + pl.repliers.push(reply); + } else { + let was_alive = running; + let res: Result<()> = endpoint.leave(now).map_err(SerfError::from); + match res { + Ok(()) if was_alive => { + pending.leave = Some(PendingLeave { + repliers: vec![reply], + deadline: now + leave_timeout, + }); + } + // Idempotent no-op (not Alive) ⇒ no `LeftCluster` will fire, or the + // call errored. Reply immediately; parking would hang. + other => { + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(other); + } + } + } + } + Command::ForceLeave(ForceLeaveCmd { + id, + prune, + now: at, + reply, + }) => { + let res = if running { + endpoint.force_leave(id, prune, at).map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(res); + } + Command::UserEvent(cmd) => { + let res = if running { + let name = cmd.name().clone(); + let payload = cmd.payload().clone(); + endpoint + .user_event(name, payload, cmd.coalesce) + .map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = cmd.reply.send(res); + } + Command::Query(cmd) => { + let res = if running { + let name = cmd.name().clone(); + let payload = cmd.payload().clone(); + let QueryCmd { + params, now: at, .. + } = &cmd; + endpoint + .query(name, payload, params.clone(), *at) + .map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = cmd.reply.send(res); + } + Command::Respond(cmd) => { + let res = if running { + let payload = cmd.payload().clone(); + endpoint + .respond(&cmd.token, payload, cmd.now) + .map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = cmd.reply.send(res); + } + Command::SetTags(SetTagsCmd { tags, reply }) => { + let res = if running { + endpoint.set_tags(tags).map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(res); + } + Command::SetEventJoinIgnore(SetEventJoinIgnoreCmd { ignore, reply }) => { + endpoint.set_event_join_ignore(ignore); + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(Ok(())); + } + #[cfg(encryption)] + Command::InstallKey(KeyCmd { + key, + now: at, + reply, + }) => { + let res = if running { + endpoint.install_key(key, at).map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(res); + } + #[cfg(encryption)] + Command::UseKey(KeyCmd { + key, + now: at, + reply, + }) => { + let res = if running { + endpoint.use_key(key, at).map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(res); + } + #[cfg(encryption)] + Command::RemoveKey(KeyCmd { + key, + now: at, + reply, + }) => { + let res = if running { + endpoint.remove_key(key, at).map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(res); + } + #[cfg(encryption)] + Command::ListKeys(ListKeysCmd { now: at, reply }) => { + let res = if running { + endpoint.list_keys(at).map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(res); + } + Command::Shutdown(ShutdownCmd { reply }) => { + // Drain every live bridge so the byte-movers observe the close and exit. + // Do NOT ack the caller here — the sockets are still bound; stash the + // reply and let the post-loop cleanup ack AFTER they drop. + for (_eid, handle) in bridges.drain() { + // Ignoring Err: bridge may have already exited; close is best-effort. + let _ = handle.out_tx.try_send(BridgeOut::Close); + } + *shutdown_reply = Some(reply); + } + } +} + +/// Route one bridge inbound message into the coordinator. +fn dispatch_bridge_inbound( + endpoint: &mut StreamEndpoint, + inbound: BridgeInbound, +) where + I: memberlist_proto::Id + Clone, + RT: StreamTransport, + G: rand::Rng, + R: rand::Rng + SeedableRng, +{ + // Each inbound carries its own `received_at` — forwarding THAT instant (not a + // fresh `Instant::now()`) ensures the stream FSM's deadline gate compares + // against the true arrival time, so a response queued before the exchange + // deadline is not retroactively marked Timeout. + match inbound { + BridgeInbound::Bytes(BridgeBytes { + eid, + bytes, + received_at, + }) => { + endpoint.handle_transport_data(eid, &bytes, false, received_at); + } + BridgeInbound::Eof(BridgeEof { eid, received_at }) => { + // Feed the read-half EOF anchor. Do NOT remove the `BridgeHandle` — for an + // inbound (server-side) push/pull bridge the read EOF arrives BEFORE the + // response is generated; the machine queues the response in this same + // call. The bridge entry stays until the matching `StreamAction::Close`. + endpoint.handle_transport_data(eid, &[], true, received_at); + } + BridgeInbound::Error(BridgeError { eid, received_at }) => { + // A transport ERROR is NOT a clean EOF: route it to `handle_transport_error` + // so the bridge fails rather than taking the benign-EOF path. + endpoint.handle_transport_error(eid, received_at); + } + } +} + +/// Decode and feed one inbound UDP gossip datagram into the coordinator, then +/// drain its memberlist ingress queue and feed every decoded message back +/// through `handle_message`. +/// +/// The codec hop is `decrypt → label-strip → decode`: with an encryption +/// backend built in, the encryption wrapper is stripped (and authenticated) +/// before the cluster label is verified; with none built in the serf gossip +/// plane carries no wire transforms, so the raw bytes ARE the label frame. A +/// compound datagram is split into its ordered messages by `parse_messages`, +/// each fed as a typed message. +fn dispatch_gossip( + endpoint: &mut StreamEndpoint, + src: SocketAddr, + datagram: &[u8], + now: Instant, + label: Option, +) where + I: memberlist_proto::Id + Clone, + RT: StreamTransport, + G: rand::Rng, + R: rand::Rng + SeedableRng, +{ + endpoint.handle_gossip(src, datagram, now); + + let decode_opts = DecodeOptions::new(label); + while let Some((from_addr, raw)) = endpoint.poll_memberlist_ingress() { + // Reverse the wire transform stack the peer applied before decoding. With an + // encryption backend built in, `decrypt_gossip` strips (and authenticates) + // the encryption wrapper — returning the frame unchanged when no keyring is + // configured, and dropping a frame the keyring cannot decrypt. With none + // built in the serf gossip plane carries no transforms, so the raw bytes are + // the plain label frame. A dropped datagram is recovered on the next gossip + // round (gossip is lossy and self-healing). + #[cfg(encryption)] + let plain = match endpoint.decrypt_gossip(&raw) { + Ok(p) => Bytes::from(p), + Err(_) => continue, + }; + #[cfg(not(encryption))] + let plain = raw; + // Strip the optional cluster label and verify it matches; a mismatched or + // absent label on a labeled cluster (or vice versa) is dropped here. + let inner = match decode_incoming(plain, &decode_opts) { + Ok(b) => b, + Err(_) => continue, + }; + // Demux plain vs compound and feed each decoded message to the coordinator. + // A malformed frame drops the whole datagram (lossy gossip; the peer + // retransmits on the next round). + let msgs = match parse_messages::(inner) { + Ok(m) => m, + Err(_) => continue, + }; + for msg in msgs { + endpoint.handle_message(from_addr, msg, now); + } + } +} + +/// Process one [`StreamAction`]. +/// +/// Connect: pre-allocate the bridge's out-channel and insert the [`BridgeHandle`] +/// BEFORE spawning the dial task, so bytes the machine surfaces on the same tick +/// as the Connect reach the bridge via the `out_rx` handed to it on dial +/// completion. Shutdown / Close / Abort signal the per-bridge channel. +fn process_one_action( + action: StreamAction, + bridges: &mut HashMap, + bridge_ready_tx: &Sender, + stream_opts: StreamTransportOptions, +) { + match action { + StreamAction::Connect(info) => { + let eid = info.id(); + let peer = info.peer(); + let (out_tx, out_rx) = mpsc::unbounded::(); + let (cancel_tx, cancel_rx) = futures_channel::oneshot::channel::<()>(); + bridges.insert(eid, BridgeHandle { out_tx, cancel_tx }); + let ready_tx = bridge_ready_tx.clone(); + let dial_timeout = stream_opts.dial_timeout(); + compio::runtime::spawn(async move { + // Bound the dial so a connect to an unreachable peer reports failure + // promptly instead of hanging on the kernel's default timeout. + let dial = TcpStream::connect(peer).fuse(); + let timeout = compio::time::sleep(dial_timeout).fuse(); + pin_mut!(dial, timeout); + let msg = select_biased! { + res = dial => match res { + Ok(stream) => BridgeReady::OutboundOk(OutboundOkReady { + eid, + stream, + out_rx, + cancel_rx, + }), + // Dropping `out_rx` / `cancel_rx` disconnects the channels; any + // bytes the pump queued during the dial are dropped — correct, the + // exchange never produced a wire to write them on. + Err(_) => BridgeReady::OutboundFail(OutboundFailReady { + eid, + received_at: Instant::now(), + }), + }, + _ = timeout => BridgeReady::OutboundFail(OutboundFailReady { + eid, + received_at: Instant::now(), + }), + }; + // Ignoring Err: pump has exited; the dial result is unobservable. + let _ = ready_tx.send_async(msg).await; + }) + .detach(); + } + StreamAction::Shutdown(eref) => { + // Half-close the send side (the push/pull half-close anchor). The bridge + // writes every queued `Bytes` ahead of `ShutdownWrite`, then shuts down + // its write half and continues reading. + if let Some(handle) = bridges.get(&eref.id()) { + // Ignoring Err: the bridge may have already exited; best-effort. + let _ = handle.out_tx.try_send(BridgeOut::ShutdownWrite); + } + } + StreamAction::Close(eref) => { + // Graceful full teardown. Removing the handle is symmetric with the + // bridge's exit — later bytes for this exchange miss the lookup in + // `drain_transport_transmits` and are dropped. + if let Some(handle) = bridges.remove(&eref.id()) { + // Ignoring Err: see Shutdown arm. + let _ = handle.out_tx.try_send(BridgeOut::Close); + } + } + StreamAction::Abort(eref) => { + // Hard teardown of a FAILED exchange. Signal `cancel_tx`: the bridge + // breaks immediately, discarding any still-queued stale bytes. + if let Some(handle) = bridges.remove(&eref.id()) { + // Ignoring Err: the bridge may have already exited; best-effort. + let _ = handle.cancel_tx.send(()); + } + } + } +} + +/// Drain every [`StreamAction`] the coordinator has queued. Returns `true` iff +/// any action was processed. +fn drain_actions( + endpoint: &mut StreamEndpoint, + bridges: &mut HashMap, + bridge_ready_tx: &Sender, + stream_opts: StreamTransportOptions, +) -> bool +where + I: memberlist_proto::Id + Clone, + RT: StreamTransport, + G: rand::Rng, + R: rand::Rng + SeedableRng, +{ + let mut progress = false; + while let Some(action) = endpoint.poll_action() { + progress = true; + process_one_action(action, bridges, bridge_ready_tx, stream_opts); + } + progress +} + +/// Drain every queued per-exchange transport-transmit and forward the bytes to +/// the matching bridge's write half. MUST run before the action queue advances +/// past a pending `Shutdown` / `Close` — the coordinator withholds the teardown +/// for an exchange until its `poll_transport_transmit` queue is empty. +fn drain_transport_transmits( + endpoint: &mut StreamEndpoint, + bridges: &HashMap, +) -> bool +where + I: memberlist_proto::Id + Clone, + RT: StreamTransport, + G: rand::Rng, + R: rand::Rng + SeedableRng, +{ + let mut progress = false; + while let Some((eid, _peer, bytes)) = endpoint.poll_transport_transmit() { + progress = true; + let Some(handle) = bridges.get(&eid) else { + // No live bridge — the dial failed and was removed, or a Close retired + // the exchange. Draining unblocks the matching Shutdown/Close. + continue; + }; + // Ignoring Err: the only failure on an unbounded channel is Disconnected + // (the bridge exited and reads no more bytes); dropping is safe. + let _ = handle.out_tx.try_send(BridgeOut::Bytes(bytes.to_vec())); + } + progress +} + +/// Drain every queued unreliable (UDP gossip) [`Transmit`] and send it on the +/// gossip socket. Outbound gossip is label-stamped (`encode_outgoing` / +/// `encode_outgoing_compound`), then — with an encryption backend built in — +/// wrapped in the encryption layer (`encrypt_gossip`) before it hits the wire. +async fn drain_transmits( + endpoint: &mut StreamEndpoint, + gossip_socket: &UdpSocket, + label: Option, +) -> bool +where + I: memberlist_proto::Id + Clone, + RT: StreamTransport, + G: rand::Rng, + R: rand::Rng + SeedableRng, +{ + let encode_opts = EncodeOptions::new(label); + let mut progress = false; + while let Some(transmit) = endpoint.poll_memberlist_transmit() { + progress = true; + let (peer, plain): (SocketAddr, Bytes) = match transmit { + Transmit::Packet(pkt) => { + let (to, msg) = pkt.into_parts(); + match encode_outgoing(&msg, &encode_opts) { + Ok(b) => (to, b), + // A locally-built message that fails to encode is dropped so one bad + // codec invocation cannot wedge the pump. + Err(_) => continue, + } + } + Transmit::Compound(cmp) => { + let (to, msgs) = cmp.into_parts(); + match encode_outgoing_compound(&msgs, &encode_opts) { + Ok(b) => (to, b), + Err(_) => continue, + } + } + }; + // Wrap the label frame in the encryption layer when an encryption backend is + // built in: `encrypt_gossip` is identity when no keyring is configured, and + // drops the datagram if a configured backend rejects it rather than emitting + // plaintext on an encrypted-cluster path. With none built in the frame goes + // out as-is. + #[allow(unused_mut)] + let mut on_wire: Vec = plain.to_vec(); + #[cfg(encryption)] + { + on_wire = match endpoint.encrypt_gossip(&on_wire) { + Ok(bytes) => bytes, + Err(_) => continue, + }; + } + let BufResult(res, _buf) = gossip_socket.send_to(on_wire, peer).await; + // Ignoring Err: a transient send error is non-fatal — gossip is lossy and + // the next probe/gossip round recovers. + let _ = res; + } + progress +} + +/// Apply one inbound [`KeyRequest`] to the driver's keyring delegate, producing +/// the [`KeyResponseArgs`] the pump forwards to `respond_key`. +/// +/// `Install` / `Use` / `Remove` carry a key (the machine enforces op-shape, so a +/// missing key is reported as a failed response rather than panicking); `List` +/// carries no key and enumerates the keyring. +#[cfg(encryption)] +fn apply_key_request( + keyring: &dyn KeyringDelegate, + req: &KeyRequest, +) -> KeyResponseArgs { + match (req.op(), req.key()) { + (KeyRequestOperation::Install, Some(key)) => keyring.install(*key), + (KeyRequestOperation::Use, Some(key)) => keyring.use_key(*key), + (KeyRequestOperation::Remove, Some(key)) => keyring.remove(*key), + (KeyRequestOperation::List, _) => keyring.list(), + (_, None) => KeyResponseArgs { + result: false, + message: "key-management request missing its required key".into(), + keys: Vec::new(), + primary_key: None, + }, + } +} + +/// Drain every queued serf [`Event`]: synchronous protocol accounting (leave +/// completion, conflict-shutdown, key requests), then hand off to the +/// observation task. NO `.await` on user delegate code. Returns `true` iff any +/// event was drained. +/// +/// Sets `*terminal` to `true` if a terminal [`Event::Shutdown`] was observed — +/// the local node lost an id-conflict vote and the pump MUST tear down. The +/// event is still delivered to subscribers before the main loop breaks. +#[allow(clippy::too_many_arguments)] +async fn drain_events( + endpoint: &mut StreamEndpoint, + obs_tx: &mpsc::Sender>, + observation_dropped: &Cell, + obs_payload_bytes: &Cell, + obs_payload_budget: Option, + pending: &mut PendingCommands, + terminal: &mut bool, + #[cfg(encryption)] keyring: &dyn KeyringDelegate, +) -> bool +where + I: memberlist_proto::Id + Clone, + RT: StreamTransport, + G: rand::Rng, + R: rand::Rng + SeedableRng, +{ + let mut drained = false; + while let Some(ev) = endpoint.poll_event() { + drained = true; + // Leave-completion resolution. `LeftCluster` fires once the leave notices + // have drained to the wire; resolving the parked waiter here — on this pump + // task, ahead of the observation task's `notify_leave` — is what makes + // `leave()` return promptly once the flush is done. + if matches!(ev, Event::LeftCluster) + && let Some(pl) = pending.leave.take() + { + pl.resolve_all(|| Ok(())).await; + } + // Conflict-shutdown enforcement. `Event::Shutdown` means the local node lost + // an id-conflict vote and MUST stop, exactly as for a `Command::Shutdown`. + // Flag it for the main loop (which breaks into teardown after this drain) + // while still delivering the event to subscribers below. + if matches!(ev, Event::Shutdown) { + *terminal = true; + } + // Key-management request enforcement. `Event::KeyRequest` requires the driver + // to apply the install/use/remove/list op to its keyring and answer the + // originator; without this the inbound key op times out and local key state + // never changes. Applied here on the pump (it mutates the endpoint through + // `respond_key`) ahead of the observation hand-off below. + #[cfg(encryption)] + if let Event::KeyRequest(req) = &ev { + let resp = apply_key_request(keyring, req); + // Ignoring Err: `respond_key` fails only when the response cannot be routed + // (originator gone / relay dropped); the key op has already applied locally. + let _ = endpoint.respond_key(req, resp, Instant::now()); + } + + let payload_bytes = observation_payload_bytes(&ev); + + // Byte backstop (bounded channels only): the count cap does not bound memory + // when an event carries a large user payload. If enqueueing would push the + // queued payload bytes over budget, yield once so the obs task can drain, + // re-check, and drop + count if still over. + if let (Some(budget), Some(bytes)) = (obs_payload_budget, payload_bytes) { + if obs_payload_bytes.get().saturating_add(bytes) > budget { + yield_once().await; + } + if obs_payload_bytes.get().saturating_add(bytes) > budget { + observation_dropped.set(observation_dropped.get() + 1); + continue; + } + } + + // Hand off to the observation task (delegate dispatch + EventStream forward, + // off this pump task), non-blocking. `Full` yields once then retries; drop + + // count only if still full. + match obs_tx.try_send(ev) { + Ok(()) => add_obs_payload(obs_payload_bytes, payload_bytes), + Err(mpsc::TrySendError::Closed(_)) => {} + Err(mpsc::TrySendError::Full(ev)) => { + yield_once().await; + match obs_tx.try_send(ev) { + Ok(()) => add_obs_payload(obs_payload_bytes, payload_bytes), + Err(_) => observation_dropped.set(observation_dropped.get() + 1), + } + } + } + } + drained +} + +/// Drain every outbound surface to quiescence in the documented order: actions, +/// transport-transmits, gossip transmits, events; repeat until no method makes +/// progress (a flushed byte queue releases a withheld Shutdown/Close that the +/// next `drain_actions` surfaces). +/// +/// Returns `true` iff a terminal [`Event::Shutdown`] was observed while +/// draining — the local node lost an id-conflict vote and the main loop MUST +/// break into teardown. +#[allow(clippy::too_many_arguments)] +async fn drain_outputs( + endpoint: &mut StreamEndpoint, + bridges: &mut HashMap, + bridge_ready_tx: &Sender, + stream_opts: StreamTransportOptions, + gossip_socket: &UdpSocket, + label: &Option, + obs_tx: &mpsc::Sender>, + observation_dropped: &Cell, + obs_payload_bytes: &Cell, + obs_payload_budget: Option, + pending: &mut PendingCommands, + #[cfg(encryption)] keyring: &dyn KeyringDelegate, +) -> bool +where + I: memberlist_proto::Id + Clone, + RT: StreamTransport, + G: rand::Rng, + R: rand::Rng + SeedableRng, +{ + let mut terminal = false; + loop { + let did_actions = drain_actions::(endpoint, bridges, bridge_ready_tx, stream_opts); + let did_transports = drain_transport_transmits::(endpoint, bridges); + let did_transmits = + drain_transmits::(endpoint, gossip_socket, label.clone()).await; + let did_events = drain_events::( + endpoint, + obs_tx, + observation_dropped, + obs_payload_bytes, + obs_payload_budget, + pending, + &mut terminal, + #[cfg(encryption)] + keyring, + ) + .await; + if !(did_actions || did_transports || did_transmits || did_events) { + break; + } + } + terminal +} + +/// Per-driver observation task: dispatch each event's [`Delegate`] hook, then +/// fan the event out to the `EventStream`, OFF the pump loop. +/// +/// Unlike a membership-only protocol, every serf [`Event`] (member transitions, +/// user events, queries, responses) is the application's observation surface, so +/// all events are forwarded to subscribers. The forward is best-effort: a full +/// queue (slow subscriber) drops the event and counts it into `events_dropped`, +/// never blocking. The task exits when `obs_rx` closes (pump dropped `obs_tx`). +async fn observation_task( + mut obs_rx: mpsc::Receiver>, + delegate: D, + events_tx: Sender>, + events_dropped: Rc>, + obs_payload_bytes: Rc>, +) where + D: Delegate, + I: Clone, +{ + while let Some(ev) = obs_rx.recv().await { + // Free the byte-backstop budget this event occupied as soon as it leaves the + // channel — before the (possibly slow) delegate hook — so the pump's enqueue + // side sees the reclaimed budget promptly. + let payload = observation_payload_bytes(&ev); + if let Some(b) = payload { + obs_payload_bytes.set(obs_payload_bytes.get().saturating_sub(b)); + } + // Contain a panicking delegate hook so the task SURVIVES and keeps releasing + // the byte-backstop reservations of still-queued events. Ignoring the unwind + // result: the panic is contained and this event is simply dropped. + let _ = std::panic::AssertUnwindSafe(dispatch_event_delegate(&delegate, &ev)) + .catch_unwind() + .await; + if events_tx + .try_send(ev) + .is_err_and(|e| matches!(e, flume::TrySendError::Full(_))) + { + events_dropped.set(events_dropped.get() + 1); + } + } +} + +/// Reap a deadline-expired graceful-leave waiter. If `pending_leave`'s deadline +/// has elapsed without `Event::LeftCluster` having resolved it, reply +/// [`SerfError::LeaveTimeout`] to every joined replier and clear the slot. +async fn reap_pending_leave(pending_leave: &mut Option, now: Instant) { + if let Some(pl) = pending_leave.as_ref() + && now >= pl.deadline + { + let pl = pending_leave.take().expect("checked Some above"); + pl.resolve_all(|| Err(SerfError::LeaveTimeout)).await; + } +} + +/// Earliest pending-leave deadline, if any — folded into the per-iteration +/// `timeout_deadline` so the timer fires even under a continuous network flood. +fn min_pending_leave_deadline(pending_leave: &Option) -> Option { + pending_leave.as_ref().map(|pl| pl.deadline) +} + +/// Drain-first timeout chokepoint for the stream pump — the single site that +/// calls `handle_timeout` on this plane. +/// +/// Both the past-due preemption branch and the main select's timer arm route +/// through here. A freshly-submitted `recv` can be pending on its first poll on a +/// completion backend (io_uring), so a biased select's timer arm winning does NOT +/// prove a genuine would-block: a near-deadline gossip Ack may be queued or +/// immediately readable when the timer fires. The order is UDP drain (each +/// datagram decoded inline by `dispatch_gossip`) → bridge-completion drain (a +/// peer's push/pull response / EOF / error and an outbound-dial completion must be +/// applied before the sweep, or an exchange whose terminating bytes are in the +/// channel is wrongly marked overdue) → deadline re-check → `handle_timeout` only +/// if still past. Returns `true` iff any work was applied. +/// +/// The caller MUST ensure no other `recv_from` SQE on `gossip_socket` is in flight +/// (the main loop drops its `recv_fut` before invoking this), so the bounded drain +/// is the sole builder of recv SQEs here. +#[allow(clippy::too_many_arguments)] +async fn fire_timeout_with_drain( + endpoint: &mut StreamEndpoint, + bridges: &mut HashMap, + bridge_inbound_tx: &mpsc::Sender, + bridge_inbound_rx: &mut mpsc::Receiver, + bridge_ready_rx: &Receiver, + gossip_socket: &UdpSocket, + recv_buf_len: usize, + label: &Option, + driver_opts: RuntimeOptions, + stream_opts: StreamTransportOptions, +) -> bool +where + I: memberlist_proto::Id + Clone, + RT: StreamTransport, + G: rand::Rng, + R: rand::Rng + SeedableRng, +{ + let mut dirty = false; + + // UDP drain FIRST: read every immediately-ready gossip datagram (bounded by the + // inbound drain cap; `.max(1)` so recv always gets at least one shot even at a + // zero cap) and decode each inline via `dispatch_gossip`, so a near-deadline Ack + // sitting BEHIND an unrelated datagram still resolves its probe before the + // suspicion sweep. Emptiness is reaped via `poll_with(ZERO)` inside the drain, + // not a time window. + let drained = drain_past_due_udp( + gossip_socket, + recv_buf_len, + driver_opts.iter_drain_cap().max(1), + |src, datagram| { + let now = Instant::now(); + // The stream pump decodes inline in `dispatch_gossip` (handle_gossip + + // memberlist-ingress drain), so a drained Ack resolves its probe deadline + // here, before the recheck below decides on `handle_timeout`. + dispatch_gossip::(endpoint, src, datagram, now, label.clone()); + // Report whether the coordinator deadline is STILL past: a drained Ack that + // resolved it ends the drain so the main select regains fairness. + now + >= endpoint + .poll_timeout() + .unwrap_or(now + driver_opts.idle_wake_interval()) + }, + ) + .await; + if drained { + dirty = true; + } + + // Bridge-completion drain (no cap): every already-queued bridge input must be + // applied before the timeout sweep. + while let Ok(inbound) = bridge_inbound_rx.try_recv() { + dispatch_bridge_inbound::(endpoint, inbound); + dirty = true; + } + while let Ok(ready) = bridge_ready_rx.try_recv() { + handle_bridge_ready::( + endpoint, + bridges, + bridge_inbound_tx, + ready, + stream_opts.bridge_recv_buf_len(), + stream_opts.close_timeout(), + ); + dirty = true; + } + + // Deadline re-check, then `handle_timeout` iff the deadline is still past: when + // the UDP drain emptied the socket a buffered Ack was already decoded inline + // above (so the deadline is no longer past), and when a flood capped the drain, + // firing for liveness is at worst a transient SWIM-refutable false Suspect + // rather than a frozen-timer stall. + let now = Instant::now(); + let after_drain_deadline = endpoint + .poll_timeout() + .unwrap_or(now + driver_opts.idle_wake_interval()); + if now >= after_drain_deadline { + endpoint.handle_timeout(now); + dirty = true; + } + dirty +} + +/// Publish a fresh [`SerfSnapshot`] of the endpoint's observable membership to +/// the snapshot cell. +/// +/// Skips the publish when the local node is not yet present in the serf +/// membership store (the local `NodeJoined` sieve has not fired): the prior +/// snapshot — seeded at construction — stays current, and `SerfSnapshot::new` +/// (which requires the local node) is never called with it absent. +fn refresh_snapshot( + endpoint: &StreamEndpoint, + snapshot: &SnapshotCell, +) where + I: memberlist_proto::Id + Clone, + RT: StreamTransport, + G: rand::Rng, + R: rand::Rng + SeedableRng, +{ + let members = endpoint.members_snapshot(); + let local_id = endpoint.local_id(); + if !members.iter().any(|m| m.node().id_ref() == local_id) { + return; + } + let snap = SerfSnapshot::new( + members, + local_id, + endpoint.state(), + LamportTime::from(endpoint.member_time()), + LamportTime::from(endpoint.event_time()), + LamportTime::from(endpoint.query_time()), + ); + *snapshot.borrow_mut() = Rc::new(snap); +} + +/// Route one [`BridgeReady`] — an outbound-dial result — into the coordinator, +/// spawning a per-bridge byte-mover on success. +fn handle_bridge_ready( + endpoint: &mut StreamEndpoint, + bridges: &mut HashMap, + bridge_inbound_tx: &mpsc::Sender, + ready: BridgeReady, + recv_buf_len: usize, + close_timeout: Duration, +) where + I: memberlist_proto::Id + Clone, + RT: StreamTransport, + G: rand::Rng, + R: rand::Rng + SeedableRng, +{ + match ready { + BridgeReady::OutboundOk(OutboundOkReady { + eid, + stream, + out_rx, + cancel_rx, + }) => { + // The handle was pre-inserted at Connect time. If it is gone the + // coordinator retired the exchange while the dial was in flight; drop the + // stream so the peer is never asked to honor an exchange we no longer track. + if !bridges.contains_key(&eid) { + drop(stream); + drop(out_rx); + drop(cancel_rx); + return; + } + spawn_bridge( + stream, + eid, + out_rx, + cancel_rx, + bridge_inbound_tx, + recv_buf_len, + close_timeout, + ); + } + BridgeReady::OutboundFail(OutboundFailReady { eid, received_at }) => { + // Remove the handle, then terminalize the exchange as a DIAL FAILURE (a + // connect that never established has no wire; a benign EOF would falsely + // complete a one-way exchange as success). + bridges.remove(&eid); + endpoint.handle_dial_failed(eid, received_at); + } + } +} + +/// Route one accepted inbound connection: allocate the exchange, register the +/// bridge handle, and spawn the byte mover. Returns `true` iff an accept was +/// processed (a state-affecting event the caller treats as dirty). +fn handle_accepted( + accepted: io::Result<(TcpStream, SocketAddr)>, + endpoint: &mut StreamEndpoint, + bridges: &mut HashMap, + bridge_inbound_tx: &mpsc::Sender, + stream_opts: StreamTransportOptions, +) -> bool +where + I: memberlist_proto::Id + Clone, + RT: StreamTransport, + G: rand::Rng, + R: rand::Rng + SeedableRng, +{ + match accepted { + Ok((stream, peer)) => { + let now = Instant::now(); + let Some(eid) = endpoint.accept_connection(peer, now) else { + // Not admitted (leaving, the inbound-stream cap is reached, or a + // record-layer config error): drop the accepted stream rather than spawn + // a byte mover for a connection the machine will never feed. + drop(stream); + return true; + }; + let (out_tx, out_rx) = mpsc::unbounded::(); + let (cancel_tx, cancel_rx) = futures_channel::oneshot::channel::<()>(); + bridges.insert(eid, BridgeHandle { out_tx, cancel_tx }); + spawn_bridge( + stream, + eid, + out_rx, + cancel_rx, + bridge_inbound_tx, + stream_opts.bridge_recv_buf_len(), + stream_opts.close_timeout(), + ); + true + } + // Transient accept error (fd pressure, peer reset mid-handshake). The kernel + // keeps the listener open; the resolved accept is re-armed at the loop top. + Err(_) => false, + } +} + +/// Spawn the [`crate::bridge::bridge_task`] byte-mover for `eid`. The caller has +/// already inserted the matching [`BridgeHandle`] so bytes queued before the +/// bridge spawned reach the wire via the `out_rx` handed in here. +fn spawn_bridge( + stream: TcpStream, + eid: ExchangeId, + out_rx: mpsc::Receiver, + cancel_rx: futures_channel::oneshot::Receiver<()>, + bridge_inbound_tx: &mpsc::Sender, + recv_buf_len: usize, + close_timeout: Duration, +) { + let inbound_tx = bridge_inbound_tx.clone(); + compio::runtime::spawn(crate::bridge::bridge_task( + stream, + eid, + out_rx, + cancel_rx, + inbound_tx, + recv_buf_len, + close_timeout, + )) + .detach(); +} + +#[cfg(test)] +mod tests; diff --git a/serf-compio/src/driver/stream/tests.rs b/serf-compio/src/driver/stream/tests.rs new file mode 100644 index 00000000..6c379bef --- /dev/null +++ b/serf-compio/src/driver/stream/tests.rs @@ -0,0 +1,147 @@ +//! Unit tests for the stream pump's past-due gossip decode. +//! +//! The bounded past-due UDP drain itself (reading every queued datagram, not +//! just the first) is shared with the QUIC pump and proven in the driver's +//! `shared::tests`. This file covers the stream-specific per-datagram step: the +//! past-due drain hands each datagram to `dispatch_gossip`, which decodes it +//! inline (unlike QUIC, which buffers and decodes in `drain_ingress`), so a +//! drained Ack resolves its probe before the deadline recheck fires +//! `handle_timeout`. + +use super::*; + +use core::num::NonZeroU8; + +use memberlist_proto::{ + Endpoint, EndpointOptions, RawRecords, + streams::{LabelOptions, StreamEndpoint as Coordinator}, +}; +use rand::rngs::StdRng; +use serf_proto::options::Options as SerfOptions; +use smol_str::SmolStr; + +/// Build a standalone plain-TCP serf `StreamEndpoint` over a memberlist stream +/// coordinator — no bound socket, no driver loop; just the composed machine, for +/// driving the gossip ingress surface directly. Mirrors the coordinator the TCP +/// transport's `run` builds. +fn build_endpoint() -> StreamEndpoint { + let local: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let inner_opts = EndpointOptions::new(SmolStr::new("node"), local) + .with_user_broadcast_tiers(NonZeroU8::new(3).expect("3 is nonzero")); + let inner = Endpoint::new(inner_opts, StdRng::seed_from_u64(1)); + let coord = Coordinator::<_, _, RawRecords, StdRng>::new( + inner, + LabelOptions::new_in(None::>, ()), + Box::new(|_: &SocketAddr| None), + Box::new(|addr: &SocketAddr| *addr), + ); + StreamEndpoint::::new_with_rng( + coord, + SerfOptions::new(), + StdRng::seed_from_u64(2), + ) +} + +/// `dispatch_gossip` must fully drain the coordinator's memberlist ingress queue +/// inline, so the stream pump's past-due drain leaves nothing buffered for a +/// subsequent `handle_timeout` to race: a drained Ack is decoded and fed back +/// through `handle_message` before the suspicion sweep. A first byte of 1 (the +/// `Compound` message tag) is buffered by `handle_gossip` and consumed by the +/// inline ingress drain inside `dispatch_gossip`. +#[test] +fn dispatch_gossip_drains_ingress_before_timeout() { + let mut endpoint = build_endpoint(); + let peer: SocketAddr = "127.0.0.1:65000".parse().expect("peer addr"); + let now = Instant::now(); + + dispatch_gossip::( + &mut endpoint, + peer, + &[1u8, 0, 0, 0], + now, + None, + ); + + assert!( + endpoint.poll_memberlist_ingress().is_none(), + "dispatch_gossip must drain memberlist ingress inline so no buffered frame is left for \ + handle_timeout to race" + ); +} + +/// The drain-first timeout chokepoint must read the gossip UDP socket BEFORE +/// deciding on `handle_timeout`: on a completion backend a freshly-submitted recv +/// is pending on first poll, so the main select's timer arm winning is NOT proof +/// of a would-block — a near-deadline Ack can be queued. `fire_timeout_with_drain` +/// is the single `handle_timeout` site, reached from both the past-due branch and +/// the main timer arm; this proves it actually invokes the folded-in UDP drain on +/// the real socket. The socket-level proof that the drain reads EVERY queued +/// datagram is in the driver's `shared::tests`. +#[compio::test] +async fn fire_timeout_with_drain_drains_socket_before_handle_timeout() { + let mut endpoint = build_endpoint(); + let any: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let driver = UdpSocket::bind(any).await.expect("bind driver socket"); + let driver_addr = driver.local_addr().expect("driver local_addr"); + let peer = UdpSocket::bind(any).await.expect("bind peer socket"); + + // A single gossip compound-tagged datagram queued in the driver's socket — the + // near-deadline "Ack" the chokepoint must consume before any suspicion. + peer + .send_to(vec![1u8, 0, 0, 0], driver_addr) + .await + .0 + .expect("queue a datagram"); + // Let loopback delivery settle so the drain's eager recv reads the datagram on + // its first poll (production's past-due Ack is already buffered). + compio::time::sleep(Duration::from_millis(100)).await; + + // The bridge plumbing the chokepoint also drains is empty here; this test + // targets the folded-in UDP drain. + let mut bridges: HashMap = HashMap::new(); + let (bridge_inbound_tx, mut bridge_inbound_rx) = mpsc::bounded::(16); + let (_bridge_ready_tx, bridge_ready_rx) = flume::unbounded::(); + + let opts = RuntimeOptions::new(); + let dirty = fire_timeout_with_drain::( + &mut endpoint, + &mut bridges, + &bridge_inbound_tx, + &mut bridge_inbound_rx, + &bridge_ready_rx, + &driver, + 64, + &None, + opts, + StreamTransportOptions::new(), + ) + .await; + assert!( + dirty, + "the chokepoint consumed the queued datagram (and/or fired handle_timeout)" + ); + + // The datagram was drained off the socket before handle_timeout: a fresh + // bounded recv now BLOCKS (the timer wins) rather than returning the + // still-queued datagram immediately — proving the chokepoint read the socket. + // Scope the recv future so it drops (releasing its borrow of `driver`) before + // the socket is closed below. + let socket_drained = { + let buf = vec![0u8; 64]; + let recv = driver.recv_from(buf).fuse(); + let timer = compio::time::sleep(Duration::from_millis(200)).fuse(); + pin_mut!(recv, timer); + select_biased! { + _ = recv => false, + _ = timer => true, + } + }; + assert!( + socket_drained, + "fire_timeout_with_drain must drain the queued datagram off the socket before handle_timeout" + ); + + // Ignoring Err: test cleanup of the probe sockets. + let _ = driver.close().await; + let _ = peer.close().await; +} diff --git a/serf-compio/src/error/mod.rs b/serf-compio/src/error/mod.rs new file mode 100644 index 00000000..90b285df --- /dev/null +++ b/serf-compio/src/error/mod.rs @@ -0,0 +1,186 @@ +//! Error types for serf-compio. + +use core::fmt; +use std::{io, net::SocketAddr}; + +pub use serf_driver::error::{GossipMtuTooSmall, InvalidOption}; + +/// Payload for [`SerfError::InvalidGossipMtu`]: the configured `gossip_mtu` +/// exceeds the largest plaintext gossip payload that can fit a single UDP +/// datagram once any encryption wrapper is added. Carries the configured value +/// and the effective ceiling. +#[derive(Debug)] +pub struct InvalidGossipMtu { + configured: usize, + ceiling: usize, +} + +impl InvalidGossipMtu { + /// Build a new payload from the configured `gossip_mtu` and the ceiling. + #[inline] + pub fn new(configured: usize, ceiling: usize) -> Self { + Self { + configured, + ceiling, + } + } + + /// The configured `gossip_mtu` that was rejected. + #[inline] + pub fn configured(&self) -> usize { + self.configured + } + + /// The effective ceiling — the largest plaintext `gossip_mtu` whose wire + /// datagram still fits a single UDP packet after any encryption wrapper. + #[inline] + pub fn ceiling(&self) -> usize { + self.ceiling + } +} + +impl fmt::Display for InvalidGossipMtu { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "gossip_mtu {} exceeds the maximum sendable plaintext gossip payload of {} bytes \ + (a gossip packet is one UDP datagram capped after the encryption wrapper); \ + a larger gossip_mtu would make near-MTU gossip packets deterministically unsendable", + self.configured, self.ceiling, + ) + } +} + +/// Payload for [`SerfError::InvalidAdvertiseAddr`]: the resolved advertise +/// address cannot serve as the local node's reachable contact identity. Two +/// independent classes are rejected: +/// +/// - NOT A USABLE UNICAST CONTACT — an unspecified IP, a multicast IP, an IPv4 +/// broadcast IP, or a zero port. Such an address is undialable. +/// - NOT REPRESENTABLE ON THE WIRE — a scoped/flow-labelled IPv6 `SocketAddr` +/// with a nonzero `scope_id` or `flowinfo` that the compact wire layout +/// (`[16B IP][2B port]`) cannot carry. +#[derive(Debug)] +pub struct InvalidAdvertiseAddr { + addr: SocketAddr, + reason: String, +} + +impl InvalidAdvertiseAddr { + /// Build a new payload from the rejected advertise address and the reason. + #[inline] + pub fn new(addr: SocketAddr, reason: String) -> Self { + Self { addr, reason } + } + + /// The advertise address that was rejected. + #[inline] + pub fn addr(&self) -> SocketAddr { + self.addr + } + + /// The reason the address cannot serve as the local node's contact identity. + #[inline] + pub fn reason(&self) -> &str { + &self.reason + } +} + +impl fmt::Display for InvalidAdvertiseAddr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "advertise address {} cannot serve as this node's reachable contact identity, \ + so peers that learn it could not route serf traffic to this node: {}", + self.addr, self.reason, + ) + } +} + +/// Errors returned by [`Serf`](crate::Serf) operations. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum SerfError { + /// I/O error from the OS, socket, or compio runtime. + #[error(transparent)] + Io(#[from] io::Error), + + /// The OS entropy source failed while seeding the gossip RNG. Surfaced in + /// the node constructor (which returns this `Result`) before the driver task + /// is spawned, so a failure is surfaced here rather than panicking in the + /// spawned task. + #[error("OS entropy source failed while seeding the gossip RNG")] + Entropy(#[source] io::Error), + + /// Address resolution failed (DNS error, etc.). + #[error("address resolution: {0}")] + Resolve(io::Error), + + /// A serf endpoint operation (join, leave, set_tags, user_event, query, …) + /// returned a machine-level error. Carries the typed + /// [`serf_proto::endpoint::Error`] so callers can dispatch on the specific + /// cause. + #[cfg(any(feature = "tcp", feature = "quic"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] + #[error(transparent)] + Proto(#[from] serf_proto::endpoint::Error), + + /// Encryption codec error from memberlist-wire. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + #[error(transparent)] + Encryption(#[from] memberlist_proto::EncryptionError), + + /// A graceful [`leave`](crate::Serf::leave) did not complete within the + /// driver's configured leave timeout. The leave was initiated but the driver + /// cannot confirm peers were notified. + #[error("leave did not complete within the configured leave timeout")] + LeaveTimeout, + + /// The driver task has shut down and is no longer accepting commands. + #[error("driver shut down")] + Shutdown, + + /// The local node has left the cluster; the operation requires an active node. + #[error("the local node has left the cluster; the operation requires a running node")] + NotRunning, + + /// The configured `gossip_mtu` exceeds the ceiling after the encryption + /// wrapper is applied. Returned at construction (fail-fast, before any socket + /// is bound) so the misconfiguration is surfaced rather than producing + /// deterministically dropped gossip. + #[error("{0}")] + InvalidGossipMtu(InvalidGossipMtu), + + /// The configured `gossip_mtu` is below the floor needed to carry the + /// mandatory single-datagram control packets the SWIM protocol always emits. + #[error("{0}")] + GossipMtuTooSmall(GossipMtuTooSmall), + + /// The resolved advertise address cannot serve as the local node's reachable + /// contact identity. + #[error("{0}")] + InvalidAdvertiseAddr(InvalidAdvertiseAddr), + + /// An operator-set driver tuning knob was given a value that would + /// deterministically break the node rather than merely degrade it. + #[error("{0}")] + InvalidOption(InvalidOption), + + /// Sending a command to the driver failed because the channel is closed. + #[error("send to driver failed (channel closed)")] + CommandSend, + + /// The driver's reply channel was dropped before a reply arrived. + #[error("driver reply channel closed")] + ReplyClosed, +} + +/// Convenience [`Result`] for [`SerfError`]. +pub type Result = core::result::Result; + +#[cfg(test)] +mod tests; diff --git a/serf-compio/src/error/tests.rs b/serf-compio/src/error/tests.rs new file mode 100644 index 00000000..246603d7 --- /dev/null +++ b/serf-compio/src/error/tests.rs @@ -0,0 +1,85 @@ +use super::*; + +#[test] +fn invalid_gossip_mtu_accessors_and_display() { + let payload = InvalidGossipMtu::new(70_000, 65_467); + assert_eq!(payload.configured(), 70_000); + assert_eq!(payload.ceiling(), 65_467); + let shown = format!("{payload}"); + assert!(!shown.is_empty()); + assert!(shown.contains("70000")); + assert!(!format!("{payload:?}").is_empty()); +} + +#[test] +fn gossip_mtu_too_small_accessors_and_display() { + let payload = GossipMtuTooSmall::new(64, 512); + assert_eq!(payload.configured(), 64); + assert_eq!(payload.minimum(), 512); + let shown = format!("{payload}"); + assert!(!shown.is_empty()); + assert!(shown.contains("512")); + assert!(!format!("{payload:?}").is_empty()); +} + +#[test] +fn invalid_advertise_addr_accessors_and_display() { + let addr: std::net::SocketAddr = "0.0.0.0:5000".parse().unwrap(); + let payload = InvalidAdvertiseAddr::new(addr, "wildcard bind".to_string()); + assert_eq!(payload.addr(), addr); + assert_eq!(payload.reason(), "wildcard bind"); + let shown = format!("{payload}"); + assert!(!shown.is_empty()); + assert!(shown.contains("wildcard bind")); + assert!(!format!("{payload:?}").is_empty()); +} + +#[test] +fn invalid_option_accessors_and_display() { + let payload = InvalidOption::new("idle_wake_interval", "must be nonzero".to_string()); + assert_eq!(payload.option(), "idle_wake_interval"); + assert_eq!(payload.reason(), "must be nonzero"); + let shown = format!("{payload}"); + assert!(shown.contains("idle_wake_interval")); + assert!(shown.contains("must be nonzero")); + assert!(!format!("{payload:?}").is_empty()); +} + +#[test] +fn every_variant_displays_and_debugs() { + let variants: &[SerfError] = &[ + SerfError::Io(io::Error::other("disk")), + SerfError::Entropy(io::Error::other("entropy")), + SerfError::Resolve(io::Error::other("dns")), + SerfError::LeaveTimeout, + SerfError::Shutdown, + SerfError::NotRunning, + SerfError::InvalidGossipMtu(InvalidGossipMtu::new(70_000, 65_467)), + SerfError::GossipMtuTooSmall(GossipMtuTooSmall::new(64, 512)), + SerfError::InvalidAdvertiseAddr(InvalidAdvertiseAddr::new( + "0.0.0.0:5000".parse().unwrap(), + "wildcard".to_string(), + )), + SerfError::InvalidOption(InvalidOption::new( + "idle_wake_interval", + "nonzero".to_string(), + )), + SerfError::CommandSend, + SerfError::ReplyClosed, + ]; + + for err in variants { + assert!( + !format!("{err}").is_empty(), + "Display non-empty for {err:?}" + ); + assert!(!format!("{err:?}").is_empty(), "Debug non-empty"); + } +} + +#[test] +fn from_io_conversion() { + let err: SerfError = io::Error::other("boom").into(); + assert!(matches!(err, SerfError::Io(_))); + assert_eq!(err.to_string(), io::Error::other("boom").to_string()); +} diff --git a/serf-compio/src/events.rs b/serf-compio/src/events.rs new file mode 100644 index 00000000..8b232260 --- /dev/null +++ b/serf-compio/src/events.rs @@ -0,0 +1,66 @@ +//! Event stream — the user-facing observation channel for serf events. + +#[cfg(any(feature = "tcp", feature = "quic"))] +use flume::r#async::RecvStream; +#[cfg(any(feature = "tcp", feature = "quic"))] +use futures_util::Stream; +#[cfg(any(feature = "tcp", feature = "quic"))] +use serf_proto::event::Event; +#[cfg(any(feature = "tcp", feature = "quic"))] +use std::{ + pin::Pin, + task::{Context, Poll}, +}; + +/// Stream of serf events. Constructed via [`Serf::events`](crate::Serf::events). +/// +/// Generic over the wire id / address types ``. +/// +/// **Concurrency model:** flume MPMC — multiple `events()` calls each return +/// an independent `EventStream`, but events ROUND-ROBIN between subscribers +/// (NOT broadcast). For single-consumer use this is the right shape. +/// +/// **Lossy under backpressure:** the events channel is bounded. When the queue +/// is full the driver drops the newest event rather than block — a slow +/// subscriber must not stall the membership FSM. +/// +/// Requires a stream or QUIC transport feature (`tcp` or `quic`). +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub struct EventStream +where + I: 'static, + A: 'static, +{ + inner: RecvStream<'static, Event>, +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +impl EventStream +where + I: 'static, + A: 'static, +{ + /// Wrap a flume receiver into an `EventStream`. + /// + /// Consumes the receiver: the resulting stream lives `'static` and owns the + /// queue handle. + pub fn new(rx: flume::Receiver>) -> Self { + Self { + inner: rx.into_stream(), + } + } +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +impl Stream for EventStream +where + I: 'static, + A: 'static, +{ + type Item = Event; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_next(cx) + } +} diff --git a/serf-compio/src/lib.rs b/serf-compio/src/lib.rs new file mode 100644 index 00000000..4a71202a --- /dev/null +++ b/serf-compio/src/lib.rs @@ -0,0 +1,170 @@ +//! compio-based async driver for the Sans-I/O serf machine. +//! +//! `serf-compio` binds the pure [`serf-proto`] Sans-I/O serf machine to the +//! [compio](https://github.com/compio-rs/compio) async runtime. It owns the +//! driver task, the command channel, the observation delegate dispatch, and the +//! transport plumbing — everything that connects the stateless machine to real +//! I/O. The machine itself (`serf-proto`) carries no runtime reference and runs +//! identically regardless of which driver crate wraps it. +//! +//! # Feature flags +//! +//! Transport (at least one required): +//! - `tcp` (default) — plain-TCP reliable coordinator. +//! - `tls` — TLS-over-TCP reliable coordinator (implies `tcp`). +//! - `quic` — QUIC coordinator. +//! - `quic-rustls-ring` — QUIC with the rustls+ring TLS backend (implies `quic`). +//! +//! Optional protocol extensions: +//! - `coordinates` — Vivaldi network coordinate estimation. +//! - `aes-gcm` — AES-256-GCM encryption backend. +//! - `chacha20-poly1305` — ChaCha20-Poly1305 encryption backend. +//! - `tag-regex` (default) — regex-backed tag-filter matching. +//! +//! Optional config layering: +//! - `serde` — `Serialize`/`Deserialize` on `*Options` types. +//! - `clap` — `clap::Args` on `*Options` types (CLI flags + env vars). +//! - `tracing` — emit `tracing` spans around public driver operations. +//! +//! Optional resolver support: +//! - `dns` — `DnsResolver` backed by `hickory-proto`. +//! - `getifs` — auto-detect the advertise address from host interfaces. + +#![deny(missing_docs)] +#![cfg_attr(docsrs, feature(doc_cfg))] + +#[cfg(feature = "tcp")] +mod bridge; +#[cfg(any(feature = "tcp", feature = "quic"))] +mod command; +#[cfg(any(feature = "tcp", feature = "quic"))] +mod delegate; +mod driver; +mod error; +mod events; +mod resolver; +#[cfg(any(feature = "tcp", feature = "quic"))] +mod snapshot; +#[cfg(any(feature = "tcp", feature = "quic"))] +mod transport; + +#[cfg(any(feature = "tcp", feature = "quic"))] +mod serf; + +#[cfg(feature = "tcp")] +mod tcp; + +#[cfg(feature = "tls")] +mod tls; + +#[cfg(feature = "quic")] +mod quic; + +use rand::{ + SeedableRng, + rngs::{StdRng, SysRng}, +}; + +/// A fresh [`StdRng`] seeded directly from the OS entropy source ([`SysRng`], +/// i.e. `getrandom`) — never from a thread-local generator, so a process that +/// forks after building a node cannot inherit a parent's RNG state and derive +/// the same gossip schedule. +/// +/// Drawn before the driver task is spawned (the result is passed to +/// [`Serf::new`], which returns a `Result`), so an OS entropy failure surfaces +/// as [`SerfError::Entropy`] rather than panicking in the spawned task after the +/// handle was already returned. +pub fn gossip_rng() -> crate::Result { + os_seeded_std_rng() +} + +/// Draw a fresh OS-seeded [`StdRng`] — the shared seed source behind both +/// [`gossip_rng`] (the memberlist gossip schedule) and each transport's +/// independent serf-core RNG seed. +/// +/// Every call draws fresh OS entropy, so two machines built in one process get +/// mutually-independent RNG streams; the serf core's RNG (which picks query IDs +/// and relay targets) is therefore never correlated with the gossip RNG or with +/// another node's, ruling out the colliding `(ltime, id)` a shared/zero seed +/// would produce. +pub(crate) fn os_seeded_std_rng() -> crate::Result { + StdRng::try_from_rng(&mut SysRng).map_err(|e| crate::SerfError::Entropy(std::io::Error::other(e))) +} + +pub use error::{ + GossipMtuTooSmall, InvalidAdvertiseAddr, InvalidGossipMtu, InvalidOption, Result, SerfError, +}; + +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub use delegate::{ + Delegate, MemberDelegate, MergeDelegate, NoopMergeDelegate, QueryDelegate, UserEventDelegate, + VoidDelegate, +}; + +#[cfg(encryption)] +#[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) +)] +pub use delegate::{KeyringDelegate, VoidKeyringDelegate}; + +/// Gossip-encryption config types re-exported from `memberlist-proto`, so a +/// caller can build a transport's `with_encryption` keyring without naming +/// `memberlist-proto` directly. `EncryptionOptions` carries an optional +/// `Keyring` (primary + secondary `SecretKey`s); attaching one enables +/// encryption, leaving it absent keeps every plane plaintext. +#[cfg(encryption)] +#[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) +)] +pub use memberlist_proto::{EncryptionOptions, Keyring, SecretKey}; + +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub use events::EventStream; + +pub use resolver::{ + AdvertiseAddrResolver, AdvertiseResolutionError, FirstAddrResolver, Ipv4PreferringResolver, + Ipv6PreferringResolver, OsResolver, Resolver, SocketAddrResolver, +}; + +#[cfg(feature = "dns")] +#[cfg_attr(docsrs, doc(cfg(feature = "dns")))] +pub use resolver::{DEFAULT_DNS_TIMEOUT, DnsResolver}; + +#[cfg(feature = "getifs")] +#[cfg_attr(docsrs, doc(cfg(feature = "getifs")))] +pub use resolver::{LocalAddrResolver, LocalAddrScope, local_advertise}; + +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub use transport::{Transport, TransportRuntime}; + +#[cfg(feature = "tcp")] +#[cfg_attr(docsrs, doc(cfg(feature = "tcp")))] +pub use tcp::{TcpTransport, TcpTransportOptions}; + +#[cfg(feature = "tls")] +#[cfg_attr(docsrs, doc(cfg(feature = "tls")))] +pub use tls::{SniProvider, TlsOptions, TlsTransport, TlsTransportOptions}; + +#[cfg(feature = "quic")] +#[cfg_attr(docsrs, doc(cfg(feature = "quic")))] +pub use quic::{QuicOptions, QuicTransport, QuicTransportOptions}; + +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub use snapshot::SerfSnapshot; + +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub use serf::Serf; + +pub use driver::options::{ + Channel, DEFAULT_BRIDGE_INBOUND_CAP, DEFAULT_BRIDGE_RECV_BUF_LEN, DEFAULT_CLOSE_TIMEOUT, + DEFAULT_CMD_FAIRNESS_BUDGET, DEFAULT_DIAL_TIMEOUT, DEFAULT_EVENT_QUEUE_CAP, + DEFAULT_IDLE_WAKE_INTERVAL, DEFAULT_ITER_DRAIN_CAP, DEFAULT_LEAVE_TIMEOUT, + DEFAULT_OBSERVATION_CHANNEL, ParseChannelError, RuntimeOptions, StreamTransportOptions, +}; diff --git a/serf-compio/src/quic/mod.rs b/serf-compio/src/quic/mod.rs new file mode 100644 index 00000000..af185472 --- /dev/null +++ b/serf-compio/src/quic/mod.rs @@ -0,0 +1,356 @@ +//! QUIC-backed serf driver — the QUIC sibling of the TCP plane. +//! +//! [`QuicTransport`] owns the bound `UdpSocket`; QUIC carries no separate TCP +//! listener — quinn-proto multiplexes the reliable push-pull streams over the +//! single UDP socket, and serf's datagram gossip rides the same socket. The +//! machine-layer `serf_proto::QuicEndpoint` is built inside +//! [`QuicTransport::run`] from the stored [`QuicOptions`] and the +//! `serf_proto::options::Options` carried by the +//! [`TransportRuntime`](crate::TransportRuntime). +//! +//! ## TLS server name +//! +//! QUIC's TLS 1.3 handshake requires a server name to verify the peer's +//! certificate against. [`QuicOptions::new`] installs a cluster-uniform string +//! used for every peer; deployments whose certs name each peer's hostname/IP +//! supply a per-peer SNI closure via `QuicOptions::new_with_sni_provider`. + +#![cfg(feature = "quic")] + +use core::num::NonZeroU8; +use std::{io::ErrorKind, net::SocketAddr}; + +use compio::net::UdpSocket; +use hostaddr::HostAddr; +use memberlist_proto::{ + CheapClone, Data, EndpointOptions, Id, MaybeResolved, QuicEndpoint as Coordinator, +}; +use rand::rngs::StdRng; +use smol_str::SmolStr; + +#[cfg(encryption)] +use memberlist_proto::EncryptionOptions; + +/// QUIC config bundle handed to [`QuicTransport`]. Re-exported from +/// `memberlist-proto` so callers don't need a direct dep. +pub use memberlist_proto::QuicOptions; + +use crate::{ + SerfError, + delegate::Delegate, + resolver::{AdvertiseAddrResolver, Resolver}, + transport::{Transport, TransportRuntime}, +}; + +/// Per-backend QUIC-specific transport options. +/// +/// Embedded into the transport constructor. Bundles the local node identifier, +/// the (possibly-unresolved) advertise address, and the caller-built +/// [`QuicOptions`] (quinn-proto `EndpointConfig` / `ServerConfig` / +/// `ClientConfig` / `TransportConfig` bundle plus SNI provider). The cluster +/// label and inbound-label-check policy are supplied via the serf `Options` +/// block (not here), feeding both planes from a single validated source. +pub struct QuicTransportOptions> { + local_id: Option, + advertise_addr: Option>, + quic_config: Option, + /// Gossip-encryption policy. The default (no keyring) leaves the gossip + /// datagrams plaintext; attaching a keyring via + /// [`with_encryption`](Self::with_encryption) makes the coordinator's + /// `encrypt_gossip`/`decrypt_gossip` AEAD-protect them. The reliable plane + /// rides quinn's own TLS, so the keyring covers only the gossip datagrams. + #[cfg(encryption)] + encryption: EncryptionOptions, +} + +impl QuicTransportOptions { + /// Construct with defaults. Caller MUST chain [`with_local_id`](Self::with_local_id), + /// [`with_advertise_addr`](Self::with_advertise_addr), and + /// [`with_quic_config`](Self::with_quic_config) before passing to + /// `QuicTransport::new`. + #[inline] + pub fn new() -> Self { + Self { + local_id: None, + advertise_addr: None, + quic_config: None, + #[cfg(encryption)] + encryption: EncryptionOptions::new(), + } + } + + /// Builder: local node identifier. + #[must_use] + #[inline] + pub fn with_local_id(mut self, id: I) -> Self { + self.local_id = Some(id); + self + } + + /// Builder: advertise address (resolved or unresolved). + #[must_use] + #[inline] + pub fn with_advertise_addr(mut self, addr: MaybeResolved) -> Self { + self.advertise_addr = Some(addr); + self + } + + /// Builder: QUIC config bundle (caller-built quinn-proto configs + SNI). + #[must_use] + #[inline] + pub fn with_quic_config(mut self, cfg: QuicOptions) -> Self { + self.quic_config = Some(cfg); + self + } + + /// Builder: gossip-encryption policy. + /// + /// The default (no keyring) keeps the gossip datagrams plaintext, so an + /// unencrypted node still builds and interoperates. Attach a keyring + /// (`EncryptionOptions::new().with_keyring(Keyring::new(primary_key))`) to + /// AEAD-protect the gossip datagrams; every node sharing the cluster MUST + /// carry the same keyring to interop. The reliable plane is quinn TLS and is + /// unaffected by this keyring. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + #[must_use] + #[inline] + pub fn with_encryption(mut self, encryption: EncryptionOptions) -> Self { + self.encryption = encryption; + self + } + + /// Local node identifier, if set. + #[inline] + pub const fn local_id(&self) -> Option<&I> { + self.local_id.as_ref() + } + + /// Advertise address, if set. + #[inline] + pub const fn advertise_addr(&self) -> Option<&MaybeResolved> { + self.advertise_addr.as_ref() + } + + /// QUIC config bundle, if set. + #[inline] + pub const fn quic_config(&self) -> Option<&QuicOptions> { + self.quic_config.as_ref() + } + + /// Gossip-encryption policy. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + #[inline] + pub const fn encryption(&self) -> &EncryptionOptions { + &self.encryption + } +} + +impl Default for QuicTransportOptions { + #[inline] + fn default() -> Self { + Self::new() + } +} + +/// QUIC-backed serf transport. +/// +/// Owns the bound `UdpSocket` only — quinn-proto multiplexes the reliable +/// push-pull streams over the single UDP socket, no separate listener, and +/// serf's datagram gossip shares the same socket. The machine-layer +/// `serf_proto::QuicEndpoint` is built inside [`Transport::run`] from the +/// stored `quic_config` and the cluster options sourced from +/// [`TransportRuntime::serf_options`](crate::TransportRuntime). +pub struct QuicTransport> { + local_id: I, + local_address: MaybeResolved, + advertise_socket: SocketAddr, + gossip_socket: UdpSocket, + quic_config: QuicOptions, + /// Independent OS-seeded seed for the serf core's RNG, drawn once per node in + /// [`Transport::new`] and consumed when [`Transport::run`] builds the + /// endpoint via `new_with_rng`. Distinct from the coordinator's gossip RNG so + /// serf's query IDs and relay choices are not correlated across nodes. + serf_rng: StdRng, + /// Gossip-encryption policy applied to the coordinator built in + /// [`Transport::run`]. Absent keyring ⇒ plaintext gossip (the default). + #[cfg(encryption)] + encryption: EncryptionOptions, +} + +impl Transport for QuicTransport +where + I: Id + CheapClone + core::fmt::Debug + core::fmt::Display + Send + Sync + 'static, + A: Data + Clone + Send + 'static, +{ + type Error = SerfError; + type Id = I; + type Address = A; + type Options = QuicTransportOptions; + + async fn new( + options: Self::Options, + resolver: &RES, + advertise_resolver: &AR, + ) -> Result + where + RES: Resolver
, + AR: AdvertiseAddrResolver, + { + let local_id = options.local_id.ok_or_else(|| { + SerfError::Io(std::io::Error::new( + ErrorKind::InvalidInput, + "local_id required", + )) + })?; + let advertise_input = options.advertise_addr.ok_or_else(|| { + SerfError::Io(std::io::Error::new( + ErrorKind::InvalidInput, + "advertise_addr required", + )) + })?; + let quic_config = options.quic_config.ok_or_else(|| { + SerfError::Io(std::io::Error::new( + ErrorKind::InvalidInput, + "quic_config required", + )) + })?; + + let advertise_socket = match &advertise_input { + MaybeResolved::Resolved(s) => *s, + MaybeResolved::Unresolved(a) => { + let candidates = resolver + .resolve(a) + .await + .map_err(|e| SerfError::Resolve(std::io::Error::other(e.to_string())))?; + advertise_resolver.pick(candidates).map_err(|e| { + SerfError::Resolve(std::io::Error::new( + ErrorKind::AddrNotAvailable, + e.to_string(), + )) + })? + } + }; + + // QUIC multiplexes streams + gossip over a single UDP socket; there is no + // separate listener to claim a port, so a plain bind suffices. + let gossip_socket = UdpSocket::bind(advertise_socket) + .await + .map_err(SerfError::Io)?; + + // The socket is now bound. `quic_post_bind_setup` reads the bound address + // back (an ephemeral `:0` resolves to a concrete OS-assigned port here, which + // the node gossips to its peers, while a wildcard `0.0.0.0:0` bind yields + // `0.0.0.0:` — rejected as an unspecified IP peers could not route serf + // traffic back to), then draws the OS-seeded serf-core RNG (so an entropy + // failure surfaces as `SerfError::Entropy` here). On ANY error the bound + // socket is closed (awaited — a plain drop is not a synchronous fd release on + // compio/Windows-IOCP) before returning, so a failed construction never leaks + // the bound UDP port to race an immediate same-address rebind into + // `AddrInUse`. + let (advertise_socket, serf_rng) = match crate::transport::quic_post_bind_setup(&gossip_socket) + { + Ok(v) => v, + Err(e) => { + // Ignoring Err: closing an abandoned construction's socket is best-effort. + let _ = gossip_socket.close().await; + return Err(e); + } + }; + + Ok(Self { + local_id, + local_address: advertise_input, + advertise_socket, + gossip_socket, + quic_config, + serf_rng, + #[cfg(encryption)] + encryption: options.encryption, + }) + } + + #[inline] + fn local_id(&self) -> &Self::Id { + &self.local_id + } + + #[inline] + fn local_address(&self) -> &MaybeResolved { + &self.local_address + } + + #[inline] + fn advertise_address(&self) -> &SocketAddr { + &self.advertise_socket + } + + async fn run(self, runtime: TransportRuntime, gossip_rng: G) + where + D: Delegate, + G: rand::Rng + Send + Unpin + 'static, + { + // `Serf::new` is generic over `T` and cannot build the QUIC endpoint (it + // needs the quinn-proto config bundle); build it here from `self`'s stored + // config. Serf ranks its user broadcasts on three tiers (intent / event / + // query → ranks 0 / 1 / 2), so the inner memberlist endpoint needs at least + // three broadcast tiers. + let inner_opts = EndpointOptions::new(self.local_id, self.advertise_socket) + .with_user_broadcast_tiers(NonZeroU8::new(3).expect("3 is nonzero")); + let inner = memberlist_proto::Endpoint::new(inner_opts, gossip_rng); + // The shared UDP socket also carries raw QUIC packets, whose size is governed + // by the quinn `EndpointConfig`'s accepted max UDP payload — which a caller + // can set above the serf gossip MTU (quinn's default 1472 already exceeds the + // 1400 default `gossip_mtu`). Read it off the config here, before it moves + // into the coordinator, so the driver can size its recv buffer for the larger + // of the two planes and not truncate a full-size QUIC packet. + let quic_max_udp_payload = self.quic_config.endpoint_ref().get_max_udp_payload_size(); + // The QUIC coordinator owns the quinn endpoint and the per-peer connection + // pool; `rng_seed = None` seeds quinn's connection-ID generator from OS + // entropy. `QuicOptions` carries the per-peer SNI plumbing internally. + #[allow(unused_mut)] + let mut coord = Coordinator::new(inner, self.quic_config); + // Install the gossip-encryption keyring so the coordinator's + // `encrypt_gossip`/`decrypt_gossip` (forwarded from the serf endpoint pump) + // AEAD-protect the gossip datagrams. A no-keyring policy is the identity + // transform; the reliable plane is quinn TLS and is unaffected either way. + #[cfg(encryption)] + coord.set_encryption_options(self.encryption); + // Serf's core RNG is seeded from its own OS-drawn entropy (`self.serf_rng`), + // independent of the coordinator's gossip RNG, so two nodes never share the + // query-ID / relay-selection stream. + let endpoint = serf_proto::QuicEndpoint::::new_with_rng( + coord, + runtime.serf_options, + self.serf_rng, + ); + + crate::driver::quic::quic_driver_loop::( + endpoint, + self.gossip_socket, + quic_max_udp_payload, + runtime.commands_rx, + runtime.events_tx, + runtime.events_dropped, + runtime.observation_dropped, + runtime.snapshot, + runtime.shutdown_flag, + runtime.driver_options, + runtime.delegate, + None, + #[cfg(encryption)] + runtime.keyring, + ) + .await; + } +} + +#[cfg(all(test, feature = "quic"))] +mod tests; diff --git a/serf-compio/src/quic/tests.rs b/serf-compio/src/quic/tests.rs new file mode 100644 index 00000000..189cca90 --- /dev/null +++ b/serf-compio/src/quic/tests.rs @@ -0,0 +1,569 @@ +//! End-to-end smoke test: two QUIC serf nodes on the loopback interface, one +//! joining the other, asserting the membership event propagates through the full +//! pump (Join command → QUIC push-pull dial → quinn handshake → coordinator merge +//! → serf `Member` event → `EventStream`). + +use core::time::Duration; +use std::{net::SocketAddr, sync::Arc}; + +use futures_util::StreamExt; +use memberlist_proto::{MaybeResolved, UnreliableTransport}; +use rustls::{ + client::danger::{HandshakeSignatureValid, ServerCertVerified}, + version::TLS13, +}; +use rustls_pki_types::{CertificateDer, PrivateKeyDer}; +use serf_proto::{ + event::{Event, MemberEventKind}, + options::Options as SerfOptions, +}; +use smol_str::SmolStr; + +use crate::{ + Channel, FirstAddrResolver, QuicOptions, QuicTransport, QuicTransportOptions, RuntimeOptions, + Serf, SerfError, SocketAddrResolver, Transport, VoidDelegate, gossip_rng, +}; + +#[cfg(encryption)] +use crate::{EncryptionOptions, Keyring, SecretKey, VoidKeyringDelegate}; + +/// A self-signed cert + key for `localhost`, for the test TLS bundle. +fn self_signed() -> (Vec>, PrivateKeyDer<'static>) { + let ck = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); + let cert = CertificateDer::from(ck.cert.der().to_vec()); + let key = PrivateKeyDer::Pkcs8(ck.signing_key.serialize_der().into()); + (vec![cert], key) +} + +fn test_endpoint_config(reset_key: &[u8]) -> quinn_proto::EndpointConfig { + let hmac = ring::hmac::Key::new(ring::hmac::HMAC_SHA256, reset_key); + quinn_proto::EndpointConfig::new(Arc::new(hmac)) +} + +fn test_server() -> quinn_proto::ServerConfig { + let (chain, key) = self_signed(); + let provider = Arc::new(rustls::crypto::ring::default_provider()); + let rustls_server = rustls::ServerConfig::builder_with_provider(provider) + .with_protocol_versions(&[&TLS13]) + .unwrap() + .with_no_client_auth() + .with_single_cert(chain, key) + .unwrap(); + let qsc = + quinn_proto::crypto::rustls::QuicServerConfig::try_from(Arc::new(rustls_server)).unwrap(); + quinn_proto::ServerConfig::with_crypto(Arc::new(qsc)) +} + +/// Accept-any server-cert verifier — test only. +#[derive(Debug)] +struct AnyServer; + +impl rustls::client::danger::ServerCertVerifier for AnyServer { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer, + _intermediates: &[CertificateDer], + _server_name: &rustls_pki_types::ServerName, + _ocsp_response: &[u8], + _now: rustls_pki_types::UnixTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + _message: &[u8], + _cert: &CertificateDer, + _dss: &rustls::DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + + fn verify_tls13_signature( + &self, + _message: &[u8], + _cert: &CertificateDer, + _dss: &rustls::DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + + fn supported_verify_schemes(&self) -> Vec { + rustls::crypto::ring::default_provider() + .signature_verification_algorithms + .supported_schemes() + } +} + +fn test_client() -> quinn_proto::ClientConfig { + let provider = Arc::new(rustls::crypto::ring::default_provider()); + let cfg = rustls::ClientConfig::builder_with_provider(provider) + .with_protocol_versions(&[&TLS13]) + .unwrap() + .dangerous() + .with_custom_certificate_verifier(Arc::new(AnyServer)) + .with_no_client_auth(); + let qcc = quinn_proto::crypto::rustls::QuicClientConfig::try_from(Arc::new(cfg)).unwrap(); + quinn_proto::ClientConfig::new(Arc::new(qcc)) +} + +/// A QUIC config bundle with a 20s idle timeout (well past a localhost handshake) +/// and datagram-mode unreliable transport. A fresh bundle is built per node so +/// each owns its own cert and quinn endpoint config. +fn test_quic_options() -> QuicOptions { + let mut transport = quinn_proto::TransportConfig::default(); + transport.max_idle_timeout(Some( + quinn_proto::IdleTimeout::try_from(Duration::from_secs(20)).unwrap(), + )); + QuicOptions::new( + test_endpoint_config(&[0x5au8; 32]), + test_server(), + test_client(), + transport, + "localhost", + UnreliableTransport::Datagram, + ) +} + +/// QUIC config bundle whose endpoint config accepts a max UDP payload (9000, +/// jumbo-frame sizing) well above the default 1400 gossip MTU. Exercises the +/// recv-buffer sizing that must cover the larger raw-QUIC plane. +fn test_quic_options_jumbo() -> QuicOptions { + let mut endpoint = test_endpoint_config(&[0x5au8; 32]); + endpoint + .max_udp_payload_size(9000) + .expect("9000 is within quinn's accepted [1200, 65527] range"); + let mut transport = quinn_proto::TransportConfig::default(); + transport.max_idle_timeout(Some( + quinn_proto::IdleTimeout::try_from(Duration::from_secs(20)).unwrap(), + )); + QuicOptions::new( + endpoint, + test_server(), + test_client(), + transport, + "localhost", + UnreliableTransport::Datagram, + ) +} + +/// Build and spawn a QUIC serf node bound to an ephemeral loopback port. +async fn spawn_node(id: &str) -> Serf { + spawn_node_with(id, test_quic_options()).await +} + +/// Build and spawn a QUIC serf node from a caller-supplied [`QuicOptions`]. +async fn spawn_node_with(id: &str, quic: QuicOptions) -> Serf { + try_spawn_node_at(id, quic, "127.0.0.1:0".parse().expect("loopback addr")) + .await + .expect("spawn serf node") +} + +/// Build a QUIC serf node bound to a specific advertise address, returning the +/// construction result so the same-address rebind regression can assert a freed +/// UDP port accepts an immediate rebind. +async fn try_spawn_node_at( + id: &str, + quic: QuicOptions, + bind: SocketAddr, +) -> Result, SerfError> { + let opts = QuicTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(bind)) + .with_quic_config(quic); + Serf::new::, SocketAddrResolver, FirstAddrResolver, _, _>( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + gossip_rng().expect("seed gossip rng"), + #[cfg(encryption)] + std::rc::Rc::new(VoidKeyringDelegate), + ) + .await +} + +/// Build VALID QUIC transport options (a real `quic_config` is supplied so the +/// runtime-option rejection, not the missing-config guard, is what fires) paired +/// with a deliberately invalid `runtime`, and assert `Serf::new` rejects it with +/// [`SerfError::InvalidOption`] — before binding a socket or spawning the +/// detached driver — rather than panicking the driver task on a zero-cap channel. +async fn assert_quic_new_rejects(runtime: RuntimeOptions) { + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let opts = QuicTransportOptions::::new() + .with_local_id(SmolStr::new("bad-opt-node")) + .with_advertise_addr(MaybeResolved::Resolved(bind)) + .with_quic_config(test_quic_options()); + let res = + Serf::new::, SocketAddrResolver, FirstAddrResolver, _, _>( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + runtime, + SerfOptions::new(), + gossip_rng().expect("seed gossip rng"), + #[cfg(encryption)] + std::rc::Rc::new(VoidKeyringDelegate), + ) + .await; + match res { + Err(SerfError::InvalidOption(_)) => {} + Err(other) => panic!("expected InvalidOption, got {other:?}"), + Ok(_) => panic!("a zero-capacity channel option must be rejected at construction"), + } +} + +/// A `Bounded(0)` observation channel (direct builder) is rejected by the QUIC +/// driver's `Serf::new` instead of panicking the detached driver task. +#[compio::test] +async fn quic_new_rejects_zero_observation_channel() { + assert_quic_new_rejects(RuntimeOptions::new().with_observation_channel(Channel::Bounded(0))) + .await; +} + +/// A zero `event_queue_cap` (direct builder) is rejected at construction. +#[compio::test] +async fn quic_new_rejects_zero_event_queue_cap() { + assert_quic_new_rejects(RuntimeOptions::new().with_event_queue_cap(0)).await; +} + +/// A zero `cmd_fairness_budget` (direct builder) starves the command drain under +/// an inbound flood, so the QUIC driver's `Serf::new` rejects it at construction. +#[compio::test] +async fn quic_new_rejects_zero_cmd_fairness_budget() { + assert_quic_new_rejects(RuntimeOptions::new().with_cmd_fairness_budget(0)).await; +} + +/// A `Bounded(0)` observation channel sourced from a serde config is rejected. +#[cfg(feature = "serde")] +#[compio::test] +async fn quic_new_rejects_zero_observation_channel_from_serde() { + let runtime: RuntimeOptions = + serde_json::from_str(r#"{"observation_channel":{"bounded":0}}"#).expect("deserialize"); + assert_quic_new_rejects(runtime).await; +} + +/// A `bounded:0` observation channel parsed from a clap flag is rejected. +#[cfg(feature = "clap")] +#[compio::test] +async fn quic_new_rejects_zero_observation_channel_from_clap() { + use clap::Parser; + + #[derive(Parser)] + struct Cli { + #[command(flatten)] + runtime: RuntimeOptions, + } + + let cli = Cli::try_parse_from(["app", "--runtime-observation-channel", "bounded:0"]) + .expect("clap parses bounded:0"); + assert_quic_new_rejects(cli.runtime).await; +} + +/// Two nodes on loopback: A joins B; A must observe B joining the cluster through +/// its event stream over a real QUIC push-pull exchange, then both shut down +/// cleanly. +#[compio::test] +async fn two_node_quic_join_observes_membership() { + let b = spawn_node("node-b").await; + let a = spawn_node("node-a").await; + let b_addr = b.advertise_address(); + let b_id = SmolStr::new("node-b"); + + // Subscribe BEFORE the join so a `Member` event cannot race ahead of the + // subscription (the channel buffers either way, but this is the clean order). + let mut a_events = a.events(); + + // Node A dials node B as its seed. + let dispatched = a.join(vec![b_addr]).await.expect("join dispatched"); + assert_eq!(dispatched, 1, "exactly one seed was dispatched"); + + // Node A should observe node B joining via a `Member(Join)` event. + let observed = compio::time::timeout(Duration::from_secs(20), async { + loop { + match a_events.next().await { + Some(Event::Member(me)) if me.kind() == MemberEventKind::Join => { + if me.members().iter().any(|m| m.node().id_ref() == &b_id) { + break true; + } + } + Some(_) => {} + None => break false, + } + } + }) + .await; + + assert!( + matches!(observed, Ok(true)), + "node A should observe node B joining the cluster within the timeout" + ); + + // Exercise the graceful shutdown command path on both nodes. + a.shutdown().await.expect("node A shuts down"); + b.shutdown().await.expect("node B shuts down"); +} + +/// The QUIC driver binds a single UDP socket and awaits its `close()` before +/// acking shutdown, so `shutdown().await` must release that bound port before it +/// resolves: a second QUIC node binding the SAME advertise address the instant +/// the first shuts down must construct successfully, not fail with `AddrInUse`. +#[compio::test] +async fn quic_shutdown_releases_bound_address_for_rebind() { + let first = spawn_node("rebind-first").await; + let addr = first.advertise_address(); + first.shutdown().await.expect("first node shuts down"); + + let second = try_spawn_node_at("rebind-second", test_quic_options(), addr) + .await + .expect("rebinding the freed UDP address must succeed, not AddrInUse"); + assert_eq!( + second.advertise_address(), + addr, + "the second node rebinds the exact freed address" + ); + second.shutdown().await.expect("second node shuts down"); +} + +/// All `Serf` handles dropping under a continuous gossip flood must still shut the +/// QUIC driver down. Under the flood the higher-priority recv arm starves the main +/// select's command arm, so the command-channel disconnect is observable ONLY by +/// the iter-top command drain; a dropped handle must therefore free the bound UDP +/// socket for an immediate same-address rebind rather than spinning forever and +/// leaking it. +#[compio::test] +async fn quic_command_disconnect_under_flood_releases_bound_port() { + let node = spawn_node("flood-drop").await; + let addr = node.advertise_address(); + + // Flood the driver's single UDP socket so the biased select's recv arm stays + // ready — the load under which the command-channel disconnect must still tear + // the driver down. A first byte of 1 is demuxed to the gossip plane (not a quinn + // packet). A detached task keyed off a stop flag so it ends with the test. + let stop = std::rc::Rc::new(std::cell::Cell::new(false)); + let flood_stop = stop.clone(); + compio::runtime::spawn(async move { + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let peer = compio::net::UdpSocket::bind(bind) + .await + .expect("bind flood peer"); + while !flood_stop.get() { + // Ignoring Err: best-effort flood; a transient send error is non-fatal. + let _ = peer.send_to(vec![1u8, 0, 0, 0], addr).await.0; + // Yield via a short sleep so the flood cannot monopolize the single-threaded + // runtime — a loopback send can complete inline, which would starve the + // driver and the rebind poll. The gossip socket's kernel queue keeps the + // recv arm ready across the gap. + compio::time::sleep(Duration::from_millis(1)).await; + } + // Ignoring Err: test cleanup of the flood socket. + let _ = peer.close().await; + }) + .detach(); + + // Drop every handle: the command channel disconnects. Under the flood the main + // select's command arm is starved, so the iter-top drain's Disconnect branch is + // the only path that can observe it and tear down. + drop(node); + + // The driver must terminate and release its bound UDP port; poll for the rebind + // under a generous timeout so a regression (driver never exits) fails as a + // timeout, not a hang. + let rebound = compio::time::timeout(Duration::from_secs(20), async { + loop { + if let Ok(gossip) = compio::net::UdpSocket::bind(addr).await { + break gossip; + } + compio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await; + + stop.set(true); + let gossip = rebound.expect( + "the QUIC driver must release its bound UDP port after a command-channel disconnect under flood", + ); + // Ignoring Err: test cleanup of the rebind probe socket. + let _ = gossip.close().await; +} + +/// Two nodes whose quinn `EndpointConfig` accepts a max UDP payload (9000) far +/// above the default 1400 gossip MTU still form a cluster: the driver sizes its +/// recv buffer for the larger raw-QUIC plane, so a QUIC packet above the gossip +/// MTU is not truncated before the coordinator demuxes it. The discriminating +/// buffer-length assertion lives in the driver's `tests.rs` +/// (`recv_buf_len_for`); this proves threading the ceiling through construction +/// keeps a real handshake working end-to-end. +#[compio::test] +async fn two_node_quic_join_with_large_max_udp_payload() { + let b = spawn_node_with("node-b", test_quic_options_jumbo()).await; + let a = spawn_node_with("node-a", test_quic_options_jumbo()).await; + let b_addr = b.advertise_address(); + let b_id = SmolStr::new("node-b"); + + let mut a_events = a.events(); + + let dispatched = a.join(vec![b_addr]).await.expect("join dispatched"); + assert_eq!(dispatched, 1, "exactly one seed was dispatched"); + + let observed = compio::time::timeout(Duration::from_secs(20), async { + loop { + match a_events.next().await { + Some(Event::Member(me)) if me.kind() == MemberEventKind::Join => { + if me.members().iter().any(|m| m.node().id_ref() == &b_id) { + break true; + } + } + Some(_) => {} + None => break false, + } + } + }) + .await; + + assert!( + matches!(observed, Ok(true)), + "a cluster with an above-MTU quinn max UDP payload should still form" + ); + + a.shutdown().await.expect("node A shuts down"); + b.shutdown().await.expect("node B shuts down"); +} + +/// Binding the wildcard `0.0.0.0:0` reads an unspecified IP back from the +/// socket; gossiping it would publish an undialable contact, so construction +/// must reject it with `InvalidAdvertiseAddr` (the `quic_config` is supplied so +/// the advertise check, not the missing-config guard, is what fires). +#[compio::test] +async fn new_rejects_wildcard_advertise() { + let wildcard: SocketAddr = "0.0.0.0:0".parse().expect("wildcard addr"); + let opts = QuicTransportOptions::::new() + .with_local_id(SmolStr::new("wild-node")) + .with_advertise_addr(MaybeResolved::Resolved(wildcard)) + .with_quic_config(test_quic_options()); + let res = + QuicTransport::::new(opts, &SocketAddrResolver, &FirstAddrResolver).await; + match res { + Err(SerfError::InvalidAdvertiseAddr(e)) => { + assert!( + e.addr().ip().is_unspecified(), + "the rejected address carries the unspecified IP read back from the wildcard bind" + ); + } + Err(other) => panic!("expected InvalidAdvertiseAddr, got {other:?}"), + Ok(_) => panic!("a wildcard advertise must be rejected, but construction succeeded"), + } +} + +/// A construction failure AFTER the socket is bound must close it (awaited) +/// before returning `Err`, or the bound UDP port leaks and a same-address rebind +/// races into `AddrInUse` (a plain drop is not a synchronous fd release on +/// compio/Windows-IOCP). A wildcard `0.0.0.0:0` advertise binds a concrete +/// OS-assigned port but is then rejected by `validate_advertise_addr` for its +/// unspecified IP; the exact freed `0.0.0.0:` must immediately re-accept a +/// UDP bind, proving the socket did not leak on the error path. A real +/// `quic_config` is supplied so the advertise rejection — not the missing-config +/// guard — is what fires. +#[compio::test] +async fn new_failure_closes_bound_socket_for_rebind() { + let opts = QuicTransportOptions::::new() + .with_local_id(SmolStr::new("rebind-fail")) + .with_advertise_addr(MaybeResolved::Resolved( + "0.0.0.0:0".parse().expect("wildcard addr"), + )) + .with_quic_config(test_quic_options()); + let res = + QuicTransport::::new(opts, &SocketAddrResolver, &FirstAddrResolver).await; + let freed = match res { + Err(SerfError::InvalidAdvertiseAddr(e)) => e.addr(), + Err(other) => panic!("expected a post-bind InvalidAdvertiseAddr failure, got {other:?}"), + Ok(_) => panic!("a post-bind failure must reject construction, but it succeeded"), + }; + + let gossip = compio::net::UdpSocket::bind(freed) + .await + .expect("the freed UDP port must rebind, not AddrInUse"); + // Ignoring Err: test cleanup of the probe socket. + let _ = gossip.close().await; +} + +/// A deterministic test secret key, selecting whichever AEAD cipher this build +/// compiled so the encrypted test works under either backend. +#[cfg(encryption)] +fn test_secret_key(fill: u8) -> SecretKey { + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes256([fill; 32]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = SecretKey::ChaCha20Poly1305([fill; 32]); + key +} + +/// Build and spawn a QUIC serf node on an ephemeral loopback port with +/// `encryption` installed as its gossip keyring policy. +#[cfg(encryption)] +async fn spawn_encrypted_node(id: &str, encryption: EncryptionOptions) -> Serf { + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let opts = QuicTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(bind)) + .with_quic_config(test_quic_options()) + .with_encryption(encryption); + Serf::new::, SocketAddrResolver, FirstAddrResolver, _, _>( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + gossip_rng().expect("seed gossip rng"), + std::rc::Rc::new(VoidKeyringDelegate), + ) + .await + .expect("spawn serf node") +} + +/// Two QUIC nodes sharing one gossip keyring: A joins B and must observe B +/// joining through its event stream. The reliable push-pull rides quinn's own +/// TLS, while the gossip datagrams are AEAD-sealed by the configured keyring — +/// so this proves the keyring reaches the QUIC coordinator and that an encrypted +/// QUIC cluster forms and interoperates end-to-end. +#[cfg(encryption)] +#[compio::test] +async fn two_node_quic_join_observes_membership_encrypted() { + let enc = EncryptionOptions::new().with_keyring(Keyring::new(test_secret_key(0x42))); + let b = spawn_encrypted_node("node-b", enc.clone()).await; + let a = spawn_encrypted_node("node-a", enc).await; + let b_addr = b.advertise_address(); + let b_id = SmolStr::new("node-b"); + + let mut a_events = a.events(); + + let dispatched = a.join(vec![b_addr]).await.expect("join dispatched"); + assert_eq!(dispatched, 1, "exactly one seed was dispatched"); + + let observed = compio::time::timeout(Duration::from_secs(20), async { + loop { + match a_events.next().await { + Some(Event::Member(me)) if me.kind() == MemberEventKind::Join => { + if me.members().iter().any(|m| m.node().id_ref() == &b_id) { + break true; + } + } + Some(_) => {} + None => break false, + } + } + }) + .await; + + assert!( + matches!(observed, Ok(true)), + "node A should observe node B joining the encrypted QUIC cluster within the timeout" + ); + + a.shutdown().await.expect("node A shuts down"); + b.shutdown().await.expect("node B shuts down"); +} diff --git a/serf-compio/src/resolver/advertise/mod.rs b/serf-compio/src/resolver/advertise/mod.rs new file mode 100644 index 00000000..25862e54 --- /dev/null +++ b/serf-compio/src/resolver/advertise/mod.rs @@ -0,0 +1,81 @@ +//! `AdvertiseAddrResolver` — picks one `SocketAddr` from a candidate set +//! during local-node advertise resolution. Called once at `Transport::new` +//! when the configured advertise address is `MaybeResolved::Unresolved(addr)` +//! and `Resolver::resolve(&addr)` returns multiple candidates. + +use std::net::SocketAddr; + +/// Picks one `SocketAddr` from a candidate set. +/// +/// The driver calls this immediately after [`super::Resolver::resolve`] returns +/// more than one result for the configured advertise address. An implementor can +/// express simple policies (prefer IPv4, prefer IPv6, take the first) or consult +/// external state. +pub trait AdvertiseAddrResolver: 'static { + /// Error type returned by [`Self::pick`]. + type Error: core::error::Error + 'static; + + /// Pick one candidate. Returns an error when `candidates` is empty. + fn pick(&self, candidates: Vec) -> Result; +} + +/// Error variants returned by the built-in [`AdvertiseAddrResolver`] impls. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum AdvertiseResolutionError { + /// The candidate set was empty — resolution returned no addresses. + #[error("advertise resolution: no candidate addresses returned")] + Empty, +} + +/// Default — returns the first candidate address in the set. +pub struct FirstAddrResolver; + +impl AdvertiseAddrResolver for FirstAddrResolver { + type Error = AdvertiseResolutionError; + + fn pick(&self, candidates: Vec) -> Result { + candidates + .into_iter() + .next() + .ok_or(AdvertiseResolutionError::Empty) + } +} + +/// Prefers the first IPv4 candidate; falls through to the first address of any +/// family if no IPv4 candidates are present. +pub struct Ipv4PreferringResolver; + +impl AdvertiseAddrResolver for Ipv4PreferringResolver { + type Error = AdvertiseResolutionError; + + fn pick(&self, candidates: Vec) -> Result { + let v4 = candidates.iter().find(|s| s.is_ipv4()).copied(); + if let Some(s) = v4 { + return Ok(s); + } + candidates + .into_iter() + .next() + .ok_or(AdvertiseResolutionError::Empty) + } +} + +/// Prefers the first IPv6 candidate; falls through to the first address of any +/// family if no IPv6 candidates are present. +pub struct Ipv6PreferringResolver; + +impl AdvertiseAddrResolver for Ipv6PreferringResolver { + type Error = AdvertiseResolutionError; + + fn pick(&self, candidates: Vec) -> Result { + let v6 = candidates.iter().find(|s| s.is_ipv6()).copied(); + if let Some(s) = v6 { + return Ok(s); + } + candidates + .into_iter() + .next() + .ok_or(AdvertiseResolutionError::Empty) + } +} diff --git a/serf-compio/src/resolver/dns/mod.rs b/serf-compio/src/resolver/dns/mod.rs new file mode 100644 index 00000000..257fed35 --- /dev/null +++ b/serf-compio/src/resolver/dns/mod.rs @@ -0,0 +1,258 @@ +//! TCP-first DNS resolver — hickory-proto codec + compio TcpStream +//! transport. Mirrors Go memberlist's `tcpLookupIP` algorithm +//! (`hashicorp/memberlist/memberlist.go:308-417`). +//! +//! Why TCP-first: UDP DNS responses are capped at 512 bytes (without EDNS), +//! which can truncate the answer list for cluster-discovery hostnames +//! resolving to many A/AAAA records. TCP-DNS has no such cap, so it gives +//! the largest possible join set on a single query. + +#![cfg(feature = "dns")] + +use crate::resolver::{OsResolver, Resolver}; +use compio::{ + buf::BufResult, + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpStream, +}; +use futures_util::{FutureExt, pin_mut, select_biased}; +use hickory_proto::{ + ProtoError, + op::{Message, Query}, + rr::{Name, RData, RecordType}, + serialize::binary::{BinEncodable, BinEncoder, DecodeError}, +}; +use hostaddr::{Host, HostAddr}; +use smol_str::SmolStr; +use std::{ + io::{self, Read}, + net::{IpAddr, SocketAddr}, + path::Path, + time::Duration, +}; + +/// Default wall-clock upper bound on a single TCP-DNS query (connect + +/// write + read length-prefix + read response). Matches the default +/// DNS query timeout used by most stub resolvers (Go's `net.Resolver` +/// uses 5s, glibc's resolver uses 5s per attempt). Configured +/// per-resolver via [`DnsResolver::with_timeout`]; without a bound +/// the query inherits the kernel's TCP timeouts (~3 minutes connect, +/// infinite read), which would let a slow or hostile nameserver hang +/// the caller's `join` future indefinitely. +pub const DEFAULT_DNS_TIMEOUT: Duration = Duration::from_secs(5); + +/// TCP-first DNS resolver — queries configured nameservers over TCP and +/// falls back to the OS resolver (which is UDP-first with TCP retry on +/// truncation) if TCP returns nothing. +/// +/// Constructed from a resolv.conf-format file. For hostnames that lack a +/// `.` (short names, likely resolved via the host's search-domain list) +/// the TCP path is skipped entirely and the OS resolver is used directly, +/// matching the upstream behavior. +pub struct DnsResolver { + servers: Vec, + fallback: OsResolver, + timeout: Duration, +} + +/// Errors returned by [`DnsResolver`]. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum DnsError { + /// I/O error from the TCP transport or the OS-resolver fallback. + #[error("I/O error: {0}")] + Io(#[from] io::Error), + + /// hickory-proto encoding error (malformed query construction). + #[error("DNS encode error: {0}")] + Encode(#[from] ProtoError), + + /// hickory-proto decoding error (malformed response from the server). + #[error("DNS decode error: {0}")] + Decode(#[from] DecodeError), + + /// The hostname could not be parsed into a wire-format DNS name. + /// Carries the hickory `ProtoError` via `#[source]` rather than `#[from]`, + /// since [`Self::Encode`] already owns the `From` conversion. + #[error("hostname parse error: {0}")] + Hostname(#[source] ProtoError), +} + +impl From for io::Error { + fn from(e: DnsError) -> Self { + Self::other(e) + } +} + +impl DnsResolver { + /// Construct from a resolv.conf-format file path. Reads the file, parses + /// the nameserver list (each pinned to port 53), and stores the OS + /// resolver as the fallback. + pub fn from_resolv_conf(path: impl AsRef) -> Result { + let mut file = std::fs::File::open(path)?; + let mut buf = String::new(); + file.read_to_string(&mut buf)?; + let cfg = resolv_conf::Config::parse(&buf) + .map_err(|e| io::Error::other(format!("resolv.conf parse: {e}")))?; + + let servers: Vec = cfg + .nameservers + .iter() + .map(|ns| SocketAddr::new(IpAddr::from(ns), 53)) + .collect(); + + Ok(Self { + servers, + fallback: OsResolver, + timeout: DEFAULT_DNS_TIMEOUT, + }) + } + + /// Construct from an explicit nameserver list. The OS resolver is used + /// as the fallback path. + pub fn from_servers(servers: Vec) -> Self { + Self { + servers, + fallback: OsResolver, + timeout: DEFAULT_DNS_TIMEOUT, + } + } + + /// Builder: override the per-query timeout. Defaults to + /// [`DEFAULT_DNS_TIMEOUT`]. + #[must_use] + #[inline] + pub const fn with_timeout(mut self, d: Duration) -> Self { + self.timeout = d; + self + } + + /// The configured per-query timeout. + #[inline] + pub const fn timeout(&self) -> Duration { + self.timeout + } + + /// Send a single TCP-DNS query for TYPE ANY against the first + /// configured nameserver and return the collected A + AAAA records. + /// Returns an empty vec when no servers are configured. + /// + /// Bounded by `self.timeout` (default [`DEFAULT_DNS_TIMEOUT`]): a + /// slow or hostile nameserver cannot hang the caller's `join` + /// future beyond this wall-clock budget. On timeout returns + /// [`DnsError::Io`]`(io::ErrorKind::TimedOut)`; the resolver's + /// caller falls back to the OS resolver via the standard error + /// path (see [`Resolver::resolve`] below). + async fn tcp_query(&self, host: &str, port: u16) -> Result, DnsError> { + let query = self.tcp_query_inner(host, port).fuse(); + let timeout = compio::time::sleep(self.timeout).fuse(); + pin_mut!(query, timeout); + select_biased! { + res = query => res, + _ = timeout => Err(DnsError::Io(io::Error::new( + io::ErrorKind::TimedOut, + "TCP-DNS query exceeded the configured timeout", + ))), + } + } + + /// Inner unbounded TCP-DNS query — invoked by [`Self::tcp_query`] + /// inside the deadline select. Kept separate so the deadline wrapper + /// owns the timer arm without complicating the protocol logic. + async fn tcp_query_inner(&self, host: &str, port: u16) -> Result, DnsError> { + let Some(&server) = self.servers.first() else { + return Ok(Vec::new()); + }; + + // Build a TYPE ANY query message. `Message::query()` initializes a + // fresh ID with the standard query flags; we add the question. + let name = Name::from_ascii(host).map_err(DnsError::Hostname)?; + let mut msg = Message::query(); + msg.add_query(Query::query(name, RecordType::ANY)); + + // Encode to bytes via BinEncoder over an owned Vec. + let mut payload: Vec = Vec::with_capacity(512); + { + let mut encoder = BinEncoder::new(&mut payload); + msg.emit(&mut encoder)?; + } + + // TCP-DNS (RFC 1035 §4.2.2) prepends a 2-byte big-endian length. + let payload_len = u16::try_from(payload.len()) + .map_err(|_| DnsError::Io(io::Error::other("DNS query exceeds 65535 bytes")))?; + let mut framed = Vec::with_capacity(2 + payload.len()); + framed.extend_from_slice(&payload_len.to_be_bytes()); + framed.extend_from_slice(&payload); + + // Connect and send the framed query. compio's IO is buffer-owning, so + // we destructure BufResult and discard the returned buffer. + let mut stream = TcpStream::connect(server).await?; + let BufResult(write_res, _) = stream.write_all(framed).await; + write_res?; + + // Read the 2-byte length prefix, then the body of exactly that length. + let len_buf = vec![0u8; 2]; + let BufResult(read_res, len_buf) = stream.read_exact(len_buf).await; + read_res?; + let response_len = u16::from_be_bytes([len_buf[0], len_buf[1]]) as usize; + + let resp_buf = vec![0u8; response_len]; + let BufResult(read_res, resp_buf) = stream.read_exact(resp_buf).await; + read_res?; + + // Decode and collect A + AAAA answers. CNAME and other RR types are + // ignored to match the upstream behavior (see Go reference above). + let response = Message::from_vec(&resp_buf)?; + let mut addrs = Vec::new(); + for record in &response.answers { + // `Record` exposes the rdata as a public field `data`; the + // same-named accessor is shadowed when the field is the same name. + match &record.data { + RData::A(ipv4) => addrs.push(SocketAddr::new(IpAddr::V4(ipv4.0), port)), + RData::AAAA(ipv6) => addrs.push(SocketAddr::new(IpAddr::V6(ipv6.0), port)), + _ => {} + } + } + Ok(addrs) + } +} + +impl Resolver for DnsResolver { + type Address = HostAddr; + type Error = DnsError; + + async fn resolve(&self, addr: &Self::Address) -> Result, Self::Error> { + let port = addr.port().unwrap_or(0); + + // IP literal: short-circuit, no DNS at all. + if let Host::Ip(ip) = addr.host() { + return Ok(vec![SocketAddr::new(*ip, port)]); + } + + let host_str: &str = match addr.host() { + Host::Domain(d) => d.as_ref(), + Host::Ip(_) => unreachable!("handled above"), + }; + + // TCP-first only for names that look fully qualified (contain a `.`) + // and only when we have at least one nameserver configured. Short + // names will be resolved through the OS resolver's search-domain list. + if host_str.contains('.') && !self.servers.is_empty() { + // Ignoring Err: TCP-first is best-effort per the upstream spec + // ("If this fails it's not fatal since this isn't a standard way to + // query DNS, and we have a fallback below.", memberlist.go:404). + // We unconditionally fall through to the OS resolver on any error + // or empty answer. + if let Ok(addrs) = self.tcp_query(host_str, port).await + && !addrs.is_empty() + { + return Ok(addrs); + } + } + + self.fallback.resolve(addr).await.map_err(DnsError::Io) + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-compio/src/resolver/dns/tests.rs b/serf-compio/src/resolver/dns/tests.rs new file mode 100644 index 00000000..4c219e2e --- /dev/null +++ b/serf-compio/src/resolver/dns/tests.rs @@ -0,0 +1,412 @@ +use super::*; +use hickory_proto::rr::Name; +use std::{ + fs, + net::{Ipv4Addr, Ipv6Addr}, + path::PathBuf, + sync::atomic::{AtomicU64, Ordering}, +}; + +/// The DNS resolver's user-facing address type — the hostname-or-IP type the +/// OS resolver also consumes. +type Address = HostAddr; + +/// A nameserver that is guaranteed never to answer: `240.0.0.1` is in the +/// reserved 240.0.0.0/4 block (RFC 1112 §4) and is unroutable. Pairing it +/// with a fast-path test proves the IP-literal branch never opens a socket +/// — a real connect would hang well past any test budget. +fn dead_server() -> SocketAddr { + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(240, 0, 0, 1)), 53) +} + +/// Build a resolver pointed at the unroutable nameserver with a sub-second +/// timeout, so any test that *did* hit the network would fail loudly +/// rather than silently fall through. +fn resolver_with_dead_server() -> DnsResolver { + DnsResolver::from_servers(vec![dead_server()]).with_timeout(Duration::from_millis(50)) +} + +/// Allocate a unique temp path for a resolv.conf fixture. Avoids a +/// `tempfile` dev-dependency; the counter + PID keep parallel test +/// threads from colliding. +fn temp_resolv_conf_path(tag: &str) -> PathBuf { + static COUNTER: AtomicU64 = AtomicU64::new(0); + let n = COUNTER.fetch_add(1, Ordering::Relaxed); + let mut p = std::env::temp_dir(); + p.push(format!( + "serf-compio-dns-{tag}-{}-{n}.conf", + std::process::id() + )); + p +} + +#[test] +fn from_servers_uses_default_timeout() { + let r = DnsResolver::from_servers(vec![dead_server()]); + assert_eq!(r.timeout(), DEFAULT_DNS_TIMEOUT); + assert_eq!(DEFAULT_DNS_TIMEOUT, Duration::from_secs(5)); +} + +#[test] +fn from_empty_servers_uses_default_timeout() { + let r = DnsResolver::from_servers(Vec::new()); + assert_eq!(r.timeout(), DEFAULT_DNS_TIMEOUT); +} + +#[test] +fn with_timeout_overrides_default() { + let custom = Duration::from_millis(250); + let r = DnsResolver::from_servers(vec![dead_server()]).with_timeout(custom); + assert_eq!(r.timeout(), custom); + + // The builder is chainable and last-write-wins. + let r = r.with_timeout(Duration::from_secs(30)); + assert_eq!(r.timeout(), Duration::from_secs(30)); +} + +#[test] +fn from_resolv_conf_parses_valid_file() { + let path = temp_resolv_conf_path("valid"); + fs::write( + &path, + "# a comment\nnameserver 8.8.8.8\nnameserver 1.1.1.1\nsearch example.com\n", + ) + .expect("write resolv.conf fixture"); + + let r = DnsResolver::from_resolv_conf(&path).expect("parse valid resolv.conf"); + // No accessor exposes the parsed server list, but a successful parse + // must still leave the default per-query timeout in place. + assert_eq!(r.timeout(), DEFAULT_DNS_TIMEOUT); + + // Ignoring Err: best-effort cleanup of the temp fixture; a leaked file + // in the OS temp dir is harmless and must not fail the test. + let _ = fs::remove_file(&path); +} + +#[test] +fn from_resolv_conf_parses_empty_file() { + let path = temp_resolv_conf_path("empty"); + fs::write(&path, "").expect("write empty resolv.conf fixture"); + + // An empty file is a valid resolv.conf with zero nameservers. + let r = DnsResolver::from_resolv_conf(&path).expect("parse empty resolv.conf"); + assert_eq!(r.timeout(), DEFAULT_DNS_TIMEOUT); + + // Ignoring Err: best-effort temp-fixture cleanup. + let _ = fs::remove_file(&path); +} + +#[test] +fn from_resolv_conf_rejects_invalid_nameserver() { + let path = temp_resolv_conf_path("invalid"); + // A `nameserver` line whose argument is not an IP makes + // `resolv_conf::Config::parse` surface an `InvalidIp` error, which the + // constructor maps to `io::Error::other`. + fs::write(&path, "nameserver not-an-ip-address\n").expect("write bad resolv.conf fixture"); + + // `DnsResolver` is not `Debug`, so map the `Ok` side to `()` before + // `expect_err` can report it. + let err = DnsResolver::from_resolv_conf(&path) + .map(|_| ()) + .expect_err("malformed resolv.conf must fail"); + assert_eq!(err.kind(), io::ErrorKind::Other); + assert!( + err.to_string().contains("resolv.conf parse"), + "error should carry the parse context, got: {err}" + ); + + // Ignoring Err: best-effort temp-fixture cleanup. + let _ = fs::remove_file(&path); +} + +#[test] +fn from_resolv_conf_missing_file_is_io_error() { + let path = temp_resolv_conf_path("does-not-exist"); + // Guard against an accidentally pre-existing path. + // Ignoring Err: removal is precautionary; absence is the desired state. + let _ = fs::remove_file(&path); + + let err = DnsResolver::from_resolv_conf(&path) + .map(|_| ()) + .expect_err("missing file must fail"); + assert_eq!(err.kind(), io::ErrorKind::NotFound); +} + +#[compio::test] +async fn resolve_ipv4_literal_skips_dns() { + // An IPv4 literal must short-circuit before any socket is opened, even + // though the resolver is pointed at an unroutable nameserver. + let r = resolver_with_dead_server(); + let addr: Address = HostAddr::from_sock_addr("127.0.0.1:7946".parse().unwrap()); + let resolved = r.resolve(&addr).await.expect("IP literal resolves"); + assert_eq!( + resolved, + vec![SocketAddr::new( + IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), + 7946 + )] + ); +} + +#[compio::test] +async fn resolve_ipv6_literal_skips_dns() { + let r = resolver_with_dead_server(); + let addr: Address = HostAddr::from_ip_addr(IpAddr::V6(Ipv6Addr::LOCALHOST)).with_port(443); + let resolved = r.resolve(&addr).await.expect("IPv6 literal resolves"); + assert_eq!( + resolved, + vec![SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 443)] + ); +} + +#[compio::test] +async fn resolve_ip_literal_without_port_uses_zero() { + // `HostAddr::port()` is `None` for a bare IP; the resolver substitutes 0. + let r = resolver_with_dead_server(); + let addr: Address = HostAddr::from_ip_addr(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 5))); + assert_eq!(addr.port(), None); + let resolved = r + .resolve(&addr) + .await + .expect("portless IP literal resolves"); + assert_eq!( + resolved, + vec![SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 5)), 0)] + ); +} + +#[compio::test] +async fn resolve_ip_literal_works_with_empty_server_list() { + // The fast path does not consult `self.servers`, so an empty list is + // irrelevant for IP literals. + let r = DnsResolver::from_servers(Vec::new()); + let addr: Address = HostAddr::from_sock_addr("192.0.2.1:8080".parse().unwrap()); + let resolved = r.resolve(&addr).await.expect("IP literal resolves"); + assert_eq!( + resolved, + vec![SocketAddr::new( + IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1)), + 8080 + )] + ); +} + +#[compio::test] +async fn resolve_short_name_bypasses_tcp_and_uses_os_fallback() { + // `localhost` has no `.`, so the TCP-first branch is skipped entirely + // and resolution goes straight to the OS resolver. `localhost` maps to + // loopback via the hosts file, so this stays off the external network + // even though the configured nameserver is unroutable. + let r = resolver_with_dead_server(); + let addr: Address = "localhost:9000".parse().expect("parse localhost:9000"); + // Confirm the test input is actually a short domain, not an IP literal. + assert!(matches!(addr.host(), Host::Domain(_))); + + let resolved = r.resolve(&addr).await.expect("localhost resolves via OS"); + assert!( + !resolved.is_empty(), + "localhost must resolve to at least one address" + ); + for sa in &resolved { + assert_eq!(sa.port(), 9000); + assert!( + sa.ip().is_loopback(), + "localhost must only map to loopback, got {sa}" + ); + } +} + +#[test] +fn dns_error_display_and_debug() { + let io_err = DnsError::Io(io::Error::new(io::ErrorKind::TimedOut, "boom")); + assert!(io_err.to_string().starts_with("I/O error:")); + assert!(io_err.to_string().contains("boom")); + assert!(!format!("{io_err:?}").is_empty()); + + // A 64-octet single label exceeds the 63-octet DNS limit, yielding a real + // hickory `ProtoError`. + let proto_err = Name::from_ascii("x".repeat(64)).unwrap_err(); + let host_err = DnsError::Hostname(proto_err); + assert!(host_err.to_string().starts_with("hostname parse error: ")); + assert!(format!("{host_err:?}").contains("Hostname")); +} + +#[test] +fn dns_error_from_io_error() { + // `#[from]` wires `io::Error` into the `Io` variant. + let converted: DnsError = io::Error::new(io::ErrorKind::ConnectionRefused, "nope").into(); + assert!(matches!(converted, DnsError::Io(_))); + assert_eq!(converted.to_string(), "I/O error: nope"); +} + +#[test] +fn dns_error_into_io_error_roundtrips() { + // `From for io::Error` wraps via `io::Error::other`, so the + // kind is `Other` and the source string is preserved. + let original = DnsError::Hostname(Name::from_ascii("x".repeat(64)).unwrap_err()); + let io_err: io::Error = original.into(); + assert_eq!(io_err.kind(), io::ErrorKind::Other); + assert!(io_err.to_string().starts_with("hostname parse error: ")); +} + +#[test] +fn dns_error_decode_variant_displays() { + // Exercise the `Decode` variant's `Display` prefix by forcing a decode + // failure from a too-short DNS message buffer. + let decode_err = Message::from_vec(&[0x00]).expect_err("truncated DNS message must fail"); + let err = DnsError::from(decode_err); + assert!( + err.to_string().starts_with("DNS decode error:"), + "got: {err}" + ); + assert!(matches!(err, DnsError::Decode(_))); +} + +/// A loopback TCP server that speaks just enough of the TCP-DNS wire protocol +/// (RFC 1035 §4.2.2) to answer ONE query: read the 2-byte length prefix + +/// query body, then write back a framed DNS response carrying the supplied +/// A and AAAA answers. Returns the bound address so the resolver can target +/// it. Stays entirely on loopback — no external DNS is contacted. +/// +/// The response echoes no relationship to the query name; the resolver's +/// `tcp_query_inner` decodes the message and harvests A/AAAA answers without +/// validating the question, so a fixed answer set exercises the full +/// frame → connect → write → read-length → read-body → decode → collect path. +async fn spawn_tcp_dns_server( + answers: Vec, +) -> (SocketAddr, compio::runtime::JoinHandle<()>) { + use compio::{ + buf::BufResult, + io::{AsyncReadExt, AsyncWriteExt}, + net::TcpListener, + }; + use hickory_proto::{ + op::{Message, OpCode}, + rr::{Name, Record}, + }; + + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind loopback DNS server"); + let addr = listener.local_addr().expect("server local_addr"); + + let handle = compio::runtime::spawn(async move { + let Ok((mut stream, _)) = listener.accept().await else { + return; + }; + + // Read the 2-byte big-endian query length, then the query body. + let len_buf = vec![0u8; 2]; + let BufResult(r, len_buf) = stream.read_exact(len_buf).await; + if r.is_err() { + return; + } + let qlen = u16::from_be_bytes([len_buf[0], len_buf[1]]) as usize; + let qbuf = vec![0u8; qlen]; + let BufResult(r, _qbuf) = stream.read_exact(qbuf).await; + if r.is_err() { + return; + } + + // Build a response message carrying the fixed answers. The id is + // irrelevant to the resolver (it does not match the query id). + let mut resp = Message::response(0, OpCode::Query); + let name = Name::from_ascii("seed.cluster.test.").expect("answer name"); + for rdata in answers { + resp.add_answer(Record::from_rdata(name.clone(), 60, rdata)); + } + let body = resp.to_vec().expect("encode response"); + let mut framed = Vec::with_capacity(2 + body.len()); + framed.extend_from_slice(&(body.len() as u16).to_be_bytes()); + framed.extend_from_slice(&body); + + // Ignoring Err: best-effort single write into a test fixture; if the + // client hung up the resolver test will surface the failure itself. + let BufResult(_w, _b) = stream.write_all(framed).await; + }); + + (addr, handle) +} + +// The TCP-first success path: a FQDN (contains a `.`) with a configured +// nameserver drives `tcp_query` → `tcp_query_inner` end to end against a +// loopback DNS server, and the harvested A + AAAA answers (carrying the +// query port) are returned without consulting the OS fallback. +#[compio::test] +async fn tcp_query_collects_a_and_aaaa_from_loopback_server() { + use hickory_proto::rr::{ + RData, + rdata::{A, AAAA}, + }; + + let answers = vec![ + RData::A(A(Ipv4Addr::new(203, 0, 113, 7))), + RData::AAAA(AAAA(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 0x1))), + // A CNAME-like non-address record is ignored by the collector; model it + // with a TXT to prove the `_ => {}` arm is taken. + RData::TXT(hickory_proto::rr::rdata::TXT::new(vec![ + "ignored".to_string(), + ])), + ]; + let (server_addr, handle) = spawn_tcp_dns_server(answers).await; + + let r = DnsResolver::from_servers(vec![server_addr]).with_timeout(Duration::from_secs(5)); + // A fully-qualified name (has a `.`) so the TCP-first branch is taken. + let addr: Address = "seed.cluster.test:8300".parse().expect("parse FQDN:port"); + assert!(matches!(addr.host(), Host::Domain(_))); + + let resolved = r.resolve(&addr).await.expect("TCP-DNS resolve"); + // Both address answers surface, each carrying the requested port; the TXT + // is dropped. + assert_eq!( + resolved.len(), + 2, + "only A + AAAA are collected: {resolved:?}" + ); + assert!(resolved.contains(&SocketAddr::new( + IpAddr::V4(Ipv4Addr::new(203, 0, 113, 7)), + 8300 + ))); + assert!(resolved.contains(&SocketAddr::new( + IpAddr::V6(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 0x1)), + 8300 + ))); + + handle.await.expect("DNS server task"); +} + +// When the TCP-DNS server returns an EMPTY answer set, the resolver treats +// the TCP attempt as unproductive and falls through to the OS fallback. With +// a short loopback name target the fallback resolves loopback, proving the +// empty-TCP-answer → fallback path (the `!addrs.is_empty()` guard going +// false) rather than surfacing the empty TCP result. +#[compio::test] +async fn tcp_query_empty_answers_falls_through_to_os_fallback() { + // Server answers with zero address records. + let (server_addr, handle) = spawn_tcp_dns_server(Vec::new()).await; + + let r = DnsResolver::from_servers(vec![server_addr]).with_timeout(Duration::from_secs(5)); + // `localhost.` is fully-qualified (trailing dot ⇒ contains `.`), so the TCP + // branch is attempted; the empty answer makes it fall through to the OS + // resolver, which maps localhost to loopback. + let addr: Address = "localhost.:9100".parse().expect("parse localhost.:9100"); + assert!(matches!(addr.host(), Host::Domain(_))); + + let resolved = r + .resolve(&addr) + .await + .expect("empty TCP answer falls back to OS resolver"); + assert!( + !resolved.is_empty(), + "OS fallback must resolve localhost to loopback" + ); + for sa in &resolved { + assert_eq!(sa.port(), 9100); + assert!( + sa.ip().is_loopback(), + "localhost maps to loopback, got {sa}" + ); + } + + handle.await.expect("DNS server task"); +} diff --git a/serf-compio/src/resolver/getifs/mod.rs b/serf-compio/src/resolver/getifs/mod.rs new file mode 100644 index 00000000..f5ddf4dd --- /dev/null +++ b/serf-compio/src/resolver/getifs/mod.rs @@ -0,0 +1,170 @@ +//! getifs-backed advertise auto-detection (feature `getifs`). +//! +//! [`LocalAddrResolver`] enumerates the host's own interface addresses when the +//! configured advertise address is a wildcard (`0.0.0.0` / `[::]`) and +//! substitutes a concrete one. This driver uses a single address for both the +//! socket bind and the advertised contact — and rejects an unspecified advertise +//! at construction — so the substituted address becomes the bind address too. +//! Unlike HashiCorp memberlist (which binds `0.0.0.0` across every interface and +//! advertises a separately-detected IP), the node binds and advertises only the +//! chosen interface address. + +use std::{ + io, + net::{IpAddr, SocketAddr}, +}; + +use crate::resolver::Resolver; + +/// Which of the host's own interface addresses to enumerate when resolving a +/// wildcard (`0.0.0.0` / `[::]`) advertise address via [`LocalAddrResolver`]. +#[cfg_attr(docsrs, doc(cfg(feature = "getifs")))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LocalAddrScope { + /// Private (RFC 1918 / RFC 4193) addresses — the usual choice for a LAN cluster. + Private, + /// Globally-routable public addresses. + Public, + /// Every interface address except loopback, unspecified, and link-local. + All, +} + +/// A [`Resolver`] that auto-detects the host's advertise address from its +/// network interfaces, via [`getifs`](https://crates.io/crates/getifs). +/// +/// `resolve` enumerates the host's own interface addresses — filtered by the +/// configured [`LocalAddrScope`], IPv4 first — **only when the input address is +/// a wildcard** (`0.0.0.0` / `[::]`). A concrete address is passed through +/// unchanged, so the same resolver also resolves `join` seeds. +/// +/// The chosen address becomes the node's single bind **and** advertise address; +/// on a multi-homed host the node binds only that one interface, not all of +/// them (this driver has no separate all-interfaces bind). +#[cfg_attr(docsrs, doc(cfg(feature = "getifs")))] +#[derive(Debug, Clone, Copy)] +pub struct LocalAddrResolver { + scope: LocalAddrScope, +} + +impl LocalAddrResolver { + /// A resolver for the given scope. + pub const fn new(scope: LocalAddrScope) -> Self { + Self { scope } + } + + /// Private addresses (the default). + pub const fn private() -> Self { + Self::new(LocalAddrScope::Private) + } + + /// Public addresses. + pub const fn public() -> Self { + Self::new(LocalAddrScope::Public) + } + + /// Every interface address except loopback, unspecified, and link-local. + pub const fn all() -> Self { + Self::new(LocalAddrScope::All) + } +} + +impl Default for LocalAddrResolver { + fn default() -> Self { + Self::private() + } +} + +impl Resolver for LocalAddrResolver { + type Address = SocketAddr; + type Error = io::Error; + + async fn resolve(&self, address: &SocketAddr) -> Result, io::Error> { + let addr = *address; + if addr.ip().is_unspecified() { + // Honor the wildcard's family: `0.0.0.0` yields only IPv4 candidates, + // `[::]` only IPv6 — never substitute the other family for the address + // the caller asked to bind + advertise. + local_socket_addrs(self.scope, addr.port(), Some(addr.is_ipv6())) + } else { + Ok(vec![addr]) + } + } +} + +/// Detect a single advertise [`SocketAddr`] directly — the first interface +/// address matching `scope` (IPv4 first), with `port` attached — for callers +/// who would rather compute the address up front and pass +/// [`MaybeResolved::Resolved`](memberlist_proto::MaybeResolved::Resolved). +/// +/// Family-agnostic: prefers IPv4. To pin a family, drive [`LocalAddrResolver`] +/// with a `0.0.0.0` / `[::]` wildcard instead. +#[cfg_attr(docsrs, doc(cfg(feature = "getifs")))] +pub fn local_advertise(scope: LocalAddrScope, port: u16) -> io::Result { + local_socket_addrs(scope, port, None)? + .into_iter() + .next() + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::AddrNotAvailable, + "no local interface address found", + ) + }) +} + +/// Enumerate the host's interface addresses matching `scope`, IPv4 first, each +/// with `port` attached. `only_ipv6` filters to one family (`Some(true)` = IPv6, +/// `Some(false)` = IPv4); `None` keeps both, IPv4 first. +fn local_socket_addrs( + scope: LocalAddrScope, + port: u16, + only_ipv6: Option, +) -> io::Result> { + // Enumerate ALL interface addresses and classify them here. getifs's own + // `private_addrs` / `public_addrs` use the broad RFC 6890 special-purpose + // registry (CGNAT, benchmarking, NAT64, ...), not the RFC 1918 / RFC 4193 + // "reachable LAN contact" set we want to advertise. + let mut ips: Vec = getifs::local_addrs()? + .iter() + .map(|n| n.addr()) + .filter(|ip| scope.accepts(*ip)) + .filter(|ip| only_ipv6.is_none_or(|v6| ip.is_ipv6() == v6)) + .collect(); + // IPv4 first — Go memberlist advertises an IPv4 private address by default. + ips.sort_by_key(|ip| ip.is_ipv6()); + Ok( + ips + .into_iter() + .map(|ip| SocketAddr::new(ip, port)) + .collect(), + ) +} + +// Classification via the `iprfc` RFC registry getifs itself uses (depended on +// directly so the version floor — iprfc 0.2.2, which fixes the RFC6890 IPv6 +// table — is enforced in our own dependency graph). Mirrored in the std +// memberlist resolvers; keep the two in sync. +impl LocalAddrScope { + /// Whether `ip` is an acceptable advertise candidate for this scope. + fn accepts(self, ip: IpAddr) -> bool { + use iprfc::{RFC1918, RFC4193, RFC6890}; + match self { + // RFC 1918 (IPv4) + RFC 4193 unique-local (IPv6). + LocalAddrScope::Private => RFC1918.contains(&ip) || RFC4193.contains(&ip), + // Globally-routable = not in the RFC 6890 special-purpose registry — the + // exact definition getifs uses for its own `public_addrs`. + LocalAddrScope::Public => !RFC6890.contains(&ip), + LocalAddrScope::All => !ip.is_loopback() && !ip.is_unspecified() && !is_link_local(ip), + } + } +} + +/// Link-local (IPv4 169.254/16, IPv6 fe80::/10) — usable only on the local link. +fn is_link_local(ip: IpAddr) -> bool { + match ip { + IpAddr::V4(a) => a.is_link_local(), + IpAddr::V6(a) => (a.segments()[0] & 0xffc0) == 0xfe80, + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-compio/src/resolver/getifs/tests.rs b/serf-compio/src/resolver/getifs/tests.rs new file mode 100644 index 00000000..1a25d1e8 --- /dev/null +++ b/serf-compio/src/resolver/getifs/tests.rs @@ -0,0 +1,129 @@ +use super::*; + +#[compio::test] +async fn concrete_address_passes_through() { + let r = LocalAddrResolver::all(); + let addr: SocketAddr = "10.0.0.5:7946".parse().unwrap(); + assert_eq!(r.resolve(&addr).await.unwrap(), vec![addr]); +} + +#[compio::test] +async fn wildcard_enumerates_interface_addrs() { + let r = LocalAddrResolver::all(); + let wild: SocketAddr = "0.0.0.0:7946".parse().unwrap(); + let out = r.resolve(&wild).await.unwrap(); + // The wildcard was replaced by real interface addresses, all on the asked + // port; none is the wildcard itself. + assert!(!out.contains(&wild)); + assert!( + out + .iter() + .all(|s| s.port() == 7946 && !s.ip().is_unspecified()) + ); +} + +#[compio::test] +async fn wildcard_respects_address_family() { + let r = LocalAddrResolver::all(); + // `0.0.0.0` must never substitute an IPv6 address, and `[::]` never an IPv4 + // one — the resolved bind+advertise address keeps the requested family. + let v4 = r.resolve(&"0.0.0.0:7946".parse().unwrap()).await.unwrap(); + assert!( + v4.iter().all(|s| s.is_ipv4()), + "0.0.0.0 must yield only IPv4" + ); + let v6 = r.resolve(&"[::]:7946".parse().unwrap()).await.unwrap(); + assert!(v6.iter().all(|s| s.is_ipv6()), "[::] must yield only IPv6"); +} + +#[test] +fn local_advertise_attaches_port() { + // Best-effort: a host may have no address in a given scope, but when one is + // found the helper attaches the requested port. + if let Ok(addr) = local_advertise(LocalAddrScope::All, 7946) { + assert_eq!(addr.port(), 7946); + } +} + +#[test] +fn private_scope_is_rfc1918_and_ula_only() { + use LocalAddrScope::Private; + for ip in [ + "10.0.0.1", + "172.16.5.4", + "192.168.1.1", + "fc00::1", + "fd12::3", + ] { + assert!( + Private.accepts(ip.parse().unwrap()), + "{ip} should be private" + ); + } + // RFC 6890 special-purpose that getifs::private_addrs would wrongly include: + for ip in [ + "100.64.0.1", // CGNAT + "198.18.0.1", // benchmarking + "192.0.2.1", // documentation + "8.8.8.8", // public + "169.254.0.1", // link-local + "2001:db8::1", // documentation v6 + "64:ff9b::1", // NAT64 + "2606:4700::1", // public v6 + "fe80::1", // link-local v6 + ] { + assert!( + !Private.accepts(ip.parse().unwrap()), + "{ip} must NOT be private" + ); + } +} + +#[test] +fn public_scope_excludes_special_purpose() { + use LocalAddrScope::Public; + assert!(Public.accepts("8.8.8.8".parse().unwrap())); + assert!(Public.accepts("2606:4700::1".parse().unwrap())); + // Real global unicast inside 2001::/16 — must be public (not the whole + // /16 special-purpose block; only 2001::/23 + 2001:db8::/32 are). + assert!(Public.accepts("2001:4860:4860::8888".parse().unwrap())); + for ip in [ + "10.0.0.1", + "100.64.0.1", // CGNAT + "198.18.0.1", // benchmarking + "192.0.2.1", // documentation + "192.0.0.1", // IETF protocol assignments + "240.0.0.1", // reserved / future use + "127.0.0.1", + "169.254.0.1", + "fc00::1", + "2001:db8::1", // documentation v6 + "2001::1", // 2001::/23 (Teredo) + "2001:2::1", // benchmarking v6 + "2001:10::1", // ORCHID + "fe80::1", + "::1", + ] { + assert!( + !Public.accepts(ip.parse().unwrap()), + "{ip} must NOT be public" + ); + } +} + +#[test] +fn all_scope_excludes_loopback_unspecified_linklocal() { + use LocalAddrScope::All; + assert!(All.accepts("10.0.0.1".parse().unwrap())); + assert!(All.accepts("8.8.8.8".parse().unwrap())); + for ip in [ + "127.0.0.1", + "0.0.0.0", + "169.254.0.1", + "::1", + "::", + "fe80::1", + ] { + assert!(!All.accepts(ip.parse().unwrap()), "{ip} must NOT be in All"); + } +} diff --git a/serf-compio/src/resolver/mod.rs b/serf-compio/src/resolver/mod.rs new file mode 100644 index 00000000..95e51c36 --- /dev/null +++ b/serf-compio/src/resolver/mod.rs @@ -0,0 +1,63 @@ +//! Address resolution for serf-compio. +//! +//! [`Resolver`] is the generic address-to-[`SocketAddr`] conversion trait. +//! [`AdvertiseAddrResolver`] picks the single advertise address from a +//! multi-candidate resolution result. +//! +//! Built-in resolvers: +//! - [`OsResolver`]: `getaddrinfo`-backed (compio `ToSocketAddrsAsync`). +//! - [`SocketAddrResolver`]: identity pass-through for already-resolved addrs. +//! +//! Optional resolvers (feature-gated): +//! - `dns` — [`DnsResolver`]: TCP-first DNS via `hickory-proto`. +//! - `getifs` — [`LocalAddrResolver`]: auto-detect the advertise address from +//! the host's own interfaces. + +mod advertise; +mod os; +mod socket_addr; + +#[cfg(feature = "dns")] +mod dns; + +#[cfg(feature = "getifs")] +mod getifs; + +pub use advertise::{ + AdvertiseAddrResolver, AdvertiseResolutionError, FirstAddrResolver, Ipv4PreferringResolver, + Ipv6PreferringResolver, +}; +pub use os::OsResolver; +pub use socket_addr::SocketAddrResolver; + +#[cfg(feature = "dns")] +#[cfg_attr(docsrs, doc(cfg(feature = "dns")))] +pub use dns::{DEFAULT_DNS_TIMEOUT, DnsResolver}; + +#[cfg(feature = "getifs")] +#[cfg_attr(docsrs, doc(cfg(feature = "getifs")))] +pub use getifs::{LocalAddrResolver, LocalAddrScope, local_advertise}; + +use std::net::SocketAddr; + +/// Resolve a user-facing address into one or more concrete [`SocketAddr`]s. +/// +/// The input address type is the implementor's choice: [`SocketAddrResolver`] +/// takes [`SocketAddr`] (identity pass-through); [`OsResolver`] takes +/// [`hostaddr::HostAddr`](hostaddr::HostAddr). Custom resolvers may +/// take any type (e.g. a service-discovery handle). +/// +/// AFIT (no `async-trait`) — compio is `!Send`-first so the trait has no +/// `Send`/`Sync` bound. Pass an instance per call; the caller owns its +/// lifetime and can reuse it across multiple joins. +#[allow(async_fn_in_trait)] +pub trait Resolver: 'static { + /// The user-facing address type this resolver consumes. + type Address; + + /// The error type returned on resolution failure. + type Error: core::error::Error + 'static; + + /// Resolve `addr` to its concrete socket addresses. + async fn resolve(&self, addr: &Self::Address) -> Result, Self::Error>; +} diff --git a/serf-compio/src/resolver/os.rs b/serf-compio/src/resolver/os.rs new file mode 100644 index 00000000..6b7a59d0 --- /dev/null +++ b/serf-compio/src/resolver/os.rs @@ -0,0 +1,32 @@ +//! OS-based resolver — delegates to compio's `getaddrinfo` equivalent. + +use crate::resolver::Resolver; +use compio::net::ToSocketAddrsAsync; +use hostaddr::Host; +use smol_str::SmolStr; +use std::{io, net::SocketAddr}; + +/// OS-based resolver. Uses compio's `ToSocketAddrsAsync` (`getaddrinfo`). +/// +/// Suitable for typical hostname lookups. For large DNS records that may be +/// truncated over UDP, use the `DnsResolver` (feature `dns`), which performs +/// TCP-first queries. +pub struct OsResolver; + +impl Resolver for OsResolver { + type Address = hostaddr::HostAddr; + type Error = io::Error; + + async fn resolve(&self, addr: &Self::Address) -> Result, Self::Error> { + let port = addr.port().unwrap_or(0); + let host_str = match addr.host() { + Host::Ip(ip) => ip.to_string(), + Host::Domain(name) => name.to_string(), + }; + let resolved: Vec = (host_str.as_str(), port) + .to_socket_addrs_async() + .await? + .collect(); + Ok(resolved) + } +} diff --git a/serf-compio/src/resolver/socket_addr.rs b/serf-compio/src/resolver/socket_addr.rs new file mode 100644 index 00000000..b6a99476 --- /dev/null +++ b/serf-compio/src/resolver/socket_addr.rs @@ -0,0 +1,21 @@ +//! Identity pass-through resolver — input is already a [`SocketAddr`]. + +use crate::resolver::Resolver; +use std::{io, net::SocketAddr}; + +/// Identity pass-through resolver. Declares +/// [`Resolver::Address`](crate::resolver::Resolver::Address)`= SocketAddr` +/// and returns the input verbatim. +/// +/// Use when seed addresses are already concrete socket addresses — no DNS +/// lookup or hostname parsing is required. +pub struct SocketAddrResolver; + +impl Resolver for SocketAddrResolver { + type Address = SocketAddr; + type Error = io::Error; + + async fn resolve(&self, addr: &Self::Address) -> Result, Self::Error> { + Ok(vec![*addr]) + } +} diff --git a/serf-compio/src/serf/mod.rs b/serf-compio/src/serf/mod.rs new file mode 100644 index 00000000..fc7b3b0b --- /dev/null +++ b/serf-compio/src/serf/mod.rs @@ -0,0 +1,463 @@ +//! The public [`Serf`] handle — a cheaply-clonable façade over the driver task. +//! +//! Construction binds a [`Transport`], builds the [`TransportRuntime`] bundle, +//! and spawns the driver pump on the compio runtime. Every clone shares the same +//! driver task; user calls flow through the command channel and reads happen via +//! the lock-free [`SerfSnapshot`]. +//! +//! This is the minimal handle surface the driver needs to be exercised +//! end-to-end (and to give every internal command a public constructor): it +//! wires one method per [`crate::command::Command`] variant. The richer +//! ergonomics — typed query/response futures, builder-style construction — are a +//! follow-up; the protocol surface here is complete. + +use std::{ + cell::{Cell, RefCell}, + net::SocketAddr, + rc::Rc, + sync::Arc, +}; + +use bytes::Bytes; +use futures_channel::oneshot; +use memberlist_proto::{Instant, Node}; +use serf_proto::{ + LamportTime, + endpoint::{QueryId, QueryParams}, + event::{Event, QueryEvent}, + members::{Member, MemberStatus, SerfState}, + options::Options as SerfOptions, + typed::Tags, +}; +use smol_str::SmolStr; + +#[cfg(encryption)] +use crate::command::{KeyCmd, ListKeysCmd}; +use crate::{ + command::{ + Command, ForceLeaveCmd, JoinCmd, LeaveCmd, QueryCmd, RespondCmd, SetEventJoinIgnoreCmd, + SetTagsCmd, ShutdownCmd, UserEventCmd, + }, + delegate::Delegate, + driver::options::RuntimeOptions, + error::{Result, SerfError}, + events::EventStream, + resolver::{AdvertiseAddrResolver, Resolver}, + snapshot::{SerfSnapshot, SnapshotCell}, + transport::{Transport, TransportRuntime}, +}; + +#[cfg(encryption)] +use memberlist_proto::SecretKey; + +#[cfg(encryption)] +use crate::delegate::KeyringDelegate; + +/// Driver-shared state every [`Serf`] clone points at. +struct Shared { + commands_tx: flume::Sender>, + events_rx: flume::Receiver>, + /// Shares the same `Rc` the driver's observation task increments. Counts + /// events dropped at the bounded user-facing channel when a slow consumer + /// lets it fill. Monotonically increasing. + events_dropped: Rc>, + /// Shares the same `Rc` the driver pump increments. Counts events dropped + /// at the bounded internal observation channel when the delegate dispatch + /// loop falls behind. Monotonically increasing. + observation_dropped: Rc>, + snapshot: SnapshotCell, + shutdown_flag: Rc>, + local_id: I, + advertise: SocketAddr, +} + +/// A cheaply-clonable handle to a running serf node. +/// +/// Construct one with [`Serf::new`]; clone it freely — every clone shares the +/// single driver task. Requires a stream or QUIC transport feature. +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub struct Serf { + shared: Rc>, +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +impl Clone for Serf { + #[inline] + fn clone(&self) -> Self { + Self { + shared: self.shared.clone(), + } + } +} + +/// Synthesize the initial published snapshot — a single-member view of the local +/// node as `Alive` with empty tags and zero clocks. The driver republishes a +/// real snapshot once the local `NodeJoined` sieve fires. +fn initial_snapshot(local_id: &I, advertise: SocketAddr) -> SerfSnapshot +where + I: Clone + PartialEq, +{ + let member = Member::new( + Node::new(local_id.clone(), advertise), + Tags::new(), + MemberStatus::Alive, + ); + SerfSnapshot::new( + vec![Arc::new(member)], + local_id, + SerfState::Alive, + LamportTime::from(0u64), + LamportTime::from(0u64), + LamportTime::from(0u64), + ) +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +impl Serf +where + I: Clone + PartialEq + 'static, +{ + /// Build a node: construct the transport `T`, spawn its driver pump, and + /// return a handle. + /// + /// `gossip_rng` seeds the memberlist gossip schedule and must be drawn from a + /// fork-safe OS entropy source (see [`crate::gossip_rng`]). + /// + /// Under an encryption backend, `keyring` is the delegate the driver applies + /// inbound key-management requests to; a node that does not manage keys can + /// pass `std::rc::Rc::new(VoidKeyringDelegate)`. + #[allow(clippy::too_many_arguments)] + pub async fn new( + options: T::Options, + resolver: &RES, + advertise_resolver: &AR, + delegate: D, + runtime_options: RuntimeOptions, + serf_options: SerfOptions, + gossip_rng: G, + #[cfg(encryption)] keyring: Rc, + ) -> core::result::Result + where + T: Transport, + RES: Resolver
, + AR: AdvertiseAddrResolver, + D: Delegate + 'static, + G: rand::Rng + Send + Unpin + 'static, + T::Error: From, + { + // Reject runtime knobs a zero capacity would deterministically break BEFORE + // binding any socket or spawning the detached driver: a `Bounded(0)` + // observation channel would panic the driver task at startup, and a zero + // `event_queue_cap` would make the event-stream channel a rendezvous the + // non-blocking forward can never deposit into, dropping every event. + runtime_options.validate()?; + + let transport = T::new(options, resolver, advertise_resolver).await?; + let local_id = transport.local_id().clone(); + let advertise = *transport.advertise_address(); + + let (commands_tx, commands_rx) = flume::unbounded::>(); + let (events_tx, events_rx) = + flume::bounded::>(runtime_options.event_queue_cap()); + let events_dropped = Rc::new(Cell::new(0u64)); + let observation_dropped = Rc::new(Cell::new(0u64)); + let shutdown_flag = Rc::new(Cell::new(false)); + let snapshot: SnapshotCell = Rc::new(RefCell::new(Rc::new(initial_snapshot( + &local_id, advertise, + )))); + + // Retain a handle-side clone of each counter before the driver takes + // ownership. Both sides share the same Cell so reads on the Serf handle + // always reflect the driver's live count. + let events_dropped_handle = events_dropped.clone(); + let observation_dropped_handle = observation_dropped.clone(); + + let runtime = TransportRuntime::::new( + delegate, + commands_rx, + events_tx, + events_dropped, + observation_dropped, + snapshot.clone(), + shutdown_flag.clone(), + runtime_options, + serf_options, + #[cfg(encryption)] + keyring, + ); + + // The driver pump owns the transport, the bound sockets, and the endpoint; + // it runs detached until a `Shutdown` command or all handles drop. + compio::runtime::spawn(transport.run(runtime, gossip_rng)).detach(); + + Ok(Self { + shared: Rc::new(Shared { + commands_tx, + events_rx, + events_dropped: events_dropped_handle, + observation_dropped: observation_dropped_handle, + snapshot, + shutdown_flag, + local_id, + advertise, + }), + }) + } + + /// The local node identifier. + #[inline] + pub fn local_id(&self) -> &I { + &self.shared.local_id + } + + /// The bound advertise address this node gossips to peers. + #[inline] + pub fn advertise_address(&self) -> SocketAddr { + self.shared.advertise + } + + /// The latest published membership snapshot (a lock-free `Rc` load). + #[inline] + pub fn snapshot(&self) -> Rc> { + self.shared.snapshot.borrow().clone() + } + + /// Number of members in the latest published snapshot. + #[inline] + pub fn num_members(&self) -> usize { + self.shared.snapshot.borrow().num_members() + } + + /// Cumulative number of [`Event`]s dropped at the bounded user-facing event + /// channel since this node started. + /// + /// Incremented by the driver's observation task each time [`Serf::events`] + /// consumers are not draining fast enough and the channel is full. Each + /// increment represents one silently discarded event. The counter is + /// monotonically increasing. + /// + /// A non-zero value indicates backpressure: drain [`Serf::events`] promptly + /// or raise `RuntimeOptions::event_queue_cap`. + #[inline] + pub fn events_dropped(&self) -> u64 { + self.shared.events_dropped.get() + } + + /// Cumulative number of [`Event`]s dropped at the bounded internal + /// observation channel since this node started. + /// + /// Incremented by the driver pump each time the delegate dispatch loop falls + /// behind and the observation queue overflows. Each increment represents one + /// event that was never delivered to the delegate or the event stream. The + /// counter is monotonically increasing. + /// + /// A non-zero value indicates a slow delegate: raise + /// `RuntimeOptions::observation_channel` capacity. + #[inline] + pub fn observation_dropped(&self) -> u64 { + self.shared.observation_dropped.get() + } + + /// Subscribe to the serf [`Event`] stream. Multiple subscribers round-robin + /// (the channel is MPMC, not broadcast). + #[inline] + pub fn events(&self) -> EventStream { + EventStream::new(self.shared.events_rx.clone()) + } + + /// Send `cmd` to the driver, failing fast if the node has shut down. + fn send(&self, cmd: Command) -> Result<()> { + if self.shared.shutdown_flag.get() { + return Err(SerfError::Shutdown); + } + self + .shared + .commands_tx + .send(cmd) + .map_err(|_| SerfError::CommandSend) + } + + /// Initiate joins to `seeds` (already-resolved addresses). + /// + /// This is **dispatch-only**: it announces the local join intent and starts a + /// push-pull to each seed, returning the count of seeds the driver dispatched + /// a push-pull to. The returned count is NOT a contact count — a seed may be + /// unreachable and its exchange fail afterward. Actual cluster membership is + /// reported through the [`Event`](serf_proto::event::Event) stream and the + /// published [`snapshot`](Self::snapshot) as peers are merged. + pub async fn join(&self, seeds: Vec) -> Result { + let (tx, rx) = oneshot::channel(); + self.send(Command::Join(JoinCmd { seeds, reply: tx }))?; + await_reply(rx).await + } + + /// Gracefully leave the cluster. Resolves once peers have been notified or the + /// configured leave timeout elapses. + pub async fn leave(&self) -> Result<()> { + let (tx, rx) = oneshot::channel(); + self.send(Command::Leave(LeaveCmd { reply: tx }))?; + await_reply(rx).await + } + + /// Force-remove `id` from the membership. With `prune`, the node is removed + /// immediately rather than after the tombstone timeout. + pub async fn force_leave(&self, id: I, prune: bool) -> Result<()> { + let (tx, rx) = oneshot::channel(); + self.send(Command::ForceLeave(ForceLeaveCmd { + id, + prune, + now: Instant::now(), + reply: tx, + }))?; + await_reply(rx).await + } + + /// Broadcast a user-defined event cluster-wide. + pub async fn user_event( + &self, + name: impl Into, + payload: Bytes, + coalesce: bool, + ) -> Result<()> { + let (tx, rx) = oneshot::channel(); + self.send(Command::UserEvent(UserEventCmd::new( + name.into(), + payload, + coalesce, + tx, + )))?; + await_reply(rx).await + } + + /// Issue a cluster-wide query; returns the [`QueryId`] identifying it. + pub async fn query( + &self, + name: impl Into, + payload: Bytes, + params: QueryParams, + ) -> Result { + let (tx, rx) = oneshot::channel(); + self.send(Command::Query(QueryCmd::new( + name.into(), + payload, + params, + Instant::now(), + tx, + )))?; + await_reply(rx).await + } + + /// Respond to an inbound query received via [`Event::Query`]. + pub async fn respond(&self, token: QueryEvent, payload: Bytes) -> Result<()> { + let (tx, rx) = oneshot::channel(); + self.send(Command::Respond(RespondCmd::new( + token, + payload, + Instant::now(), + tx, + )))?; + await_reply(rx).await + } + + /// Replace the local node's advertised tags. + pub async fn set_tags(&self, tags: Tags) -> Result<()> { + let (tx, rx) = oneshot::channel(); + self.send(Command::SetTags(SetTagsCmd { tags, reply: tx }))?; + await_reply(rx).await + } + + /// Enable or disable suppression of member-join events in the observation + /// stream. + pub async fn set_event_join_ignore(&self, ignore: bool) -> Result<()> { + let (tx, rx) = oneshot::channel(); + self.send(Command::SetEventJoinIgnore(SetEventJoinIgnoreCmd { + ignore, + reply: tx, + }))?; + await_reply(rx).await + } + + /// Issue a cluster-wide install-key query; returns the issued [`QueryId`]. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub async fn install_key(&self, key: SecretKey) -> Result { + let (tx, rx) = oneshot::channel(); + self.send(Command::InstallKey(KeyCmd { + key, + now: Instant::now(), + reply: tx, + }))?; + await_reply(rx).await + } + + /// Issue a cluster-wide use-key query to promote `key` to primary. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub async fn use_key(&self, key: SecretKey) -> Result { + let (tx, rx) = oneshot::channel(); + self.send(Command::UseKey(KeyCmd { + key, + now: Instant::now(), + reply: tx, + }))?; + await_reply(rx).await + } + + /// Issue a cluster-wide remove-key query. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub async fn remove_key(&self, key: SecretKey) -> Result { + let (tx, rx) = oneshot::channel(); + self.send(Command::RemoveKey(KeyCmd { + key, + now: Instant::now(), + reply: tx, + }))?; + await_reply(rx).await + } + + /// Issue a cluster-wide list-keys query. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub async fn list_keys(&self) -> Result { + let (tx, rx) = oneshot::channel(); + self.send(Command::ListKeys(ListKeysCmd { + now: Instant::now(), + reply: tx, + }))?; + await_reply(rx).await + } + + /// Gracefully shut the driver down, releasing the bound ports before this + /// resolves so an immediate rebind on the same address succeeds. + pub async fn shutdown(&self) -> Result<()> { + let (tx, rx) = oneshot::channel(); + self.send(Command::Shutdown(ShutdownCmd { reply: tx }))?; + await_reply(rx).await + } +} + +/// Await a driver reply, mapping a dropped reply channel to +/// [`SerfError::ReplyClosed`]. +async fn await_reply(rx: oneshot::Receiver>) -> Result { + match rx.await { + Ok(res) => res, + Err(_) => Err(SerfError::ReplyClosed), + } +} + +#[cfg(all(test, feature = "tcp"))] +mod tests; diff --git a/serf-compio/src/serf/tests.rs b/serf-compio/src/serf/tests.rs new file mode 100644 index 00000000..19591368 --- /dev/null +++ b/serf-compio/src/serf/tests.rs @@ -0,0 +1,481 @@ +//! End-to-end smoke test: two TCP serf nodes on the loopback interface, one +//! joining the other, asserting the membership event propagates through the full +//! pump (Join command → push-pull dial → coordinator merge → serf `Member` event +//! → `EventStream`). + +use core::time::Duration; +use std::net::SocketAddr; + +use bytes::Bytes; +use futures_util::StreamExt; +use memberlist_proto::MaybeResolved; +use serf_proto::{ + event::{Event, MemberEventKind}, + options::Options as SerfOptions, +}; +use smol_str::SmolStr; + +use crate::{ + Channel, FirstAddrResolver, RuntimeOptions, Serf, SerfError, SocketAddrResolver, TcpTransport, + TcpTransportOptions, VoidDelegate, gossip_rng, +}; + +#[cfg(encryption)] +use crate::{EncryptionOptions, Keyring, SecretKey, VoidKeyringDelegate}; + +/// Build and spawn a TCP serf node bound to an ephemeral loopback port. +async fn spawn_node(id: &str) -> Serf { + try_spawn_node_at(id, "127.0.0.1:0".parse().expect("loopback addr")) + .await + .expect("spawn serf node") +} + +/// Build a TCP serf node bound to a specific advertise address, returning the +/// construction result so the same-address rebind regression can assert a freed +/// port accepts an immediate rebind. +async fn try_spawn_node_at(id: &str, bind: SocketAddr) -> Result, SerfError> { + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(bind)); + Serf::new::, SocketAddrResolver, FirstAddrResolver, _, _>( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + gossip_rng().expect("seed gossip rng"), + #[cfg(encryption)] + std::rc::Rc::new(VoidKeyringDelegate), + ) + .await +} + +/// `shutdown().await` must release the bound TCP listener and UDP gossip socket +/// before it resolves: a second node binding the SAME advertise address the +/// instant the first shuts down must construct successfully, not fail with +/// `AddrInUse`. A plain drop of the compio listener does not guarantee its fd is +/// closed synchronously, so the driver awaits an explicit `close()` on both +/// sockets before acking the shutdown caller. +#[compio::test] +async fn tcp_shutdown_releases_bound_address_for_rebind() { + let first = spawn_node("rebind-first").await; + let addr = first.advertise_address(); + first.shutdown().await.expect("first node shuts down"); + + let second = try_spawn_node_at("rebind-second", addr) + .await + .expect("rebinding the freed address must succeed, not AddrInUse"); + assert_eq!( + second.advertise_address(), + addr, + "the second node rebinds the exact freed address" + ); + second.shutdown().await.expect("second node shuts down"); +} + +/// All `Serf` handles dropping under a continuous gossip flood must still shut the +/// driver down. Under the flood the higher-priority recv arm starves the main +/// select's command arm, so the command-channel disconnect is observable ONLY by +/// the iter-top command drain; a dropped handle must therefore free the bound TCP +/// listener and UDP gossip socket for an immediate same-address rebind rather than +/// spinning forever and leaking them. +#[compio::test] +async fn tcp_command_disconnect_under_flood_releases_bound_ports() { + let node = spawn_node("flood-drop").await; + let addr = node.advertise_address(); + + // Flood the driver's gossip UDP socket so the biased select's recv arm stays + // ready — the load under which the command-channel disconnect must still tear + // the driver down. A detached task keyed off a stop flag so it ends with the + // test. + let stop = std::rc::Rc::new(std::cell::Cell::new(false)); + let flood_stop = stop.clone(); + compio::runtime::spawn(async move { + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let peer = compio::net::UdpSocket::bind(bind) + .await + .expect("bind flood peer"); + while !flood_stop.get() { + // Ignoring Err: best-effort flood; a transient send error is non-fatal. + let _ = peer.send_to(vec![1u8, 0, 0, 0], addr).await.0; + // Yield via a short sleep so the flood cannot monopolize the single-threaded + // runtime — a loopback send can complete inline, which would starve the + // driver and the rebind poll. The gossip socket's kernel queue keeps the + // recv arm ready across the gap. + compio::time::sleep(Duration::from_millis(1)).await; + } + // Ignoring Err: test cleanup of the flood socket. + let _ = peer.close().await; + }) + .detach(); + + // Drop every handle: the command channel disconnects. Under the flood the main + // select's command arm is starved, so the iter-top drain's Disconnect branch is + // the only path that can observe it and tear down. + drop(node); + + // The driver must terminate and release BOTH bound ports; poll for the rebind + // under a generous timeout so a regression (driver never exits) fails as a + // timeout, not a hang. + let rebound = compio::time::timeout(Duration::from_secs(20), async { + loop { + if let Ok(listener) = compio::net::TcpListener::bind(addr).await { + if let Ok(gossip) = compio::net::UdpSocket::bind(addr).await { + break (listener, gossip); + } + // Teardown closes the listener before the UDP socket; release the + // just-bound listener and retry until the UDP port frees too. + // Ignoring Err: discarding the probe listener. + let _ = listener.close().await; + } + compio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await; + + stop.set(true); + let (listener, gossip) = rebound.expect( + "the driver must release its bound ports after a command-channel disconnect under flood", + ); + // Ignoring Err: test cleanup of the rebind probe sockets. + let _ = listener.close().await; + let _ = gossip.close().await; +} + +/// Build VALID TCP transport options paired with a deliberately invalid +/// `runtime`, and assert `Serf::new` rejects it with [`SerfError::InvalidOption`] +/// — before binding a socket or spawning the detached driver — rather than +/// returning `Ok` and later panicking the driver task on a zero-capacity channel. +async fn assert_tcp_new_rejects(runtime: RuntimeOptions) { + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new("bad-opt-node")) + .with_advertise_addr(MaybeResolved::Resolved(bind)); + let res = + Serf::new::, SocketAddrResolver, FirstAddrResolver, _, _>( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + runtime, + SerfOptions::new(), + gossip_rng().expect("seed gossip rng"), + #[cfg(encryption)] + std::rc::Rc::new(VoidKeyringDelegate), + ) + .await; + match res { + Err(SerfError::InvalidOption(_)) => {} + Err(other) => panic!("expected InvalidOption, got {other:?}"), + Ok(_) => panic!("a zero-capacity channel option must be rejected at construction"), + } +} + +/// A `Bounded(0)` observation channel (direct builder) is rejected by the stream +/// driver's `Serf::new` instead of panicking the detached driver task. +#[compio::test] +async fn tcp_new_rejects_zero_observation_channel() { + assert_tcp_new_rejects(RuntimeOptions::new().with_observation_channel(Channel::Bounded(0))).await; +} + +/// A zero `event_queue_cap` (direct builder) is rejected at construction. +#[compio::test] +async fn tcp_new_rejects_zero_event_queue_cap() { + assert_tcp_new_rejects(RuntimeOptions::new().with_event_queue_cap(0)).await; +} + +/// A zero `cmd_fairness_budget` (direct builder) starves the command drain under +/// an inbound flood, so `Serf::new` rejects it at construction rather than +/// spawning a driver whose commands could never make progress. +#[compio::test] +async fn tcp_new_rejects_zero_cmd_fairness_budget() { + assert_tcp_new_rejects(RuntimeOptions::new().with_cmd_fairness_budget(0)).await; +} + +/// A `Bounded(0)` observation channel sourced from a serde config is rejected. +#[cfg(feature = "serde")] +#[compio::test] +async fn tcp_new_rejects_zero_observation_channel_from_serde() { + let runtime: RuntimeOptions = + serde_json::from_str(r#"{"observation_channel":{"bounded":0}}"#).expect("deserialize"); + assert_tcp_new_rejects(runtime).await; +} + +/// A `bounded:0` observation channel parsed from a clap flag is rejected. +#[cfg(feature = "clap")] +#[compio::test] +async fn tcp_new_rejects_zero_observation_channel_from_clap() { + use clap::Parser; + + #[derive(Parser)] + struct Cli { + #[command(flatten)] + runtime: RuntimeOptions, + } + + let cli = Cli::try_parse_from(["app", "--runtime-observation-channel", "bounded:0"]) + .expect("clap parses bounded:0"); + assert_tcp_new_rejects(cli.runtime).await; +} + +/// Two nodes on loopback: A joins B; A must observe B joining the cluster +/// through its event stream, then both shut down cleanly. +#[compio::test] +async fn two_node_tcp_join_observes_membership() { + let b = spawn_node("node-b").await; + let a = spawn_node("node-a").await; + let b_addr = b.advertise_address(); + let b_id = SmolStr::new("node-b"); + + // Subscribe BEFORE the join so a `Member` event cannot race ahead of the + // subscription (the channel buffers either way, but this is the clean order). + let mut a_events = a.events(); + + // Node A dials node B as its seed. + let dispatched = a.join(vec![b_addr]).await.expect("join dispatched"); + assert_eq!(dispatched, 1, "exactly one seed was dispatched"); + + // Node A should observe node B joining via a `Member(Join)` event. + let observed = compio::time::timeout(Duration::from_secs(20), async { + loop { + match a_events.next().await { + Some(Event::Member(me)) if me.kind() == MemberEventKind::Join => { + if me.members().iter().any(|m| m.node().id_ref() == &b_id) { + break true; + } + } + Some(_) => {} + None => break false, + } + } + }) + .await; + + assert!( + matches!(observed, Ok(true)), + "node A should observe node B joining the cluster within the timeout" + ); + + // Exercise the graceful shutdown command path on both nodes. + a.shutdown().await.expect("node A shuts down"); + b.shutdown().await.expect("node B shuts down"); +} + +/// A deterministic test secret key, selecting whichever AEAD cipher this build +/// compiled so the encrypted tests work under either backend. +#[cfg(encryption)] +fn test_secret_key(fill: u8) -> SecretKey { + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes256([fill; 32]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = SecretKey::ChaCha20Poly1305([fill; 32]); + key +} + +/// Build and spawn a TCP serf node on an ephemeral loopback port with `encryption` +/// installed as its gossip-and-reliable keyring policy. +#[cfg(encryption)] +async fn spawn_encrypted_node(id: &str, encryption: EncryptionOptions) -> Serf { + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(bind)) + .with_encryption(encryption); + Serf::new::, SocketAddrResolver, FirstAddrResolver, _, _>( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + gossip_rng().expect("seed gossip rng"), + std::rc::Rc::new(VoidKeyringDelegate), + ) + .await + .expect("spawn serf node") +} + +/// Two nodes sharing one keyring: A joins B over an AEAD-sealed plain-TCP +/// reliable push-pull (and encrypted gossip), and A must still observe B joining +/// through its event stream. Proves the keyring reaches the coordinator and that +/// `encrypt_gossip`/`decrypt_gossip` round-trip end-to-end rather than running +/// as identity transforms when no keyring is wired. +#[cfg(encryption)] +#[compio::test] +async fn two_node_tcp_join_observes_membership_encrypted() { + let enc = EncryptionOptions::new().with_keyring(Keyring::new(test_secret_key(0x42))); + let b = spawn_encrypted_node("node-b", enc.clone()).await; + let a = spawn_encrypted_node("node-a", enc).await; + let b_addr = b.advertise_address(); + let b_id = SmolStr::new("node-b"); + + let mut a_events = a.events(); + + let dispatched = a.join(vec![b_addr]).await.expect("join dispatched"); + assert_eq!(dispatched, 1, "exactly one seed was dispatched"); + + let observed = compio::time::timeout(Duration::from_secs(20), async { + loop { + match a_events.next().await { + Some(Event::Member(me)) if me.kind() == MemberEventKind::Join => { + if me.members().iter().any(|m| m.node().id_ref() == &b_id) { + break true; + } + } + Some(_) => {} + None => break false, + } + } + }) + .await; + + assert!( + matches!(observed, Ok(true)), + "node A should observe node B joining the encrypted cluster within the timeout" + ); + + a.shutdown().await.expect("node A shuts down"); + b.shutdown().await.expect("node B shuts down"); +} + +/// A node holding one keyring and a node holding a DIFFERENT keyring must NOT +/// exchange membership: the plain-TCP reliable push-pull units and the gossip +/// datagrams are both AEAD-sealed under disjoint keys, so neither side can +/// authenticate the other and the join never merges. Proves the encryption is +/// real enforcement, not an identity pass-through. +#[cfg(encryption)] +#[compio::test] +async fn mismatched_keyring_nodes_do_not_exchange_membership() { + let b = spawn_encrypted_node( + "node-b", + EncryptionOptions::new().with_keyring(Keyring::new(test_secret_key(0x42))), + ) + .await; + let a = spawn_encrypted_node( + "node-a", + EncryptionOptions::new().with_keyring(Keyring::new(test_secret_key(0x43))), + ) + .await; + let b_addr = b.advertise_address(); + let b_id = SmolStr::new("node-b"); + + let mut a_events = a.events(); + + let dispatched = a.join(vec![b_addr]).await.expect("join dispatched"); + assert_eq!(dispatched, 1, "exactly one seed was dispatched"); + + // Absence probe: A must never surface a Join carrying node-b. A short window + // covers several gossip / probe / push-pull rounds on loopback — the positive + // test forms its cluster within ~1-2s, so a clean 3s window is decisive. + let observed = compio::time::timeout(Duration::from_secs(3), async { + loop { + match a_events.next().await { + Some(Event::Member(me)) if me.kind() == MemberEventKind::Join => { + if me.members().iter().any(|m| m.node().id_ref() == &b_id) { + break true; + } + } + Some(_) => {} + None => break false, + } + } + }) + .await; + + assert!( + !matches!(observed, Ok(true)), + "node A must NOT observe node B across a mismatched keyring" + ); + + a.shutdown().await.expect("node A shuts down"); + b.shutdown().await.expect("node B shuts down"); +} + +/// Build a TCP serf node with a custom `RuntimeOptions`. +async fn spawn_node_with_runtime(id: &str, runtime_options: RuntimeOptions) -> Serf { + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(bind)); + Serf::new::, SocketAddrResolver, FirstAddrResolver, _, _>( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + runtime_options, + SerfOptions::new(), + gossip_rng().expect("seed gossip rng"), + #[cfg(encryption)] + std::rc::Rc::new(VoidKeyringDelegate), + ) + .await + .expect("spawn serf node") +} + +/// With a cap-1 event queue, the `events_dropped` counter on the `Serf` +/// handle becomes non-zero once more than one event arrives while no consumer +/// is draining the channel. Both the stream (TCP) and QUIC transports feed +/// the same shared counter via the same `Rc>` that `Serf::new` +/// retains on the handle, so a TCP test covers both code paths. +/// +/// The test drains A's event stream only until the initial Member(Join) event +/// confirms the push-pull completed, then stops polling. User events gossiped +/// from B subsequently fill the cap=1 slot and overflow it, incrementing +/// `events_dropped`. +#[compio::test] +async fn tcp_events_dropped_counter_observable_under_backpressure() { + // Node B provides the seed. Node A uses event_queue_cap=1 so the bounded + // channel fills after a single unread event and every subsequent delivery + // is dropped and counted. + let b = spawn_node("drop-b").await; + let b_addr = b.advertise_address(); + let b_id = SmolStr::new("drop-b"); + let a = spawn_node_with_runtime("drop-a", RuntimeOptions::new().with_event_queue_cap(1)).await; + + // Subscribe before the join so we can drain until completion is confirmed. + let mut events = a.events(); + + a.join(vec![b_addr]).await.expect("join dispatched"); + + // Drain A's event stream only until Member(Join, [B]) confirms the + // push-pull completed. After breaking, `events` is alive but never polled + // again: subsequent events fill the cap=1 slot and overflow it. + compio::time::timeout(Duration::from_secs(20), async { + loop { + match events.next().await { + Some(Event::Member(me)) if me.kind() == MemberEventKind::Join => { + if me.members().iter().any(|m| m.node().id_ref() == &b_id) { + break; + } + } + Some(_) => {} + None => panic!("event stream closed before join was observed"), + } + } + }) + .await + .expect("A must observe B joining within timeout"); + + // Flood A with user events gossiped from B. With cap=1 and nobody draining, + // the slot fills after one delivery and each subsequent try_send returns + // Full, incrementing events_dropped on the Serf handle. + for i in 0u32..10 { + b.user_event(format!("drop-{i}"), Bytes::new(), false) + .await + .expect("user event dispatched"); + } + + // Give gossip time to propagate across the loopback. + compio::time::sleep(Duration::from_secs(3)).await; + + let dropped = a.events_dropped(); + + a.shutdown().await.expect("drop-a shuts down"); + b.shutdown().await.expect("drop-b shuts down"); + + assert!( + dropped > 0, + "events_dropped must be > 0 when event_queue_cap=1 and events are not drained (got {dropped})" + ); +} diff --git a/serf-compio/src/snapshot.rs b/serf-compio/src/snapshot.rs new file mode 100644 index 00000000..700cf9cc --- /dev/null +++ b/serf-compio/src/snapshot.rs @@ -0,0 +1,14 @@ +//! Snapshot of serf membership state, published into a single-owner cell. + +use core::net::SocketAddr; +use std::{cell::RefCell, rc::Rc}; + +pub use serf_driver::SerfSnapshot; + +/// The published-snapshot cell shared by the handle and the driver. Single-owner +/// per thread (thread-per-core): the driver swaps in a fresh +/// `Rc` after each state-affecting tick, and the handle (and the +/// driver) read the current one. The `RefCell` is a borrow check, not a lock — +/// handle and driver share one thread and never borrow it at overlapping times. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) type SnapshotCell = Rc>>>; diff --git a/serf-compio/src/tcp/mod.rs b/serf-compio/src/tcp/mod.rs new file mode 100644 index 00000000..1e339774 --- /dev/null +++ b/serf-compio/src/tcp/mod.rs @@ -0,0 +1,382 @@ +//! TCP-backed serf driver — the first end-to-end usable surface. +//! +//! [`TcpTransport`] owns the bound UDP gossip socket and TCP reliable listener. +//! The machine-layer `serf_proto::StreamEndpoint` is +//! built inside [`TcpTransport::run`] from the stored stream knobs and the +//! `serf_proto::options::Options` carried by the +//! [`TransportRuntime`](crate::TransportRuntime). + +#![cfg(feature = "tcp")] + +use core::num::NonZeroU8; +use std::{io::ErrorKind, net::SocketAddr}; + +use compio::net::{TcpListener, UdpSocket}; +use hostaddr::HostAddr; +use memberlist_proto::{ + CheapClone, Data, Endpoint, EndpointOptions, Id, MaybeResolved, RawRecords, + streams::{LabelOptions, StreamEndpoint as Coordinator}, +}; +use rand::rngs::StdRng; +use smol_str::SmolStr; + +#[cfg(encryption)] +use memberlist_proto::EncryptionOptions; + +use crate::{ + SerfError, + delegate::Delegate, + driver::options::StreamTransportOptions, + resolver::{AdvertiseAddrResolver, Resolver}, + transport::{Transport, TransportRuntime}, +}; + +/// Per-backend TCP-specific transport options. +/// +/// Embedded into the transport constructor. Bundles the local node identifier, +/// the (possibly-unresolved) advertise address, and the stream-transport tuning +/// knobs. The cluster label and inbound-label-check policy are supplied via the +/// serf `Options` block (not here), feeding both planes from a single validated +/// source. +pub struct TcpTransportOptions> { + local_id: Option, + advertise_addr: Option>, + stream: StreamTransportOptions, + /// Gossip-and-reliable encryption policy. The default (no keyring) leaves + /// both planes plaintext; attaching a keyring via + /// [`with_encryption`](Self::with_encryption) makes the coordinator's + /// `encrypt_gossip`/`decrypt_gossip` (and the plain-TCP reliable record + /// layer) AEAD-protect every datagram and stream unit. + #[cfg(encryption)] + encryption: EncryptionOptions, +} + +impl TcpTransportOptions { + /// Construct with defaults. Caller MUST chain [`with_local_id`](Self::with_local_id) + /// and [`with_advertise_addr`](Self::with_advertise_addr) before passing to + /// `TcpTransport::new`. + #[inline] + pub fn new() -> Self { + Self { + local_id: None, + advertise_addr: None, + stream: StreamTransportOptions::new(), + #[cfg(encryption)] + encryption: EncryptionOptions::new(), + } + } + + /// Builder: local node identifier. + #[must_use] + #[inline] + pub fn with_local_id(mut self, id: I) -> Self { + self.local_id = Some(id); + self + } + + /// Builder: advertise address (resolved or unresolved). + #[must_use] + #[inline] + pub fn with_advertise_addr(mut self, addr: MaybeResolved) -> Self { + self.advertise_addr = Some(addr); + self + } + + /// Builder: stream-transport tuning knobs. + #[must_use] + #[inline] + pub fn with_stream(mut self, opts: StreamTransportOptions) -> Self { + self.stream = opts; + self + } + + /// Builder: gossip-and-reliable encryption policy. + /// + /// The default (no keyring) keeps both planes plaintext, so an unencrypted + /// node still builds and interoperates. Attach a keyring + /// (`EncryptionOptions::new().with_keyring(Keyring::new(primary_key))`) to + /// AEAD-protect the gossip datagrams and the plain-TCP reliable record layer + /// — every node sharing the cluster MUST carry the same keyring to interop. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + #[must_use] + #[inline] + pub fn with_encryption(mut self, encryption: EncryptionOptions) -> Self { + self.encryption = encryption; + self + } + + /// Local node identifier, if set. + #[inline] + pub const fn local_id(&self) -> Option<&I> { + self.local_id.as_ref() + } + + /// Advertise address, if set. + #[inline] + pub const fn advertise_addr(&self) -> Option<&MaybeResolved> { + self.advertise_addr.as_ref() + } + + /// Stream-transport tuning knobs. + #[inline] + pub const fn stream(&self) -> &StreamTransportOptions { + &self.stream + } + + /// Gossip-and-reliable encryption policy. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + #[inline] + pub const fn encryption(&self) -> &EncryptionOptions { + &self.encryption + } +} + +impl Default for TcpTransportOptions { + #[inline] + fn default() -> Self { + Self::new() + } +} + +/// TCP-backed serf transport. +/// +/// Owns the bound `UdpSocket` (gossip unreliable plane) and `TcpListener` +/// (reliable coordinator). The machine-layer +/// `serf_proto::StreamEndpoint` is built inside +/// [`Transport::run`] from the cluster options sourced from +/// [`TransportRuntime::serf_options`](crate::TransportRuntime). +pub struct TcpTransport> { + local_id: I, + local_address: MaybeResolved, + advertise_socket: SocketAddr, + gossip_socket: UdpSocket, + tcp_listener: TcpListener, + stream_options: StreamTransportOptions, + /// Independent OS-seeded seed for the serf core's RNG, drawn once per node in + /// [`Transport::new`] and consumed when [`Transport::run`] builds the + /// endpoint via `new_with_rng`. Distinct from the coordinator's gossip RNG so + /// serf's query IDs and relay choices are not correlated across nodes. + serf_rng: StdRng, + /// Gossip-and-reliable encryption policy applied to the coordinator built in + /// [`Transport::run`]. Absent keyring ⇒ plaintext (the default). + #[cfg(encryption)] + encryption: EncryptionOptions, +} + +impl Transport for TcpTransport +where + I: Id + CheapClone + core::fmt::Debug + core::fmt::Display + Send + Sync + 'static, + A: Data + Clone + Send + 'static, +{ + type Error = SerfError; + type Id = I; + type Address = A; + type Options = TcpTransportOptions; + + async fn new( + options: Self::Options, + resolver: &RES, + advertise_resolver: &AR, + ) -> Result + where + RES: Resolver
, + AR: AdvertiseAddrResolver, + { + // Validate stream knobs that would deterministically break the backend + // (e.g. a zero `bridge_recv_buf_len` makes every bridge read return a + // false EOF) BEFORE binding any socket. + options.stream.validate()?; + + let local_id = options.local_id.ok_or_else(|| { + SerfError::Io(std::io::Error::new( + ErrorKind::InvalidInput, + "local_id required", + )) + })?; + let advertise_input = options.advertise_addr.ok_or_else(|| { + SerfError::Io(std::io::Error::new( + ErrorKind::InvalidInput, + "advertise_addr required", + )) + })?; + + let advertise_socket = match &advertise_input { + MaybeResolved::Resolved(s) => *s, + MaybeResolved::Unresolved(a) => { + let candidates = resolver + .resolve(a) + .await + .map_err(|e| SerfError::Resolve(std::io::Error::other(e.to_string())))?; + advertise_resolver.pick(candidates).map_err(|e| { + SerfError::Resolve(std::io::Error::new( + ErrorKind::AddrNotAvailable, + e.to_string(), + )) + })? + } + }; + + // Bind the TCP listener first to claim a port, then bind the UDP gossip + // socket to the same port. For an ephemeral (`:0`) advertise address, + // retry the pair on a fresh port if the UDP bind fails transiently: + // AddrInUse from the TCP/UDP port-space race, or PermissionDenied on + // Windows when the TCP-assigned port falls in a UDP-excluded range. + const EPHEMERAL_BIND_RETRIES: usize = 16; + let ephemeral = advertise_socket.port() == 0; + let (tcp_listener, advertise_socket, gossip_socket) = { + let mut attempt = 0usize; + loop { + let tcp_listener = TcpListener::bind(advertise_socket) + .await + .map_err(SerfError::Io)?; + // Every post-listener-bind fallible step — the `local_addr` readback + // (which resolves an ephemeral `:0` to a concrete port) and the paired + // UDP bind — runs in this one fallible scope, so the single close site + // below releases the just-bound listener on ANY non-success outcome. No + // post-bind step can `?`-return past the close and leak the bound port. + let paired: Result<(SocketAddr, UdpSocket), std::io::Error> = async { + let bound = tcp_listener.local_addr()?; + let gossip_socket = UdpSocket::bind(bound).await?; + Ok((bound, gossip_socket)) + } + .await; + match paired { + Ok((bound, gossip_socket)) => break (tcp_listener, bound, gossip_socket), + Err(e) => { + // Release the just-bound listener (awaited — a plain drop is not a + // synchronous fd release on compio/Windows-IOCP) before retrying or + // aborting, so its port cannot leak into a same-address rebind. + // Ignoring Err: closing an abandoned construction's listener. + let _ = tcp_listener.close().await; + // A transient ephemeral collision (the TCP/UDP port-space race, or a + // Windows UDP-excluded port) retries a fresh pair; any other error + // (readback failure, fatal UDP bind) aborts construction. + if ephemeral + && attempt < EPHEMERAL_BIND_RETRIES + && matches!(e.kind(), ErrorKind::AddrInUse | ErrorKind::PermissionDenied) + { + attempt += 1; + continue; + } + return Err(SerfError::Io(e)); + } + } + } + }; + + // Both sockets are now bound. Group every remaining fallible step so that on + // ANY error BOTH bound sockets are closed (awaited — a plain drop is not a + // synchronous fd release on compio/Windows-IOCP) before returning, so a + // failed construction never leaks a bound port to race an immediate + // same-address rebind into `AddrInUse`. + // + // The readback above resolves an ephemeral `:0` to a concrete port but keeps + // an unspecified IP (a wildcard `0.0.0.0:0` bind yields `0.0.0.0:`): + // `post_bind_setup` rejects an advertise address peers could not route serf + // traffic back to, then draws the OS-seeded serf-core RNG before the driver + // task is spawned so an entropy failure surfaces as `SerfError::Entropy` + // here. Either failure closes BOTH bound sockets before returning. + let serf_rng = match crate::transport::post_bind_setup(&advertise_socket) { + Ok(rng) => rng, + Err(e) => { + crate::transport::close_stream_sockets(tcp_listener, gossip_socket).await; + return Err(e); + } + }; + + Ok(Self { + local_id, + local_address: advertise_input, + advertise_socket, + gossip_socket, + tcp_listener, + stream_options: options.stream, + serf_rng, + #[cfg(encryption)] + encryption: options.encryption, + }) + } + + #[inline] + fn local_id(&self) -> &Self::Id { + &self.local_id + } + + #[inline] + fn local_address(&self) -> &MaybeResolved { + &self.local_address + } + + #[inline] + fn advertise_address(&self) -> &SocketAddr { + &self.advertise_socket + } + + async fn run(self, runtime: TransportRuntime, gossip_rng: G) + where + D: Delegate, + G: rand::Rng + Send + Unpin + 'static, + { + // `Serf::new` is generic over `T` and cannot build the record-layer-specific + // endpoint; build it here from `self`'s stored config. Serf ranks its user + // broadcasts on three tiers (intent / event / query → ranks 0 / 1 / 2), so + // the inner memberlist endpoint needs at least three broadcast tiers. + let inner_opts = EndpointOptions::new(self.local_id, self.advertise_socket) + .with_user_broadcast_tiers(NonZeroU8::new(3).expect("3 is nonzero")); + let inner = Endpoint::new(inner_opts, gossip_rng); + // Plain TCP has no SNI (`|_| None`) and a membership address that IS the + // transport socket (`|addr| *addr`). No cluster label at this stage. + #[allow(unused_mut)] + let mut coord = Coordinator::<_, _, RawRecords, G>::new( + inner, + LabelOptions::new_in(None::>, ()), + Box::new(|_: &SocketAddr| None), + Box::new(|addr: &SocketAddr| *addr), + ); + // Install the gossip-encryption keyring so the coordinator's + // `encrypt_gossip`/`decrypt_gossip` (forwarded from the serf endpoint pump) + // and the plain-TCP reliable record layer become real. A no-keyring policy + // is the identity transform, so an unencrypted node is unaffected. + #[cfg(encryption)] + coord.set_encryption_options(self.encryption); + // Serf's core RNG is seeded from its own OS-drawn entropy (`self.serf_rng`), + // independent of the coordinator's gossip RNG, so two nodes never share the + // query-ID / relay-selection stream. + let endpoint = + serf_proto::StreamEndpoint::::new_with_rng( + coord, + runtime.serf_options, + self.serf_rng, + ); + + crate::driver::stream::stream_driver_loop::( + endpoint, + self.gossip_socket, + self.tcp_listener, + runtime.commands_rx, + runtime.events_tx, + runtime.events_dropped, + runtime.observation_dropped, + runtime.snapshot, + runtime.shutdown_flag, + runtime.driver_options, + self.stream_options, + runtime.delegate, + None, + #[cfg(encryption)] + runtime.keyring, + ) + .await; + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-compio/src/tcp/tests.rs b/serf-compio/src/tcp/tests.rs new file mode 100644 index 00000000..d3547577 --- /dev/null +++ b/serf-compio/src/tcp/tests.rs @@ -0,0 +1,125 @@ +//! Construction-time guards for the TCP transport: advertise-address validation +//! (a wildcard bind must not be gossiped as a contact) and the per-node serf +//! RNG seeding (two fresh nodes must draw independent entropy). + +use core::time::Duration; +use std::net::SocketAddr; + +use memberlist_proto::MaybeResolved; +use rand::Rng; +use smol_str::SmolStr; + +use crate::{ + FirstAddrResolver, SerfError, SocketAddrResolver, StreamTransportOptions, TcpTransport, + TcpTransportOptions, Transport, +}; + +/// Binding the wildcard `0.0.0.0:0` reads an unspecified IP back from the +/// socket; gossiping it would publish an undialable contact, so construction +/// must reject it with `InvalidAdvertiseAddr` rather than join as an unreachable +/// member. +#[compio::test] +async fn new_rejects_wildcard_advertise() { + let wildcard: SocketAddr = "0.0.0.0:0".parse().expect("wildcard addr"); + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new("wild-node")) + .with_advertise_addr(MaybeResolved::Resolved(wildcard)); + let res = + TcpTransport::::new(opts, &SocketAddrResolver, &FirstAddrResolver).await; + match res { + Err(SerfError::InvalidAdvertiseAddr(e)) => { + assert!( + e.addr().ip().is_unspecified(), + "the rejected address carries the unspecified IP read back from the wildcard bind" + ); + } + Err(other) => panic!("expected InvalidAdvertiseAddr, got {other:?}"), + Ok(_) => panic!("a wildcard advertise must be rejected, but construction succeeded"), + } +} + +/// A zero `dial_timeout` makes every outbound dial resolve as an immediate +/// biased-select timeout — and since serf's `join` is dispatch-only it would +/// return `Ok` while no reliable exchange ever completes — so `TcpTransport::new` +/// rejects it with `InvalidOption` at the top of `new`, before binding any socket. +#[compio::test] +async fn new_rejects_zero_dial_timeout() { + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new("zero-dial-node")) + .with_advertise_addr(MaybeResolved::Resolved(bind)) + .with_stream(StreamTransportOptions::new().with_dial_timeout(Duration::ZERO)); + let res = + TcpTransport::::new(opts, &SocketAddrResolver, &FirstAddrResolver).await; + match res { + Err(SerfError::InvalidOption(_)) => {} + Err(other) => panic!("expected InvalidOption, got {other:?}"), + Ok(_) => panic!("a zero dial_timeout must be rejected at construction"), + } +} + +/// Construct a TCP transport bound to an ephemeral loopback port (the per-node +/// construction path that draws the serf-core RNG). +async fn build_node(id: &str) -> TcpTransport { + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(bind)); + TcpTransport::::new(opts, &SocketAddrResolver, &FirstAddrResolver) + .await + .expect("construct transport") +} + +/// A construction failure AFTER the sockets are bound must close them (awaited) +/// before returning `Err`, or the bound port leaks and a same-address rebind +/// races into `AddrInUse` (a plain drop is not a synchronous fd release on +/// compio/Windows-IOCP). A wildcard `0.0.0.0:0` advertise binds a concrete +/// OS-assigned port (free for both the TCP listener and the UDP socket) but is +/// then rejected by `validate_advertise_addr` for its unspecified IP; the exact +/// freed `0.0.0.0:` must immediately re-accept the SAME listener + UDP +/// socket the transport bound, proving neither leaked on the error path. +#[compio::test] +async fn new_failure_closes_bound_sockets_for_rebind() { + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new("rebind-fail")) + .with_advertise_addr(MaybeResolved::Resolved( + "0.0.0.0:0".parse().expect("wildcard addr"), + )); + let res = + TcpTransport::::new(opts, &SocketAddrResolver, &FirstAddrResolver).await; + let freed = match res { + Err(SerfError::InvalidAdvertiseAddr(e)) => e.addr(), + Err(other) => panic!("expected a post-bind InvalidAdvertiseAddr failure, got {other:?}"), + Ok(_) => panic!("a post-bind failure must reject construction, but it succeeded"), + }; + + let listener = compio::net::TcpListener::bind(freed) + .await + .expect("the freed TCP port must rebind, not AddrInUse"); + let gossip = compio::net::UdpSocket::bind(freed) + .await + .expect("the freed UDP port must rebind, not AddrInUse"); + // Ignoring Err: test cleanup of the probe sockets. + let _ = listener.close().await; + let _ = gossip.close().await; +} + +/// Two freshly-constructed nodes must seed their serf-core RNGs from independent +/// OS entropy, not a shared/zero seed: a shared seed makes two nodes with the +/// same op history emit identical `(ltime, id)` for concurrent queries, which +/// the dedup ring drops. Drawing several words from each and comparing the +/// streams asserts the seeds diverge (the field is reachable from this child +/// module; `Transport::run` consumes it into the endpoint via `new_with_rng`). +#[compio::test] +async fn freshly_constructed_nodes_have_independent_serf_rngs() { + let mut a = build_node("rng-a").await; + let mut b = build_node("rng-b").await; + + let sample_a: [u64; 4] = core::array::from_fn(|_| a.serf_rng.next_u64()); + let sample_b: [u64; 4] = core::array::from_fn(|_| b.serf_rng.next_u64()); + + assert_ne!( + sample_a, sample_b, + "two fresh nodes must hold independently OS-seeded serf RNGs, not a shared stream" + ); +} diff --git a/serf-compio/src/tls/mod.rs b/serf-compio/src/tls/mod.rs new file mode 100644 index 00000000..ac428acf --- /dev/null +++ b/serf-compio/src/tls/mod.rs @@ -0,0 +1,469 @@ +//! TLS-backed serf driver — the TLS sibling of the TCP plane. +//! +//! [`TlsTransport`] owns the bound UDP gossip socket and TCP reliable listener, +//! exactly like [`TcpTransport`](crate::TcpTransport); it differs only in the +//! reliable record layer, which wraps every accepted/dialed byte stream in a +//! rustls session. The machine-layer +//! `serf_proto::StreamEndpoint>` is built +//! inside [`TlsTransport::run`] from the stored +//! [`TlsOptions`](memberlist_proto::TlsOptions) (cert/key/verifier bundle) and +//! the per-peer SNI provider; the gossip datagram plane stays plain UDP. +//! +//! TLS secures the *reliable* push-pull plane. The unreliable gossip plane is +//! still AEAD-protected by the optional encryption keyring (carried through to +//! the coordinator the same way the TCP plane carries it), so an encrypted +//! cluster protects both planes. +//! +//! ## Server name +//! +//! TLS verifies the peer's certificate against a server name. The +//! `sni_provider` closure on [`TlsTransportOptions`] is called per dial with the +//! peer's membership address; it must return `Some(name)` matching the peer +//! cert's SAN/CN. Returning `None` aborts the dial before the handshake. The +//! default closure returns `Some("localhost".to_string())` for every peer — +//! matching the bundled smoke test's self-signed localhost-SAN cert. Production +//! operators supply a closure mapping each peer to its actual DNS name or SAN. + +#![cfg(feature = "tls")] + +use core::num::NonZeroU8; +use std::{io::ErrorKind, net::SocketAddr}; + +use compio::net::{TcpListener, UdpSocket}; +use hostaddr::HostAddr; +use memberlist_proto::{ + CheapClone, Data, Endpoint, EndpointOptions, Id, MaybeResolved, TlsRecords, + streams::{LabelOptions, Labeled, StreamEndpoint as Coordinator}, +}; +use rand::rngs::StdRng; +use smol_str::SmolStr; + +/// TLS machine-options bundle (server + client `rustls` config) handed to +/// [`TlsTransport`]. Re-exported from `memberlist-proto` so callers don't need a +/// direct dep on it. +pub use memberlist_proto::TlsOptions; + +#[cfg(encryption)] +use memberlist_proto::EncryptionOptions; + +use crate::{ + SerfError, + delegate::Delegate, + driver::options::StreamTransportOptions, + resolver::{AdvertiseAddrResolver, Resolver}, + transport::{Transport, TransportRuntime}, +}; + +/// Boxed SNI provider closure: maps a peer's `SocketAddr` to the expected TLS +/// server name used for certificate verification. Returns `None` to abort the +/// dial before the handshake. +pub type SniProvider = Box Option + Send + Sync>; + +/// Per-backend TLS-specific transport options. +/// +/// Embedded into the transport constructor. Bundles the local node identifier, +/// the (possibly-unresolved) advertise address, the stream-transport tuning +/// knobs, the per-peer SNI provider closure, the machine-layer [`TlsOptions`] +/// bundle (cert/key/verifier), and the optional gossip-encryption policy. The +/// cluster label and inbound-label-check policy are supplied via the serf +/// `Options` block (not here), feeding both planes from a single validated +/// source. +pub struct TlsTransportOptions> { + local_id: Option, + advertise_addr: Option>, + stream: StreamTransportOptions, + sni_provider: SniProvider, + tls_options: Option, + /// Gossip encryption policy. The default (no keyring) leaves the gossip + /// datagrams plaintext; attaching a keyring via + /// [`with_encryption`](Self::with_encryption) makes the coordinator's + /// `encrypt_gossip`/`decrypt_gossip` AEAD-protect them. The reliable plane + /// rides the TLS session, so the keyring covers only the gossip datagrams. + #[cfg(encryption)] + encryption: EncryptionOptions, +} + +impl TlsTransportOptions { + /// Construct with defaults. Caller MUST chain [`with_local_id`](Self::with_local_id), + /// [`with_advertise_addr`](Self::with_advertise_addr), and + /// [`with_tls_options`](Self::with_tls_options) before passing to + /// `TlsTransport::new`. The default `sni_provider` returns + /// `Some("localhost".to_string())` for every peer — matching the bundled + /// smoke test's self-signed localhost-SAN cert. + #[inline] + pub fn new() -> Self { + Self { + local_id: None, + advertise_addr: None, + stream: StreamTransportOptions::new(), + sni_provider: Box::new(|_addr: &SocketAddr| Some("localhost".to_string())), + tls_options: None, + #[cfg(encryption)] + encryption: EncryptionOptions::new(), + } + } + + /// Builder: local node identifier. + #[must_use] + #[inline] + pub fn with_local_id(mut self, id: I) -> Self { + self.local_id = Some(id); + self + } + + /// Builder: advertise address (resolved or unresolved). + #[must_use] + #[inline] + pub fn with_advertise_addr(mut self, addr: MaybeResolved) -> Self { + self.advertise_addr = Some(addr); + self + } + + /// Builder: stream-transport tuning knobs. + #[must_use] + #[inline] + pub fn with_stream(mut self, opts: StreamTransportOptions) -> Self { + self.stream = opts; + self + } + + /// Builder: SNI provider closure. Default returns + /// `Some("localhost".to_string())` for every peer; for deployments with + /// per-peer SAN certs, supply a closure mapping each dialed `SocketAddr` to + /// its expected SNI string. Returning `None` for a peer causes the outbound + /// TLS dial to fail before the handshake. + #[must_use] + #[inline] + pub fn with_sni_provider(mut self, f: SniProvider) -> Self { + self.sni_provider = f; + self + } + + /// Builder: TLS machine options (cert/key/verifier). Must be set before + /// `TlsTransport::new`. + #[must_use] + #[inline] + pub fn with_tls_options(mut self, opts: TlsOptions) -> Self { + self.tls_options = Some(opts); + self + } + + /// Builder: gossip-encryption policy. + /// + /// The default (no keyring) keeps the gossip datagrams plaintext, so an + /// unencrypted node still builds and interoperates. Attach a keyring + /// (`EncryptionOptions::new().with_keyring(Keyring::new(primary_key))`) to + /// AEAD-protect the gossip datagrams — every node sharing the cluster MUST + /// carry the same keyring to interop. The reliable plane is secured by the + /// TLS session independently of this keyring. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + #[must_use] + #[inline] + pub fn with_encryption(mut self, encryption: EncryptionOptions) -> Self { + self.encryption = encryption; + self + } + + /// Local node identifier, if set. + #[inline] + pub const fn local_id(&self) -> Option<&I> { + self.local_id.as_ref() + } + + /// Advertise address, if set. + #[inline] + pub const fn advertise_addr(&self) -> Option<&MaybeResolved> { + self.advertise_addr.as_ref() + } + + /// Stream-transport tuning knobs. + #[inline] + pub const fn stream(&self) -> &StreamTransportOptions { + &self.stream + } + + /// SNI provider closure. + #[inline] + pub fn sni_provider(&self) -> &(dyn Fn(&SocketAddr) -> Option + Send + Sync) { + self.sni_provider.as_ref() + } + + /// TLS machine options bundle, if set. + #[inline] + pub const fn tls_options(&self) -> Option<&TlsOptions> { + self.tls_options.as_ref() + } + + /// Gossip-encryption policy. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + #[inline] + pub const fn encryption(&self) -> &EncryptionOptions { + &self.encryption + } +} + +impl Default for TlsTransportOptions { + #[inline] + fn default() -> Self { + Self::new() + } +} + +/// TLS-backed serf transport. +/// +/// Owns the bound `UdpSocket` (gossip unreliable plane) and `TcpListener` +/// (reliable coordinator; TLS handshake-on-accept). The machine-layer +/// `serf_proto::StreamEndpoint>` is built +/// inside [`Transport::run`] from the stored config (`tls_options` + +/// `sni_provider`) and the cluster options sourced from +/// [`TransportRuntime::serf_options`](crate::TransportRuntime). +pub struct TlsTransport> { + local_id: I, + local_address: MaybeResolved, + advertise_socket: SocketAddr, + gossip_socket: UdpSocket, + tcp_listener: TcpListener, + stream_options: StreamTransportOptions, + sni_provider: SniProvider, + tls_options: TlsOptions, + /// Independent OS-seeded seed for the serf core's RNG, drawn once per node in + /// [`Transport::new`] and consumed when [`Transport::run`] builds the + /// endpoint via `new_with_rng`. Distinct from the coordinator's gossip RNG so + /// serf's query IDs and relay choices are not correlated across nodes. + serf_rng: StdRng, + /// Gossip-encryption policy applied to the coordinator built in + /// [`Transport::run`]. Absent keyring ⇒ plaintext gossip (the default). + #[cfg(encryption)] + encryption: EncryptionOptions, +} + +impl Transport for TlsTransport +where + I: Id + CheapClone + core::fmt::Debug + core::fmt::Display + Send + Sync + 'static, + A: Data + Clone + Send + 'static, +{ + type Error = SerfError; + type Id = I; + type Address = A; + type Options = TlsTransportOptions; + + async fn new( + options: Self::Options, + resolver: &RES, + advertise_resolver: &AR, + ) -> Result + where + RES: Resolver
, + AR: AdvertiseAddrResolver, + { + // Validate stream knobs that would deterministically break the backend + // (e.g. a zero `bridge_recv_buf_len` makes every bridge read return a + // false EOF) BEFORE binding any socket. + options.stream.validate()?; + + let local_id = options.local_id.ok_or_else(|| { + SerfError::Io(std::io::Error::new( + ErrorKind::InvalidInput, + "local_id required", + )) + })?; + let advertise_input = options.advertise_addr.ok_or_else(|| { + SerfError::Io(std::io::Error::new( + ErrorKind::InvalidInput, + "advertise_addr required", + )) + })?; + let tls_options = options.tls_options.ok_or_else(|| { + SerfError::Io(std::io::Error::new( + ErrorKind::InvalidInput, + "tls_options required", + )) + })?; + + let advertise_socket = match &advertise_input { + MaybeResolved::Resolved(s) => *s, + MaybeResolved::Unresolved(a) => { + let candidates = resolver + .resolve(a) + .await + .map_err(|e| SerfError::Resolve(std::io::Error::other(e.to_string())))?; + advertise_resolver.pick(candidates).map_err(|e| { + SerfError::Resolve(std::io::Error::new( + ErrorKind::AddrNotAvailable, + e.to_string(), + )) + })? + } + }; + + // Bind the TCP listener first to claim a port, then bind the UDP gossip + // socket to the same port. For an ephemeral (`:0`) advertise address, + // retry the pair on a fresh port if the UDP bind fails transiently: + // AddrInUse from the TCP/UDP port-space race, or PermissionDenied on + // Windows when the TCP-assigned port falls in a UDP-excluded range. + const EPHEMERAL_BIND_RETRIES: usize = 16; + let ephemeral = advertise_socket.port() == 0; + let (tcp_listener, advertise_socket, gossip_socket) = { + let mut attempt = 0usize; + loop { + let tcp_listener = TcpListener::bind(advertise_socket) + .await + .map_err(SerfError::Io)?; + // Every post-listener-bind fallible step — the `local_addr` readback + // (which resolves an ephemeral `:0` to a concrete port) and the paired + // UDP bind — runs in this one fallible scope, so the single close site + // below releases the just-bound listener on ANY non-success outcome. No + // post-bind step can `?`-return past the close and leak the bound port. + let paired: Result<(SocketAddr, UdpSocket), std::io::Error> = async { + let bound = tcp_listener.local_addr()?; + let gossip_socket = UdpSocket::bind(bound).await?; + Ok((bound, gossip_socket)) + } + .await; + match paired { + Ok((bound, gossip_socket)) => break (tcp_listener, bound, gossip_socket), + Err(e) => { + // Release the just-bound listener (awaited — a plain drop is not a + // synchronous fd release on compio/Windows-IOCP) before retrying or + // aborting, so its port cannot leak into a same-address rebind. + // Ignoring Err: closing an abandoned construction's listener. + let _ = tcp_listener.close().await; + // A transient ephemeral collision (the TCP/UDP port-space race, or a + // Windows UDP-excluded port) retries a fresh pair; any other error + // (readback failure, fatal UDP bind) aborts construction. + if ephemeral + && attempt < EPHEMERAL_BIND_RETRIES + && matches!(e.kind(), ErrorKind::AddrInUse | ErrorKind::PermissionDenied) + { + attempt += 1; + continue; + } + return Err(SerfError::Io(e)); + } + } + } + }; + + // Both sockets are now bound. Group every remaining fallible step so that on + // ANY error BOTH bound sockets are closed (awaited — a plain drop is not a + // synchronous fd release on compio/Windows-IOCP) before returning, so a + // failed construction never leaks a bound port to race an immediate + // same-address rebind into `AddrInUse`. + // + // The readback above resolves an ephemeral `:0` to a concrete port but keeps + // an unspecified IP (a wildcard `0.0.0.0:0` bind yields `0.0.0.0:`): + // `post_bind_setup` rejects an advertise address peers could not route serf + // traffic back to, then draws the OS-seeded serf-core RNG before the driver + // task is spawned so an entropy failure surfaces as `SerfError::Entropy` + // here. Either failure closes BOTH bound sockets before returning. + let serf_rng = match crate::transport::post_bind_setup(&advertise_socket) { + Ok(rng) => rng, + Err(e) => { + crate::transport::close_stream_sockets(tcp_listener, gossip_socket).await; + return Err(e); + } + }; + + Ok(Self { + local_id, + local_address: advertise_input, + advertise_socket, + gossip_socket, + tcp_listener, + stream_options: options.stream, + sni_provider: options.sni_provider, + tls_options, + serf_rng, + #[cfg(encryption)] + encryption: options.encryption, + }) + } + + #[inline] + fn local_id(&self) -> &Self::Id { + &self.local_id + } + + #[inline] + fn local_address(&self) -> &MaybeResolved { + &self.local_address + } + + #[inline] + fn advertise_address(&self) -> &SocketAddr { + &self.advertise_socket + } + + async fn run(self, runtime: TransportRuntime, gossip_rng: G) + where + D: Delegate, + G: rand::Rng + Send + Unpin + 'static, + { + // `Serf::new` is generic over `T` and cannot build the record-layer-specific + // endpoint; build it here from `self`'s stored config. Serf ranks its user + // broadcasts on three tiers (intent / event / query → ranks 0 / 1 / 2), so + // the inner memberlist endpoint needs at least three broadcast tiers. + let inner_opts = EndpointOptions::new(self.local_id, self.advertise_socket) + .with_user_broadcast_tiers(NonZeroU8::new(3).expect("3 is nonzero")); + let inner = Endpoint::new(inner_opts, gossip_rng); + // The TLS coordinator carries the per-peer SNI provider and the cert/key + // bundle (ridden as the inner options on `LabelOptions`); the membership + // address IS the transport socket (`|addr| *addr`). Like the plain-TCP + // plane this stage carries no cluster label (`None`) — TLS isolation comes + // from the record-layer cert verification and SNI. + #[allow(unused_mut)] + let mut coord = Coordinator::<_, _, Labeled, G>::new( + inner, + LabelOptions::new_in(None::>, self.tls_options), + self.sni_provider, + Box::new(|addr: &SocketAddr| *addr), + ); + // Install the gossip-encryption keyring so the coordinator's + // `encrypt_gossip`/`decrypt_gossip` (forwarded from the serf endpoint pump) + // become real on the unreliable plane. A no-keyring policy is the identity + // transform, so an unencrypted node is unaffected. The reliable plane is + // secured by the TLS session regardless. + #[cfg(encryption)] + coord.set_encryption_options(self.encryption); + // Serf's core RNG is seeded from its own OS-drawn entropy (`self.serf_rng`), + // independent of the coordinator's gossip RNG, so two nodes never share the + // query-ID / relay-selection stream. + let endpoint = serf_proto::StreamEndpoint::< + Self::Id, + SocketAddr, + Labeled, + G, + StdRng, + >::new_with_rng(coord, runtime.serf_options, self.serf_rng); + + crate::driver::stream::stream_driver_loop::, D, G, StdRng>( + endpoint, + self.gossip_socket, + self.tcp_listener, + runtime.commands_rx, + runtime.events_tx, + runtime.events_dropped, + runtime.observation_dropped, + runtime.snapshot, + runtime.shutdown_flag, + runtime.driver_options, + self.stream_options, + runtime.delegate, + None, + #[cfg(encryption)] + runtime.keyring, + ) + .await; + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-compio/src/tls/tests.rs b/serf-compio/src/tls/tests.rs new file mode 100644 index 00000000..155b9ab2 --- /dev/null +++ b/serf-compio/src/tls/tests.rs @@ -0,0 +1,457 @@ +//! End-to-end smoke test: two TLS serf nodes on the loopback interface, one +//! joining the other over a real rustls handshake, asserting the membership +//! event propagates through the full pump (Join command → push-pull dial → TLS +//! handshake → coordinator merge → serf `Member` event → `EventStream`). + +use core::time::Duration; +use std::{io::ErrorKind, net::SocketAddr, sync::Arc}; + +use futures_util::StreamExt; +use memberlist_proto::MaybeResolved; +use rustls::{ + client::danger::{HandshakeSignatureValid, ServerCertVerified}, + crypto::CryptoProvider, + pki_types::CertificateDer, + version::TLS13, +}; +use serf_proto::{ + event::{Event, MemberEventKind}, + options::Options as SerfOptions, +}; +use smol_str::SmolStr; + +use crate::{ + Channel, FirstAddrResolver, RuntimeOptions, Serf, SerfError, SocketAddrResolver, + StreamTransportOptions, TlsOptions, TlsTransport, TlsTransportOptions, Transport, VoidDelegate, + gossip_rng, +}; + +#[cfg(encryption)] +use crate::{EncryptionOptions, Keyring, SecretKey, VoidKeyringDelegate}; + +/// Accept-any server-cert verifier for the loopback smoke test. +/// +/// The two nodes share a self-signed localhost-SAN cert; the client side +/// accepts whatever the server presents so the handshake completes without a +/// real trust anchor. NEVER use this outside a test. +#[derive(Debug)] +struct AcceptAnyServer(Arc); + +impl rustls::client::danger::ServerCertVerifier for AcceptAnyServer { + fn verify_server_cert( + &self, + _e: &CertificateDer<'_>, + _i: &[CertificateDer<'_>], + _n: &rustls::pki_types::ServerName<'_>, + _o: &[u8], + _t: rustls::pki_types::UnixTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } + fn verify_tls12_signature( + &self, + _m: &[u8], + _c: &CertificateDer<'_>, + _d: &rustls::DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + fn verify_tls13_signature( + &self, + _m: &[u8], + _c: &CertificateDer<'_>, + _d: &rustls::DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + fn supported_verify_schemes(&self) -> Vec { + self.0.signature_verification_algorithms.supported_schemes() + } +} + +fn crypto_provider() -> Arc { + CryptoProvider::get_default() + .cloned() + .unwrap_or_else(|| Arc::new(rustls::crypto::ring::default_provider())) +} + +/// Build a self-signed localhost-SAN `ServerConfig` + accept-any `ClientConfig` +/// bundle. A fresh bundle is built per node so each owns its own cert. +fn test_tls_options() -> TlsOptions { + let ck = rcgen::generate_simple_self_signed(vec!["localhost".into()]) + .expect("rcgen generate_simple_self_signed"); + let chain = vec![CertificateDer::from(ck.cert.der().to_vec())]; + let key = rustls::pki_types::PrivateKeyDer::Pkcs8(ck.signing_key.serialize_der().into()); + + let provider = crypto_provider(); + + let server_cfg = rustls::ServerConfig::builder_with_provider(provider.clone()) + .with_protocol_versions(&[&TLS13]) + .expect("TLS 1.3 supported") + .with_no_client_auth() + .with_single_cert(chain, key) + .expect("valid self-signed cert"); + + let client_cfg = rustls::ClientConfig::builder_with_provider(provider.clone()) + .with_protocol_versions(&[&TLS13]) + .expect("TLS 1.3 supported") + .dangerous() + .with_custom_certificate_verifier(Arc::new(AcceptAnyServer(provider))) + .with_no_client_auth(); + + TlsOptions::new(server_cfg, client_cfg) +} + +/// Build and spawn a TLS serf node bound to an ephemeral loopback port. The +/// default SNI provider (`Some("localhost")`) matches the self-signed cert SAN. +async fn spawn_node(id: &str) -> Serf { + try_spawn_node_at(id, "127.0.0.1:0".parse().expect("loopback addr")) + .await + .expect("spawn serf node") +} + +/// Build a TLS serf node bound to a specific advertise address, returning the +/// construction result so the same-address rebind regression can assert a freed +/// port accepts an immediate rebind. A fresh self-signed bundle is built per +/// node, matching `spawn_node`. +async fn try_spawn_node_at(id: &str, bind: SocketAddr) -> Result, SerfError> { + let opts = TlsTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(bind)) + .with_tls_options(test_tls_options()); + Serf::new::, SocketAddrResolver, FirstAddrResolver, _, _>( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + gossip_rng().expect("seed gossip rng"), + #[cfg(encryption)] + std::rc::Rc::new(VoidKeyringDelegate), + ) + .await +} + +/// TLS rides the same stream driver as plain TCP, so it inherits the same +/// bound-port release guarantee: `shutdown().await` must close the TCP listener +/// and UDP gossip socket before it resolves, so a second TLS node binding the +/// SAME advertise address the instant the first shuts down must construct +/// successfully, not fail with `AddrInUse`. +#[compio::test] +async fn tls_shutdown_releases_bound_address_for_rebind() { + let first = spawn_node("rebind-first").await; + let addr = first.advertise_address(); + first.shutdown().await.expect("first node shuts down"); + + let second = try_spawn_node_at("rebind-second", addr) + .await + .expect("rebinding the freed address must succeed, not AddrInUse"); + assert_eq!( + second.advertise_address(), + addr, + "the second node rebinds the exact freed address" + ); + second.shutdown().await.expect("second node shuts down"); +} + +/// A `Bounded(0)` observation channel is rejected by the TLS stream driver's +/// `Serf::new` with [`SerfError::InvalidOption`] — before binding a socket or +/// spawning the detached driver — rather than panicking the driver task. TLS +/// rides the same stream driver as plain TCP; a real `tls_options` is supplied so +/// the runtime-option rejection, not the missing-config guard, is what fires. +#[compio::test] +async fn tls_new_rejects_zero_observation_channel() { + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let opts = TlsTransportOptions::::new() + .with_local_id(SmolStr::new("bad-opt-node")) + .with_advertise_addr(MaybeResolved::Resolved(bind)) + .with_tls_options(test_tls_options()); + let res = + Serf::new::, SocketAddrResolver, FirstAddrResolver, _, _>( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new().with_observation_channel(Channel::Bounded(0)), + SerfOptions::new(), + gossip_rng().expect("seed gossip rng"), + #[cfg(encryption)] + std::rc::Rc::new(VoidKeyringDelegate), + ) + .await; + match res { + Err(SerfError::InvalidOption(_)) => {} + Err(other) => panic!("expected InvalidOption, got {other:?}"), + Ok(_) => panic!("a zero-capacity observation channel must be rejected at construction"), + } +} + +/// Two nodes on loopback: A joins B over a real TLS push-pull exchange; A must +/// observe B joining the cluster through its event stream, then both shut down +/// cleanly. +#[compio::test] +async fn two_node_tls_join_observes_membership() { + let b = spawn_node("node-b").await; + let a = spawn_node("node-a").await; + let b_addr = b.advertise_address(); + let b_id = SmolStr::new("node-b"); + + // Subscribe BEFORE the join so a `Member` event cannot race ahead of the + // subscription (the channel buffers either way, but this is the clean order). + let mut a_events = a.events(); + + // Node A dials node B as its seed. + let dispatched = a.join(vec![b_addr]).await.expect("join dispatched"); + assert_eq!(dispatched, 1, "exactly one seed was dispatched"); + + // Node A should observe node B joining via a `Member(Join)` event. + let observed = compio::time::timeout(Duration::from_secs(20), async { + loop { + match a_events.next().await { + Some(Event::Member(me)) if me.kind() == MemberEventKind::Join => { + if me.members().iter().any(|m| m.node().id_ref() == &b_id) { + break true; + } + } + Some(_) => {} + None => break false, + } + } + }) + .await; + + assert!( + matches!(observed, Ok(true)), + "node A should observe node B joining the cluster within the timeout" + ); + + // Exercise the graceful shutdown command path on both nodes. + a.shutdown().await.expect("node A shuts down"); + b.shutdown().await.expect("node B shuts down"); +} + +/// The `TlsTransportOptions` getters reflect what the builders set, including +/// the SNI provider closure and the TLS options bundle. The pre-build state is +/// `None` for the required fields the `new()` `ok_or_else` checks arm. +#[test] +fn options_accessors_reflect_builders() { + let addr: SocketAddr = "127.0.0.1:7946".parse().unwrap(); + // Default options: required fields unset, but the SNI provider has a default + // (`localhost`) so `sni_provider()` returns a live closure even before build. + let empty = TlsTransportOptions::::new(); + assert!(empty.local_id().is_none()); + assert!(empty.advertise_addr().is_none()); + assert!(empty.tls_options().is_none()); + assert!(empty.stream().validate().is_ok()); + assert_eq!((empty.sni_provider())(&addr), Some("localhost".to_string())); + + let opts = TlsTransportOptions::::new() + .with_local_id(SmolStr::new("acc-node")) + .with_advertise_addr(MaybeResolved::Resolved(addr)) + .with_tls_options(test_tls_options()) + .with_sni_provider(Box::new(|_| Some("peer.example".to_string()))); + assert_eq!(opts.local_id().map(|s| s.as_str()), Some("acc-node")); + match opts.advertise_addr() { + Some(MaybeResolved::Resolved(s)) => assert_eq!(*s, addr), + other => panic!("expected a resolved advertise addr, got {other:?}"), + } + assert!(opts.tls_options().is_some()); + // The custom SNI provider overrides the default for every peer. + assert_eq!( + (opts.sni_provider())(&addr), + Some("peer.example".to_string()) + ); +} + +/// `Default` is the `new()` state: required fields `None`, default SNI provider +/// installed. +#[test] +fn default_matches_new() { + let addr: SocketAddr = "127.0.0.1:0".parse().unwrap(); + let d = TlsTransportOptions::::default(); + assert!(d.local_id().is_none()); + assert!(d.advertise_addr().is_none()); + assert!(d.tls_options().is_none()); + assert_eq!((d.sni_provider())(&addr), Some("localhost".to_string())); +} + +/// Binding the wildcard `0.0.0.0:0` reads an unspecified IP back from the +/// socket; gossiping it would publish an undialable contact, so construction +/// must reject it with `InvalidAdvertiseAddr` (the `tls_options` are supplied so +/// the advertise check, not the missing-config guard, is what fires). +#[compio::test] +async fn new_rejects_wildcard_advertise() { + let wildcard: SocketAddr = "0.0.0.0:0".parse().expect("wildcard addr"); + let opts = TlsTransportOptions::::new() + .with_local_id(SmolStr::new("wild-node")) + .with_advertise_addr(MaybeResolved::Resolved(wildcard)) + .with_tls_options(test_tls_options()); + let res = + TlsTransport::::new(opts, &SocketAddrResolver, &FirstAddrResolver).await; + match res { + Err(SerfError::InvalidAdvertiseAddr(e)) => { + assert!( + e.addr().ip().is_unspecified(), + "the rejected address carries the unspecified IP read back from the wildcard bind" + ); + } + Err(other) => panic!("expected InvalidAdvertiseAddr, got {other:?}"), + Ok(_) => panic!("a wildcard advertise must be rejected, but construction succeeded"), + } +} + +/// A construction failure AFTER the sockets are bound must close them (awaited) +/// before returning `Err`, or the bound port leaks and a same-address rebind +/// races into `AddrInUse` (a plain drop is not a synchronous fd release on +/// compio/Windows-IOCP). TLS rides the same stream constructor as plain TCP: a +/// wildcard `0.0.0.0:0` advertise binds a concrete OS-assigned port (free for +/// both the TCP listener and the UDP socket) but is then rejected by +/// `validate_advertise_addr` for its unspecified IP; the exact freed +/// `0.0.0.0:` must immediately re-accept the SAME listener + UDP socket, +/// proving neither leaked on the error path. A real `tls_options` is supplied so +/// the advertise rejection — not the missing-config guard — is what fires. +#[compio::test] +async fn new_failure_closes_bound_sockets_for_rebind() { + let opts = TlsTransportOptions::::new() + .with_local_id(SmolStr::new("rebind-fail")) + .with_advertise_addr(MaybeResolved::Resolved( + "0.0.0.0:0".parse().expect("wildcard addr"), + )) + .with_tls_options(test_tls_options()); + let res = + TlsTransport::::new(opts, &SocketAddrResolver, &FirstAddrResolver).await; + let freed = match res { + Err(SerfError::InvalidAdvertiseAddr(e)) => e.addr(), + Err(other) => panic!("expected a post-bind InvalidAdvertiseAddr failure, got {other:?}"), + Ok(_) => panic!("a post-bind failure must reject construction, but it succeeded"), + }; + + let listener = compio::net::TcpListener::bind(freed) + .await + .expect("the freed TCP port must rebind, not AddrInUse"); + let gossip = compio::net::UdpSocket::bind(freed) + .await + .expect("the freed UDP port must rebind, not AddrInUse"); + // Ignoring Err: test cleanup of the probe sockets. + let _ = listener.close().await; + let _ = gossip.close().await; +} + +/// A zero `dial_timeout` is rejected by `TlsTransport::new` with `InvalidOption`. +/// `options.stream.validate()` runs at the top of `new` — before any socket bind +/// — so a real `tls_options` is supplied to show the dial-timeout rejection is +/// what fires, not a missing-config guard. TLS rides the same stream driver as +/// plain TCP, so the same zero-dial footgun applies. +#[compio::test] +async fn new_rejects_zero_dial_timeout() { + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let opts = TlsTransportOptions::::new() + .with_local_id(SmolStr::new("zero-dial-node")) + .with_advertise_addr(MaybeResolved::Resolved(bind)) + .with_tls_options(test_tls_options()) + .with_stream(StreamTransportOptions::new().with_dial_timeout(Duration::ZERO)); + let res = + TlsTransport::::new(opts, &SocketAddrResolver, &FirstAddrResolver).await; + match res { + Err(SerfError::InvalidOption(_)) => {} + Err(other) => panic!("expected InvalidOption, got {other:?}"), + Ok(_) => panic!("a zero dial_timeout must be rejected at construction"), + } +} + +/// `new` rejects a missing `tls_options` with `InvalidInput`. +#[compio::test] +async fn new_without_tls_options_errors() { + let opts = TlsTransportOptions::::new() + .with_local_id(SmolStr::new("no-tls")) + .with_advertise_addr(MaybeResolved::Resolved("127.0.0.1:0".parse().unwrap())); + let res = + TlsTransport::::new(opts, &SocketAddrResolver, &FirstAddrResolver).await; + match res { + Err(SerfError::Io(e)) => { + assert_eq!(e.kind(), ErrorKind::InvalidInput); + assert!(e.to_string().contains("tls_options")); + } + Err(other) => panic!("expected InvalidInput(tls_options), got {other:?}"), + Ok(_) => panic!("a missing tls_options must be rejected, but construction succeeded"), + } +} + +/// A deterministic test secret key, selecting whichever AEAD cipher this build +/// compiled so the encrypted test works under either backend. +#[cfg(encryption)] +fn test_secret_key(fill: u8) -> SecretKey { + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes256([fill; 32]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = SecretKey::ChaCha20Poly1305([fill; 32]); + key +} + +/// Build and spawn a TLS serf node on an ephemeral loopback port with +/// `encryption` installed as its gossip keyring policy. +#[cfg(encryption)] +async fn spawn_encrypted_node(id: &str, encryption: EncryptionOptions) -> Serf { + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let opts = TlsTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(bind)) + .with_tls_options(test_tls_options()) + .with_encryption(encryption); + Serf::new::, SocketAddrResolver, FirstAddrResolver, _, _>( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + gossip_rng().expect("seed gossip rng"), + std::rc::Rc::new(VoidKeyringDelegate), + ) + .await + .expect("spawn serf node") +} + +/// Two TLS nodes sharing one gossip keyring: A joins B and must observe B +/// joining through its event stream. The reliable push-pull rides the TLS +/// session, while the gossip datagrams are AEAD-sealed by the configured +/// keyring — proving the keyring reaches the TLS coordinator and that an +/// encrypted TLS cluster forms and interoperates end-to-end. +#[cfg(encryption)] +#[compio::test] +async fn two_node_tls_join_observes_membership_encrypted() { + let enc = EncryptionOptions::new().with_keyring(Keyring::new(test_secret_key(0x42))); + let b = spawn_encrypted_node("node-b", enc.clone()).await; + let a = spawn_encrypted_node("node-a", enc).await; + let b_addr = b.advertise_address(); + let b_id = SmolStr::new("node-b"); + + let mut a_events = a.events(); + + let dispatched = a.join(vec![b_addr]).await.expect("join dispatched"); + assert_eq!(dispatched, 1, "exactly one seed was dispatched"); + + let observed = compio::time::timeout(Duration::from_secs(20), async { + loop { + match a_events.next().await { + Some(Event::Member(me)) if me.kind() == MemberEventKind::Join => { + if me.members().iter().any(|m| m.node().id_ref() == &b_id) { + break true; + } + } + Some(_) => {} + None => break false, + } + } + }) + .await; + + assert!( + matches!(observed, Ok(true)), + "node A should observe node B joining the encrypted TLS cluster within the timeout" + ); + + a.shutdown().await.expect("node A shuts down"); + b.shutdown().await.expect("node B shuts down"); +} diff --git a/serf-compio/src/transport/mod.rs b/serf-compio/src/transport/mod.rs new file mode 100644 index 00000000..221ccdbf --- /dev/null +++ b/serf-compio/src/transport/mod.rs @@ -0,0 +1,211 @@ +//! `Transport` trait — abstracts a per-backend serf driver (TCP/TLS/QUIC). +//! +//! Concrete impls live in `src/{tcp,tls,quic}.rs`. The Sans-I/O machine +//! endpoint (`serf_proto::StreamEndpoint` for TCP/TLS; +//! `serf_proto::QuicEndpoint` for QUIC) is built inside `T::run` from the +//! transport's stored config — a generic `Serf::new` cannot build the +//! backend's private record-layer config + dial closures, so the endpoint +//! never flows through the [`TransportRuntime`] bundle. + +use core::future::Future; +use std::net::SocketAddr; + +use crate::{ + delegate::Delegate, + resolver::{AdvertiseAddrResolver, Resolver}, +}; +use memberlist_proto::MaybeResolved; + +pub mod runtime; +pub use runtime::TransportRuntime; + +/// Abstracts a per-backend serf driver. The trait owns construction, +/// resource ownership (bound sockets / TCP listener / quinn endpoint), +/// the Sans-I/O machine endpoint, and the I/O event loop. +/// +/// `Self::Error` is bounded by `From` — convertible from +/// the OS layer. +/// +/// `Resolver` and `AdvertiseAddrResolver` are call-site arguments to +/// `Self::new`, NOT associated types, so users can swap resolvers without +/// changing the `Transport` type. The compatibility bound +/// `RES: Resolver
` enforces type alignment. +/// +/// `Self::run`'s future is `!Send` — compio is thread-per-core, +/// `!Send`-first. A future driver crate built on a `Send`-required runtime +/// defines its own `Transport` trait with a `Send` bound. +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub trait Transport: Sized + 'static { + /// Per-backend error type. + type Error: core::error::Error + From + Send + Sync + 'static; + + /// Node identifier type. + type Id; + + /// User-facing unresolved address type (e.g. `hostaddr::HostAddr`). + type Address; + + /// Per-backend transport-knobs block embedded into `Options`. + type Options; + + /// Construct the transport. Resolves `options.advertise_addr` via the + /// caller-supplied resolvers if it is `MaybeResolved::Unresolved(…)`; + /// binds the UDP gossip socket, the TCP listener, or the QUIC endpoint + /// and stores them. + fn new( + options: Self::Options, + resolver: &RES, + advertise_resolver: &AR, + ) -> impl Future> + where + RES: Resolver
, + AR: AdvertiseAddrResolver; + + /// Local node identifier. + fn local_id(&self) -> &Self::Id; + + /// Original advertise input form — `Resolved` if the user supplied a + /// concrete `SocketAddr`; `Unresolved` if a hostname was resolved at + /// construction. + fn local_address(&self) -> &MaybeResolved; + + /// Bound advertise `SocketAddr` — what the local node gossips to peers + /// and what the UDP / QUIC socket is bound to. + fn advertise_address(&self) -> &SocketAddr; + + /// Run the I/O event loop. Consumes `self` (sockets and listener move + /// into the loop), the `TransportRuntime` bundle (channels, + /// snapshot, delegate, tuning knobs), and the gossip RNG `gossip_rng` the + /// node constructor drew; the body builds the machine endpoint from + /// `self`'s stored config. Returns when shutdown is requested. + fn run(self, runtime: TransportRuntime, gossip_rng: G) -> impl Future + where + D: Delegate, + G: rand::Rng + Send + Unpin + 'static; +} + +/// Validate that the resolved advertise address can serve as the local node's +/// reachable, wire-representable contact identity. +/// +/// Each transport reads the advertise `SocketAddr` back from its bound socket +/// after construction and gossips it as the node's published contact: every +/// peer that learns this node aims its UDP probes and reliable dials at this +/// address. An address that the codec encodes fine but that is undialable — +/// classically the wildcard `0.0.0.0:0` / `[::]:0`, whose `local_addr()` +/// readback keeps the unspecified IP — would let the node join a cluster as a +/// member no peer can route to, so peers eventually suspect and reap it. +/// +/// Rejected with [`SerfError::InvalidAdvertiseAddr`] (not clamped — the operator +/// supplies a concrete reachable address, or auto-resolves one from the host's +/// interfaces via the `getifs` resolver) for either class: +/// +/// - NOT A USABLE UNICAST CONTACT — an unspecified IP (`0.0.0.0` / `::`), a +/// multicast IP, an IPv4 broadcast IP (`255.255.255.255`), or a zero port. +/// - NOT REPRESENTABLE ON THE WIRE — a scoped/flow-labelled IPv6 address with a +/// nonzero `scope_id` or `flowinfo`, which the compact `[16B IP][2B port]` +/// wire layout carries neither field of, so peers could not decode a routable +/// contact for this node. +/// +/// Loopback, private, and global unicast addresses stay valid. Called from each +/// transport's `new` on the post-readback advertise address, before any driver +/// task is spawned; on `Err` the caller explicitly closes (awaited, not a plain +/// drop) every already-bound socket before returning, so the rejected port is +/// released for an immediate rebind. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) fn validate_advertise_addr(advertise_addr: &SocketAddr) -> Result<(), crate::SerfError> { + let reject = |reason: &str| { + Err(crate::SerfError::InvalidAdvertiseAddr( + crate::error::InvalidAdvertiseAddr::new(*advertise_addr, reason.to_string()), + )) + }; + + let ip = advertise_addr.ip(); + if ip.is_unspecified() { + return reject( + "an unspecified IP (0.0.0.0 / ::) is the wildcard-bind address, not a routable contact \ + — peers cannot dial it (set a concrete advertise address, or resolve one from the \ + host's interfaces, when binding the wildcard)", + ); + } + if ip.is_multicast() { + return reject("a multicast IP is a group address, not a single peer's unicast contact"); + } + // IPv4 broadcast (255.255.255.255) is a v4-only concept; match the variant. + if let SocketAddr::V4(v4) = advertise_addr + && v4.ip().is_broadcast() + { + return reject("an IPv4 broadcast IP (255.255.255.255) is not a unicast contact"); + } + if advertise_addr.port() == 0 { + return reject( + "a zero port is undialable — the bound socket's local_addr() readback must carry a \ + concrete port", + ); + } + // The compact `[16B IP][2B port]` wire layout carries neither scope_id nor + // flowinfo, so a scoped/flow-labelled IPv6 advertise address (e.g. a + // link-local `fe80::1%scope`) cannot be encoded as a routable contact. + if let SocketAddr::V6(v6) = advertise_addr + && (v6.scope_id() != 0 || v6.flowinfo() != 0) + { + return reject( + "a scoped/flow-labelled IPv6 address (nonzero scope_id or flowinfo) is not representable \ + on the compact `[16B IP][2B port]` wire layout, so peers could not decode a routable \ + contact for this node", + ); + } + Ok(()) +} + +/// Run the post-bind construction steps common to every transport: reject an +/// undialable advertise address, then draw the OS-seeded serf-core RNG. +/// +/// Both are fallible AFTER the transport's socket(s) are already bound, so the +/// caller closes its bound socket(s) (awaited) before returning the `Err` this +/// produces — see [`close_stream_sockets`] / the QUIC single-socket close. +/// Grouping them keeps each transport's error path to a single close site +/// rather than one per fallible step. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) fn post_bind_setup( + advertise_addr: &SocketAddr, +) -> Result { + validate_advertise_addr(advertise_addr)?; + crate::os_seeded_std_rng() +} + +/// The QUIC variant of [`post_bind_setup`]: read the bound socket's address back +/// (an ephemeral `:0` resolves to a concrete port here), then run the shared +/// advertise validation + RNG draw, returning both. +/// +/// Borrows the socket immutably and returns owned values, so the caller can +/// close the socket (awaited) on `Err` or move it into the transport on `Ok`. +#[cfg(feature = "quic")] +pub(crate) fn quic_post_bind_setup( + gossip: &compio::net::UdpSocket, +) -> Result<(SocketAddr, rand::rngs::StdRng), crate::SerfError> { + let advertise_addr = gossip.local_addr().map_err(crate::SerfError::Io)?; + let serf_rng = post_bind_setup(&advertise_addr)?; + Ok((advertise_addr, serf_rng)) +} + +/// Close a stream transport's already-bound TCP listener and UDP gossip socket +/// (both awaited) on a construction error path. +/// +/// A plain drop is NOT a synchronous fd release on compio (Windows IOCP closes +/// asynchronously), so a failed `new` that merely dropped its already-bound +/// sockets could race an immediate same-address rebind into `AddrInUse`. +/// Awaiting `close()` drains each fd to release before the `Err` propagates, +/// mirroring the awaited close the stream driver's teardown and the ephemeral +/// retry path already use. `tls` implies `tcp`, so this single `tcp`-gated +/// helper serves both stream transports. +#[cfg(feature = "tcp")] +pub(crate) async fn close_stream_sockets( + listener: compio::net::TcpListener, + gossip: compio::net::UdpSocket, +) { + // Ignoring Err: a close error on an abandoned construction is unactionable — + // the bound fds are being released regardless. + let _ = listener.close().await; + let _ = gossip.close().await; +} diff --git a/serf-compio/src/transport/runtime.rs b/serf-compio/src/transport/runtime.rs new file mode 100644 index 00000000..84eca6bd --- /dev/null +++ b/serf-compio/src/transport/runtime.rs @@ -0,0 +1,97 @@ +//! `TransportRuntime` — the bundle handed to `T::run(self, runtime)`. +//! +//! Carries the serf observation delegate, command receiver, events sender, +//! snapshot cell, and driver / serf tuning knobs. The concrete machine +//! endpoint is NOT carried here: a generic `Serf::new` cannot build the +//! backend's record-layer config + dial closures, so each `T::run` body +//! builds its own endpoint from the transport's stored config and then +//! drives the shared stream or QUIC driver loop. +use std::{cell::Cell, net::SocketAddr, rc::Rc}; + +use flume::{Receiver, Sender}; + +use serf_proto::{event::Event, options::Options as SerfOptions}; + +use crate::{ + command::Command, delegate::Delegate, driver::options::RuntimeOptions, snapshot::SnapshotCell, + transport::Transport, +}; + +#[cfg(encryption)] +use crate::delegate::KeyringDelegate; + +/// Type alias for the command channel receiver, parameterised over the +/// transport so `Command::Respond` can carry `Node` without an +/// extra generic on `TransportRuntime`. +type CommandReceiver = Receiver::Id, SocketAddr>>; + +/// Bundle handed to `Transport::run(self, runtime)`. +/// +/// Carries the serf observation delegate, command receiver, events sender, +/// snapshot cell, and driver / serf tuning knobs. The machine endpoint is +/// built inside `T::run` (it needs the backend's private record-layer config), +/// so it is deliberately absent from this bundle. +/// +/// Requires a stream or QUIC transport feature. +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub struct TransportRuntime +where + T: Transport, + D: Delegate, +{ + pub(crate) delegate: D, + pub(crate) commands_rx: CommandReceiver, + pub(crate) events_tx: Sender>, + /// Counter for events dropped at the `EventStream` fan-out when the + /// subscriber queue is full (slow consumer) — recoverable membership gaps. + pub(crate) events_dropped: Rc>, + /// Counter for events dropped at the delegate observation channel when the + /// delegate fell behind — may include unrecoverable app-data. + pub(crate) observation_dropped: Rc>, + pub(crate) snapshot: SnapshotCell, + pub(crate) shutdown_flag: Rc>, + pub(crate) driver_options: RuntimeOptions, + pub(crate) serf_options: SerfOptions, + /// The driver's keyring delegate, applied to inbound key-management requests. + /// Present only under an encryption backend. + #[cfg(encryption)] + pub(crate) keyring: Rc, +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +impl TransportRuntime +where + T: Transport, + D: Delegate, +{ + /// Construct the runtime bundle. Called by the `Serf` handle constructor. + #[allow(clippy::too_many_arguments)] + #[inline] + pub(crate) fn new( + delegate: D, + commands_rx: CommandReceiver, + events_tx: Sender>, + events_dropped: Rc>, + observation_dropped: Rc>, + snapshot: SnapshotCell, + shutdown_flag: Rc>, + driver_options: RuntimeOptions, + serf_options: SerfOptions, + #[cfg(encryption)] keyring: Rc, + ) -> Self { + Self { + delegate, + commands_rx, + events_tx, + events_dropped, + observation_dropped, + snapshot, + shutdown_flag, + driver_options, + serf_options, + #[cfg(encryption)] + keyring, + } + } +} From af58a794b8a8123ff844b578fc851cc1e3d97630 Mon Sep 17 00:00:00 2001 From: Al Liu Date: Sun, 28 Jun 2026 20:18:53 +1200 Subject: [PATCH 023/128] feat(serf): await-result address-set join, per-exchange ignore_old, watermark-guarded leave (#65) --- serf-compio/Cargo.toml | 1 + serf-compio/src/command/mod.rs | 90 +++- serf-compio/src/command/tests.rs | 39 +- serf-compio/src/driver/options/mod.rs | 55 +++ serf-compio/src/driver/options/tests.rs | 46 ++ serf-compio/src/driver/quic/mod.rs | 329 ++++++++++++-- serf-compio/src/driver/quic/tests.rs | 145 ++++++ serf-compio/src/driver/stream/mod.rs | 368 ++++++++++++++-- serf-compio/src/driver/stream/tests.rs | 171 ++++++++ serf-compio/src/error/mod.rs | 10 +- serf-compio/src/error/tests.rs | 1 + serf-compio/src/quic/tests.rs | 165 ++++++- serf-compio/src/serf/mod.rs | 316 +++++++++++-- serf-compio/src/serf/tests.rs | 438 ++++++++++++++++++- serf-compio/src/tls/tests.rs | 14 +- serf-driver/src/error/mod.rs | 50 +++ serf-driver/src/error/tests.rs | 15 +- serf-proto/src/endpoint/mod.rs | 308 ++++++++++--- serf-proto/src/endpoint/serf_parity_tests.rs | 153 +++++-- serf-proto/src/endpoint/tests.rs | 346 +++++++++++---- serf-proto/src/event/mod.rs | 14 +- serf-proto/src/lib.rs | 4 + serf-proto/src/quic_endpoint/mod.rs | 85 +++- serf-proto/src/quic_endpoint/tests.rs | 6 +- serf-proto/src/stream_endpoint/mod.rs | 89 +++- serf-proto/src/stream_endpoint/tests.rs | 112 ++++- 26 files changed, 3028 insertions(+), 342 deletions(-) diff --git a/serf-compio/Cargo.toml b/serf-compio/Cargo.toml index 7855a5b8..555005ef 100644 --- a/serf-compio/Cargo.toml +++ b/serf-compio/Cargo.toml @@ -44,6 +44,7 @@ memberlist-proto = { git = "https://github.com/al8n/memberlist", branch = "main" rand = { workspace = true } smol_str.workspace = true +smallvec.workspace = true bytes = { workspace = true } thiserror.workspace = true futures-channel.workspace = true diff --git a/serf-compio/src/command/mod.rs b/serf-compio/src/command/mod.rs index 996ee8a7..86acdccd 100644 --- a/serf-compio/src/command/mod.rs +++ b/serf-compio/src/command/mod.rs @@ -11,21 +11,77 @@ use memberlist_proto::Instant; #[cfg(encryption)] use memberlist_proto::SecretKey; #[cfg(any(feature = "tcp", feature = "quic"))] +use smallvec::SmallVec; +#[cfg(any(feature = "tcp", feature = "quic"))] +use std::net::SocketAddr; + +#[cfg(any(feature = "tcp", feature = "quic"))] +use crate::error::SerfError; +#[cfg(any(feature = "tcp", feature = "quic"))] use serf_proto::{ endpoint::{QueryId, QueryParams}, event::QueryEvent, typed::Tags, }; +/// Address-set reply for [`Command::Join`]. +/// +/// Both join kinds reply through this single type (mirroring how the memberlist +/// driver unifies its join reply into one channel type): `Ok(set)` carries the +/// dispatched set ([`JoinKind::Dispatch`]) or the contacted set +/// ([`JoinKind::WaitForCompletion`] success); `Err((set, err))` is the legacy +/// partial-success tuple, surfacing the reached-so-far set alongside the error. +/// Every error this driver produces — `NotRunning`, `Shutdown`, and +/// `JoinAllFailed` — resolves before any contact is accumulated, so the tuple's +/// set is empty in practice; it is carried for the legacy `join_many` shape. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) type JoinReply = + core::result::Result, (SmallVec<[SocketAddr; 1]>, SerfError)>; + +/// Payload for [`JoinKind::WaitForCompletion`]. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) struct WaitForCompletionArgs { + /// Wall-clock instant past which the driver replies with whatever contacted + /// set it has accumulated (an empty set surfaces as `JoinAllFailed`). + pub(crate) deadline: Instant, +} + +/// Semantic of a [`Command::Join`] dispatch. +/// +/// Both kinds share the same `start_push_pull` fan-out (one outbound exchange +/// per resolved seed); the kind only affects WHEN the reply fires and WHAT it +/// carries: +/// - `Dispatch`: reply immediately with the dispatched seed set +/// (fire-and-forget; the caller does not wait for any exchange to terminate). +/// - `WaitForCompletion`: reply once every dispatched exchange has terminated +/// (an [`Event::ExchangeCompleted`](serf_proto::event::Event) with +/// `kind == ExchangeKind::PushPull` for each) OR the deadline elapses, +/// whichever comes first; the reply carries the contacted set (an empty set +/// surfaces as `JoinAllFailed`). +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) enum JoinKind { + /// Reply immediately with the dispatched seed set. + Dispatch, + /// Reply once every dispatched exchange has terminated OR the deadline expires. + WaitForCompletion(WaitForCompletionArgs), +} + /// Payload for [`Command::Join`]. +#[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) struct JoinCmd { - /// Pre-resolved socket addresses of the seed peers to contact. - pub(crate) seeds: Vec, - /// One-shot reply channel delivering the count of seeds the driver dispatched - /// a push-pull to. Join is dispatch-only: the reply reports how many exchanges - /// were initiated, not how many seeds were reached — actual joins surface as - /// membership [`Event`](serf_proto::event::Event)s. - pub(crate) reply: Sender>, + /// Pre-resolved socket addresses of the seed peers to contact. The handle + /// resolves every `MaybeResolved` seed through the caller's resolver before + /// sending the command, so the driver only ever sees concrete addresses. + pub(crate) seeds: Vec, + /// Dispatch semantic — see [`JoinKind`]. + pub(crate) kind: JoinKind, + /// When `true`, each seed's join push/pull is started as an `ignore_old` join + /// so the machine records that exchange's `StreamId` and suppresses replay of + /// the seed's pre-join user events. Keyed per-EXCHANGE and one-shot — see + /// `serf_proto::StreamEndpoint::start_join_push_pull`. + pub(crate) ignore_old: bool, + /// One-shot reply channel delivering the address-set result. See [`JoinReply`]. + pub(crate) reply: Sender, } /// Payload for [`Command::Leave`]. @@ -185,14 +241,6 @@ pub(crate) struct SetTagsCmd { pub(crate) reply: Sender>, } -/// Payload for [`Command::SetEventJoinIgnore`]. -pub(crate) struct SetEventJoinIgnoreCmd { - /// When `true`, the machine suppresses `Event::Member(Join)` events. - pub(crate) ignore: bool, - /// One-shot reply channel (always `Ok(())`; the setter never fails). - pub(crate) reply: Sender>, -} - /// Payload for [`Command::InstallKey`], [`Command::UseKey`], and /// [`Command::RemoveKey`]. /// @@ -236,9 +284,11 @@ pub(crate) struct ShutdownCmd { /// `I` is the node-id type; `A` is the resolved peer-address type (typically /// `std::net::SocketAddr`). All variants are newtype-over-payload-struct. pub(crate) enum Command { - /// Initiate joins to the given seed peers (addresses already resolved). The - /// reply carries the count of seeds the driver dispatched a push-pull to, not - /// the count reached; actual joins surface as membership events. + /// Initiate joins to the given resolved seeds. The reply carries the + /// address-set result: a dispatched or contacted set on success, or the + /// legacy partial-success tuple on failure (see [`JoinReply`]). + #[cfg(any(feature = "tcp", feature = "quic"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] Join(JoinCmd), /// Begin a graceful leave from the cluster. @@ -270,10 +320,6 @@ pub(crate) enum Command { #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] SetTags(SetTagsCmd), - /// Enable or disable suppression of member-join events in the driver's - /// observation stream. - SetEventJoinIgnore(SetEventJoinIgnoreCmd), - /// Issue a cluster-wide install-key query. /// /// Requires the `aes-gcm` or `chacha20-poly1305` feature. diff --git a/serf-compio/src/command/tests.rs b/serf-compio/src/command/tests.rs index f91fde05..e5df6f9f 100644 --- a/serf-compio/src/command/tests.rs +++ b/serf-compio/src/command/tests.rs @@ -9,25 +9,42 @@ fn unit_reply() -> Sender> { futures_channel::oneshot::channel::>().0 } +#[cfg(any(feature = "tcp", feature = "quic"))] #[test] -fn join_cmd_carries_seeds_and_reply() { - let (tx, _rx) = futures_channel::oneshot::channel::>(); +fn join_cmd_carries_seeds_kind_and_reply() { + use memberlist_proto::Instant; + + let (tx, _rx) = futures_channel::oneshot::channel::(); let cmd = JoinCmd { seeds: vec![addr()], + kind: JoinKind::Dispatch, + ignore_old: false, reply: tx, }; assert_eq!(cmd.seeds.len(), 1); assert_eq!(cmd.seeds[0], addr()); + assert!(matches!(cmd.kind, JoinKind::Dispatch)); + assert!(!cmd.ignore_old); - // The enum variant wraps the payload transparently. - let (tx2, _rx2) = futures_channel::oneshot::channel::>(); + // The enum variant wraps the payload transparently; the await kind carries + // its deadline, and the ignore_old flag rides along. + let (tx2, _rx2) = futures_channel::oneshot::channel::(); let Command::::Join(c) = Command::Join(JoinCmd { seeds: vec![addr(), addr()], + kind: JoinKind::WaitForCompletion(WaitForCompletionArgs { + deadline: Instant::now(), + }), + ignore_old: true, reply: tx2, }) else { panic!("wrong Command variant"); }; assert_eq!(c.seeds.len(), 2); + assert!(matches!(c.kind, JoinKind::WaitForCompletion(_))); + assert!( + c.ignore_old, + "ignore_old must round-trip through the command" + ); } #[test] @@ -38,20 +55,6 @@ fn leave_cmd_constructs() { assert!(matches!(cmd, Command::Leave(_))); } -#[test] -fn set_event_join_ignore_cmd_round_trips_flag() { - for &flag in &[true, false] { - let cmd = Command::::SetEventJoinIgnore(SetEventJoinIgnoreCmd { - ignore: flag, - reply: unit_reply(), - }); - assert!(matches!(cmd, Command::SetEventJoinIgnore(_))); - if let Command::SetEventJoinIgnore(c) = cmd { - assert_eq!(c.ignore, flag); - } - } -} - #[test] fn shutdown_cmd_constructs() { let cmd = Command::::Shutdown(ShutdownCmd { diff --git a/serf-compio/src/driver/options/mod.rs b/serf-compio/src/driver/options/mod.rs index baebd67b..332893b1 100644 --- a/serf-compio/src/driver/options/mod.rs +++ b/serf-compio/src/driver/options/mod.rs @@ -14,6 +14,15 @@ use crate::error::{InvalidOption, SerfError}; #[cfg(feature = "clap")] use humantime::parse_duration; +/// Default per-call deadline for the await-result +/// [`Serf::join`](crate::Serf::join) / [`Serf::join_many`](crate::Serf::join_many). +/// +/// Past this deadline the driver replies with whatever seeds it has contacted so +/// far (an empty contact set surfaces as +/// [`SerfError::JoinAllFailed`](crate::SerfError::JoinAllFailed)). Mirrors the +/// memberlist driver's `DEFAULT_JOIN_DEADLINE`. +pub const DEFAULT_JOIN_DEADLINE: Duration = Duration::from_secs(10); + /// Default per-call deadline for [`Serf::leave`](crate::Serf::leave). pub const DEFAULT_LEAVE_TIMEOUT: Duration = Duration::from_secs(5); @@ -118,6 +127,8 @@ pub const DEFAULT_OBSERVATION_CHANNEL: Channel = Channel::Bounded(1024); #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(feature = "serde", serde(default, deny_unknown_fields))] pub struct RuntimeOptions { + #[cfg_attr(feature = "serde", serde(with = "humantime_serde"))] + join_deadline: Duration, #[cfg_attr(feature = "serde", serde(with = "humantime_serde"))] leave_timeout: Duration, #[cfg_attr(feature = "serde", serde(with = "humantime_serde"))] @@ -142,6 +153,15 @@ const _: () = { #[derive(Args)] struct RuntimeOptionsCli { + #[arg( + id = "runtime-join-deadline", + long = "runtime-join-deadline", + env = "SERF_RUNTIME_JOIN_DEADLINE", + value_parser = parse_duration, + // The humantime spelling of DEFAULT_JOIN_DEADLINE. + default_value = "10s", + )] + join_deadline: Duration, #[arg( id = "runtime-leave-timeout", long = "runtime-leave-timeout", @@ -193,6 +213,7 @@ const _: () = { impl From for RuntimeOptions { fn from(c: RuntimeOptionsCli) -> Self { Self { + join_deadline: c.join_deadline, leave_timeout: c.leave_timeout, idle_wake_interval: c.idle_wake_interval, iter_drain_cap: c.iter_drain_cap, @@ -234,6 +255,7 @@ const _: () = { } }; } + take!("runtime-join-deadline", join_deadline, Duration); take!("runtime-leave-timeout", leave_timeout, Duration); take!("runtime-idle-wake-interval", idle_wake_interval, Duration); take!("runtime-iter-drain-cap", iter_drain_cap, usize); @@ -250,6 +272,7 @@ impl RuntimeOptions { #[inline] pub const fn new() -> Self { Self { + join_deadline: DEFAULT_JOIN_DEADLINE, leave_timeout: DEFAULT_LEAVE_TIMEOUT, idle_wake_interval: DEFAULT_IDLE_WAKE_INTERVAL, iter_drain_cap: DEFAULT_ITER_DRAIN_CAP, @@ -259,6 +282,15 @@ impl RuntimeOptions { } } + /// Builder: per-call deadline for the await-result + /// [`Serf::join`](crate::Serf::join) / [`join_many`](crate::Serf::join_many). + #[must_use] + #[inline] + pub const fn with_join_deadline(mut self, d: Duration) -> Self { + self.join_deadline = d; + self + } + /// Builder: per-call deadline for [`Serf::leave`](crate::Serf::leave). #[must_use] #[inline] @@ -308,6 +340,13 @@ impl RuntimeOptions { self } + /// Per-call deadline for the await-result [`Serf::join`](crate::Serf::join) / + /// [`join_many`](crate::Serf::join_many). + #[inline] + pub const fn join_deadline(&self) -> Duration { + self.join_deadline + } + /// Per-call deadline for [`Serf::leave`](crate::Serf::leave). #[inline] pub const fn leave_timeout(&self) -> Duration { @@ -353,6 +392,12 @@ impl RuntimeOptions { /// (TCP/TLS/QUIC) routes through the one `Serf::new` path, so this single call /// covers all three. /// + /// - `join_deadline == 0`: the await-result join parks a waiter until either + /// every dispatched push/pull terminates or this deadline elapses. A zero + /// deadline is past-due the instant the waiter is parked, so the next reap + /// sweep replies before any `ExchangeCompleted` can be observed — every + /// await-result join would spuriously surface `JoinAllFailed` even against a + /// reachable seed. /// - `idle_wake_interval == 0`: the driver loop's fallback sleep when the /// coordinator's `poll_timeout` has no nearer deadline /// (`poll_timeout().unwrap_or(now + idle_wake_interval)`). Zero makes a @@ -380,6 +425,16 @@ impl RuntimeOptions { /// loud immediate [`SerfError::LeaveTimeout`] rather than a silent break. #[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) fn validate(&self) -> Result<(), SerfError> { + if self.join_deadline.is_zero() { + return Err(SerfError::InvalidOption(InvalidOption::new( + "join_deadline", + "the await-result join deadline must be nonzero: a parked join waiter is past-due the \ + instant it is parked, so the next reap sweep replies before any push/pull \ + ExchangeCompleted can be observed — every await-result join would spuriously surface \ + JoinAllFailed even against a reachable seed" + .to_string(), + ))); + } if self.idle_wake_interval.is_zero() { return Err(SerfError::InvalidOption(InvalidOption::new( "idle_wake_interval", diff --git a/serf-compio/src/driver/options/tests.rs b/serf-compio/src/driver/options/tests.rs index 8e3e3b4a..7e6ae97d 100644 --- a/serf-compio/src/driver/options/tests.rs +++ b/serf-compio/src/driver/options/tests.rs @@ -5,6 +5,7 @@ use core::time::Duration; #[test] fn runtime_options_defaults_are_accessible() { let opts = RuntimeOptions::new(); + assert_eq!(opts.join_deadline(), DEFAULT_JOIN_DEADLINE); assert_eq!(opts.leave_timeout(), DEFAULT_LEAVE_TIMEOUT); assert_eq!(opts.idle_wake_interval(), DEFAULT_IDLE_WAKE_INTERVAL); assert_eq!(opts.iter_drain_cap(), DEFAULT_ITER_DRAIN_CAP); @@ -123,6 +124,19 @@ fn runtime_validate_rejects_zero_idle_wake_interval() { )); } +// A zero `join_deadline` makes every parked await-join waiter past-due on insert, +// so the reaper replies `JoinAllFailed` before any push/pull can complete; +// `validate` rejects it at construction. +#[cfg(any(feature = "tcp", feature = "quic"))] +#[test] +fn runtime_validate_rejects_zero_join_deadline() { + let opts = RuntimeOptions::new().with_join_deadline(Duration::ZERO); + assert!(matches!( + opts.validate(), + Err(crate::SerfError::InvalidOption(_)) + )); +} + // `iter_drain_cap == 0` (the per-iteration batch cap; the select arms and the // uncapped timeout drain still make one-per-pass forward progress) and // `leave_timeout == 0` (a loud immediate `LeaveTimeout`) degrade-but-function, @@ -212,6 +226,18 @@ fn runtime_validate_rejects_zero_idle_wake_interval_from_serde() { )); } +#[cfg(all(feature = "serde", any(feature = "tcp", feature = "quic")))] +#[test] +fn runtime_validate_rejects_zero_join_deadline_from_serde() { + let opts: RuntimeOptions = + serde_json::from_str(r#"{"join_deadline":"0s"}"#).expect("deserialize"); + assert_eq!(opts.join_deadline(), Duration::ZERO); + assert!(matches!( + opts.validate(), + Err(crate::SerfError::InvalidOption(_)) + )); +} + // A zero capacity sourced via a clap-parsed flag is rejected the same way. #[cfg(all(feature = "clap", any(feature = "tcp", feature = "quic")))] #[test] @@ -275,6 +301,26 @@ fn runtime_validate_rejects_zero_idle_wake_interval_from_clap() { )); } +#[cfg(all(feature = "clap", any(feature = "tcp", feature = "quic")))] +#[test] +fn runtime_validate_rejects_zero_join_deadline_from_clap() { + use clap::Parser; + + #[derive(Parser)] + struct Cli { + #[command(flatten)] + runtime: RuntimeOptions, + } + + let cli = Cli::try_parse_from(["app", "--runtime-join-deadline", "0s"]) + .expect("clap parses join-deadline 0s"); + assert_eq!(cli.runtime.join_deadline(), Duration::ZERO); + assert!(matches!( + cli.runtime.validate(), + Err(crate::SerfError::InvalidOption(_)) + )); +} + #[cfg(feature = "serde")] #[test] fn runtime_options_serde_round_trip_and_partial() { diff --git a/serf-compio/src/driver/quic/mod.rs b/serf-compio/src/driver/quic/mod.rs index 4bdc290e..2ee4c741 100644 --- a/serf-compio/src/driver/quic/mod.rs +++ b/serf-compio/src/driver/quic/mod.rs @@ -18,7 +18,7 @@ #![cfg(feature = "quic")] -use std::{cell::Cell, net::SocketAddr, rc::Rc}; +use std::{cell::Cell, collections::HashSet, net::SocketAddr, rc::Rc}; use core::time::Duration; @@ -29,13 +29,16 @@ use futures_channel::oneshot; use futures_util::{FutureExt, pin_mut, select_biased}; use lochan::mpsc; use memberlist_proto::{ - Instant, PushPullKind, Rng, SeedableRng, Transmit, + Instant, Rng, SeedableRng, StreamId, Transmit, codec::{ DecodeOptions, EncodeOptions, decode_incoming, encode_outgoing, encode_outgoing_compound, parse_messages, }, }; -use serf_proto::{LamportTime, QuicEndpoint, event::Event, members::SerfState}; +use serf_proto::{ + ExchangeKind, ExchangeStatus, LamportTime, QuicEndpoint, event::Event, members::SerfState, +}; +use smallvec::SmallVec; #[cfg(encryption)] use crate::command::{KeyCmd, ListKeysCmd}; @@ -44,23 +47,147 @@ use crate::delegate::KeyringDelegate; use crate::{ Channel, command::{ - Command, ForceLeaveCmd, JoinCmd, LeaveCmd, QueryCmd, RespondCmd, SetEventJoinIgnoreCmd, - SetTagsCmd, ShutdownCmd, + Command, ForceLeaveCmd, JoinCmd, JoinKind, JoinReply, LeaveCmd, QueryCmd, RespondCmd, + SetTagsCmd, ShutdownCmd, WaitForCompletionArgs, }, delegate::Delegate, driver::{ options::RuntimeOptions, shared::{ - add_obs_payload, dispatch_event_delegate, drain_past_due_udp, observation_payload_bytes, - yield_once, + ExchangeId, add_obs_payload, dispatch_event_delegate, drain_past_due_udp, + observation_payload_bytes, yield_once, }, }, - error::{Result, SerfError}, + error::{JoinFailed, Result, SerfError}, snapshot::{SerfSnapshot, SnapshotCell}, }; #[cfg(encryption)] use serf_proto::{KeyRequestOperation, KeyResponseArgs, event::KeyRequest}; +/// Driver-side state for one outstanding await-result join call. +/// +/// Mirrors the stream driver's `PendingJoin`. A [`Command::Join`] carrying +/// [`JoinKind::WaitForCompletion`] dispatches one push/pull per resolved seed +/// and parks the per-call state here. The QUIC coordinator services the dial +/// in-band, so each `start_push_pull`'s returned machine `StreamId` coerces +/// directly into the [`ExchangeId`] domain (via `From`) — the same +/// value the bridge-reap path stamps onto its [`Event::ExchangeCompleted`]. +/// Contact accounting is per-OUTBOUND-EXCHANGE, filtered to +/// [`ExchangeKind::PushPull`]: each successful exchange pushes its peer into +/// `contacted`; duplicate seeds count independently. +/// +/// Reply resolution and ignore-stream cleanup are SEPARATE terminal states. The +/// caller's reply resolves on all-exchanges-done OR `deadline` (whichever first), +/// consuming `reply`. The ignore-stream cleanup must wait until every dispatched +/// exchange has completed — i.e. `pending` is empty — because a `StreamId` +/// recorded for a still-live exchange must stay in the machine's ignore set so a +/// late merge still suppresses the peer's pre-join user events. The waiter is +/// removed from `joins` only once BOTH terminals are reached (reply sent and +/// `pending` empty); on a deadline that fires with exchanges still live it +/// replies and LINGERS, holding its `ignore_streams` until they complete. +struct PendingJoin { + /// Outbound exchange ids this waiter dispatched and is still awaiting a + /// terminal `ExchangeCompleted` for. Removed on completion; when empty the + /// ignore-stream cleanup runs and the waiter is reaped. + pending: HashSet, + /// Peer addresses of the dispatched exchanges that terminated `Succeeded`. + contacted: SmallVec<[SocketAddr; 1]>, + /// The `StreamId`s this join recorded in the machine's per-exchange ignore set + /// (non-empty only for an `ignore_old` join). Each is consumed by its own merge + /// on the success path; any that did NOT merge are cleared via + /// `clear_ignore_join_stream` only once every dispatched exchange has completed + /// (`pending` empty), so a failed `ignore_old` exchange never leaks its + /// `StreamId` and a still-live one is never prematurely cleared. + ignore_streams: SmallVec<[StreamId; 1]>, + /// Total outbound-exchange count this call dispatched — the `JoinAllFailed` + /// denominator on a zero-contact resolution. + requested: usize, + /// Wall-clock instant past which the driver replies with whatever `contacted` + /// set it has accumulated even if `pending` is non-empty. + deadline: Instant, + /// One-shot reply channel back to the caller, taken when the reply resolves + /// (all-exchanges-done or `deadline`). `None` once resolved; the waiter then + /// lingers — only to drive ignore-stream cleanup — until `pending` empties. + /// See [`JoinReply`]. + reply: Option>, +} + +impl PendingJoin { + /// Resolve the caller's reply once, from the current `contacted` set. Idempotent: + /// after the first call `reply` is `None` and this is a no-op, so the deadline + /// path and the all-exchanges-done path never double-send. A zero-contact + /// resolution is the `JoinAllFailed` the reaper would otherwise have produced. + fn resolve_reply(&mut self) { + if let Some(reply) = self.reply.take() { + let result = if self.contacted.is_empty() { + Err(( + SmallVec::new(), + SerfError::JoinAllFailed(JoinFailed::new(self.requested, 0)), + )) + } else { + Ok(self.contacted.clone()) + }; + // Ignoring Err: caller dropped the reply receiver (the join future was + // cancelled). + let _ = reply.send(result); + } + } + + /// This waiter has reached both terminal states — its reply resolved AND every + /// dispatched exchange completed — so it can be removed and its ignore-stream + /// cleanup run. + fn is_done(&self) -> bool { + self.reply.is_none() && self.pending.is_empty() + } +} + +/// Apply one terminal `ExchangeCompleted` to its await-result join waiter (if +/// any), driving both decoupled terminals: remove `eid` from the waiter's +/// `pending` and, on success, push the peer into `contacted`; resolve the +/// caller's reply the instant `pending` empties (ahead of the observation +/// hand-off, so a slow delegate cannot delay it); and once the waiter is fully +/// done (reply sent AND `pending` empty) clear its still-recorded ignore-join +/// streams and reap it. A `StreamId` the success-path merge already consumed is +/// absent, so the clear removes only the streams whose exchange did not merge. +/// +/// Shared by the live drain and unit tests; takes the decoded +/// `(eid, peer, succeeded)` rather than the `Event` so it is callable without +/// constructing a coordinator-internal `ExchangeCompleted`. +fn complete_join_exchange( + endpoint: &mut QuicEndpoint, + pending_joins: &mut Vec, + eid: ExchangeId, + peer: SocketAddr, + succeeded: bool, +) where + I: memberlist_proto::Id + Clone, + G: Rng, + R: Rng + SeedableRng, +{ + let Some(idx) = pending_joins + .iter() + .position(|pj| pj.pending.contains(&eid)) + else { + return; + }; + let pj = &mut pending_joins[idx]; + pj.pending.remove(&eid); + if succeeded { + pj.contacted.push(peer); + } + // Resolve the reply the moment every dispatched exchange has terminated. If the + // deadline already replied, `reply` is `None` and this is a no-op. + if pj.pending.is_empty() { + pj.resolve_reply(); + } + if pending_joins[idx].is_done() { + let pj = pending_joins.swap_remove(idx); + for s in &pj.ignore_streams { + endpoint.clear_ignore_join_stream(*s); + } + } +} + /// Driver-side state for the single in-flight graceful-leave operation. /// /// A [`Command::Leave`] that finds the endpoint `Alive` initiates the machine's @@ -103,6 +230,8 @@ impl PendingLeave { /// Pump-loop-local state tracking outstanding commands awaiting completion. struct PendingCommands { + /// Outstanding await-result join waiters. See [`PendingJoin`]. + joins: Vec, /// Outstanding graceful-leave waiter (at most one at a time). See [`PendingLeave`]. leave: Option, } @@ -208,7 +337,10 @@ pub(crate) async fn quic_driver_loop( // closes the gossip socket so the bound port is free when the caller resumes // from `shutdown.await`. let mut shutdown_reply: Option>> = None; - let mut pending = PendingCommands { leave: None }; + let mut pending = PendingCommands { + joins: Vec::new(), + leave: None, + }; // Per-pump UDP recv buffer size, derived once at entry as the larger of the // gossip plane (`gossip_mtu` + AEAD wrapper) and the raw-QUIC plane (quinn's @@ -293,6 +425,7 @@ pub(crate) async fn quic_driver_loop( &*keyring, ) .await; + reap_pending_joins(&mut endpoint, &mut pending.joins, Instant::now()).await; reap_pending_leave(&mut pending.leave, Instant::now()).await; refresh_snapshot::(&endpoint, &snapshot); break; @@ -308,6 +441,7 @@ pub(crate) async fn quic_driver_loop( .unwrap_or(setup_now + driver_opts.idle_wake_interval()); let timeout_deadline = [ Some(endpoint_deadline), + min_pending_join_deadline(&pending.joins), min_pending_leave_deadline(&pending.leave), ] .into_iter() @@ -351,6 +485,7 @@ pub(crate) async fn quic_driver_loop( &*keyring, ) .await; + reap_pending_joins(&mut endpoint, &mut pending.joins, Instant::now()).await; reap_pending_leave(&mut pending.leave, Instant::now()).await; if dirty { refresh_snapshot::(&endpoint, &snapshot); @@ -380,6 +515,7 @@ pub(crate) async fn quic_driver_loop( &*keyring, ) .await; + reap_pending_joins(&mut endpoint, &mut pending.joins, Instant::now()).await; reap_pending_leave(&mut pending.leave, Instant::now()).await; refresh_snapshot::(&endpoint, &snapshot); dirty = false; @@ -487,6 +623,7 @@ pub(crate) async fn quic_driver_loop( { exit = true; } + reap_pending_joins(&mut endpoint, &mut pending.joins, Instant::now()).await; reap_pending_leave(&mut pending.leave, Instant::now()).await; if dirty { @@ -507,6 +644,21 @@ pub(crate) async fn quic_driver_loop( reply_shutdown(c); } drop(commands); + // Reply Err(Shutdown) to every parked await-result join waiter whose reply has + // not yet resolved — their reply receivers would otherwise hang forever (the + // loop's reap path is gone and the task is exiting). Clear EVERY remaining + // waiter's ignore-join streams (including a lingering waiter that already + // replied on its deadline but was awaiting exchange completion): after shutdown + // no late merge can arrive, so nothing is left to suppress. + for pj in pending.joins.drain(..) { + for s in &pj.ignore_streams { + endpoint.clear_ignore_join_stream(*s); + } + if let Some(reply) = pj.reply { + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(Err((SmallVec::new(), SerfError::Shutdown))); + } + } if let Some(pl) = pending.leave.take() { pl.resolve_all(|| Err(SerfError::Shutdown)).await; } @@ -527,14 +679,11 @@ fn reply_shutdown(c: Command) { // Ignoring Err on each send: caller dropped the reply receiver. match c { Command::Join(JoinCmd { reply, .. }) => { - let _ = reply.send(Err(SerfError::Shutdown)); + let _ = reply.send(Err((SmallVec::new(), SerfError::Shutdown))); } Command::Leave(LeaveCmd { reply }) | Command::Shutdown(ShutdownCmd { reply }) => { let _ = reply.send(Err(SerfError::Shutdown)); } - Command::SetEventJoinIgnore(SetEventJoinIgnoreCmd { reply, .. }) => { - let _ = reply.send(Err(SerfError::Shutdown)); - } Command::ForceLeave(ForceLeaveCmd { reply, .. }) => { let _ = reply.send(Err(SerfError::Shutdown)); } @@ -584,34 +733,81 @@ async fn dispatch_command( { let running = endpoint.state() == SerfState::Alive; match cmd { - Command::Join(JoinCmd { seeds, reply }) => { + Command::Join(JoinCmd { + seeds, + kind, + ignore_old, + reply, + }) => { // Gate on a running node: `leave()` is terminal (it stops the periodic // schedulers), so a join after leave would leave the node non-participating. if !running { // Ignoring Err: caller dropped the reply receiver. - let _ = reply.send(Err(SerfError::NotRunning)); + let _ = reply.send(Err((SmallVec::new(), SerfError::NotRunning))); return; } // Announce the serf-level join intent so peers learn the local join ltime // without waiting for the next anti-entropy round. if let Err(e) = endpoint.join() { // Ignoring Err: caller dropped the reply receiver. - let _ = reply.send(Err(SerfError::from(e))); + let _ = reply.send(Err((SmallVec::new(), SerfError::from(e)))); return; } - // Dial every seed via a coordinator push-pull (the driver owns the inner- - // memberlist join). The QUIC coordinator services the dial + flushes the - // outbound queue in-band; the handshake packets surface on the next - // `poll_transmit` drain. - let count = seeds.len(); - for seed in seeds { - let _sid = endpoint.start_push_pull(seed, PushPullKind::Join, now); + // The QUIC coordinator services the dial + flushes the outbound queue + // in-band, so each `start_push_pull`'s returned machine `StreamId` coerces + // directly into the `ExchangeId` the bridge-reap path will stamp onto its + // `ExchangeCompleted` — no inline action drain / capture is needed. + match kind { + JoinKind::Dispatch => { + let mut dispatched: SmallVec<[SocketAddr; 1]> = SmallVec::new(); + for seed in seeds { + // Ignoring StreamId return: the Dispatch arm tracks no per-exchange + // waiter state — completion / failure surfaces through `poll_event`. + let _sid = endpoint.start_join_push_pull(seed, ignore_old, now); + dispatched.push(seed); + } + // Ignoring Err: caller dropped the reply receiver (the fire-and-forget + // `dispatch_join` future was cancelled). + let _ = reply.send(Ok(dispatched)); + } + JoinKind::WaitForCompletion(WaitForCompletionArgs { deadline }) => { + let requested = seeds.len(); + let mut exchange_ids: HashSet = HashSet::with_capacity(requested); + // An `ignore_old` join records every seed's `StreamId` in the machine; + // the driver owns clearing any that fail to merge. A plain join records + // nothing, so this stays empty. + let mut ignore_streams: SmallVec<[StreamId; 1]> = SmallVec::new(); + for seed in seeds { + let sid = endpoint.start_join_push_pull(seed, ignore_old, now); + if ignore_old { + ignore_streams.push(sid); + } + exchange_ids.insert(ExchangeId::from(sid)); + } + if exchange_ids.is_empty() { + // No seed produced an exchange (only reachable with a zero-length + // `seeds`, which the handle never sends for an await join). Resolve + // now — parking would hang with no terminal event incoming. + for s in &ignore_streams { + endpoint.clear_ignore_join_stream(*s); + } + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(Err(( + SmallVec::new(), + SerfError::JoinAllFailed(JoinFailed::new(requested, 0)), + ))); + } else { + pending.joins.push(PendingJoin { + pending: exchange_ids, + contacted: SmallVec::new(), + ignore_streams, + requested, + deadline, + reply: Some(reply), + }); + } + } } - // serf surfaces a push-pull's outcome as the internal `RemoteStateReceived` - // sieve, not an `ExchangeCompleted` event, so the pump reports the count of - // seeds dispatched rather than parking for per-exchange contact accounting. - // Ignoring Err: caller dropped the reply receiver. - let _ = reply.send(Ok(count)); } Command::Leave(LeaveCmd { reply }) => { // Leave is a SHARED in-flight operation. If one is in flight, JOIN it (do @@ -704,11 +900,6 @@ async fn dispatch_command( // Ignoring Err: caller dropped the reply receiver. let _ = reply.send(res); } - Command::SetEventJoinIgnore(SetEventJoinIgnoreCmd { ignore, reply }) => { - endpoint.set_event_join_ignore(ignore); - // Ignoring Err: caller dropped the reply receiver. - let _ = reply.send(Ok(())); - } #[cfg(encryption)] Command::InstallKey(KeyCmd { key, @@ -1004,6 +1195,23 @@ where let mut drained = false; while let Some(ev) = endpoint.poll_event() { drained = true; + // Await-result join resolution. `ExchangeCompleted` fires for every outbound + // bridge kind; an await-join waiter consumes only `PushPull` completions. + // `complete_join_exchange` drives both decoupled terminals: it resolves the + // caller's reply the moment `pending` empties (here on the pump task, ahead of + // the observation hand-off, so a slow delegate cannot delay it) and clears the + // ignore-join streams only once the waiter is fully done. + if let Event::ExchangeCompleted(ref c) = ev + && c.kind() == ExchangeKind::PushPull + { + complete_join_exchange( + endpoint, + &mut pending.joins, + c.eid(), + *c.peer(), + matches!(c.outcome(), ExchangeStatus::Succeeded), + ); + } // Leave-completion resolution. `LeftCluster` fires once the leave notices // have drained to the wire; resolving the parked waiter here — on this pump // task, ahead of the observation task's `notify_leave` — is what makes @@ -1182,6 +1390,61 @@ async fn observation_task( } } +/// Reap await-result join waiters on the deadline timer. This drives ONLY the +/// reply terminal: a waiter whose `deadline` has elapsed replies its partial +/// `contacted` (the same timeout semantics the caller always had). Reply +/// resolution is decoupled from ignore-stream cleanup — on a deadline that fires +/// with exchanges still live (`pending` non-empty) the waiter replies and +/// LINGERS, keeping its `ignore_streams` recorded so a late merge for a still-live +/// exchange still suppresses; the lingering waiter is then reaped by +/// [`complete_join_exchange`] when its last exchange completes. The all-exchanges- +/// done case (including a zero-exchange degenerate waiter) is reaped there too. +/// +/// A waiter is removed (and its still-recorded ignore-join streams cleared) here +/// only once BOTH terminals are reached — reply resolved AND `pending` empty — +/// covering any timer/event ordering. A stream the success-path merge already +/// consumed is absent, so the clear removes only the streams whose exchange did +/// not merge. Mirrors the stream driver's `reap_pending_joins`; `swap_remove` is +/// sound because `joins` has no ordering. +async fn reap_pending_joins( + endpoint: &mut QuicEndpoint, + pending_joins: &mut Vec, + now: Instant, +) where + I: memberlist_proto::Id + Clone, + G: Rng, + R: Rng + SeedableRng, +{ + let mut i = 0; + while i < pending_joins.len() { + if now >= pending_joins[i].deadline { + // Deadline reached: resolve the reply with the partial `contacted`. This is + // a no-op if the all-exchanges-done path already replied. + pending_joins[i].resolve_reply(); + } + if pending_joins[i].is_done() { + // Reply resolved AND every dispatched exchange completed — clear the + // ignore-join streams that did not merge and reap the waiter. + let pj = pending_joins.swap_remove(i); + for s in &pj.ignore_streams { + endpoint.clear_ignore_join_stream(*s); + } + } else { + i += 1; + } + } +} + +/// Earliest pending-join deadline, if any — folded into the per-iteration +/// `timeout_deadline` so the timer fires by the first expiring join's deadline. +fn min_pending_join_deadline(pending_joins: &[PendingJoin]) -> Option { + pending_joins + .iter() + .filter(|pj| pj.reply.is_some()) + .map(|pj| pj.deadline) + .min() +} + /// Reap a deadline-expired graceful-leave waiter. If `pending_leave`'s deadline /// has elapsed without `Event::LeftCluster` having resolved it, reply /// [`SerfError::LeaveTimeout`] to every joined replier and clear the slot. diff --git a/serf-compio/src/driver/quic/tests.rs b/serf-compio/src/driver/quic/tests.rs index 5bb7fe16..c57a6df6 100644 --- a/serf-compio/src/driver/quic/tests.rs +++ b/serf-compio/src/driver/quic/tests.rs @@ -265,3 +265,148 @@ async fn fire_quic_timeout_drains_socket_before_handle_timeout() { let _ = driver.close().await; let _ = peer.close().await; } + +/// Park one `ignore_old` await-result `PendingJoin` whose single exchange is +/// still live, with its `StreamId` recorded in the machine's ignore set and a +/// `deadline` already in the past. Returns the join's `(StreamId, ExchangeId)` +/// and the oneshot receiver the caller awaits. +fn park_ignore_old_join( + endpoint: &mut QuicEndpoint, + pending_joins: &mut Vec, + deadline: Instant, +) -> (StreamId, ExchangeId, oneshot::Receiver) { + let seed: SocketAddr = "127.0.0.1:7946".parse().expect("seed addr"); + // `ignore_old = true` records the returned `StreamId` in the machine's + // per-exchange ignore set — the entry whose premature clear is the bug. + let sid = endpoint.start_join_push_pull(seed, true, Instant::now()); + let eid = ExchangeId::from(sid); + let (tx, rx) = oneshot::channel::(); + pending_joins.push(PendingJoin { + pending: core::iter::once(eid).collect(), + contacted: SmallVec::new(), + ignore_streams: core::iter::once(sid).collect(), + requested: 1, + deadline, + reply: Some(tx), + }); + (sid, eid, rx) +} + +/// The deadline reaper must NOT clear an `ignore_old` join's ignore `StreamId` +/// while its push/pull exchange is still live: it replies to the caller (deadline +/// path) but the waiter LINGERS with its `StreamId` recorded, so a late merge for +/// that still-live exchange still suppresses the peer's pre-join user events. Only +/// when the exchange finally completes is the waiter reaped and its ignore stream +/// cleared. This is the premature-clear / cancellation-safety regression. +#[compio::test] +async fn deadline_reap_keeps_ignore_stream_until_exchange_completes() { + let mut endpoint = build_endpoint(); + let mut joins: Vec = Vec::new(); + // Deadline already elapsed; the exchange is still pending. + let past = Instant::now() - Duration::from_secs(1); + let (sid, eid, rx) = park_ignore_old_join(&mut endpoint, &mut joins, past); + + // Reap on the elapsed deadline. Reply resolution is decoupled from ignore-stream + // cleanup: the caller is answered, but the waiter must linger. + reap_pending_joins::(&mut endpoint, &mut joins, Instant::now()).await; + + // The caller got the deadline reply (zero contacts -> JoinAllFailed) ... + match rx.await { + Ok(Err((_set, SerfError::JoinAllFailed(_)))) => {} + other => panic!("deadline reap must reply JoinAllFailed, got {other:?}"), + } + // ... but the waiter LINGERED rather than being removed, and its ignore + // `StreamId` was NOT cleared — it stays recorded for the still-live exchange so + // a late merge for that exchange still suppresses the peer's pre-join user + // events. Removing the waiter and clearing the stream here would let that late + // merge replay them. + assert_eq!( + joins.len(), + 1, + "the waiter must linger past its reply while the exchange is still live" + ); + assert!( + joins[0].reply.is_none(), + "the deadline reply must have resolved (reply taken)" + ); + assert_eq!( + joins[0].ignore_streams.as_slice(), + &[sid], + "the ignore StreamId must stay recorded for the still-live exchange" + ); + assert!( + joins[0].pending.contains(&eid), + "the live exchange is still pending" + ); + + // The delayed terminal `ExchangeCompleted` (a failure / timeout / decode error + // all surface as `Failed`) finally arrives: NOW the waiter is reaped and its + // ignore stream cleared. No second reply is sent (the deadline already replied). + let peer: SocketAddr = "127.0.0.1:7946".parse().expect("peer addr"); + complete_join_exchange::(&mut endpoint, &mut joins, eid, peer, false); + assert!( + joins.is_empty(), + "the waiter must be reaped once its last exchange completes" + ); +} + +/// A deadline-reaped join lingers with `reply == None` while its exchange is +/// still live. `min_pending_join_deadline` must exclude it: the lingering waiter +/// has already replied to its caller and is cleaned up by `complete_join_exchange` +/// when its `ExchangeCompleted` arrives (I/O-driven), NOT by a timer. Contributing +/// a past deadline here causes a CPU busy-spin for the remaining stream-timeout +/// window. +#[compio::test] +async fn resolved_lingering_join_excluded_from_min_deadline() { + let mut endpoint = build_endpoint(); + let mut joins: Vec = Vec::new(); + // Past deadline — the join will be reaped by the timer, leaving a lingering + // waiter (reply == None, exchange still pending). + let past = Instant::now() - Duration::from_secs(1); + let (_sid, _eid, _rx) = park_ignore_old_join(&mut endpoint, &mut joins, past); + + // Reap: the reply resolves (reply taken -> None), waiter lingers. + reap_pending_joins::(&mut endpoint, &mut joins, Instant::now()).await; + + assert_eq!(joins.len(), 1, "the lingering waiter must still be present"); + assert!( + joins[0].reply.is_none(), + "the deadline reply must have resolved (reply taken)" + ); + + // The lingering waiter (reply == None) must NOT contribute a timer deadline — + // returning None here is what prevents the busy-spin. + assert_eq!( + min_pending_join_deadline(&joins), + None, + "a resolved-lingering waiter must not contribute a timer deadline" + ); +} + +/// A success-path merge consumes the ignore `StreamId` BEFORE the exchange's +/// `ExchangeCompleted`, so the count->0 clear in `complete_join_exchange` is a +/// no-op for it: the all-exchanges-done path resolves the reply with the contact +/// and reaps the waiter. This guards the non-deadline terminal that shares the +/// reap path with the deadline-linger case. +#[compio::test] +async fn all_exchanges_done_resolves_and_reaps() { + let mut endpoint = build_endpoint(); + let mut joins: Vec = Vec::new(); + // A far-future deadline: this resolution is driven by exchange completion, not + // the timer. + let future = Instant::now() + Duration::from_secs(60); + let (_sid, eid, rx) = park_ignore_old_join(&mut endpoint, &mut joins, future); + + // The exchange completes Succeeded; its peer enters `contacted` and, with no + // exchanges left pending, the reply resolves and the waiter is reaped. + let peer: SocketAddr = "127.0.0.1:7946".parse().expect("peer addr"); + complete_join_exchange::(&mut endpoint, &mut joins, eid, peer, true); + assert!( + joins.is_empty(), + "an all-exchanges-done waiter is reaped on completion, not left for the timer" + ); + match rx.await { + Ok(Ok(contacted)) => assert_eq!(contacted.as_slice(), &[peer]), + other => panic!("a successful exchange must reply Ok(contacted), got {other:?}"), + } +} diff --git a/serf-compio/src/driver/stream/mod.rs b/serf-compio/src/driver/stream/mod.rs index 6bebb887..94592bee 100644 --- a/serf-compio/src/driver/stream/mod.rs +++ b/serf-compio/src/driver/stream/mod.rs @@ -14,7 +14,13 @@ //! gossip socket are explicitly closed (awaited) when the loop exits so the //! bound ports are released before shutdown returns. -use std::{cell::Cell, collections::HashMap, io, net::SocketAddr, rc::Rc}; +use std::{ + cell::Cell, + collections::{HashMap, HashSet}, + io, + net::SocketAddr, + rc::Rc, +}; use core::{ task::{Context, Poll, Waker}, @@ -30,14 +36,17 @@ use flume::{Receiver, Sender}; use futures_util::{FutureExt, future::FusedFuture, pin_mut, select_biased}; use lochan::mpsc; use memberlist_proto::{ - Instant, PushPullKind, SeedableRng, Transmit, + Instant, SeedableRng, StreamId, Transmit, codec::{ DecodeOptions, EncodeOptions, decode_incoming, encode_outgoing, encode_outgoing_compound, parse_messages, }, streams::{StreamAction, StreamTransport}, }; -use serf_proto::{LamportTime, StreamEndpoint, event::Event, members::SerfState}; +use serf_proto::{ + ExchangeKind, ExchangeStatus, LamportTime, StreamEndpoint, event::Event, members::SerfState, +}; +use smallvec::SmallVec; #[cfg(encryption)] use crate::command::{KeyCmd, ListKeysCmd}; @@ -46,8 +55,8 @@ use crate::delegate::KeyringDelegate; use crate::{ Channel, command::{ - Command, ForceLeaveCmd, JoinCmd, LeaveCmd, QueryCmd, RespondCmd, SetEventJoinIgnoreCmd, - SetTagsCmd, ShutdownCmd, + Command, ForceLeaveCmd, JoinCmd, JoinKind, JoinReply, LeaveCmd, QueryCmd, RespondCmd, + SetTagsCmd, ShutdownCmd, WaitForCompletionArgs, }, delegate::Delegate, driver::{ @@ -57,12 +66,139 @@ use crate::{ observation_payload_bytes, yield_once, }, }, - error::{Result, SerfError}, + error::{JoinFailed, Result, SerfError}, snapshot::{SerfSnapshot, SnapshotCell}, }; #[cfg(encryption)] use serf_proto::{KeyRequestOperation, KeyResponseArgs, event::KeyRequest}; +/// Driver-side state for one outstanding await-result join call. +/// +/// A [`Command::Join`] carrying [`JoinKind::WaitForCompletion`] dispatches one +/// push/pull per resolved seed and parks the per-call state here. Contact +/// accounting is strictly per-OUTBOUND-EXCHANGE, observed via the machine's +/// [`Event::ExchangeCompleted`] filtered to [`ExchangeKind::PushPull`]: each +/// `start_push_pull` allocates a fresh [`ExchangeId`]; the driver tracks every +/// dispatched id in `pending` and pushes the peer address into `contacted` for +/// each that terminates [`ExchangeStatus::Succeeded`]. Tracking by `ExchangeId` +/// (not by `SocketAddr`) keeps duplicate-seed semantics: passing the same +/// address twice yields two exchanges counted independently. +/// +/// Reply resolution and ignore-stream cleanup are SEPARATE terminal states. The +/// caller's reply resolves on all-exchanges-done OR `deadline` (whichever first), +/// consuming `reply`. The ignore-stream cleanup must wait until every dispatched +/// exchange has completed — i.e. `pending` is empty — because a `StreamId` +/// recorded for a still-live exchange must stay in the machine's ignore set so a +/// late merge still suppresses the peer's pre-join user events. The waiter is +/// removed from `joins` only once BOTH terminals are reached (reply sent and +/// `pending` empty); on a deadline that fires with exchanges still live it +/// replies and LINGERS, holding its `ignore_streams` until they complete. +struct PendingJoin { + /// Outbound exchange ids this waiter dispatched and is still awaiting a + /// terminal `ExchangeCompleted` for. An id is removed when its completion + /// arrives (success or failure); when this set empties the ignore-stream + /// cleanup runs and the waiter is reaped. + pending: HashSet, + /// Peer addresses of the dispatched exchanges that terminated `Succeeded`. + /// Duplicate seeds contribute one entry per successful exchange. + contacted: SmallVec<[SocketAddr; 1]>, + /// The `StreamId`s this join recorded in the machine's per-exchange ignore set + /// (non-empty only for an `ignore_old` join). Each is consumed by its own merge + /// on the success path; any that did NOT merge (dial failure / timeout / empty + /// body) are cleared via `clear_ignore_join_stream` only once every dispatched + /// exchange has completed (`pending` empty), so a failed `ignore_old` exchange + /// never leaks its `StreamId` and a still-live one is never prematurely cleared. + ignore_streams: SmallVec<[StreamId; 1]>, + /// Total outbound-exchange count this call dispatched — the `JoinAllFailed` + /// denominator on a zero-contact resolution. + requested: usize, + /// Wall-clock instant past which the driver replies with whatever `contacted` + /// set it has accumulated even if `pending` is non-empty. + deadline: Instant, + /// One-shot reply channel back to the caller, taken when the reply resolves + /// (all-exchanges-done or `deadline`). `None` once resolved; the waiter then + /// lingers — only to drive ignore-stream cleanup — until `pending` empties. + /// See [`JoinReply`]. + reply: Option>, +} + +impl PendingJoin { + /// Resolve the caller's reply once, from the current `contacted` set. Idempotent: + /// after the first call `reply` is `None` and this is a no-op, so the deadline + /// path and the all-exchanges-done path never double-send. A zero-contact + /// resolution is the `JoinAllFailed` the reaper would otherwise have produced. + fn resolve_reply(&mut self) { + if let Some(reply) = self.reply.take() { + let result = if self.contacted.is_empty() { + Err(( + SmallVec::new(), + SerfError::JoinAllFailed(JoinFailed::new(self.requested, 0)), + )) + } else { + Ok(self.contacted.clone()) + }; + // Ignoring Err: caller dropped the reply receiver (the join future was + // cancelled). + let _ = reply.send(result); + } + } + + /// This waiter has reached both terminal states — its reply resolved AND every + /// dispatched exchange completed — so it can be removed and its ignore-stream + /// cleanup run. + fn is_done(&self) -> bool { + self.reply.is_none() && self.pending.is_empty() + } +} + +/// Apply one terminal `ExchangeCompleted` to its await-result join waiter (if +/// any), driving both decoupled terminals: remove `eid` from the waiter's +/// `pending` and, on success, push the peer into `contacted`; resolve the +/// caller's reply the instant `pending` empties (ahead of the observation +/// hand-off, so a slow delegate cannot delay it); and once the waiter is fully +/// done (reply sent AND `pending` empty) clear its still-recorded ignore-join +/// streams and reap it. A `StreamId` the success-path merge already consumed is +/// absent, so the clear removes only the streams whose exchange did not merge. +/// +/// Shared by the live drain and unit tests; takes the decoded +/// `(eid, peer, succeeded)` rather than the `Event` so it is callable without +/// constructing a coordinator-internal `ExchangeCompleted`. +fn complete_join_exchange( + endpoint: &mut StreamEndpoint, + pending_joins: &mut Vec, + eid: ExchangeId, + peer: SocketAddr, + succeeded: bool, +) where + I: memberlist_proto::Id + Clone, + RT: StreamTransport, + G: rand::Rng, + R: rand::Rng + SeedableRng, +{ + let Some(idx) = pending_joins + .iter() + .position(|pj| pj.pending.contains(&eid)) + else { + return; + }; + let pj = &mut pending_joins[idx]; + pj.pending.remove(&eid); + if succeeded { + pj.contacted.push(peer); + } + // Resolve the reply the moment every dispatched exchange has terminated. If the + // deadline already replied, `reply` is `None` and this is a no-op. + if pj.pending.is_empty() { + pj.resolve_reply(); + } + if pending_joins[idx].is_done() { + let pj = pending_joins.swap_remove(idx); + for s in &pj.ignore_streams { + endpoint.clear_ignore_join_stream(*s); + } + } +} + /// Driver-side state for the single in-flight graceful-leave operation. /// /// A [`Command::Leave`] that finds the endpoint `Alive` initiates the machine's @@ -106,6 +242,8 @@ impl PendingLeave { /// Pump-loop-local state tracking outstanding commands awaiting completion. struct PendingCommands { + /// Outstanding await-result join waiters. See [`PendingJoin`]. + joins: Vec, /// Outstanding graceful-leave waiter (at most one at a time). See [`PendingLeave`]. leave: Option, } @@ -326,7 +464,10 @@ pub(crate) async fn stream_driver_loop( // cleanup closes the listener and gossip socket so the bound ports are free // when the caller resumes from `shutdown.await`. let mut shutdown_reply: Option>> = None; - let mut pending = PendingCommands { leave: None }; + let mut pending = PendingCommands { + joins: Vec::new(), + leave: None, + }; // Per-pump UDP recv buffer size, derived once at entry from the coordinator's // `gossip_mtu` (fixed for the endpoint lifetime). @@ -484,6 +625,7 @@ pub(crate) async fn stream_driver_loop( &*keyring, ) .await; + reap_pending_joins(&mut endpoint, &mut pending.joins, Instant::now()).await; reap_pending_leave(&mut pending.leave, Instant::now()).await; refresh_snapshot::(&endpoint, &snapshot); break; @@ -499,6 +641,7 @@ pub(crate) async fn stream_driver_loop( .unwrap_or(setup_now + driver_opts.idle_wake_interval()); let timeout_deadline = [ Some(endpoint_deadline), + min_pending_join_deadline(&pending.joins), min_pending_leave_deadline(&pending.leave), ] .into_iter() @@ -550,6 +693,7 @@ pub(crate) async fn stream_driver_loop( &*keyring, ) .await; + reap_pending_joins(&mut endpoint, &mut pending.joins, Instant::now()).await; reap_pending_leave(&mut pending.leave, Instant::now()).await; if dirty { refresh_snapshot::(&endpoint, &snapshot); @@ -582,6 +726,7 @@ pub(crate) async fn stream_driver_loop( &*keyring, ) .await; + reap_pending_joins(&mut endpoint, &mut pending.joins, Instant::now()).await; reap_pending_leave(&mut pending.leave, Instant::now()).await; refresh_snapshot::(&endpoint, &snapshot); dirty = false; @@ -734,6 +879,7 @@ pub(crate) async fn stream_driver_loop( { exit = true; } + reap_pending_joins(&mut endpoint, &mut pending.joins, Instant::now()).await; reap_pending_leave(&mut pending.leave, Instant::now()).await; if dirty { @@ -755,6 +901,21 @@ pub(crate) async fn stream_driver_loop( reply_shutdown(c); } drop(commands); + // Reply Err(Shutdown) to every parked await-result join waiter whose reply has + // not yet resolved — their reply receivers would otherwise hang forever (the + // loop's reap path is gone and the task is exiting). Clear EVERY remaining + // waiter's ignore-join streams (including a lingering waiter that already + // replied on its deadline but was awaiting exchange completion): after shutdown + // no late merge can arrive, so nothing is left to suppress. + for pj in pending.joins.drain(..) { + for s in &pj.ignore_streams { + endpoint.clear_ignore_join_stream(*s); + } + if let Some(reply) = pj.reply { + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(Err((SmallVec::new(), SerfError::Shutdown))); + } + } if let Some(pl) = pending.leave.take() { pl.resolve_all(|| Err(SerfError::Shutdown)).await; } @@ -790,14 +951,11 @@ fn reply_shutdown(c: Command) { // Ignoring Err on each send: caller dropped the reply receiver. match c { Command::Join(JoinCmd { reply, .. }) => { - let _ = reply.send(Err(SerfError::Shutdown)); + let _ = reply.send(Err((SmallVec::new(), SerfError::Shutdown))); } Command::Leave(LeaveCmd { reply }) | Command::Shutdown(ShutdownCmd { reply }) => { let _ = reply.send(Err(SerfError::Shutdown)); } - Command::SetEventJoinIgnore(SetEventJoinIgnoreCmd { reply, .. }) => { - let _ = reply.send(Err(SerfError::Shutdown)); - } Command::ForceLeave(ForceLeaveCmd { reply, .. }) => { let _ = reply.send(Err(SerfError::Shutdown)); } @@ -852,36 +1010,103 @@ async fn dispatch_command( { let running = endpoint.state() == SerfState::Alive; match cmd { - Command::Join(JoinCmd { seeds, reply }) => { + Command::Join(JoinCmd { + seeds, + kind, + ignore_old, + reply, + }) => { // Gate on a running node: `leave()` is terminal (it stops the periodic // schedulers), so a join after leave would leave the node non-participating. if !running { // Ignoring Err: caller dropped the reply receiver. - let _ = reply.send(Err(SerfError::NotRunning)); + let _ = reply.send(Err((SmallVec::new(), SerfError::NotRunning))); return; } // Announce the serf-level join intent so peers learn the local join ltime // without waiting for the next anti-entropy round. if let Err(e) = endpoint.join() { // Ignoring Err: caller dropped the reply receiver. - let _ = reply.send(Err(SerfError::from(e))); + let _ = reply.send(Err((SmallVec::new(), SerfError::from(e)))); return; } - // Dial every seed via a coordinator push-pull (the driver owns the - // inner-memberlist join). Each `start_push_pull` queues a `Connect` the - // inline drain routes to its bridge before the next seed dials. - let count = seeds.len(); - for seed in seeds { - let _sid = endpoint.start_push_pull(seed, PushPullKind::Join, now); - while let Some(action) = endpoint.poll_action() { - process_one_action(action, bridges, bridge_ready_tx, stream_opts); + // Both kinds share the same `start_push_pull` fan-out; the kind only + // affects WHEN the reply fires. Each `start_push_pull` queues a `Connect` + // the inline drain routes to its bridge before the next seed dials. + match kind { + JoinKind::Dispatch => { + let mut dispatched: SmallVec<[SocketAddr; 1]> = SmallVec::new(); + for seed in seeds { + // Ignoring StreamId return: the Dispatch arm tracks no per-exchange + // waiter state — completion / failure surfaces through `poll_event`. + let _sid = endpoint.start_join_push_pull(seed, ignore_old, now); + while let Some(action) = endpoint.poll_action() { + process_one_action(action, bridges, bridge_ready_tx, stream_opts, None); + } + dispatched.push(seed); + } + // Ignoring Err: caller dropped the reply receiver (the fire-and-forget + // `dispatch_join` future was cancelled). + let _ = reply.send(Ok(dispatched)); + } + JoinKind::WaitForCompletion(WaitForCompletionArgs { deadline }) => { + // Capture the resolved seed count BEFORE the loop consumes `seeds`: + // this is the `JoinAllFailed` denominator. A seed that retires before + // producing a `Connect` (TLS sni miss, dialer error, elapsed dial + // deadline) never enters `exchange_ids`, so deriving `requested` from + // the captured-exchange count would undercount the requested seeds. + let requested = seeds.len(); + let mut exchange_ids: HashSet = HashSet::with_capacity(requested); + // The `StreamId`s this join's `start_push_pull` calls returned; the + // Connect capture keys on this set (not the peer) so a same-peer dial + // flushed for another subsystem is never misattributed to this join. + let mut started: HashSet = HashSet::with_capacity(requested); + for seed in seeds { + let sid = endpoint.start_join_push_pull(seed, ignore_old, now); + started.insert(sid); + while let Some(action) = endpoint.poll_action() { + process_one_action( + action, + bridges, + bridge_ready_tx, + stream_opts, + Some((&started, &mut exchange_ids)), + ); + } + } + // An `ignore_old` join recorded every seed's `StreamId` in the machine + // (via `start_join_push_pull`); the driver owns clearing any that fail + // to merge. A plain join recorded nothing, so this stays empty. + let ignore_streams: SmallVec<[StreamId; 1]> = if ignore_old { + started.iter().copied().collect() + } else { + SmallVec::new() + }; + if exchange_ids.is_empty() { + // Every seed retired before a `Connect`, so no exchange will ever + // surface a terminal `ExchangeCompleted` (nor a merge). Parking would + // idle the waiter until `deadline`; resolve now with the all-failed + // outcome, and clear the recorded ignore streams that will never merge. + for s in &ignore_streams { + endpoint.clear_ignore_join_stream(*s); + } + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(Err(( + SmallVec::new(), + SerfError::JoinAllFailed(JoinFailed::new(requested, 0)), + ))); + } else { + pending.joins.push(PendingJoin { + pending: exchange_ids, + contacted: SmallVec::new(), + ignore_streams, + requested, + deadline, + reply: Some(reply), + }); + } } } - // serf surfaces a push-pull's outcome as the internal `RemoteStateReceived` - // sieve, not an `ExchangeCompleted` event, so the pump reports the count of - // seeds dispatched rather than parking for per-exchange contact accounting. - // Ignoring Err: caller dropped the reply receiver. - let _ = reply.send(Ok(count)); } Command::Leave(LeaveCmd { reply }) => { // Leave is a SHARED in-flight operation. If one is in flight, JOIN it (do @@ -974,11 +1199,6 @@ async fn dispatch_command( // Ignoring Err: caller dropped the reply receiver. let _ = reply.send(res); } - Command::SetEventJoinIgnore(SetEventJoinIgnoreCmd { ignore, reply }) => { - endpoint.set_event_join_ignore(ignore); - // Ignoring Err: caller dropped the reply receiver. - let _ = reply.send(Ok(())); - } #[cfg(encryption)] Command::InstallKey(KeyCmd { key, @@ -1146,16 +1366,26 @@ fn dispatch_gossip( /// BEFORE spawning the dial task, so bytes the machine surfaces on the same tick /// as the Connect reach the bridge via the `out_rx` handed to it on dial /// completion. Shutdown / Close / Abort signal the per-bridge channel. +/// +/// `capture` is `Some` only on the await-result join path: when the Connect's +/// originating [`StreamId`] is in the `started` set, its [`ExchangeId`] is +/// recorded into the waiter's pending set, binding it to that join alone. fn process_one_action( action: StreamAction, bridges: &mut HashMap, bridge_ready_tx: &Sender, stream_opts: StreamTransportOptions, + capture: Option<(&HashSet, &mut HashSet)>, ) { match action { StreamAction::Connect(info) => { let eid = info.id(); let peer = info.peer(); + if let Some((started, pending_exchanges)) = capture + && started.contains(&info.stream_id()) + { + pending_exchanges.insert(eid); + } let (out_tx, out_rx) = mpsc::unbounded::(); let (cancel_tx, cancel_rx) = futures_channel::oneshot::channel::<()>(); bridges.insert(eid, BridgeHandle { out_tx, cancel_tx }); @@ -1239,7 +1469,7 @@ where let mut progress = false; while let Some(action) = endpoint.poll_action() { progress = true; - process_one_action(action, bridges, bridge_ready_tx, stream_opts); + process_one_action(action, bridges, bridge_ready_tx, stream_opts, None); } progress } @@ -1385,6 +1615,23 @@ where let mut drained = false; while let Some(ev) = endpoint.poll_event() { drained = true; + // Await-result join resolution. `ExchangeCompleted` fires for every outbound + // bridge kind; an await-join waiter consumes only `PushPull` completions. + // `complete_join_exchange` drives both decoupled terminals: it resolves the + // caller's reply the moment `pending` empties (here on the pump task, ahead of + // the observation hand-off, so a slow delegate cannot delay it) and clears the + // ignore-join streams only once the waiter is fully done. + if let Event::ExchangeCompleted(ref c) = ev + && c.kind() == ExchangeKind::PushPull + { + complete_join_exchange( + endpoint, + &mut pending.joins, + c.eid(), + *c.peer(), + matches!(c.outcome(), ExchangeStatus::Succeeded), + ); + } // Leave-completion resolution. `LeftCluster` fires once the leave notices // have drained to the wire; resolving the parked waiter here — on this pump // task, ahead of the observation task's `notify_leave` — is what makes @@ -1543,6 +1790,61 @@ async fn observation_task( } } +/// Reap await-result join waiters on the deadline timer. This drives ONLY the +/// reply terminal: a waiter whose `deadline` has elapsed replies its partial +/// `contacted` (the same timeout semantics the caller always had). Reply +/// resolution is decoupled from ignore-stream cleanup — on a deadline that fires +/// with exchanges still live (`pending` non-empty) the waiter replies and +/// LINGERS, keeping its `ignore_streams` recorded so a late merge for a still-live +/// exchange still suppresses; the lingering waiter is then reaped by +/// [`complete_join_exchange`] when its last exchange completes. The all-exchanges- +/// done case (including a zero-exchange degenerate waiter) is reaped there too. +/// +/// A waiter is removed (and its still-recorded ignore-join streams cleared) here +/// only once BOTH terminals are reached — reply resolved AND `pending` empty — +/// covering any timer/event ordering. A stream the success-path merge already +/// consumed is absent, so the clear removes only the streams whose exchange did +/// not merge. `swap_remove` is sound because `joins` has no ordering. +async fn reap_pending_joins( + endpoint: &mut StreamEndpoint, + pending_joins: &mut Vec, + now: Instant, +) where + I: memberlist_proto::Id + Clone, + RT: StreamTransport, + G: rand::Rng, + R: rand::Rng + SeedableRng, +{ + let mut i = 0; + while i < pending_joins.len() { + if now >= pending_joins[i].deadline { + // Deadline reached: resolve the reply with the partial `contacted`. This is + // a no-op if the all-exchanges-done path already replied. + pending_joins[i].resolve_reply(); + } + if pending_joins[i].is_done() { + // Reply resolved AND every dispatched exchange completed — clear the + // ignore-join streams that did not merge and reap the waiter. + let pj = pending_joins.swap_remove(i); + for s in &pj.ignore_streams { + endpoint.clear_ignore_join_stream(*s); + } + } else { + i += 1; + } + } +} + +/// Earliest pending-join deadline, if any — folded into the per-iteration +/// `timeout_deadline` so the timer fires by the first expiring join's deadline. +fn min_pending_join_deadline(pending_joins: &[PendingJoin]) -> Option { + pending_joins + .iter() + .filter(|pj| pj.reply.is_some()) + .map(|pj| pj.deadline) + .min() +} + /// Reap a deadline-expired graceful-leave waiter. If `pending_leave`'s deadline /// has elapsed without `Event::LeftCluster` having resolved it, reply /// [`SerfError::LeaveTimeout`] to every joined replier and clear the slot. diff --git a/serf-compio/src/driver/stream/tests.rs b/serf-compio/src/driver/stream/tests.rs index 6c379bef..bee1e58f 100644 --- a/serf-compio/src/driver/stream/tests.rs +++ b/serf-compio/src/driver/stream/tests.rs @@ -145,3 +145,174 @@ async fn fire_timeout_with_drain_drains_socket_before_handle_timeout() { let _ = driver.close().await; let _ = peer.close().await; } + +/// Park one `ignore_old` await-result `PendingJoin` whose single exchange is +/// still live, with its `StreamId` recorded in the machine's ignore set and a +/// `deadline` already in the past. Returns the join's `(StreamId, ExchangeId)` +/// and the oneshot receiver the caller awaits. +fn park_ignore_old_join( + endpoint: &mut StreamEndpoint, + pending_joins: &mut Vec, + deadline: Instant, +) -> ( + StreamId, + ExchangeId, + futures_channel::oneshot::Receiver, +) { + let seed: SocketAddr = "127.0.0.1:7946".parse().expect("seed addr"); + // `ignore_old = true` records the returned `StreamId` in the machine's + // per-exchange ignore set — the entry whose premature clear is the bug. + let sid = endpoint.start_join_push_pull(seed, true, Instant::now()); + let eid = ExchangeId::from(sid); + let (tx, rx) = futures_channel::oneshot::channel::(); + pending_joins.push(PendingJoin { + pending: core::iter::once(eid).collect(), + contacted: SmallVec::new(), + ignore_streams: core::iter::once(sid).collect(), + requested: 1, + deadline, + reply: Some(tx), + }); + (sid, eid, rx) +} + +/// The deadline reaper must NOT clear an `ignore_old` join's ignore `StreamId` +/// while its push/pull exchange is still live: it replies to the caller (deadline +/// path) but the waiter LINGERS with its `StreamId` recorded, so a late merge for +/// that still-live exchange still suppresses the peer's pre-join user events. Only +/// when the exchange finally completes is the waiter reaped and its ignore stream +/// cleared. This is the premature-clear / cancellation-safety regression. +#[compio::test] +async fn deadline_reap_keeps_ignore_stream_until_exchange_completes() { + let mut endpoint = build_endpoint(); + let mut joins: Vec = Vec::new(); + // Deadline already elapsed; the exchange is still pending. + let past = Instant::now() - Duration::from_secs(1); + let (sid, eid, rx) = park_ignore_old_join(&mut endpoint, &mut joins, past); + + // Reap on the elapsed deadline. Reply resolution is decoupled from ignore-stream + // cleanup: the caller is answered, but the waiter must linger. + reap_pending_joins::( + &mut endpoint, + &mut joins, + Instant::now(), + ) + .await; + + // The caller got the deadline reply (zero contacts -> JoinAllFailed) ... + match rx.await { + Ok(Err((_set, SerfError::JoinAllFailed(_)))) => {} + other => panic!("deadline reap must reply JoinAllFailed, got {other:?}"), + } + // ... but the waiter LINGERED rather than being removed, and its ignore + // `StreamId` was NOT cleared — it stays recorded for the still-live exchange so + // a late merge for that exchange still suppresses the peer's pre-join user + // events. Removing the waiter and clearing the stream here would let that late + // merge replay them. + assert_eq!( + joins.len(), + 1, + "the waiter must linger past its reply while the exchange is still live" + ); + assert!( + joins[0].reply.is_none(), + "the deadline reply must have resolved (reply taken)" + ); + assert_eq!( + joins[0].ignore_streams.as_slice(), + &[sid], + "the ignore StreamId must stay recorded for the still-live exchange" + ); + assert!( + joins[0].pending.contains(&eid), + "the live exchange is still pending" + ); + + // The delayed terminal `ExchangeCompleted` (a failure / timeout / decode error + // all surface as `Failed`) finally arrives: NOW the waiter is reaped and its + // ignore stream cleared. No second reply is sent (the deadline already replied). + let peer: SocketAddr = "127.0.0.1:7946".parse().expect("peer addr"); + complete_join_exchange::( + &mut endpoint, + &mut joins, + eid, + peer, + false, + ); + assert!( + joins.is_empty(), + "the waiter must be reaped once its last exchange completes" + ); +} + +/// A deadline-reaped join lingers with `reply == None` while its exchange is +/// still live. `min_pending_join_deadline` must exclude it: the lingering waiter +/// has already replied to its caller and is cleaned up by `complete_join_exchange` +/// when its `ExchangeCompleted` arrives (I/O-driven), NOT by a timer. Contributing +/// a past deadline here causes a CPU busy-spin for the remaining stream-timeout +/// window. +#[compio::test] +async fn resolved_lingering_join_excluded_from_min_deadline() { + let mut endpoint = build_endpoint(); + let mut joins: Vec = Vec::new(); + // Past deadline — the join will be reaped by the timer, leaving a lingering + // waiter (reply == None, exchange still pending). + let past = Instant::now() - Duration::from_secs(1); + let (_sid, _eid, _rx) = park_ignore_old_join(&mut endpoint, &mut joins, past); + + // Reap: the reply resolves (reply taken -> None), waiter lingers. + reap_pending_joins::( + &mut endpoint, + &mut joins, + Instant::now(), + ) + .await; + + assert_eq!(joins.len(), 1, "the lingering waiter must still be present"); + assert!( + joins[0].reply.is_none(), + "the deadline reply must have resolved (reply taken)" + ); + + // The lingering waiter (reply == None) must NOT contribute a timer deadline — + // returning None here is what prevents the busy-spin. + assert_eq!( + min_pending_join_deadline(&joins), + None, + "a resolved-lingering waiter must not contribute a timer deadline" + ); +} + +/// A success-path merge consumes the ignore `StreamId` BEFORE the exchange's +/// `ExchangeCompleted`, so the count->0 clear in `complete_join_exchange` is a +/// no-op for it: the all-exchanges-done path resolves the reply with the contact +/// and reaps the waiter. This guards the non-deadline terminal that shares the +/// reap path with the deadline-linger case. +#[compio::test] +async fn all_exchanges_done_resolves_and_reaps() { + let mut endpoint = build_endpoint(); + let mut joins: Vec = Vec::new(); + // A far-future deadline: this resolution is driven by exchange completion, not + // the timer. + let future = Instant::now() + Duration::from_secs(60); + let (_sid, eid, rx) = park_ignore_old_join(&mut endpoint, &mut joins, future); + + // The exchange completes Succeeded; its peer enters `contacted` and, with no + // exchanges left pending, the reply resolves and the waiter is reaped. + let peer: SocketAddr = "127.0.0.1:7946".parse().expect("peer addr"); + complete_join_exchange::( + &mut endpoint, + &mut joins, + eid, + peer, + true, + ); + assert!( + joins.is_empty(), + "an all-exchanges-done waiter is reaped on completion, not left for the timer" + ); + match rx.await { + Ok(Ok(contacted)) => assert_eq!(contacted.as_slice(), &[peer]), + other => panic!("a successful exchange must reply Ok(contacted), got {other:?}"), + } +} diff --git a/serf-compio/src/error/mod.rs b/serf-compio/src/error/mod.rs index 90b285df..cf800e0c 100644 --- a/serf-compio/src/error/mod.rs +++ b/serf-compio/src/error/mod.rs @@ -3,7 +3,7 @@ use core::fmt; use std::{io, net::SocketAddr}; -pub use serf_driver::error::{GossipMtuTooSmall, InvalidOption}; +pub use serf_driver::error::{GossipMtuTooSmall, InvalidOption, JoinFailed}; /// Payload for [`SerfError::InvalidGossipMtu`]: the configured `gossip_mtu` /// exceeds the largest plaintext gossip payload that can fit a single UDP @@ -148,6 +148,14 @@ pub enum SerfError { #[error("the local node has left the cluster; the operation requires a running node")] NotRunning, + /// An await-result [`join`](crate::Serf::join) dispatched a push/pull to one + /// or more resolved seeds but none was contacted before the call resolved — + /// the typical "all seeds unreachable" cluster-bootstrap failure. A non-empty + /// seed input that resolved to zero addresses surfaces here too, rather than + /// collapsing to a silent success. + #[error(transparent)] + JoinAllFailed(JoinFailed), + /// The configured `gossip_mtu` exceeds the ceiling after the encryption /// wrapper is applied. Returned at construction (fail-fast, before any socket /// is bound) so the misconfiguration is surfaced rather than producing diff --git a/serf-compio/src/error/tests.rs b/serf-compio/src/error/tests.rs index 246603d7..2ee94e47 100644 --- a/serf-compio/src/error/tests.rs +++ b/serf-compio/src/error/tests.rs @@ -54,6 +54,7 @@ fn every_variant_displays_and_debugs() { SerfError::LeaveTimeout, SerfError::Shutdown, SerfError::NotRunning, + SerfError::JoinAllFailed(JoinFailed::new(3, 0)), SerfError::InvalidGossipMtu(InvalidGossipMtu::new(70_000, 65_467)), SerfError::GossipMtuTooSmall(GossipMtuTooSmall::new(64, 512)), SerfError::InvalidAdvertiseAddr(InvalidAdvertiseAddr::new( diff --git a/serf-compio/src/quic/tests.rs b/serf-compio/src/quic/tests.rs index 189cca90..ff678e3c 100644 --- a/serf-compio/src/quic/tests.rs +++ b/serf-compio/src/quic/tests.rs @@ -20,10 +20,29 @@ use serf_proto::{ use smol_str::SmolStr; use crate::{ - Channel, FirstAddrResolver, QuicOptions, QuicTransport, QuicTransportOptions, RuntimeOptions, - Serf, SerfError, SocketAddrResolver, Transport, VoidDelegate, gossip_rng, + Channel, FirstAddrResolver, QuicOptions, QuicTransport, QuicTransportOptions, Resolver, + RuntimeOptions, Serf, SerfError, SocketAddrResolver, Transport, VoidDelegate, gossip_rng, }; +/// A loopback address with a port nothing listens on — its QUIC push/pull dial +/// never completes a handshake, so its exchange fails. The port is below the OS +/// ephemeral range, so a `:0` test bind never collides with it. +fn blackhole_addr() -> SocketAddr { + "127.0.0.1:7214".parse().expect("loopback addr") +} + +/// Resolver that always resolves to an empty address list. +struct EmptyResolver; + +impl Resolver for EmptyResolver { + type Address = String; + type Error = std::io::Error; + + async fn resolve(&self, _addr: &String) -> Result, std::io::Error> { + Ok(Vec::new()) + } +} + #[cfg(encryption)] use crate::{EncryptionOptions, Keyring, SecretKey, VoidKeyringDelegate}; @@ -277,9 +296,13 @@ async fn two_node_quic_join_observes_membership() { // subscription (the channel buffers either way, but this is the clean order). let mut a_events = a.events(); - // Node A dials node B as its seed. - let dispatched = a.join(vec![b_addr]).await.expect("join dispatched"); - assert_eq!(dispatched, 1, "exactly one seed was dispatched"); + // Node A joins node B (await-result): the call returns the reached address + // once the QUIC push-pull to B's seed completes. + let reached = a + .join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + assert_eq!(reached, b_addr, "join returns the reached seed address"); // Node A should observe node B joining via a `Member(Join)` event. let observed = compio::time::timeout(Duration::from_secs(20), async { @@ -406,8 +429,11 @@ async fn two_node_quic_join_with_large_max_udp_payload() { let mut a_events = a.events(); - let dispatched = a.join(vec![b_addr]).await.expect("join dispatched"); - assert_eq!(dispatched, 1, "exactly one seed was dispatched"); + let reached = a + .join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + assert_eq!(reached, b_addr, "join returns the reached seed address"); let observed = compio::time::timeout(Duration::from_secs(20), async { loop { @@ -541,8 +567,11 @@ async fn two_node_quic_join_observes_membership_encrypted() { let mut a_events = a.events(); - let dispatched = a.join(vec![b_addr]).await.expect("join dispatched"); - assert_eq!(dispatched, 1, "exactly one seed was dispatched"); + let reached = a + .join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + assert_eq!(reached, b_addr, "join returns the reached seed address"); let observed = compio::time::timeout(Duration::from_secs(20), async { loop { @@ -567,3 +596,121 @@ async fn two_node_quic_join_observes_membership_encrypted() { a.shutdown().await.expect("node A shuts down"); b.shutdown().await.expect("node B shuts down"); } + +/// Build and spawn a QUIC serf node with a custom `RuntimeOptions` (used by the +/// blackhole join tests to shorten the await-join deadline — a QUIC dial to a +/// closed UDP port has no fast reset, so its exchange resolves only at the +/// deadline reaper). +async fn spawn_node_with_runtime(id: &str, runtime: RuntimeOptions) -> Serf { + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let opts = QuicTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(bind)) + .with_quic_config(test_quic_options()); + Serf::new::, SocketAddrResolver, FirstAddrResolver, _, _>( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + runtime, + SerfOptions::new(), + gossip_rng().expect("seed gossip rng"), + #[cfg(encryption)] + std::rc::Rc::new(VoidKeyringDelegate), + ) + .await + .expect("spawn serf node") +} + +/// `join_many` over two QUIC seeds — one reachable (node B), one a blackhole — +/// returns only the reached seed. B's handshake succeeds quickly; the blackhole +/// exchange stays pending until the (shortened) deadline, at which the waiter is +/// reaped with the reachable seed already in its contacted set. +#[compio::test] +async fn quic_join_many_returns_only_reached_seeds() { + let b = spawn_node("jm-b").await; + let a = spawn_node_with_runtime( + "jm-a", + RuntimeOptions::new().with_join_deadline(Duration::from_secs(6)), + ) + .await; + let b_addr = b.advertise_address(); + + let reached = a + .join_many( + &SocketAddrResolver, + [ + MaybeResolved::Resolved(b_addr), + MaybeResolved::Resolved(blackhole_addr()), + ] + .into_iter(), + false, + ) + .await + .expect("join_many reaches the reachable seed"); + + assert_eq!(reached.len(), 1, "only the reachable seed is contacted"); + assert_eq!( + reached[0], b_addr, + "the reached set carries node B's address" + ); + + a.shutdown().await.expect("jm-a shuts down"); + b.shutdown().await.expect("jm-b shuts down"); +} + +/// An await-result `join` against an unreachable blackhole QUIC seed surfaces +/// `SerfError::JoinAllFailed { requested: 1, contacted: 0 }` at the deadline. +#[compio::test] +async fn quic_join_unreachable_seed_surfaces_join_all_failed() { + let a = spawn_node_with_runtime( + "blackhole-joiner", + RuntimeOptions::new().with_join_deadline(Duration::from_secs(3)), + ) + .await; + + let err = a + .join( + &SocketAddrResolver, + MaybeResolved::Resolved(blackhole_addr()), + false, + ) + .await + .expect_err("join against a blackhole must fail"); + + match err { + SerfError::JoinAllFailed(payload) => { + assert_eq!(payload.requested(), 1, "one seed requested"); + assert_eq!(payload.contacted(), 0, "no seed contacted"); + } + other => panic!("expected JoinAllFailed, got {other:?}"), + } + + a.shutdown().await.expect("joiner shuts down"); +} + +/// A non-empty `join` whose resolver returns zero addresses surfaces +/// `JoinAllFailed` rather than a silent success (no command is even dispatched). +#[compio::test] +async fn quic_join_zero_resolution_surfaces_join_all_failed() { + let a = spawn_node("empty-resolve-joiner").await; + + let err = a + .join( + &EmptyResolver, + MaybeResolved::Unresolved("svc-a".into()), + false, + ) + .await + .expect_err("a seed resolving to zero addresses must fail"); + + match err { + SerfError::JoinAllFailed(payload) => { + assert_eq!(payload.requested(), 1, "one input seed requested"); + assert_eq!(payload.contacted(), 0); + } + other => panic!("expected JoinAllFailed, got {other:?}"), + } + + a.shutdown().await.expect("joiner shuts down"); +} diff --git a/serf-compio/src/serf/mod.rs b/serf-compio/src/serf/mod.rs index fc7b3b0b..3d654f86 100644 --- a/serf-compio/src/serf/mod.rs +++ b/serf-compio/src/serf/mod.rs @@ -11,6 +11,7 @@ //! ergonomics — typed query/response futures, builder-style construction — are a //! follow-up; the protocol surface here is complete. +use core::time::Duration; use std::{ cell::{Cell, RefCell}, net::SocketAddr, @@ -20,7 +21,7 @@ use std::{ use bytes::Bytes; use futures_channel::oneshot; -use memberlist_proto::{Instant, Node}; +use memberlist_proto::{Instant, MaybeResolved, Node}; use serf_proto::{ LamportTime, endpoint::{QueryId, QueryParams}, @@ -29,18 +30,19 @@ use serf_proto::{ options::Options as SerfOptions, typed::Tags, }; +use smallvec::SmallVec; use smol_str::SmolStr; #[cfg(encryption)] use crate::command::{KeyCmd, ListKeysCmd}; use crate::{ command::{ - Command, ForceLeaveCmd, JoinCmd, LeaveCmd, QueryCmd, RespondCmd, SetEventJoinIgnoreCmd, - SetTagsCmd, ShutdownCmd, UserEventCmd, + Command, ForceLeaveCmd, JoinCmd, JoinKind, JoinReply, LeaveCmd, QueryCmd, RespondCmd, + SetTagsCmd, ShutdownCmd, UserEventCmd, WaitForCompletionArgs, }, delegate::Delegate, driver::options::RuntimeOptions, - error::{Result, SerfError}, + error::{JoinFailed, Result, SerfError}, events::EventStream, resolver::{AdvertiseAddrResolver, Resolver}, snapshot::{SerfSnapshot, SnapshotCell}, @@ -69,6 +71,14 @@ struct Shared { shutdown_flag: Rc>, local_id: I, advertise: SocketAddr, + /// Per-call deadline applied to await-result joins, cached from + /// [`RuntimeOptions::join_deadline`] so the handle can stamp each + /// `WaitForCompletion` command's absolute deadline before sending it. + join_deadline: Duration, + /// Cached serf options for computing [`Serf::default_query_timeout`] without + /// a driver round-trip. Cloned from the caller's options at construction + /// before they are moved into the driver. + serf_options: SerfOptions, } /// A cheaply-clonable handle to a running serf node. @@ -153,6 +163,11 @@ where // non-blocking forward can never deposit into, dropping every event. runtime_options.validate()?; + // Cache the join deadline on the handle BEFORE `runtime_options` is moved + // into the driver bundle, so each await-result join can stamp its absolute + // `WaitForCompletion` deadline from `Instant::now() + join_deadline`. + let join_deadline = runtime_options.join_deadline(); + let transport = T::new(options, resolver, advertise_resolver).await?; let local_id = transport.local_id().clone(); let advertise = *transport.advertise_address(); @@ -172,6 +187,10 @@ where // always reflect the driver's live count. let events_dropped_handle = events_dropped.clone(); let observation_dropped_handle = observation_dropped.clone(); + // Clone the serf options before they are moved into the driver so the + // handle can compute `default_query_timeout` / `default_query_param` + // without a driver round-trip. + let serf_options_handle = serf_options.clone(); let runtime = TransportRuntime::::new( delegate, @@ -201,6 +220,8 @@ where shutdown_flag, local_id, advertise, + join_deadline, + serf_options: serf_options_handle, }), }) } @@ -229,6 +250,89 @@ where self.shared.snapshot.borrow().num_members() } + /// All known cluster members at the latest published snapshot instant. + /// + /// Returns every member the local node knows about — alive, leaving, left, + /// and failed within the reap window. Mirrors legacy `Serf::members()` + /// (api.rs:136). + #[inline] + pub fn members(&self) -> Vec>> { + self.snapshot().members().to_vec() + } + + /// The local node's full membership view at the latest snapshot instant. + /// + /// Returns the same `Arc` that lives at `local_index` in `members()`, so + /// the result is always consistent with the member view. Mirrors legacy + /// `Serf::local_member()` (api.rs:202). + #[inline] + pub fn local_member(&self) -> Arc> { + self.snapshot().local() + } + + /// The lifecycle state of the local serf endpoint. + /// + /// Derived from the latest published snapshot. Mirrors legacy `Serf::state()` + /// (api.rs:130). + #[inline] + pub fn state(&self) -> SerfState { + self.snapshot().state() + } + + /// The local node as an `(id, advertise-address)` pair. + /// + /// Composes `local_id()` and `advertise_address()` into a [`Node`]. Mirrors + /// legacy `Serf::advertise_node()` (api.rs:106). + #[inline] + pub fn advertise_node(&self) -> Node { + Node::new(self.shared.local_id.clone(), self.shared.advertise) + } + + /// Force-remove a failed node immediately without pruning the tombstone. + /// + /// Thin alias for `force_leave(id, false)`. Serf will stop attempting to + /// reconnect to this node. Mirrors legacy `Serf::remove_failed_node()` + /// (api.rs:505). + pub async fn remove_failed_node(&self, id: I) -> Result<()> { + self.force_leave(id, false).await + } + + /// Force-remove a failed node immediately and prune the tombstone. + /// + /// Thin alias for `force_leave(id, true)`. The node is removed immediately + /// rather than waiting for the tombstone timeout. Mirrors legacy + /// `Serf::remove_failed_node_prune()` (api.rs:513). + pub async fn remove_failed_node_prune(&self, id: I) -> Result<()> { + self.force_leave(id, true).await + } + + /// Default query timeout derived from the current snapshot member count. + /// + /// Computed as `200ms × query_timeout_mult × ⌈log₁₀(N+1)⌉` where N is the + /// snapshot member count. Matches the machine's own zero-timeout resolution + /// (endpoint/mod.rs:2665) and mirrors legacy `Serf::default_query_timeout()` + /// (query.rs:421). + pub fn default_query_timeout(&self) -> Duration { + let n = self.num_members(); + let mult = self.shared.serf_options.query_timeout_mult(); + let log_factor = ((n as f64 + 1.0).log10().ceil() as u32).max(1); + Duration::from_millis(200) * mult as u32 * log_factor + } + + /// Default query parameters derived from the current snapshot. + /// + /// Returns a [`QueryParams`] with no filters, no relay, no ACK, and a + /// timeout from [`Self::default_query_timeout`]. Mirrors legacy + /// `Serf::default_query_param()` (query.rs:430). + pub fn default_query_param(&self) -> QueryParams { + QueryParams { + filters: Vec::new(), + relay_factor: 0, + request_ack: false, + timeout: self.default_query_timeout(), + } + } + /// Cumulative number of [`Event`]s dropped at the bounded user-facing event /// channel since this node started. /// @@ -278,18 +382,153 @@ where .map_err(|_| SerfError::CommandSend) } - /// Initiate joins to `seeds` (already-resolved addresses). + /// Join an existing cluster through a single seed, waiting for the seed to be + /// contacted. Returns the resolved [`SocketAddr`] of the seed actually reached. /// - /// This is **dispatch-only**: it announces the local join intent and starts a - /// push-pull to each seed, returning the count of seeds the driver dispatched - /// a push-pull to. The returned count is NOT a contact count — a seed may be - /// unreachable and its exchange fail afterward. Actual cluster membership is - /// reported through the [`Event`](serf_proto::event::Event) stream and the - /// published [`snapshot`](Self::snapshot) as peers are merged. - pub async fn join(&self, seeds: Vec) -> Result { + /// `node` is resolved through `resolver` (an already-resolved + /// [`MaybeResolved::Resolved`] passes straight through). The call dispatches a + /// push/pull to every resolved address and resolves once one succeeds (returns + /// that address) or the configured + /// [`join_deadline`](crate::RuntimeOptions::with_join_deadline) elapses with no + /// contact ([`SerfError::JoinAllFailed`]). A `node` that resolves to zero + /// addresses surfaces `JoinAllFailed` rather than a silent success. + /// + /// When `ignore_old` is `true`, replay of the seed's pre-join user events is + /// suppressed: the machine records each join exchange's `StreamId` as a + /// one-shot ignore-join target consumed at that exchange's own merge. Dropping + /// the join future cannot leak the suppression onto a later join, and a + /// concurrent join — even to the SAME seed — is unaffected because it is a + /// distinct exchange. + pub async fn join( + &self, + resolver: &RES, + node: MaybeResolved, + ignore_old: bool, + ) -> Result + where + RES: Resolver, + { + let addrs = resolve_seeds(resolver, core::slice::from_ref(&node)).await?; + match self.join_await(addrs, 1, ignore_old).await { + // The driver only replies `Ok` with a non-empty contacted set, so `next` + // is the single reached address; the fallback can never fire. + Ok(reached) => reached + .into_iter() + .next() + .ok_or_else(|| SerfError::JoinAllFailed(JoinFailed::new(1, 0))), + Err((_, e)) => Err(e), + } + } + + /// Join an existing cluster through several seeds, waiting for the join to + /// complete. Returns the set of seed addresses actually contacted on success, + /// or the legacy partial-success tuple `(reached_so_far, error)` on failure. + /// + /// Each `existing` seed is resolved through `resolver` and a push/pull is + /// dispatched to every resolved address; the call resolves once every + /// dispatched exchange terminates or the configured + /// [`join_deadline`](crate::RuntimeOptions::with_join_deadline) elapses. At + /// least one contact yields `Ok(contacted)`; zero contacts yields + /// `Err((SmallVec::new(), SerfError::JoinAllFailed(..)))`. An empty `existing` + /// iterator is a trivial `Ok(empty)` (no command is sent); a non-empty input + /// resolving to zero addresses surfaces `JoinAllFailed`. + /// + /// `ignore_old` behaves as in [`join`](Self::join). + pub async fn join_many( + &self, + resolver: &RES, + existing: impl Iterator>, + ignore_old: bool, + ) -> core::result::Result, (SmallVec<[SocketAddr; 1]>, SerfError)> + where + RES: Resolver, + { + let seeds: Vec> = existing.collect(); + // Empty input is a trivial caller-side request — `Ok(empty)` without + // sending a command (mirrors the memberlist join's empty short-circuit). + if seeds.is_empty() { + return Ok(SmallVec::new()); + } + let requested = seeds.len(); + let addrs = match resolve_seeds(resolver, &seeds).await { + Ok(a) => a, + Err(e) => return Err((SmallVec::new(), e)), + }; + self.join_await(addrs, requested, ignore_old).await + } + + /// Fire-and-forget join: resolve `seeds`, dispatch a push/pull against each, + /// and return the dispatched-exchange count immediately without waiting for + /// any to complete. + /// + /// Unlike [`join`](Self::join) / [`join_many`](Self::join_many) this never + /// waits for contact: the returned count is the number of resolved seed + /// addresses handed to the driver, NOT the number reached. Actual membership + /// surfaces through the [`events`](Self::events) stream and the + /// [`snapshot`](Self::snapshot). Intended for long-lived background + /// re-discovery loops that observe membership separately. + pub async fn dispatch_join( + &self, + resolver: &RES, + seeds: &[MaybeResolved], + ) -> Result + where + RES: Resolver, + { + let addrs = resolve_seeds(resolver, seeds).await?; let (tx, rx) = oneshot::channel(); - self.send(Command::Join(JoinCmd { seeds, reply: tx }))?; - await_reply(rx).await + self.send(Command::Join(JoinCmd { + seeds: addrs, + kind: JoinKind::Dispatch, + // Fire-and-forget joins never ignore old events. + ignore_old: false, + reply: tx, + }))?; + match await_join_reply(rx).await { + Ok(dispatched) => Ok(dispatched.len()), + Err((_, e)) => Err(e), + } + } + + /// Drive an await-result join over already-resolved `addrs`, threading + /// `ignore_old` to each seed's join push/pull. + /// + /// `requested_if_empty` is the JoinAllFailed denominator used only when + /// `addrs` is empty (the input seed count, since no exchange is dispatched); + /// when `addrs` is non-empty the driver derives `requested` from the + /// dispatched-exchange count itself. + /// + /// No serialising lock is needed: `ignore_old` is recorded per-EXCHANGE (keyed + /// by the join's `StreamId`) in the machine and consumed one-shot at that + /// exchange's own merge (or cleared by the driver when the join terminates + /// without merging), so concurrent joins — even to the SAME seed, and a + /// concurrent `dispatch_join` — never interfere. + async fn join_await( + &self, + addrs: Vec, + requested_if_empty: usize, + ignore_old: bool, + ) -> JoinReply { + // A non-empty seed input that resolved to zero addresses is NOT a silent + // success — surface `JoinAllFailed` so a bootstrap / discovery outage is + // never reported as a healthy zero-contact join. + if addrs.is_empty() { + return Err(( + SmallVec::new(), + SerfError::JoinAllFailed(JoinFailed::new(requested_if_empty, 0)), + )); + } + let deadline = Instant::now() + self.shared.join_deadline; + let (tx, rx) = oneshot::channel(); + match self.send(Command::Join(JoinCmd { + seeds: addrs, + kind: JoinKind::WaitForCompletion(WaitForCompletionArgs { deadline }), + ignore_old, + reply: tx, + })) { + Ok(()) => await_join_reply(rx).await, + Err(e) => Err((SmallVec::new(), e)), + } } /// Gracefully leave the cluster. Resolves once peers have been notified or the @@ -367,17 +606,6 @@ where await_reply(rx).await } - /// Enable or disable suppression of member-join events in the observation - /// stream. - pub async fn set_event_join_ignore(&self, ignore: bool) -> Result<()> { - let (tx, rx) = oneshot::channel(); - self.send(Command::SetEventJoinIgnore(SetEventJoinIgnoreCmd { - ignore, - reply: tx, - }))?; - await_reply(rx).await - } - /// Issue a cluster-wide install-key query; returns the issued [`QueryId`]. #[cfg(encryption)] #[cfg_attr( @@ -459,5 +687,43 @@ async fn await_reply(rx: oneshot::Receiver>) -> Result { } } +/// Await an address-set join reply, mapping a dropped reply channel to the +/// [`SerfError::ReplyClosed`] tuple form. +async fn await_join_reply(rx: oneshot::Receiver) -> JoinReply { + match rx.await { + Ok(res) => res, + Err(_) => Err((SmallVec::new(), SerfError::ReplyClosed)), + } +} + +/// Resolve a slice of [`MaybeResolved`] seeds into a flat `Vec`. +/// +/// [`Resolved`](MaybeResolved::Resolved) entries pass through directly; +/// [`Unresolved`](MaybeResolved::Unresolved) entries run through `resolver` and +/// their results are appended. A resolver failure surfaces as +/// [`SerfError::Resolve`]. Mirrors the memberlist driver's `resolve_seeds`. +async fn resolve_seeds( + resolver: &RES, + seeds: &[MaybeResolved], +) -> Result> +where + RES: Resolver, +{ + let mut addrs: Vec = Vec::new(); + for seed in seeds { + match seed { + MaybeResolved::Resolved(s) => addrs.push(*s), + MaybeResolved::Unresolved(a) => { + let resolved = resolver + .resolve(a) + .await + .map_err(|e| SerfError::Resolve(std::io::Error::other(e.to_string())))?; + addrs.extend(resolved); + } + } + } + Ok(addrs) +} + #[cfg(all(test, feature = "tcp"))] mod tests; diff --git a/serf-compio/src/serf/tests.rs b/serf-compio/src/serf/tests.rs index 19591368..23873c71 100644 --- a/serf-compio/src/serf/tests.rs +++ b/serf-compio/src/serf/tests.rs @@ -7,19 +7,41 @@ use core::time::Duration; use std::net::SocketAddr; use bytes::Bytes; -use futures_util::StreamExt; +use futures_util::{StreamExt, future}; use memberlist_proto::MaybeResolved; use serf_proto::{ event::{Event, MemberEventKind}, + members::SerfState, options::Options as SerfOptions, }; use smol_str::SmolStr; use crate::{ - Channel, FirstAddrResolver, RuntimeOptions, Serf, SerfError, SocketAddrResolver, TcpTransport, - TcpTransportOptions, VoidDelegate, gossip_rng, + Channel, FirstAddrResolver, Resolver, RuntimeOptions, Serf, SerfError, SocketAddrResolver, + TcpTransport, TcpTransportOptions, VoidDelegate, gossip_rng, }; +/// A loopback address with a port nothing listens on — `connect()` returns +/// `ECONNREFUSED` immediately, so its push/pull exchange fails fast. The port is +/// below the OS ephemeral range, so a `:0` test bind never collides with it. +fn blackhole_addr() -> SocketAddr { + "127.0.0.1:7213".parse().expect("loopback addr") +} + +/// Resolver that always resolves to an empty address list — models a +/// service-discovery resolver that finds no live endpoints under a configured +/// service key. +struct EmptyResolver; + +impl Resolver for EmptyResolver { + type Address = String; + type Error = std::io::Error; + + async fn resolve(&self, _addr: &String) -> Result, std::io::Error> { + Ok(Vec::new()) + } +} + #[cfg(encryption)] use crate::{EncryptionOptions, Keyring, SecretKey, VoidKeyringDelegate}; @@ -232,9 +254,13 @@ async fn two_node_tcp_join_observes_membership() { // subscription (the channel buffers either way, but this is the clean order). let mut a_events = a.events(); - // Node A dials node B as its seed. - let dispatched = a.join(vec![b_addr]).await.expect("join dispatched"); - assert_eq!(dispatched, 1, "exactly one seed was dispatched"); + // Node A joins node B (await-result): the call returns the reached address + // once the push-pull to B's seed completes. + let reached = a + .join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + assert_eq!(reached, b_addr, "join returns the reached seed address"); // Node A should observe node B joining via a `Member(Join)` event. let observed = compio::time::timeout(Duration::from_secs(20), async { @@ -312,8 +338,11 @@ async fn two_node_tcp_join_observes_membership_encrypted() { let mut a_events = a.events(); - let dispatched = a.join(vec![b_addr]).await.expect("join dispatched"); - assert_eq!(dispatched, 1, "exactly one seed was dispatched"); + let reached = a + .join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B over the encrypted reliable plane"); + assert_eq!(reached, b_addr, "join returns the reached seed address"); let observed = compio::time::timeout(Duration::from_secs(20), async { loop { @@ -362,7 +391,14 @@ async fn mismatched_keyring_nodes_do_not_exchange_membership() { let mut a_events = a.events(); - let dispatched = a.join(vec![b_addr]).await.expect("join dispatched"); + // Fire-and-forget dispatch: the dial is queued regardless of whether the + // reliable exchange can authenticate. An await-result `join` would instead + // fail here (the mismatched-key push-pull never completes); the absence probe + // below is what proves membership never merges. + let dispatched = a + .dispatch_join(&SocketAddrResolver, &[MaybeResolved::Resolved(b_addr)]) + .await + .expect("join dispatched"); assert_eq!(dispatched, 1, "exactly one seed was dispatched"); // Absence probe: A must never surface a Join carrying node-b. A short window @@ -436,7 +472,9 @@ async fn tcp_events_dropped_counter_observable_under_backpressure() { // Subscribe before the join so we can drain until completion is confirmed. let mut events = a.events(); - a.join(vec![b_addr]).await.expect("join dispatched"); + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); // Drain A's event stream only until Member(Join, [B]) confirms the // push-pull completed. After breaking, `events` is alive but never polled @@ -479,3 +517,383 @@ async fn tcp_events_dropped_counter_observable_under_backpressure() { "events_dropped must be > 0 when event_queue_cap=1 and events are not drained (got {dropped})" ); } + +/// `join_many` over two seeds — one reachable (node B), one a blackhole port — +/// returns only the reached seed's address. The reachable exchange succeeds and +/// the blackhole exchange fails fast; once both terminate the call resolves +/// `Ok([b_addr])`. +#[compio::test] +async fn tcp_join_many_returns_only_reached_seeds() { + let b = spawn_node("jm-b").await; + let a = spawn_node("jm-a").await; + let b_addr = b.advertise_address(); + let blackhole = blackhole_addr(); + + let reached = a + .join_many( + &SocketAddrResolver, + [ + MaybeResolved::Resolved(b_addr), + MaybeResolved::Resolved(blackhole), + ] + .into_iter(), + false, + ) + .await + .expect("join_many reaches the reachable seed"); + + assert_eq!(reached.len(), 1, "only the reachable seed is contacted"); + assert_eq!( + reached[0], b_addr, + "the reached set carries node B's address" + ); + + a.shutdown().await.expect("jm-a shuts down"); + b.shutdown().await.expect("jm-b shuts down"); +} + +/// An await-result `join` against an unreachable blackhole seed surfaces +/// `SerfError::JoinAllFailed { requested: 1, contacted: 0 }` once the dial fails +/// fast — well before the join deadline. +#[compio::test] +async fn tcp_join_unreachable_seed_surfaces_join_all_failed() { + let a = spawn_node("blackhole-joiner").await; + + let err = a + .join( + &SocketAddrResolver, + MaybeResolved::Resolved(blackhole_addr()), + false, + ) + .await + .expect_err("join against a blackhole must fail"); + + match err { + SerfError::JoinAllFailed(payload) => { + assert_eq!(payload.requested(), 1, "one seed requested"); + assert_eq!(payload.contacted(), 0, "no seed contacted"); + } + other => panic!("expected JoinAllFailed, got {other:?}"), + } + + a.shutdown().await.expect("joiner shuts down"); +} + +/// A non-empty `join` whose resolver returns zero addresses surfaces +/// `JoinAllFailed`, NOT a silent success — a service-discovery resolver that +/// finds no endpoints must never be reported as a healthy zero-contact join. +#[compio::test] +async fn tcp_join_zero_resolution_surfaces_join_all_failed() { + let a = spawn_node("empty-resolve-joiner").await; + + let err = a + .join( + &EmptyResolver, + MaybeResolved::Unresolved("svc-a".into()), + false, + ) + .await + .expect_err("a seed resolving to zero addresses must fail"); + + match err { + SerfError::JoinAllFailed(payload) => { + assert_eq!(payload.requested(), 1, "one input seed requested"); + assert_eq!(payload.contacted(), 0); + } + other => panic!("expected JoinAllFailed, got {other:?}"), + } + + // An empty `join_many` input is a trivial `Ok(empty)` — no command is sent. + let empty: Vec> = Vec::new(); + let reached = a + .join_many(&EmptyResolver, empty.into_iter(), false) + .await + .expect("empty input is a trivial success"); + assert!(reached.is_empty(), "empty input contacts nothing"); + + a.shutdown().await.expect("joiner shuts down"); +} + +/// After a two-node join, the snapshot forwarders on the joined node must reflect +/// the two-member cluster: `members()` returns both nodes, `local_member()` returns +/// this node's own `Member`, and `state()` returns `SerfState::Alive`. +/// +/// `broadcast_join` materialises the local node in members.states immediately, so +/// the snapshot update races the join return: we drain until `Member(Join, [B])` +/// confirms the push-pull completed, at which point the snapshot is already +/// current (refresh_snapshot is called before the join reply is delivered). +#[compio::test] +async fn tcp_snapshot_forwarders_reflect_joined_cluster() { + let b = spawn_node("snap-b").await; + let a = spawn_node("snap-a").await; + let b_addr = b.advertise_address(); + let b_id = SmolStr::new("snap-b"); + let a_id = SmolStr::new("snap-a"); + + // Subscribe before the join so a Member event does not race the subscription. + let mut a_events = a.events(); + + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + + // Drain until B's join event confirms the push-pull completed. The event + // stream may carry Member(Join, [A]) first (local node materialised in + // members.states during broadcast_join), so we skip non-B join events. + compio::time::timeout(Duration::from_secs(20), async { + loop { + match a_events.next().await { + Some(Event::Member(me)) if me.kind() == MemberEventKind::Join => { + if me.members().iter().any(|m| m.node().id_ref() == &b_id) { + break; + } + } + Some(_) => {} + None => panic!("event stream closed before join was observed"), + } + } + }) + .await + .expect("A must observe B joining within timeout"); + + // The snapshot is refreshed before the join reply is sent, so num_members() + // reflects the 2-member cluster immediately. A brief poll guards against any + // marginal scheduling jitter on slow CI hosts. + compio::time::timeout(Duration::from_secs(5), async { + loop { + if a.num_members() == 2 { + break; + } + compio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("snapshot must show 2 members immediately after B's join event"); + + // members() must reflect both nodes. + let members = a.members(); + assert_eq!( + members.len(), + 2, + "two-node cluster: members() must return 2" + ); + let ids: Vec<&SmolStr> = members.iter().map(|m| m.node().id_ref()).collect(); + assert!(ids.contains(&&a_id), "members() must include local node A"); + assert!(ids.contains(&&b_id), "members() must include peer node B"); + + // local_member() must return A's own membership record. + let local = a.local_member(); + assert_eq!( + local.node().id_ref(), + &a_id, + "local_member() must return local node A" + ); + + // state() must be Alive after a successful join. + assert_eq!( + a.state(), + SerfState::Alive, + "state() must be Alive after join" + ); + + // advertise_node() must compose the local id and advertise address. + let anode = a.advertise_node(); + assert_eq!( + anode.id_ref(), + &a_id, + "advertise_node() id matches local_id()" + ); + assert_eq!( + anode.addr_ref(), + &a.advertise_address(), + "advertise_node() addr matches advertise_address()" + ); + + // default_query_timeout() must be a positive duration for a 2-member cluster. + let qt = a.default_query_timeout(); + assert!( + qt > Duration::ZERO, + "default_query_timeout() must be positive" + ); + + // default_query_param() must carry that timeout with no filters/relay/ack. + let qp = a.default_query_param(); + assert_eq!( + qp.timeout, qt, + "default_query_param().timeout matches default_query_timeout()" + ); + assert!( + qp.filters.is_empty(), + "default_query_param() has no filters" + ); + assert!(!qp.request_ack, "default_query_param() has no ack"); + assert_eq!(qp.relay_factor, 0, "default_query_param() has no relay"); + + a.shutdown().await.expect("snap-a shuts down"); + b.shutdown().await.expect("snap-b shuts down"); +} + +/// `remove_failed_node` is a thin alias for `force_leave(id, false)`. Calling it +/// on a valid node-id in the cluster must complete without error — the driver +/// processes the forced leave broadcast unconditionally. +#[compio::test] +async fn tcp_remove_failed_node_alias_succeeds() { + let b = spawn_node("rfn-b").await; + let a = spawn_node("rfn-a").await; + let b_addr = b.advertise_address(); + let b_id = SmolStr::new("rfn-b"); + + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + + // remove_failed_node is force_leave(id, false); must not error. + a.remove_failed_node(b_id.clone()) + .await + .expect("remove_failed_node must not error"); + + // remove_failed_node_prune is force_leave(id, true); must not error either. + a.remove_failed_node_prune(b_id) + .await + .expect("remove_failed_node_prune must not error"); + + a.shutdown().await.expect("rfn-a shuts down"); + b.shutdown().await.expect("rfn-b shuts down"); +} + +/// Two concurrently-polled `join` calls with `ignore_old=true` on cloned handles +/// must both complete without panicking or deadlocking. +/// +/// The per-exchange ignore mechanism records each join exchange's `StreamId` +/// independently and consumes it one-shot at that exchange's own merge, so +/// concurrent ignore_old joins are safe WITHOUT the former `join_lock` — there is +/// no shared flag to serialize. This is the concurrency-correctness property that +/// replaces the old lock-serialization contract. +#[compio::test] +async fn tcp_concurrent_ignore_old_joins_coexist() { + let b = spawn_node("cji-b").await; + let b_addr = b.advertise_address(); + let a1 = spawn_node("cji-a").await; + let a2 = a1.clone(); + + // Run both join futures concurrently on the same compio task; with no lock, + // they interleave freely and must both still resolve. + let (r1, r2) = future::join( + a1.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), true), + a2.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), true), + ) + .await; + + let contacted = r1.is_ok() as usize + r2.is_ok() as usize; + assert!( + contacted >= 1, + "at least one concurrent ignore_old join must reach node B (got r1={r1:?}, r2={r2:?})" + ); + + a1.shutdown().await.expect("cji-a shuts down"); + b.shutdown().await.expect("cji-b shuts down"); +} + +/// The same-seed hole the per-exchange key closes: an `ignore_old` join AND a +/// plain (non-ignore) join to the SAME seed, polled concurrently, must coexist. +/// Each is a distinct exchange with a distinct `StreamId`, so the ignore_old +/// suppression targets ONLY its own merge — the plain join's same-seed merge is +/// never wrongly suppressed, and neither call consumes the other's token. A +/// per-PEER key could mis-route whichever merge landed first. (The deterministic +/// per-exchange suppression precision is proven by the serf-proto +/// `non_ignore_join_to_same_seed_is_not_suppressed` test; here we assert the +/// driver-level coexistence and convergence.) +#[compio::test] +async fn tcp_concurrent_ignore_old_and_plain_join_same_seed_coexist() { + let b = spawn_node("cssj-b").await; + let b_addr = b.advertise_address(); + let a = spawn_node("cssj-a").await; + + // Same seed B, two concurrent joins: one ignore_old, one plain. + let (r_ignore, r_plain) = future::join( + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), true), + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false), + ) + .await; + + assert!( + r_ignore.is_ok() || r_plain.is_ok(), + "at least one same-seed join must reach B (ignore={r_ignore:?}, plain={r_plain:?})" + ); + + // A converges to a 2-member cluster (self + B): neither same-seed join blocked + // or dropped the other. + compio::time::timeout(Duration::from_secs(10), async { + loop { + if a.num_members() == 2 { + break; + } + compio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("A must converge to a 2-member cluster (self + B)"); + + a.shutdown().await.expect("cssj-a shuts down"); + b.shutdown().await.expect("cssj-b shuts down"); +} + +/// A `dispatch_join` to one peer running concurrently with a +/// `join_many(ignore_old=true)` to a DIFFERENT peer must not interfere: both +/// resolve and the node converges to a 3-member cluster. The ignore_old join's +/// per-exchange suppression targets only its own `StreamId`, so the concurrent +/// dispatch_join's exchange is never wrongly suppressed (the machine-level +/// `non_ignore_join_to_same_seed_is_not_suppressed` test proves the +/// merge-suppression precision deterministically). +#[compio::test] +async fn tcp_concurrent_dispatch_join_and_ignore_old_join_coexist() { + let b = spawn_node("cdj-b").await; + let c = spawn_node("cdj-c").await; + let b_addr = b.advertise_address(); + let c_addr = c.advertise_address(); + let a = spawn_node("cdj-a").await; + + // Concurrently: an ignore_old await-join to B, and a fire-and-forget + // dispatch_join to C. Neither shares state with the other. + let (join_b, dispatch_c) = future::join( + a.join_many( + &SocketAddrResolver, + core::iter::once(MaybeResolved::Resolved(b_addr)), + true, + ), + a.dispatch_join(&SocketAddrResolver, &[MaybeResolved::Resolved(c_addr)]), + ) + .await; + + join_b.expect("the ignore_old join_many to B must reach B"); + assert_eq!( + dispatch_c.expect("dispatch_join to C must dispatch"), + 1, + "dispatch_join reports the single dispatched seed" + ); + + // Both peers must converge into A's membership — the concurrent ignore_old + // join to B did not block or drop the dispatch_join to C. + compio::time::timeout(Duration::from_secs(10), async { + loop { + if a.num_members() == 3 { + break; + } + compio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("A must converge to a 3-member cluster (self + B + C)"); + + let ids: Vec = a + .members() + .iter() + .map(|m| m.node().id_ref().clone()) + .collect(); + assert!(ids.iter().any(|id| id == "cdj-b"), "members must include B"); + assert!(ids.iter().any(|id| id == "cdj-c"), "members must include C"); + + a.shutdown().await.expect("cdj-a shuts down"); + b.shutdown().await.expect("cdj-b shuts down"); + c.shutdown().await.expect("cdj-c shuts down"); +} diff --git a/serf-compio/src/tls/tests.rs b/serf-compio/src/tls/tests.rs index 155b9ab2..5ce83d03 100644 --- a/serf-compio/src/tls/tests.rs +++ b/serf-compio/src/tls/tests.rs @@ -202,8 +202,11 @@ async fn two_node_tls_join_observes_membership() { let mut a_events = a.events(); // Node A dials node B as its seed. - let dispatched = a.join(vec![b_addr]).await.expect("join dispatched"); - assert_eq!(dispatched, 1, "exactly one seed was dispatched"); + let reached = a + .join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + assert_eq!(reached, b_addr, "join returns the reached seed address"); // Node A should observe node B joining via a `Member(Join)` event. let observed = compio::time::timeout(Duration::from_secs(20), async { @@ -429,8 +432,11 @@ async fn two_node_tls_join_observes_membership_encrypted() { let mut a_events = a.events(); - let dispatched = a.join(vec![b_addr]).await.expect("join dispatched"); - assert_eq!(dispatched, 1, "exactly one seed was dispatched"); + let reached = a + .join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + assert_eq!(reached, b_addr, "join returns the reached seed address"); let observed = compio::time::timeout(Duration::from_secs(20), async { loop { diff --git a/serf-driver/src/error/mod.rs b/serf-driver/src/error/mod.rs index 973a68d5..18df6827 100644 --- a/serf-driver/src/error/mod.rs +++ b/serf-driver/src/error/mod.rs @@ -91,5 +91,55 @@ impl core::fmt::Display for InvalidOption { impl std::error::Error for InvalidOption {} +/// Payload for the join-all-failed error: an await-result join dispatched a +/// push/pull to one or more resolved seeds but none was contacted before the +/// call resolved. Carries the number of seeds the call requested +/// (post-resolution) and the number actually contacted — `contacted` is always +/// `0` for this error (a non-zero contact count resolves the join `Ok`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct JoinFailed { + requested: usize, + contacted: usize, +} + +impl JoinFailed { + /// Build a new payload from the requested-seed count and the contacted count. + #[inline] + pub const fn new(requested: usize, contacted: usize) -> Self { + Self { + requested, + contacted, + } + } + + /// The number of seed addresses the join requested (post-resolution): the + /// count of outbound push/pull exchanges the driver dispatched. + #[must_use] + #[inline] + pub const fn requested(&self) -> usize { + self.requested + } + + /// The number of seeds actually contacted before the join resolved. Always + /// `0` for this error variant — a non-zero contact count resolves `Ok`. + #[must_use] + #[inline] + pub const fn contacted(&self) -> usize { + self.contacted + } +} + +impl core::fmt::Display for JoinFailed { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!( + f, + "join reached {} of {} seed(s)", + self.contacted, self.requested, + ) + } +} + +impl std::error::Error for JoinFailed {} + #[cfg(test)] mod tests; diff --git a/serf-driver/src/error/tests.rs b/serf-driver/src/error/tests.rs index 11121027..756e4313 100644 --- a/serf-driver/src/error/tests.rs +++ b/serf-driver/src/error/tests.rs @@ -1,4 +1,4 @@ -use super::{GossipMtuTooSmall, InvalidOption}; +use super::{GossipMtuTooSmall, InvalidOption, JoinFailed}; #[test] fn gossip_mtu_too_small_fields_and_display() { @@ -21,3 +21,16 @@ fn invalid_option_fields_and_display() { assert!(s.contains("must be nonzero"), "missing reason: {s}"); let _: &dyn std::error::Error = &opt; } + +#[test] +fn join_failed_fields_and_display() { + let jf = JoinFailed::new(3, 0); + assert_eq!(jf.requested(), 3); + assert_eq!(jf.contacted(), 0); + let s = format!("{jf}"); + assert!(s.contains('3'), "missing requested count: {s}"); + assert!(s.contains('0'), "missing contacted count: {s}"); + let _: &dyn std::error::Error = &jf; + // Copy + Eq are derived. + assert_eq!(jf, JoinFailed::new(3, 0)); +} diff --git a/serf-proto/src/endpoint/mod.rs b/serf-proto/src/endpoint/mod.rs index 9af46c86..000a309d 100644 --- a/serf-proto/src/endpoint/mod.rs +++ b/serf-proto/src/endpoint/mod.rs @@ -60,7 +60,7 @@ use std::collections::VecDeque; use bytes::Bytes; use memberlist_proto::{ - CheapClone, Data, Id, Instant, Node, PushPullKind, Rng, SeedableRng, SmallRng, + CheapClone, Data, Id, Instant, Node, PushPullKind, Rng, SeedableRng, SmallRng, StreamId, typed::{Meta, NodeState}, }; @@ -497,6 +497,17 @@ pub enum Error { /// when already `Leaving` or `Shutdown`. #[error("leave called from invalid state: {0}")] BadLeaveState(SerfState), + /// `leave()` could not stamp a leave intent because the member Lamport clock + /// has reached the `LTIME_MAX` integrity floor. + /// + /// The post-incremented leave ltime would land at or above `LTIME_MAX`, which + /// every node (including this one) rejects, so broadcasting it or starting the + /// inner leave would desync local membership. The endpoint parks + /// degraded-but-safe: it stays `Alive` and consistent, emits no invalid + /// intent, and starts no inner leave. Unreachable in any finite cluster + /// lifetime (2^63 membership events). + #[error("leave clock exhausted: member clock reached the LTIME_MAX integrity floor")] + LeaveClockExhausted, /// `join()` was called while the local endpoint is not `Alive`. /// /// serf only announces its own join intent from the `Alive` state; a @@ -761,13 +772,28 @@ where /// `Endpoint` always ships a current serf snapshot on the next push-pull /// egress — not the snapshot from the last explicit sync. local_state_dirty: bool, - /// When `true`, a push-pull join bumps `event_buffer.min_time` to the - /// remote `event_ltime` (H8/G4 / Go serf `eventJoinIgnore`). - /// - /// Set by the driver via `set_event_join_ignore(true)` before issuing a - /// join that should suppress replay of the peer's buffered events. The - /// ingress `merge_remote_state` checks this on `is_join == true`. - event_join_ignore: bool, + /// Outbound exchange [`StreamId`]s serf started as an `ignore_old` join + /// push/pull. + /// + /// When the driver issues an `ignore_old` join, the composing super-machine + /// records the exchange's `StreamId` here (via + /// [`Endpoint::note_ignore_join_stream`]) — the same `StreamId` that + /// `start_push_pull` returned. The matching merge arrives as + /// `RemoteStateReceived` carrying that exchange's `originating_stream_id`; when + /// it equals a recorded entry the entry is consumed (one-shot) and + /// `event_buffer.min_time` is bumped to the remote `event_ltime`, dropping the + /// peer's pre-join user events (H8/G4 / Go serf `eventJoinIgnore`). A failed, + /// timed-out, or dropped `ignore_old` join never merges, so the driver removes + /// its `StreamId` (via [`Endpoint::clear_ignore_join_stream`]) when the join + /// reaches its terminal — a stale entry can never leak. + /// + /// Keyed per-EXCHANGE, not per-peer: two joins to the SAME seed (e.g. an + /// `ignore_old` join racing a `dispatch_join`) get distinct `StreamId`s, so + /// only the `ignore_old` join's own merge is suppressed — whichever merge lands + /// first can no longer consume the wrong token. The set is tiny (the in-flight + /// `ignore_old` exchanges), so a `Vec` with linear lookup is the right + /// structure. + ignore_join_streams: Vec, /// The address of the most recently reconnect-dialled peer. /// /// Set by `fire_reconnect` each time a dial is initiated. Test helpers @@ -843,7 +869,7 @@ where // The snapshot starts dirty so the first push-pull always ships a fresh // body even if no explicit API call has been made yet. local_state_dirty: true, - event_join_ignore: false, + ignore_join_streams: Vec::new(), #[cfg(feature = "coordinates")] coord_client, #[cfg(feature = "coordinates")] @@ -1138,9 +1164,19 @@ where self.handle_user_packet(t, from, data, now); } IE::RemoteStateReceived(r) => { + // Correlate the merge to the exchange that produced it by its + // `originating_stream_id` — for an outbound join this is exactly the + // `StreamId` `start_push_pull` returned and that the driver recorded. + let sid = r.originating_stream_id(); let (_peer, user_data, is_join) = r.into_parts(); if !user_data.is_empty() { - self.merge_remote_state(t, user_data, is_join); + // Consume the one-shot ignore-join entry for THIS exchange iff this is + // a join: the `ignore_old` join's own merge suppresses its pre-join + // user events. A refresh, or any other exchange to the same peer (a + // concurrent `dispatch_join`, or an inbound join we did not ignore), + // carries a different `StreamId` and leaves the set untouched. + let suppress = is_join && self.consume_ignore_join_stream(sid); + self.merge_remote_state(t, user_data, suppress); } } @@ -1187,8 +1223,17 @@ where // ── drop with no serf-level action ─────────────────────────────────── // PingFailed: no serf action; the inner already records the failure. IE::PingFailed(_) => {} - // ExchangeCompleted: driver bookkeeping only; serf takes no action. - IE::ExchangeCompleted(_) => {} + // Surface the terminal exchange outcome so drivers awaiting a Join + // push/pull (or any other reliable exchange) can resolve without + // inferring completion from membership-state side effects. + IE::ExchangeCompleted(p) => { + // An ignore-join's `StreamId` is NOT cleaned here: `ExchangeCompleted` + // carries the coordinator-allocated `eid`, which on the stream backend + // is a separate domain from the `StreamId` the ignore set is keyed by. + // The driver clears a terminated `ignore_old` join's `StreamId` from its + // own per-join bookkeeping instead (see `clear_ignore_join_stream`). + self.pending_events.push_back(Event::ExchangeCompleted(p)); + } // DecodeError: the inner has already logged / tracked the error; // serf takes no action on undecodable inner messages. IE::DecodeError(_) => {} @@ -1306,16 +1351,49 @@ where } } - /// Set the `event_join_ignore` flag. + /// Record outbound exchange `id` as an `ignore_old` join. /// - /// When `true`, an incoming push-pull marked as a join bumps - /// `event_buffer.min_time` to the remote `event_ltime` so that - /// pre-join events from that peer are not replayed locally (H8/G4). + /// The composing super-machine calls this when the driver starts an + /// `ignore_old` join push/pull, passing the `StreamId` `start_push_pull` + /// returned, so the matching merge (whose `originating_stream_id` equals `id`) + /// bumps `event_buffer.min_time` and suppresses replay of the peer's pre-join + /// user events (H8/G4). The entry is one-shot: it is consumed at the join merge + /// (see [`Self::consume_ignore_join_stream`]) or removed by the driver via + /// [`Self::clear_ignore_join_stream`] when the join terminates without a merge. + /// Idempotent — a duplicate `StreamId` is not stored twice. + pub(crate) fn note_ignore_join_stream(&mut self, id: StreamId) { + if !self.ignore_join_streams.contains(&id) { + self.ignore_join_streams.push(id); + } + } + + /// Consume the one-shot ignore-join entry for exchange `id`, returning whether + /// one was present. /// - /// The driver sets this before calling `inner.join` to suppress historic - /// user events from the seed peers. - pub fn set_event_join_ignore(&mut self, v: bool) { - self.event_join_ignore = v; + /// Called on a join push/pull merge whose `originating_stream_id` is `id` + /// (suppression is applied iff this returns `true`). Removing the entry here is + /// what makes the ignore per-exchange one-shot: a second merge on the same + /// stream, or any other exchange to the same peer, is unaffected. + fn consume_ignore_join_stream(&mut self, id: StreamId) -> bool { + if let Some(idx) = self.ignore_join_streams.iter().position(|s| *s == id) { + self.ignore_join_streams.swap_remove(idx); + true + } else { + false + } + } + + /// Remove exchange `id` from the ignore-join set without applying suppression. + /// + /// The driver calls this when an `ignore_old` join reaches its terminal without + /// a merge having consumed the entry (dial failure, timeout, empty push/pull + /// body, or a dropped join future), so a stale `StreamId` can never linger. + /// Idempotent: a `StreamId` the merge already consumed on the success path is + /// simply absent, making this a no-op there. `ExchangeCompleted` cannot drive + /// this on the stream backend because its `eid` is a different domain from the + /// `StreamId`, so the cleanup is driver-side per-join bookkeeping. + pub(crate) fn clear_ignore_join_stream(&mut self, id: StreamId) { + self.consume_ignore_join_stream(id); } /// Update the local node's tags, re-advertise them via the coordinator, and @@ -1406,11 +1484,12 @@ where /// the causal ordering intact). /// /// **G4 — `eventJoinIgnore` bumps `event_buffer.min_time`:** - /// If `is_join` is `true` AND `self.event_join_ignore` is set, the - /// `event_buffer.min_time` is raised to `max(min_time, event_ltime)`. - /// This suppresses all buffered user events from the remote peer that - /// pre-date the current event-clock (prevents re-emitting stale events - /// on a fresh join exchange). + /// If `suppress_pre_join_events` is set (the caller saw a join push/pull from + /// a peer it started an `ignore_old` join against and consumed the one-shot + /// per-peer entry), the `event_buffer.min_time` is raised to + /// `max(min_time, event_ltime)`. This suppresses all buffered user events + /// from the remote peer that pre-date the current event-clock (prevents + /// re-emitting stale events on a fresh join exchange). /// /// After the intent passes, every buffered user event in the push-pull /// body is replayed via `handle_user_event`. Events older than @@ -1419,7 +1498,7 @@ where /// /// Decoding errors in the `user_data` bytes are silently dropped — /// the machine must not panic on bad network input. - fn merge_remote_state(&mut self, t: &mut T, user_data: Bytes, is_join: bool) + fn merge_remote_state(&mut self, t: &mut T, user_data: Bytes, suppress_pre_join_events: bool) where T: Reliable, I: Clone + Data, @@ -1518,10 +1597,12 @@ where self.handle_node_join_intent(*ltime, node_id, now); } - // G4: if is_join && event_join_ignore, bump event_buffer.min_time to - // max(min_time, remote event_ltime). This prevents pre-join events - // from being replayed on a fresh join exchange. - if is_join && self.event_join_ignore && event_ltime > self.event_buffer.min_time { + // G4: when this join push/pull came from a peer we started an `ignore_old` + // join against (the caller consumed the one-shot per-peer entry and passed + // `suppress_pre_join_events`), bump event_buffer.min_time to + // max(min_time, remote event_ltime). This prevents the peer's pre-join + // events from being replayed on a fresh join exchange. + if suppress_pre_join_events && event_ltime > self.event_buffer.min_time { self.event_buffer.min_time = event_ltime; self.mark_local_state_dirty(); } @@ -2171,18 +2252,23 @@ where /// - `Alive` → proceeds with the leave chain below. /// /// Leave chain (decision 5 / oracle `api.go` leave()): - /// 1. Set `state = Leaving`. - /// 2. Handle the local leave intent (`handle_node_leave_intent` for the - /// local id), which marks the local node as `Leaving` in the membership - /// store and queues a join-refute suppression. - /// 3. Increment the member clock. - /// 4. Enqueue a leave-intent broadcast on the intent tier (rank 0) so + /// 1. Post-increment the member clock to stamp the leave ltime — but only + /// after the [`LTIME_MAX`] integrity-floor gate: if the stamp would land at + /// or above `LTIME_MAX` (a clock driven near the floor), return + /// [`Error::LeaveClockExhausted`] without mutating any state. No invalid + /// intent is emitted and no inner leave starts; the endpoint stays a + /// consistent `Alive` member (degraded-but-safe). + /// 2. Set `state = Leaving`, then handle the local leave intent + /// (`handle_node_leave_intent` for the local id), which marks the local + /// node as `Leaving` in the membership store and queues a join-refute + /// suppression. + /// 3. Enqueue a leave-intent broadcast on the intent tier (rank 0) so /// peers learn about the leave. (`FIX`: no `on_finished` callback — /// flushing is bounded by `broadcast_timeout` deadline instead.) - /// 5. Call inner `leave(now)` to begin the memberlist dead-self fan-out. + /// 4. Call inner `leave(now)` to begin the memberlist dead-self fan-out. /// The inner will eventually emit `Event::LeftCluster` once all dead-self /// packets are drained via `poll_transmit`. - /// 6. Arm `leave_broadcast_deadline = now + broadcast_timeout`. The + /// 5. Arm `leave_broadcast_deadline = now + broadcast_timeout`. The /// driver can short-circuit by watching `user_broadcast_queue_len()`. /// /// The `Leaving → Left` transition happens later in `handle_timeout` when @@ -2202,26 +2288,53 @@ where SerfState::Alive => {} } - // 1. Transition to Leaving. - self.state = SerfState::Leaving; - - // 2. Local leave intent — marks the local node as Leaving in the store - // and witnesses the member clock. let local_id = t.endpoint_ref().local_id_ref().clone(); - // next_ltime stamps the current clock value (clamped to < LTIME_MAX) and - // advances the clock in one atomic step, closing the local-emission hole. - let ltime = LamportTime(next_ltime(&mut self.clock)); + + // Post-increment: advance first, then stamp the new value. This matches + // Go serf's clock.Increment() (returns the new value), and is required so + // the leave ltime (1 on a fresh node) is strictly greater than the + // self-join status_time (0), preventing a stale-intent rejection. + // + // Compute the prospective stamp WITHOUT committing the clock yet: near the + // LTIME_MAX integrity floor (a member clock driven there by a corrupt + // snapshot or crafted peer value), the post-incremented stamp lands at or + // above LTIME_MAX, which handle_node_leave_intent and every peer reject. + // Committing such a clock would also poison our push-pull snapshot — peers + // drop a whole body whose top-level ltime is out of range — so we park + // degraded-but-safe BEFORE any mutation: stay Alive and consistent, emit no + // invalid intent, and start no inner leave. Unreachable in any finite + // cluster lifetime (2^63 membership events). + let stamp = self.clock.saturating_add(1); + if !ltime_is_acceptable(stamp) { + return Err(Error::LeaveClockExhausted); + } + + // The stamp is acceptable: commit it and transition to Leaving. + self.clock = stamp; + let ltime = LamportTime(self.clock); self.mark_local_state_dirty(); - // We are setting state = Leaving above so the self-refute guard in - // handle_node_leave_intent will NOT fire (it only fires when state == Alive). - self.handle_node_leave_intent(t, ltime, &local_id, false, now); - // 4. Broadcast the leave intent on the intent tier (rank 0) so peers learn + // 1. Transition to Leaving BEFORE applying the local intent so the + // self-refute guard in handle_node_leave_intent does NOT fire (it only + // fires when state == Alive). + self.state = SerfState::Leaving; + + // 2. Local leave intent — marks the local node as Leaving in the store. The + // acceptability gate above guarantees this applies; defensively revert to + // a consistent Alive state and bail if it somehow did not, so we never + // broadcast an intent no node can apply nor start an inconsistent inner + // leave. + if !self.handle_node_leave_intent(t, ltime, &local_id, false, now) { + self.state = SerfState::Alive; + return Err(Error::LeaveClockExhausted); + } + + // 3. Broadcast the leave intent on the intent tier (rank 0) so peers learn // the local node is leaving without waiting for anti-entropy. The // driver bounds the flush via the broadcast deadline below. self.broadcast_leave(t, ltime, local_id, false); - // 5. Arm the broadcast-timeout deadline so the driver always has a finite + // 4. Arm the broadcast-timeout deadline so the driver always has a finite // wait; it can short-circuit by watching `user_broadcast_queue_len()`. self.leave_broadcast_deadline = Some(now + self.opts.broadcast_timeout()); @@ -2256,13 +2369,28 @@ where return Err(Error::BadLeaveState(self.state)); } - // Advance first, then stamp the new value so the forced leave outranks any - // prior status the target node has (post-increment semantics, unlike leave() - // which stamps pre-increment). saturating_add is the no-UB backstop. - self.clock = self.clock.saturating_add(1); + // Compute the prospective stamp WITHOUT committing the clock yet: near the + // LTIME_MAX integrity floor, the post-incremented stamp lands at or above + // LTIME_MAX, which handle_node_leave_intent and every peer reject. + // Committing such a clock would also poison push-pull snapshots, so we park + // degraded-but-safe BEFORE any mutation: advance no clock, apply no local + // intent, and broadcast nothing. Mirrors the identical guard in leave(). + let stamp = self.clock.saturating_add(1); + if !ltime_is_acceptable(stamp) { + return Err(Error::LeaveClockExhausted); + } + + // The stamp is acceptable: commit it. + self.clock = stamp; let ltime = LamportTime(self.clock); self.mark_local_state_dirty(); - self.handle_node_leave_intent(t, ltime, &id, prune, now); + + // Apply the local leave intent; if for any reason it does not apply (e.g. + // the target is unknown), skip the broadcast so we do not emit an intent + // no node can apply. + if !self.handle_node_leave_intent(t, ltime, &id, prune, now) { + return Ok(()); + } // Broadcast the leave intent (carrying the prune flag) so peers apply the // same forced removal. @@ -4036,10 +4164,27 @@ where self.local_state_dirty } - /// Set the `event_join_ignore` flag (test adapter). + /// Record outbound exchange `id` as an `ignore_old` join (test adapter). + #[cfg(test)] + pub(crate) fn test_note_ignore_join_stream(&mut self, id: StreamId) { + self.note_ignore_join_stream(id); + } + + /// Whether exchange `id` is currently a recorded `ignore_old` join (test + /// adapter for cancellation / one-shot-consume / per-exchange assertions). + #[cfg(test)] + pub(crate) fn test_has_ignore_join_stream(&self, id: StreamId) -> bool { + self.ignore_join_streams.contains(&id) + } + + /// Mirror the driver's terminal cleanup (test adapter): drop the pending + /// ignore-join entry for exchange `id`, exactly as + /// [`Self::clear_ignore_join_stream`] does when a join terminates without a + /// merge — a dial failure, timeout, or a dropped join future — so the + /// cancellation-safety property can be asserted directly. #[cfg(test)] - pub(crate) fn test_set_event_join_ignore(&mut self, v: bool) { - self.event_join_ignore = v; + pub(crate) fn test_clear_ignore_join_stream(&mut self, id: StreamId) { + self.clear_ignore_join_stream(id); } /// Read the current `event_buffer.min_time` (test adapter for G4 assertions). @@ -4050,19 +4195,56 @@ where // ── Push-pull / merge test helpers ─────────────────────────────────────── - /// Directly invoke `merge_remote_state` with raw `user_data` bytes (test adapter). + /// Directly invoke `merge_remote_state` with raw `user_data` bytes and no + /// ignore-old suppression (test adapter). /// /// Sets `drain_now = Instant::ORIGIN` before the call so intent handlers /// receive a stable `now`. Use `test_set_drain_now` to override the - /// timestamp when wall-clock values matter. + /// timestamp when wall-clock values matter. For the ignore-old / G4 path use + /// [`Self::test_merge_remote_state_with_stream`] (per-exchange consume) or + /// [`Self::test_merge_remote_state_suppressed`] (suppression applied directly). #[cfg(test)] - pub(crate) fn test_merge_remote_state(&mut self, t: &mut T, user_data: Bytes, is_join: bool) + pub(crate) fn test_merge_remote_state(&mut self, t: &mut T, user_data: Bytes) where T: Reliable, I: Clone + Data, A: Data, { - self.merge_remote_state(t, user_data, is_join); + self.merge_remote_state(t, user_data, false); + } + + /// Invoke `merge_remote_state` with suppression forced on (test adapter for the + /// G4 `eventJoinIgnore` watermark): bumps `event_buffer.min_time` to the remote + /// `event_ltime` and drops the body's pre-join user events, exactly as a + /// consumed ignore-join merge does — without minting an exchange `StreamId`. + #[cfg(test)] + pub(crate) fn test_merge_remote_state_suppressed(&mut self, t: &mut T, user_data: Bytes) + where + T: Reliable, + I: Clone + Data, + A: Data, + { + self.merge_remote_state(t, user_data, true); + } + + /// Drive the full `RemoteStateReceived` ignore-old path for a merge whose + /// `originating_stream_id` is `sid` (test adapter): consume the one-shot + /// ignore-join entry for `sid` iff `is_join`, then merge. Mirrors + /// `on_inner_event`'s per-exchange suppression logic. + #[cfg(test)] + pub(crate) fn test_merge_remote_state_with_stream( + &mut self, + t: &mut T, + user_data: Bytes, + is_join: bool, + sid: StreamId, + ) where + T: Reliable, + I: Clone + Data, + A: Data, + { + let suppress = is_join && self.consume_ignore_join_stream(sid); + self.merge_remote_state(t, user_data, suppress); } /// Return the `ltime` of the most recently buffered intent for `id` of `kind`, diff --git a/serf-proto/src/endpoint/serf_parity_tests.rs b/serf-proto/src/endpoint/serf_parity_tests.rs index 7b1537d4..0d038f58 100644 --- a/serf-proto/src/endpoint/serf_parity_tests.rs +++ b/serf-proto/src/endpoint/serf_parity_tests.rs @@ -30,7 +30,14 @@ fn ep() -> StreamEndpoint { Box::new(|_addr: &std::net::SocketAddr| None), Box::new(|addr: &std::net::SocketAddr| *addr), ); - StreamEndpoint::new(coord, Options::new()) + let mut e = StreamEndpoint::new(coord, Options::new()); + let _ = e.poll_event(); + e +} + +/// A distinct loopback peer address for the per-peer ignore-join tests. +fn peer(port: u16) -> std::net::SocketAddr { + std::net::SocketAddr::from(([127, 0, 0, 1], port)) } // ── base.rs handle_node_join invariants ─────────────────────────────────────── @@ -249,7 +256,7 @@ fn push_pull_body( fn merge_witnesses_clocks_at_ltime_minus_one() { let mut e = ep(); let body = push_pull_body(10, vec![], vec![], 8, vec![], 6); - e.test_merge_remote_state(body, false); + e.test_merge_remote_state(body); // witness(9) → clock = 10; witness(7) → event_clock = 8; witness(5) → query_clock = 6 assert_eq!(e.member_time(), 10, "member clock: witness(9) → 10"); assert_eq!(e.event_time(), 8, "event clock: witness(7) → 8"); @@ -263,7 +270,7 @@ fn merge_witnesses_clocks_at_ltime_minus_one() { fn merge_zero_clocks_are_not_witnessed() { let mut e = ep(); let body = push_pull_body(0, vec![], vec![], 0, vec![], 0); - e.test_merge_remote_state(body, false); + e.test_merge_remote_state(body); assert_eq!(e.member_time(), 0, "zero ltime must not witness"); assert_eq!(e.event_time(), 0, "zero event_ltime must not witness"); assert_eq!(e.query_time(), 0, "zero query_ltime must not witness"); @@ -281,7 +288,7 @@ fn merge_processes_left_members_before_joins() { let mut e = ep(); // Node 2 in status_ltimes (ltime=4) AND in left_members. let body = push_pull_body(5, vec![(2u32, 4)], vec![2u32], 0, vec![], 0); - e.test_merge_remote_state(body, false); + e.test_merge_remote_state(body); // The join pass must have skipped node 2 (left_set guard). // The leave intent at synthetic ltime=5 must be buffered (node 2 not in states yet). assert_eq!( @@ -304,7 +311,7 @@ fn merge_join_intent_buffered_for_non_left_node() { let mut e = ep(); // Node 3 appears only in status_ltimes, NOT in left_members. let body = push_pull_body(5, vec![(3u32, 7)], vec![], 0, vec![], 0); - e.test_merge_remote_state(body, false); + e.test_merge_remote_state(body); // A join intent must be buffered for node 3 (it is not in states yet). assert_eq!( e.test_intent_ltime(3, IntentKind::Join), @@ -313,46 +320,94 @@ fn merge_join_intent_buffered_for_non_left_node() { ); } -/// delegate.rs:528-534 — G4: `eventJoinIgnore` + `is_join` bumps `event_buffer.min_time` -/// to max(min_time, event_ltime). +/// delegate.rs:528-534 — G4: a join merge whose `originating_stream_id` matches a +/// recorded ignore-join exchange bumps `event_buffer.min_time` to +/// max(min_time, event_ltime), and CONSUMES the one-shot per-exchange entry. #[test] fn join_with_event_join_ignore_bumps_event_min_time() { let mut e = ep(); - e.test_set_event_join_ignore(true); + // Start a real `ignore_old` join to mint and record this exchange's StreamId. + let s = e.start_join_push_pull(peer(5001), /*ignore_old*/ true, Instant::ORIGIN); + assert!( + e.test_has_ignore_join_stream(s), + "an ignore_old join must record its exchange StreamId" + ); let body = push_pull_body(0, vec![], vec![], 42, vec![], 0); - e.test_merge_remote_state(body, /*is_join*/ true); + e.test_merge_remote_state_with_stream(body, /*is_join*/ true, s); assert_eq!( e.test_event_min_time(), 42, - "event_join_ignore + is_join must set min_time to remote event_ltime" + "an ignore-join exchange's merge must set min_time to remote event_ltime" + ); + assert!( + !e.test_has_ignore_join_stream(s), + "the ignore-join entry must be consumed (one-shot) at the join merge" ); } -/// G4 inverse: `is_join = false` must NOT bump event_buffer.min_time. +/// G4 inverse: a REFRESH (non-join) exchange on a recorded ignore-join stream must +/// NOT bump min_time and must NOT consume the entry — only a join merge does. #[test] fn refresh_exchange_does_not_bump_event_min_time() { let mut e = ep(); - e.test_set_event_join_ignore(true); + let s = e.start_join_push_pull(peer(5002), /*ignore_old*/ true, Instant::ORIGIN); let body = push_pull_body(0, vec![], vec![], 99, vec![], 0); - e.test_merge_remote_state(body, /*is_join*/ false); + e.test_merge_remote_state_with_stream(body, /*is_join*/ false, s); assert_eq!( e.test_event_min_time(), 0, "non-join exchange must not bump event_buffer.min_time" ); + assert!( + e.test_has_ignore_join_stream(s), + "a refresh must not consume the exchange's pending ignore-join entry" + ); } -/// G4 inverse: flag not set even with is_join must NOT bump min_time. +/// G4 same-peer concurrency-correctness (the per-exchange precision the StreamId +/// key buys): an `ignore_old` join and a non-ignore join to the SAME seed get +/// DISTINCT `StreamId`s, so the non-ignore join's merge must NOT be suppressed and +/// must NOT consume the ignore-join's still-pending entry. A per-PEER key could +/// not tell these apart — whichever merge landed first would consume the wrong +/// token. #[test] -fn join_without_event_join_ignore_does_not_bump_min_time() { +fn non_ignore_join_to_same_seed_is_not_suppressed() { let mut e = ep(); - // event_join_ignore is false (default) + let seed = peer(5003); + // Two concurrent joins to the SAME seed: one ignore_old, one plain. + let ignored = e.start_join_push_pull(seed, /*ignore_old*/ true, Instant::ORIGIN); + let plain = e.start_join_push_pull(seed, /*ignore_old*/ false, Instant::ORIGIN); + assert_ne!(ignored, plain, "same-seed joins get distinct StreamIds"); + assert!( + e.test_has_ignore_join_stream(ignored), + "the ignore_old join recorded its exchange" + ); + assert!( + !e.test_has_ignore_join_stream(plain), + "the plain join to the same seed is NOT recorded" + ); + + // The plain join's merge lands first: it must not be suppressed and must leave + // the ignore_old join's entry intact. let body = push_pull_body(0, vec![], vec![], 99, vec![], 0); - e.test_merge_remote_state(body, /*is_join*/ true); + e.test_merge_remote_state_with_stream(body, /*is_join*/ true, plain); assert_eq!( e.test_event_min_time(), 0, - "event_join_ignore=false: min_time must not be bumped" + "a non-ignore join to the same seed must not bump event_buffer.min_time" + ); + assert!( + e.test_has_ignore_join_stream(ignored), + "the plain join's merge must not consume the ignore_old join's entry" + ); + + // The ignore_old join's own merge still suppresses, keyed by ITS StreamId. + let body = push_pull_body(0, vec![], vec![], 99, vec![], 0); + e.test_merge_remote_state_with_stream(body, /*is_join*/ true, ignored); + assert_eq!( + e.test_event_min_time(), + 99, + "the ignore_old join's own merge (its StreamId) bumps min_time" ); } @@ -369,7 +424,7 @@ fn merge_replays_buffered_user_events() { }], }]; let body = push_pull_body(4, vec![], vec![], 4, events, 0); - e.test_merge_remote_state(body, false); + e.test_merge_remote_state(body); // The replayed user event must surface. let ev = e.poll_event().expect("replayed user event must surface"); assert!( @@ -378,15 +433,19 @@ fn merge_replays_buffered_user_events() { ); } -/// G4 + replay: when event_join_ignore bumps min_time to the remote event_ltime, -/// events strictly below the new min_time must be suppressed on replay. +/// G4 + replay: when an ignore-join peer's merge bumps min_time to the remote +/// event_ltime, events strictly below the new min_time must be suppressed on +/// replay. /// /// G4 sets `min_time = event_ltime`. An event at `ltime=4` with `min_time=5` /// satisfies `ltime < min_time`, so it is suppressed. #[test] fn join_with_event_join_ignore_suppresses_event_replay() { let mut e = ep(); - e.test_set_event_join_ignore(true); + let s = e.start_join_push_pull(peer(5005), /*ignore_old*/ true, Instant::ORIGIN); + // Starting the dial surfaces no serf event, but drain defensively so the + // assertion below only observes replay (or its suppression). + while e.poll_event().is_some() {} // event_ltime = 5 → G4 bumps min_time to 5. Event at ltime=4 < 5 must be dropped. let events = vec![UserEvents { ltime: LamportTime::new(4), @@ -396,7 +455,7 @@ fn join_with_event_join_ignore_suppresses_event_replay() { }], }]; let body = push_pull_body(0, vec![], vec![], 5, events, 0); - e.test_merge_remote_state(body, /*is_join*/ true); + e.test_merge_remote_state_with_stream(body, /*is_join*/ true, s); // min_time = 5; event at ltime=4 must be suppressed. assert!( e.poll_event().is_none(), @@ -404,6 +463,50 @@ fn join_with_event_join_ignore_suppresses_event_replay() { ); } +/// Cancellation safety: an ignore-join whose exchange terminates WITHOUT a merge +/// (a dropped/failed join future) is cleaned up by the driver at the join's +/// terminal via `clear_ignore_join_stream`, so a later merge on that stream is NOT +/// suppressed — the entry cannot leak. +#[test] +fn ignore_join_cleared_on_exchange_termination_does_not_leak() { + let mut e = ep(); + let s = e.start_join_push_pull(peer(5006), /*ignore_old*/ true, Instant::ORIGIN); + assert!( + e.test_has_ignore_join_stream(s), + "the ignore_old join recorded its exchange" + ); + // The ignore-old join terminates without ever merging (the join future was + // dropped / the dial failed): the driver clears its StreamId. + e.test_clear_ignore_join_stream(s); + assert!( + !e.test_has_ignore_join_stream(s), + "a terminated ignore-join exchange must drop its pending entry" + ); + while e.poll_event().is_some() {} + + // A later merge carrying a buffered event must replay: because the entry was + // cleared, the merge is not suppressed (no min_time bump, event surfaces). + let events = vec![UserEvents { + ltime: LamportTime::new(3), + events: vec![UserEvent { + name: "should-replay".into(), + payload: Bytes::from_static(b"z"), + }], + }]; + let body = push_pull_body(4, vec![], vec![], 4, events, 0); + e.test_merge_remote_state_with_stream(body, /*is_join*/ true, s); + assert_eq!( + e.test_event_min_time(), + 0, + "a cleared ignore must NOT bump min_time on a later merge" + ); + let ev = e.poll_event().expect("the later merge's event must replay"); + assert!( + matches!(ev, Event::User(ref u) if u.name == "should-replay"), + "expected the un-suppressed event to surface, got: {ev:?}" + ); +} + /// Empty user_data must be silently ignored without panicking. #[test] fn merge_empty_user_data_is_silently_dropped() { @@ -411,7 +514,7 @@ fn merge_empty_user_data_is_silently_dropped() { // Simulate the RemoteStateReceived path: empty bytes are dropped before // merge_remote_state is called (the sieve arm guards `!user_data.is_empty()`). // Here we test that a zero-byte body passed directly does not panic. - e.test_merge_remote_state(Bytes::new(), false); + e.test_merge_remote_state(Bytes::new()); assert!(e.poll_event().is_none()); } @@ -419,7 +522,7 @@ fn merge_empty_user_data_is_silently_dropped() { #[test] fn merge_malformed_body_is_silently_dropped() { let mut e = ep(); - e.test_merge_remote_state(Bytes::from_static(b"\xff\xfe\xfd"), false); + e.test_merge_remote_state(Bytes::from_static(b"\xff\xfe\xfd")); assert_eq!( e.member_time(), 0, diff --git a/serf-proto/src/endpoint/tests.rs b/serf-proto/src/endpoint/tests.rs index e21dd7a0..16866acb 100644 --- a/serf-proto/src/endpoint/tests.rs +++ b/serf-proto/src/endpoint/tests.rs @@ -42,7 +42,11 @@ fn ep() -> StreamEndpoint { memberlist_proto::Instant::ORIGIN, SmallRng::seed_from_u64(0), ); - StreamEndpoint::new(coord(inner), Options::new()) + let mut e = StreamEndpoint::new(coord(inner), Options::new()); + // The inner memberlist emits NodeJoined(self) on construction; drain it so + // every test starts from the post-self-join-drained state (self is a member). + let _ = e.poll_event(); + e } /// Build a serf `Endpoint` with coordinates enabled (for coordinate-gated tests). @@ -56,7 +60,9 @@ fn ep_with_coords() -> StreamEndpoint { SmallRng::seed_from_u64(0), ); let opts = Options::new().with_disable_coordinates(false); - StreamEndpoint::new(coord(inner), opts) + let mut e = StreamEndpoint::new(coord(inner), opts); + let _ = e.poll_event(); + e } #[test] @@ -66,7 +72,8 @@ fn new_endpoint_starts_alive_with_zero_clocks() { assert_eq!(e.member_time(), 0); assert_eq!(e.event_time(), 0); assert_eq!(e.query_time(), 0); - assert_eq!(e.num_members(), 0); + // After the construction self-join is drained, the local node is the sole member. + assert_eq!(e.num_members(), 1); } #[test] @@ -376,6 +383,70 @@ fn leave_from_alive_transitions_to_leaving() { assert!(e.state().is_leaving(), "leave() must set state to Leaving"); } +/// Integrity floor: a `leave()` whose post-incremented stamp would reach +/// `LTIME_MAX` (member clock at `LTIME_MAX - 1`) parks degraded-but-safe — it +/// returns `LeaveClockExhausted`, advances no clock, queues NO invalid intent, +/// starts no inner leave, and leaves the lifecycle AND self-membership a +/// consistent `Alive`. +#[test] +fn leave_at_ltime_watermark_parks_degraded_but_safe() { + let mut e = ep(); + // Drive the member clock to the floor so the next leave stamp would be LTIME_MAX. + e.test_set_clocks(LTIME_MAX - 1, 0, 0); + let q_before = e.user_broadcast_queue_len(); + + let err = e + .leave(memberlist_proto::Instant::ORIGIN) + .expect_err("a leave whose stamp reaches LTIME_MAX must be refused"); + assert!( + matches!(err, Error::LeaveClockExhausted), + "expected LeaveClockExhausted, got {err:?}" + ); + + // No invalid intent was queued, the clock did not advance into the rejected + // range, and both the lifecycle state and the local member stay a consistent + // Alive — nothing entered an inconsistent Leaving half-state. + assert_eq!( + e.user_broadcast_queue_len(), + q_before, + "the degraded leave must not queue a leave intent" + ); + assert_eq!( + e.member_time(), + LTIME_MAX - 1, + "the degraded leave must not advance the member clock" + ); + assert!(e.state().is_alive(), "the endpoint must remain Alive"); + assert_eq!( + e.test_member_status(1), + Some(MemberStatus::Alive), + "self-membership must stay consistent (Alive, not a half-Leaving)" + ); +} + +/// Boundary: one tick below the floor (stamp lands at `LTIME_MAX - 1`, still +/// acceptable), `leave()` proceeds normally — the gate is exactly at the floor, +/// not off-by-one. +#[test] +fn leave_one_below_watermark_succeeds() { + let mut e = ep(); + e.test_set_clocks(LTIME_MAX - 2, 0, 0); + e.leave(memberlist_proto::Instant::ORIGIN) + .expect("a leave whose stamp is LTIME_MAX - 1 must still apply"); + // The intent (ltime = LTIME_MAX - 1) is acceptable, so the leave applies + // fully: both the lifecycle and self-membership transition to Leaving — the + // exact contrast with the degraded watermark path, which stays Alive. + assert!( + e.state().is_leaving(), + "the boundary leave must transition to Leaving" + ); + assert_eq!( + e.test_member_status(1), + Some(MemberStatus::Leaving), + "self-membership must transition to Leaving on the boundary leave" + ); +} + #[test] fn double_leave_is_rejected() { let mut e = ep(); @@ -454,11 +525,14 @@ fn inner_left_cluster_drives_serf_to_left_and_emits_left_cluster_event() { e.state().is_left(), "state must be Left after propagation delay" ); - // The next poll_event must yield Event::LeftCluster. - let ev = e.poll_event().expect("LeftCluster event must be pending"); + // Member(Leave, self) appears when the inner's NodeLeft(self) is drained + // (Leaving → Left transition). Event::LeftCluster follows once the + // leave_propagate_delay deadline fires. Drain until LeftCluster is found. + let found_left_cluster = + core::iter::from_fn(|| e.poll_event()).any(|ev| matches!(ev, Event::LeftCluster)); assert!( - matches!(ev, Event::LeftCluster), - "expected Event::LeftCluster, got {ev:?}" + found_left_cluster, + "Event::LeftCluster must be emitted after the leave propagation delay" ); } @@ -470,9 +544,16 @@ fn leave_complete_deadline_not_fired_before_delay() { // Tick to just before the propagation deadline (< 1s). let before_delay = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_millis(500); e.handle_timeout(before_delay); - // Still Leaving; no event yet. + // SerfState must still be Leaving (the deadline hasn't fired yet). assert!(e.state().is_leaving(), "must still be Leaving before delay"); - assert!(e.poll_event().is_none(), "no LeftCluster before deadline"); + // Member(Leave, self) from the inner's NodeLeft drain may appear, but + // Event::LeftCluster must NOT appear before the propagation deadline. + let no_left_cluster = + core::iter::from_fn(|| e.poll_event()).all(|ev| !matches!(ev, Event::LeftCluster)); + assert!( + no_left_cluster, + "LeftCluster must not appear before the propagation deadline" + ); } #[test] @@ -491,8 +572,14 @@ fn shutdown_prevents_leaving_to_left_transition() { e.state().is_shutdown(), "Shutdown should not transition to Left" ); - // No LeftCluster event. - assert!(e.poll_event().is_none()); + // Event::LeftCluster must not appear (the leave chain was interrupted by + // Shutdown). Member(Leave, self) from the inner NodeLeft drain may appear. + let no_left_cluster = + core::iter::from_fn(|| e.poll_event()).all(|ev| !matches!(ev, Event::LeftCluster)); + assert!( + no_left_cluster, + "LeftCluster must not be emitted when Shutdown interrupts the leave chain" + ); } #[test] @@ -533,6 +620,64 @@ fn force_leave_transitions_alive_member_to_leaving() { ); } +/// `force_leave` at `clock == LTIME_MAX - 1` parks degraded-but-safe — it +/// returns `LeaveClockExhausted`, advances no clock, queues NO invalid intent, +/// and leaves the target member and clock state consistent. +#[test] +fn force_leave_at_ltime_watermark_parks_degraded_but_safe() { + let mut e = ep(); + e.test_seed_member(2, MemberStatus::Alive, LamportTime::new(0)); + // Drive the member clock to the floor so the next stamp would be LTIME_MAX. + e.test_set_clocks(LTIME_MAX - 1, 0, 0); + let q_before = e.user_broadcast_queue_len(); + + let err = e + .force_leave(2u32, false, memberlist_proto::Instant::ORIGIN) + .expect_err("a force_leave whose stamp reaches LTIME_MAX must be refused"); + assert!( + matches!(err, Error::LeaveClockExhausted), + "expected LeaveClockExhausted, got {err:?}" + ); + + // No invalid intent was queued, the clock did not advance into the rejected + // range, the target member stayed Alive, and the local endpoint state is + // unchanged — nothing entered an inconsistent half-state. + assert_eq!( + e.user_broadcast_queue_len(), + q_before, + "the degraded force_leave must not queue a leave intent" + ); + assert_eq!( + e.member_time(), + LTIME_MAX - 1, + "the degraded force_leave must not advance the member clock" + ); + assert_eq!( + e.test_member_status(2), + Some(MemberStatus::Alive), + "the target member must stay Alive after a degraded force_leave" + ); +} + +/// Boundary: one tick below the floor (stamp lands at `LTIME_MAX - 1`, still +/// acceptable), `force_leave()` proceeds normally — the gate is exactly at the +/// floor, not off-by-one. +#[test] +fn force_leave_one_below_watermark_succeeds() { + let mut e = ep(); + e.test_seed_member(2, MemberStatus::Alive, LamportTime::new(0)); + e.test_set_clocks(LTIME_MAX - 2, 0, 0); + e.force_leave(2u32, false, memberlist_proto::Instant::ORIGIN) + .expect("a force_leave whose stamp is LTIME_MAX - 1 must still apply"); + // The intent is acceptable, so the target transitions to Leaving — the exact + // contrast with the degraded watermark path, which stays Alive. + assert_eq!( + e.test_member_status(2), + Some(MemberStatus::Leaving), + "the boundary force_leave must transition the target to Leaving" + ); +} + #[test] fn leave_witnesses_and_advances_member_clock() { let mut e = ep(); @@ -1117,15 +1262,30 @@ fn drain_inner_calls_resync_when_dirty() { } #[test] -fn event_join_ignore_flag_can_be_set() { - // Accessor smoke-test: set_event_join_ignore and test_set_event_join_ignore - // both write the same field. +fn ignore_join_stream_recorded_and_consumed_one_shot() { + // Accessor smoke-test: an ignore_old join records its exchange StreamId + // (idempotently); the matching merge consumes the one-shot entry, and an + // unrecorded stream is never present. let mut e = ep(); - assert!(!e.test_is_dirty() || true); // just ensure no panic - e.set_event_join_ignore(true); - // test_set_event_join_ignore is also wired, confirm symmetry. - e.test_set_event_join_ignore(false); - // Confirm the flag can be toggled (no panic, no side-effects without a merge). + let p = std::net::SocketAddr::from(([127, 0, 0, 1], 6100)); + let s = e.start_join_push_pull( + p, + /*ignore_old*/ true, + memberlist_proto::Instant::ORIGIN, + ); + // Idempotent: re-noting the same StreamId must not double-store. + e.test_note_ignore_join_stream(s); + assert!( + e.test_has_ignore_join_stream(s), + "recorded exchange must be present" + ); + // A join merge on this stream consumes the single entry (a second entry, had + // the record not been idempotent, would survive this one consume). + e.test_merge_remote_state_with_stream(bytes::Bytes::new(), true, s); + assert!( + !e.test_has_ignore_join_stream(s), + "a join merge consumes the one-shot ignore-join entry" + ); } #[test] @@ -1832,8 +1992,8 @@ fn relay_node(port: u16) -> memberlist_proto::Node { #[test] fn relay_response_is_silent_noop_when_too_few_members() { - // With 0 non-self members, relay_factor=2 requires at least 3 total members - // (relay_factor + 1 = 3) but we have 0 → silent no-op: no RelayDropped event, + // With only self in membership, relay_factor=2 requires at least 3 total members + // (relay_factor + 1 = 3) but we have 1 → silent no-op: no RelayDropped event, // no directed send. let mut e = ep(); let querier = relay_node(2000); @@ -3419,7 +3579,7 @@ fn merge_remote_state_max_member_clock_is_ignored() { let encoded = AnyMessage::::PushPull(pp) .encode() .expect("encode must succeed"); - e.test_merge_remote_state(encoded, false); + e.test_merge_remote_state(encoded); assert_eq!( e.member_time(), @@ -3570,11 +3730,12 @@ fn inbound_user_event_too_large_is_dropped() { #[test] fn key_query_num_nodes_equals_member_count_at_issue_time() { let mut e = ep(); - // Seed 3 distinct members so num_members() == 3 at issue time. + // Seed 3 distinct members; including self (drained at construction) the total + // membership at issue time is 4. for id in [10u32, 11, 12] { e.test_seed_member(id, MemberStatus::Alive, LamportTime::new(1)); } - assert_eq!(e.num_members(), 3); + assert_eq!(e.num_members(), 4); let far_future = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(9999); let _query_id = e.test_register_key_query(far_future); @@ -3584,7 +3745,7 @@ fn key_query_num_nodes_equals_member_count_at_issue_time() { .test_last_pending_query_num_nodes() .expect("pending query must exist"); assert_eq!( - num_nodes, 3, + num_nodes, 4, "num_nodes must equal the member count at issue time" ); @@ -3598,7 +3759,7 @@ fn key_query_num_nodes_equals_member_count_at_issue_time() { match ev { Event::KeyResponse(kr) => { assert_eq!( - kr.num_nodes, 3, + kr.num_nodes, 4, "KeyResponse.num_nodes must equal the queried member count" ); assert_eq!(kr.num_resp, 0, "no responses were folded before timeout"); @@ -3613,8 +3774,9 @@ fn key_query_num_nodes_equals_member_count_at_issue_time() { #[test] fn key_query_num_nodes_is_captured_at_issue_not_at_close() { let mut e = ep(); + // Self is already a member after construction drain; seed one more. e.test_seed_member(10u32, MemberStatus::Alive, LamportTime::new(1)); - assert_eq!(e.num_members(), 1); + assert_eq!(e.num_members(), 2); let far_future = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(9999); let _query_id = e.test_register_key_query(far_future); @@ -3622,9 +3784,9 @@ fn key_query_num_nodes_is_captured_at_issue_not_at_close() { // Now add 2 more members after the query was issued. e.test_seed_member(11u32, MemberStatus::Alive, LamportTime::new(1)); e.test_seed_member(12u32, MemberStatus::Alive, LamportTime::new(1)); - assert_eq!(e.num_members(), 3); + assert_eq!(e.num_members(), 4); - // Close the query: num_nodes must reflect membership AT ISSUE TIME (1), not now (3). + // Close the query: num_nodes must reflect membership AT ISSUE TIME (2), not now (4). let after_deadline = far_future + std::time::Duration::from_nanos(1); e.test_fire_due_query_closes(after_deadline); @@ -3632,8 +3794,8 @@ fn key_query_num_nodes_is_captured_at_issue_not_at_close() { match ev { Event::KeyResponse(kr) => { assert_eq!( - kr.num_nodes, 1, - "num_nodes must be the count at issue time (1), not the current count (3)" + kr.num_nodes, 2, + "num_nodes must be the count at issue time (2), not the current count (4)" ); } other => panic!("expected Event::KeyResponse, got {other:?}"), @@ -3810,7 +3972,7 @@ fn merge_remote_state_max_minus_one_member_clock_drops_entire_message() { let encoded = AnyMessage::::PushPull(pp) .encode() .expect("encode must succeed"); - e.test_merge_remote_state(encoded, false); + e.test_merge_remote_state(encoded); // No state mutation: clock unchanged, dirty flag not set by this call. assert_eq!( @@ -3852,7 +4014,7 @@ fn merge_remote_state_max_minus_one_event_clock_drops_entire_message() { let encoded = AnyMessage::::PushPull(pp) .encode() .expect("encode must succeed"); - e.test_merge_remote_state(encoded, false); + e.test_merge_remote_state(encoded); assert_eq!( e.event_time(), @@ -4073,7 +4235,7 @@ fn zero_event_buffer_size_does_not_panic_on_merge_remote_state() { .expect("encode must succeed"); // Must NOT panic. - e.test_merge_remote_state(encoded, false); + e.test_merge_remote_state(encoded); } // ── Clock integrity + overflow regression tests ─────────────────────────────── @@ -4277,6 +4439,8 @@ fn inbound_query_oversized_is_dropped_before_state_mutation() { let opts = crate::options::Options::new().with_query_size_limit(64); let mut e: StreamEndpoint = StreamEndpoint::new(coord(inner), opts); + // Drain the construction self-join so it does not appear as a spurious event. + let _ = e.poll_event(); // Encode a QueryMessage whose payload pushes the wire encoding over 64 bytes. let oversized_payload = bytes::Bytes::from(vec![0u8; 100]); @@ -4586,7 +4750,7 @@ fn merge_remote_state_with_trailing_junk_is_dropped() { let clock_before = e.member_time(); let dirty_before = e.test_is_dirty(); - e.test_merge_remote_state(padded, false); + e.test_merge_remote_state(padded); assert_eq!( e.member_time(), @@ -4803,6 +4967,8 @@ fn user_event_total_packet_with_junk_exceeding_size_limit_is_dropped() { let opts = crate::options::Options::new().with_max_user_event_size(32); let mut e: StreamEndpoint = StreamEndpoint::new(coord(inner), opts); + // Drain the construction self-join so it does not appear as a spurious event. + let _ = e.poll_event(); // A small valid UserEvent that fits within 32 bytes on its own. let valid = AnyMessage::::UserEvent(UserEventMessage { @@ -4850,6 +5016,8 @@ fn pre_decode_fence_drops_oversized_valid_query_frame() { let opts = crate::options::Options::new().with_query_size_limit(64); let mut e: StreamEndpoint = StreamEndpoint::new(coord(inner), opts); + // Drain the construction self-join so it does not appear as a spurious event. + let _ = e.poll_event(); // Construct a syntactically valid Query whose encoded frame exceeds 64 bytes. let q = QueryMessage:: { @@ -4916,6 +5084,8 @@ fn pre_decode_fence_drops_oversized_valid_user_event_frame() { let opts = crate::options::Options::new().with_max_user_event_size(32); let mut e: StreamEndpoint = StreamEndpoint::new(coord(inner), opts); + // Drain the construction self-join so it does not appear as a spurious event. + let _ = e.poll_event(); // Construct a syntactically valid UserEvent whose encoded frame exceeds 32 bytes. let frame = AnyMessage::::UserEvent(UserEventMessage { @@ -5378,7 +5548,7 @@ fn stale_push_pull_does_not_set_dirty() { let encoded = AnyMessage::::PushPull(pp) .encode() .expect("encode PushPull"); - e.test_merge_remote_state(encoded, false); + e.test_merge_remote_state(encoded); assert!( !e.test_is_dirty(), @@ -5407,7 +5577,7 @@ fn clock_advancing_push_pull_sets_dirty() { let encoded = AnyMessage::::PushPull(pp) .encode() .expect("encode PushPull"); - e.test_merge_remote_state(encoded, false); + e.test_merge_remote_state(encoded); assert!( e.test_is_dirty(), @@ -5932,7 +6102,7 @@ fn next_ltime_integrity_floor_near_watermark() { e.event_time() ); - // The leave path uses next_ltime(&mut self.clock). + // The leave path uses post-increment (saturating_add then stamp the new value). let mut e2 = ep(); e2.test_set_clocks(LTIME_MAX - 1, LTIME_MAX - 1, LTIME_MAX - 1); let now = memberlist_proto::Instant::ORIGIN; @@ -6272,7 +6442,7 @@ fn left_member_replay_at_watermark_boundary_skips_leave() { .encode() .expect("encode must succeed"); // Must not panic. - e.test_merge_remote_state(encoded, false); + e.test_merge_remote_state(encoded); // The leave is skipped (derived LTIME_MAX rejected); member stays Alive. let status = e.test_member_status(99u32); @@ -6302,6 +6472,12 @@ fn left_member_replay_at_watermark_boundary_skips_leave() { /// and every push-pull emission must satisfy the integrity floor (not 0, not /// u64::MAX, no wrap). /// +/// witness(LTIME_MAX - 1) advances the stored member clock to LTIME_MAX (post- +/// increment semantics), so force_leave's prospective stamp = LTIME_MAX + 1, +/// which is unacceptable — the watermark guard returns LeaveClockExhausted +/// without mutating the clock. member_clock stays at LTIME_MAX — not 0, not +/// u64::MAX. +/// /// After load_snapshot with last_event_clock = LTIME_MAX - 1, the stored /// event_clock is LTIME_MAX - 1 (from witness) and event_buffer.min_time is /// LTIME_MAX (saturating_add(1)). user_event() stamps LTIME_MAX - 1 via @@ -6379,11 +6555,25 @@ fn all_clock_derived_values_satisfy_integrity_floor_after_near_watermark_snapsho "query_buffer.min_time must not be u64::MAX" ); - // force_leave stamps clock.saturating_add(1): no UB. + // After load_snapshot, witness(LTIME_MAX - 1) advances the stored clock to + // LTIME_MAX (witness semantics: stored = t + 1). The next force_leave stamp + // would be LTIME_MAX + 1, which is not acceptable — the watermark guard + // returns LeaveClockExhausted without mutating anything further. No invalid + // intent is emitted and the clock stays at LTIME_MAX. + let clock_after_snapshot = e.member_time(); + assert_eq!( + clock_after_snapshot, LTIME_MAX, + "after witnessing LTIME_MAX-1, stored clock must be LTIME_MAX" + ); let fl = e.force_leave(99u32, false, now); assert!( - fl.is_ok(), - "force_leave must succeed after near-watermark snapshot: {fl:?}" + matches!(fl, Err(Error::LeaveClockExhausted)), + "force_leave at LTIME_MAX clock must return LeaveClockExhausted: {fl:?}" + ); + assert_eq!( + e.member_time(), + LTIME_MAX, + "member_clock must remain LTIME_MAX after refused force_leave" ); // resync_local_state must emit push-pull with non-0 / non-u64::MAX clocks. @@ -6473,7 +6663,7 @@ fn push_pull_near_watermark_status_time_integrity_floor() { let encoded = AnyMessage::::PushPull(pp) .encode() .expect("encode must succeed"); - e.test_merge_remote_state(encoded, false); + e.test_merge_remote_state(encoded); // The inner NodeJoined event materialises node 99. e.test_inner_node_joined(99u32, now); @@ -6629,8 +6819,6 @@ fn push_pull_near_watermark_event_floor_integrity_and_delivery() { use crate::typed::PushPullMessage; let mut e = ep(); - // Set event_join_ignore = true so the G4 path fires on is_join = true. - e.set_event_join_ignore(true); // LTIME_MAX - 1 passes ltime_is_acceptable. let near_max = LTIME_MAX - 1; @@ -6650,8 +6838,9 @@ fn push_pull_near_watermark_event_floor_integrity_and_delivery() { let encoded = AnyMessage::::PushPull(pp) .encode() .expect("encode must succeed"); - // is_join = true triggers the G4 event_buffer.min_time update path. - e.test_merge_remote_state(encoded, true); + // A suppressed (ignore-join) merge triggers the G4 event_buffer.min_time + // update path. + e.test_merge_remote_state_suppressed(encoded); // Integrity floor: min_time is the ingress value, not 0, not u64::MAX. assert_ne!( @@ -6742,27 +6931,34 @@ fn set_tags_local_member_state_is_observable_without_poll_event() { ); } -/// When `set_tags` is called before the local `NodeJoined` has been drained, -/// no phantom entry must appear in `members.states`. The new tags are written -/// to the coordinator's meta store via `update_meta` and will be decoded from -/// there when `handle_node_join` materializes the member. +/// After the construction self-join is drained (the local `NodeJoined` has been +/// processed), `set_tags` must update the existing local member in-place and +/// must not create a duplicate or phantom entry. The tags must also be reflected +/// in the coordinator's meta store. #[test] fn set_tags_does_not_materialize_absent_local_member() { use crate::typed::Tags; - let mut e = ep(); // local member absent from members.states + // ep() drains the construction NodeJoined, so the local member IS already in + // members.states as Alive when this test begins. + let mut e = ep(); let tags: Tags = [("env", "staging")].into_iter().collect(); - e.set_tags(tags) - .expect("set_tags must succeed even before the local node is in members.states"); + e.set_tags(tags.clone()) + .expect("set_tags must succeed when the local node is in members.states"); - // No phantom entry must be inserted into members.states. - assert!( - e.test_local_tags().is_none(), - "set_tags must not fabricate a local member when absent from members.states" + // set_tags must have updated the existing local member's tags in-place. + let local_tags = e.test_local_tags(); + assert_eq!( + local_tags + .as_ref() + .and_then(|t| t.0.get("env")) + .map(|s| s.as_str()), + Some("staging"), + "set_tags must update the local member's tags in members.states" ); - // The tags must have been advertised via the coordinator's meta store. + // The tags must also have been advertised via the coordinator's meta store. let meta = e .test_local_meta() .expect("coordinator must track local node meta"); @@ -6771,7 +6967,7 @@ fn set_tags_does_not_materialize_absent_local_member() { assert_eq!( decoded.0.get("env").map(|s| s.as_str()), Some("staging"), - "new tags must be reflected in the coordinator meta even when local member is absent" + "new tags must be reflected in the coordinator meta" ); } @@ -6797,16 +6993,19 @@ fn set_tags_does_not_mark_local_state_dirty() { ); } -/// When `set_tags` is called before the local member exists in `members.states`, -/// the coordinator's queued `NodeUpdated` event must be suppressed (because the -/// member is absent) and must not surface as `Member(Update)` before the -/// `Member(Join)` that a subsequent `NodeJoined` would produce. If no `Join` -/// event appears in the drain, no `Update` event must appear either. +/// After the construction self-join is drained (the local member is already in +/// `members.states`), `set_tags` triggers a `NodeUpdated` in the inner +/// coordinator. The invariant is that no `Member(Update)` ever appears BEFORE a +/// `Member(Join)` in the same event-stream segment. When the Join already +/// occurred at construction (before this collection), an Update appearing alone +/// (with no Join in the current segment) is valid — the prior Join satisfies the +/// ordering constraint. #[test] fn set_tags_before_join_does_not_emit_update_before_join() { use crate::typed::Tags; - let mut e = ep(); // local member absent + // ep() drains the construction NodeJoined, so the local member IS present. + let mut e = ep(); let tags: Tags = [("role", "cache")].into_iter().collect(); e.set_tags(tags).expect("set_tags must succeed"); @@ -6820,7 +7019,9 @@ fn set_tags_before_join_does_not_emit_update_before_join() { evs.push(ev); } - // If a Join appears, no Update may precede it. + // Within this collection: if both a Join and an Update appear, the Join must + // come first. An Update without a Join in this segment is also valid because + // the Join already happened at construction (before this collection). let join_pos = evs .iter() .position(|ev| matches!(ev, Event::Member(me) if me.kind() == MemberEventKind::Join)); @@ -6828,11 +7029,10 @@ fn set_tags_before_join_does_not_emit_update_before_join() { .iter() .position(|ev| matches!(ev, Event::Member(me) if me.kind() == MemberEventKind::Update)); - match (join_pos, update_pos) { - (None, Some(_)) => panic!("Member(Update) appeared without a preceding Member(Join): {evs:?}"), - (Some(j), Some(u)) if u < j => { - panic!("Member(Update) at index {u} appeared before Member(Join) at index {j}: {evs:?}") - } - _ => {} + if let (Some(j), Some(u)) = (join_pos, update_pos) { + assert!( + u >= j, + "Member(Update) at index {u} must not precede Member(Join) at index {j}: {evs:?}" + ); } } diff --git a/serf-proto/src/event/mod.rs b/serf-proto/src/event/mod.rs index f7a1dbe5..7f541ca4 100644 --- a/serf-proto/src/event/mod.rs +++ b/serf-proto/src/event/mod.rs @@ -9,7 +9,7 @@ use std::sync::Arc; use bytes::Bytes; #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] use memberlist_proto::SecretKey; -use memberlist_proto::{Instant, Node, StreamId}; +use memberlist_proto::{Instant, Node, StreamId, event::ExchangeCompleted}; use smol_str::SmolStr; use crate::{LamportTime, UserEventMessage, members::Member}; @@ -493,6 +493,18 @@ pub enum Event { doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] KeyRequest(KeyRequest), + /// The terminal outcome of a reliable exchange initiated by this node. + /// + /// Emitted when the coordinator's bridge-reap path fires for an outbound + /// exchange. The payload carries the opaque `eid` (correlates with the + /// `ExchangeId` returned by the coordinator's `start_push_pull` / + /// `accept_connection`), the `peer` address, the `outcome` + /// ([`memberlist_proto::event::ExchangeStatus`]), and the `kind` + /// ([`memberlist_proto::event::ExchangeKind`]) that identifies the + /// initiator. A driver awaiting a Join push/pull resolves when + /// `kind() == ExchangeKind::PushPull`. Inbound (peer-initiated) + /// exchanges do NOT emit this event — only outbound ones do. + ExchangeCompleted(ExchangeCompleted), } #[cfg(test)] diff --git a/serf-proto/src/lib.rs b/serf-proto/src/lib.rs index 4cc880ee..61c34bf6 100644 --- a/serf-proto/src/lib.rs +++ b/serf-proto/src/lib.rs @@ -130,3 +130,7 @@ pub use event::{KeyRequest, KeyRequestOperation, KeyResponseArgs}; doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] pub use memberlist_proto::SecretKey; + +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub use memberlist_proto::event::{ExchangeCompleted, ExchangeId, ExchangeKind, ExchangeStatus}; diff --git a/serf-proto/src/quic_endpoint/mod.rs b/serf-proto/src/quic_endpoint/mod.rs index 8e41a2f2..996b3ed7 100644 --- a/serf-proto/src/quic_endpoint/mod.rs +++ b/serf-proto/src/quic_endpoint/mod.rs @@ -265,6 +265,42 @@ where id } + /// Initiate an outbound **join** push-pull dial to `peer`, returning the + /// exchange's [`StreamId`]. + /// + /// Like [`Self::start_push_pull`] with [`PushPullKind::Join`], but when + /// `ignore_old` is set it records the returned `StreamId` as a per-EXCHANGE + /// ignore-join target on the serf core, so the resulting join merge (whose + /// `originating_stream_id` equals this `StreamId`) suppresses replay of the + /// peer's pre-join user events (H8/G4). The driver uses this for the seed joins + /// of an `ignore_old` join, and must hand the returned `StreamId` to + /// [`Self::clear_ignore_join_stream`] if the join terminates without merging. + /// Reconnect-driven joins go through the coordinator directly and never ignore + /// old events. + pub fn start_join_push_pull( + &mut self, + peer: SocketAddr, + ignore_old: bool, + now: Instant, + ) -> StreamId { + let id = self.start_push_pull(peer, PushPullKind::Join, now); + if ignore_old { + self.core.note_ignore_join_stream(id); + } + id + } + + /// Remove a terminated `ignore_old` join's exchange `id` from the serf core's + /// ignore set. + /// + /// The driver calls this when an `ignore_old` join reaches its terminal without + /// a merge having consumed the entry (dial failure, timeout, empty push/pull + /// body, or a dropped join future). Idempotent: a `StreamId` the success-path + /// merge already consumed is simply absent. + pub fn clear_ignore_join_stream(&mut self, id: StreamId) { + self.core.clear_ignore_join_stream(id); + } + /// Feed one already-decoded gossip [`Message`] into the coordinator, then /// sieve the resulting inner events into serf. /// @@ -465,11 +501,6 @@ where self.core.resync_local_state(&mut self.transport) } - /// Forwards to [`Endpoint::set_event_join_ignore`]. - pub fn set_event_join_ignore(&mut self, v: bool) { - self.core.set_event_join_ignore(v) - } - /// Update the local node's tags, re-advertise them via the coordinator, and /// synchronously refresh the local member in the membership store. /// @@ -885,9 +916,19 @@ where .map(|ns| ns.meta_ref().clone()) } - /// Forwards to [`Endpoint::test_set_event_join_ignore`]. - pub(crate) fn test_set_event_join_ignore(&mut self, v: bool) { - self.core.test_set_event_join_ignore(v) + /// Forwards to [`Endpoint::test_note_ignore_join_stream`]. + pub(crate) fn test_note_ignore_join_stream(&mut self, id: StreamId) { + self.core.test_note_ignore_join_stream(id) + } + + /// Forwards to [`Endpoint::test_has_ignore_join_stream`]. + pub(crate) fn test_has_ignore_join_stream(&self, id: StreamId) -> bool { + self.core.test_has_ignore_join_stream(id) + } + + /// Forwards to [`Endpoint::test_clear_ignore_join_stream`]. + pub(crate) fn test_clear_ignore_join_stream(&mut self, id: StreamId) { + self.core.test_clear_ignore_join_stream(id) } /// Forwards to [`Endpoint::test_event_min_time`]. @@ -896,13 +937,37 @@ where } /// Forwards to [`Endpoint::test_merge_remote_state`]. - pub(crate) fn test_merge_remote_state(&mut self, user_data: Bytes, is_join: bool) + pub(crate) fn test_merge_remote_state(&mut self, user_data: Bytes) + where + I: Clone + Data, + { + self + .core + .test_merge_remote_state(&mut self.transport, user_data) + } + + /// Forwards to [`Endpoint::test_merge_remote_state_suppressed`]. + pub(crate) fn test_merge_remote_state_suppressed(&mut self, user_data: Bytes) where I: Clone + Data, { self .core - .test_merge_remote_state(&mut self.transport, user_data, is_join) + .test_merge_remote_state_suppressed(&mut self.transport, user_data) + } + + /// Forwards to [`Endpoint::test_merge_remote_state_with_stream`]. + pub(crate) fn test_merge_remote_state_with_stream( + &mut self, + user_data: Bytes, + is_join: bool, + sid: StreamId, + ) where + I: Clone + Data, + { + self + .core + .test_merge_remote_state_with_stream(&mut self.transport, user_data, is_join, sid) } /// Forwards to [`Endpoint::test_intent_ltime`]. diff --git a/serf-proto/src/quic_endpoint/tests.rs b/serf-proto/src/quic_endpoint/tests.rs index 007c583d..4de72431 100644 --- a/serf-proto/src/quic_endpoint/tests.rs +++ b/serf-proto/src/quic_endpoint/tests.rs @@ -145,7 +145,9 @@ fn ep(id: u32, port: u16) -> QuicEndpoint { test_quic_options(), Some(seed), ); - QuicEndpoint::new(coord, Options::new()) + let mut e = QuicEndpoint::new(coord, Options::new()); + let _ = e.poll_event(); + e } #[test] @@ -224,7 +226,7 @@ fn merge_remote_state_folds_remote_clocks() { .encode() .expect("encode push-pull body"); - e.test_merge_remote_state(encoded, false); + e.test_merge_remote_state(encoded); assert_eq!( e.member_time(), diff --git a/serf-proto/src/stream_endpoint/mod.rs b/serf-proto/src/stream_endpoint/mod.rs index 1f69b0d4..b8449d72 100644 --- a/serf-proto/src/stream_endpoint/mod.rs +++ b/serf-proto/src/stream_endpoint/mod.rs @@ -300,6 +300,40 @@ where id } + /// Initiate an outbound **join** push-pull dial to `peer`, returning the + /// exchange's [`StreamId`]. + /// + /// Like [`Self::start_push_pull`] with [`PushPullKind::Join`], but when + /// `ignore_old` is set it records the returned `StreamId` as a per-EXCHANGE + /// ignore-join target on the serf core, so the resulting join merge (whose + /// `originating_stream_id` equals this `StreamId`) suppresses replay of the + /// peer's pre-join user events (H8/G4). The driver uses this for the seed joins + /// of an `ignore_old` join, and must hand the returned `StreamId` to + /// [`Self::clear_ignore_join_stream`] if the join terminates without merging. + /// Reconnect-driven joins go through the coordinator directly and never ignore + /// old events. + pub fn start_join_push_pull(&mut self, peer: A, ignore_old: bool, now: Instant) -> StreamId { + let id = self.start_push_pull(peer, PushPullKind::Join, now); + if ignore_old { + self.core.note_ignore_join_stream(id); + } + id + } + + /// Remove a terminated `ignore_old` join's exchange `id` from the serf core's + /// ignore set. + /// + /// The driver calls this when an `ignore_old` join reaches its terminal without + /// a merge having consumed the entry (dial failure, timeout, empty push/pull + /// body, or a dropped join future). Idempotent: a `StreamId` the success-path + /// merge already consumed is simply absent. Required because + /// [`memberlist_proto::event::ExchangeCompleted`]'s `eid` is a different domain + /// from the `StreamId` on the stream backend, so the machine cannot self-clean + /// the no-merge case — the driver owns the per-join terminal bookkeeping. + pub fn clear_ignore_join_stream(&mut self, id: StreamId) { + self.core.clear_ignore_join_stream(id); + } + /// Feed one already-decoded gossip [`Message`] into the coordinator, then /// sieve the resulting inner events into serf. /// @@ -488,11 +522,6 @@ where self.core.resync_local_state(&mut self.transport) } - /// Forwards to [`Endpoint::set_event_join_ignore`]. - pub fn set_event_join_ignore(&mut self, v: bool) { - self.core.set_event_join_ignore(v) - } - /// Update the local node's tags, re-advertise them via the coordinator, and /// synchronously refresh the local member in the membership store. /// @@ -924,10 +953,22 @@ where .map(|ns| ns.meta_ref().cheap_clone()) } - /// Forwards to [`Endpoint::test_set_event_join_ignore`]. + /// Forwards to [`Endpoint::test_note_ignore_join_stream`]. #[cfg(test)] - pub(crate) fn test_set_event_join_ignore(&mut self, v: bool) { - self.core.test_set_event_join_ignore(v) + pub(crate) fn test_note_ignore_join_stream(&mut self, id: StreamId) { + self.core.test_note_ignore_join_stream(id) + } + + /// Forwards to [`Endpoint::test_has_ignore_join_stream`]. + #[cfg(test)] + pub(crate) fn test_has_ignore_join_stream(&self, id: StreamId) -> bool { + self.core.test_has_ignore_join_stream(id) + } + + /// Forwards to [`Endpoint::test_clear_ignore_join_stream`]. + #[cfg(test)] + pub(crate) fn test_clear_ignore_join_stream(&mut self, id: StreamId) { + self.core.test_clear_ignore_join_stream(id) } /// Forwards to [`Endpoint::test_event_min_time`]. @@ -938,14 +979,42 @@ where /// Forwards to [`Endpoint::test_merge_remote_state`]. #[cfg(test)] - pub(crate) fn test_merge_remote_state(&mut self, user_data: Bytes, is_join: bool) + pub(crate) fn test_merge_remote_state(&mut self, user_data: Bytes) where I: Clone + Data, A: Data, { self .core - .test_merge_remote_state(&mut self.transport, user_data, is_join) + .test_merge_remote_state(&mut self.transport, user_data) + } + + /// Forwards to [`Endpoint::test_merge_remote_state_suppressed`]. + #[cfg(test)] + pub(crate) fn test_merge_remote_state_suppressed(&mut self, user_data: Bytes) + where + I: Clone + Data, + A: Data, + { + self + .core + .test_merge_remote_state_suppressed(&mut self.transport, user_data) + } + + /// Forwards to [`Endpoint::test_merge_remote_state_with_stream`]. + #[cfg(test)] + pub(crate) fn test_merge_remote_state_with_stream( + &mut self, + user_data: Bytes, + is_join: bool, + sid: StreamId, + ) where + I: Clone + Data, + A: Data, + { + self + .core + .test_merge_remote_state_with_stream(&mut self.transport, user_data, is_join, sid) } /// Forwards to [`Endpoint::test_intent_ltime`]. diff --git a/serf-proto/src/stream_endpoint/tests.rs b/serf-proto/src/stream_endpoint/tests.rs index 5e949c6e..6cb729a3 100644 --- a/serf-proto/src/stream_endpoint/tests.rs +++ b/serf-proto/src/stream_endpoint/tests.rs @@ -42,7 +42,9 @@ fn ep(id: u32, port: u16) -> StreamEndpoint { Box::new(|_addr: &SocketAddr| None), Box::new(|addr: &SocketAddr| *addr), ); - StreamEndpoint::new(coord, Options::new()) + let mut e = StreamEndpoint::new(coord, Options::new()); + let _ = e.poll_event(); + e } #[test] @@ -125,7 +127,7 @@ fn merge_remote_state_folds_remote_clocks() { .encode() .expect("encode push-pull body"); - e.test_merge_remote_state(encoded, false); + e.test_merge_remote_state(encoded); assert_eq!( e.member_time(), @@ -259,3 +261,109 @@ fn loopback_push_pull_converges_member_clock() { "the acceptor learned the dialer (node 1) as a member over the exchange" ); } + +/// A completed Join push/pull surfaces `Event::ExchangeCompleted` with +/// `kind() == ExchangeKind::PushPull` on the dialer side. +/// +/// The inner coordinator emits `memberlist_proto::Event::ExchangeCompleted` +/// when the outbound bridge is reaped; serf's `on_inner_event` must forward +/// it rather than swallowing it so that a driver awaiting a join can resolve +/// directly from the event stream. +#[test] +fn completed_join_push_pull_surfaces_exchange_completed_event() { + use crate::{ExchangeKind, ExchangeStatus, event::Event}; + + let now = Instant::ORIGIN; + let mut dialer = ep(1, 7946); + let mut acceptor = ep(2, 7000); + + // Dialer: start a Join push/pull and grab the Connect action's exchange id. + let dial_exchange = { + dialer + .transport_mut() + .start_push_pull(sa(7000), PushPullKind::Join, now); + match dialer.poll_action() { + Some(StreamAction::Connect(c)) => c.id(), + other => panic!("dialer must surface a Connect, got {other:?}"), + } + }; + while dialer.poll_action().is_some() {} + + // Acceptor: admit the inbound connection. + let accept_exchange = acceptor + .accept_connection(sa(7946), now) + .expect("acceptor admits the inbound connection"); + + // Shuttle bytes and half-close signals until the exchange completes. + for _ in 0..256 { + let mut moved = false; + + let mut to_acceptor = Vec::new(); + while let Some((id, _peer, bytes)) = dialer.poll_transport_transmit() { + if id == dial_exchange { + to_acceptor.extend_from_slice(&bytes); + } + } + if !to_acceptor.is_empty() { + acceptor.handle_transport_data(accept_exchange, &to_acceptor, false, now); + moved = true; + } + while let Some(action) = dialer.poll_action() { + if let StreamAction::Shutdown(r) | StreamAction::Close(r) | StreamAction::Abort(r) = action { + if r.id() == dial_exchange { + acceptor.handle_transport_data(accept_exchange, &[], true, now); + moved = true; + } + } + } + + let mut to_dialer = Vec::new(); + while let Some((id, _peer, bytes)) = acceptor.poll_transport_transmit() { + if id == accept_exchange { + to_dialer.extend_from_slice(&bytes); + } + } + if !to_dialer.is_empty() { + dialer.handle_transport_data(dial_exchange, &to_dialer, false, now); + moved = true; + } + while let Some(action) = acceptor.poll_action() { + if let StreamAction::Shutdown(r) | StreamAction::Close(r) | StreamAction::Abort(r) = action { + if r.id() == accept_exchange { + dialer.handle_transport_data(dial_exchange, &[], true, now); + moved = true; + } + } + } + + dialer.handle_timeout(now); + acceptor.handle_timeout(now); + + // Look for Event::ExchangeCompleted on the dialer side. + while let Some(ev) = dialer.poll_event() { + if let Event::ExchangeCompleted(ref c) = ev { + assert_eq!( + c.kind(), + ExchangeKind::PushPull, + "ExchangeCompleted kind must be PushPull for a Join push/pull" + ); + assert_eq!( + c.outcome(), + ExchangeStatus::Succeeded, + "Join push/pull outcome must be Succeeded" + ); + return; + } + } + while acceptor.poll_event().is_some() {} + + if !moved { + break; + } + } + + panic!( + "no Event::ExchangeCompleted(kind=PushPull) surfaced from the dialer after a \ + completed loopback Join push/pull" + ); +} From caec31eaf4573fb6b2c72bd121fe725e52fb7d26 Mon Sep 17 00:00:00 2001 From: Al Liu Date: Mon, 29 Jun 2026 19:43:24 +1200 Subject: [PATCH 024/128] chore(deps): bump buffa and compio to latest (#66) --- Cargo.toml | 2 +- serf-proto/Cargo.toml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index c87460f2..683ee488 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,7 +40,7 @@ tracing = "0.1" clap = { version = "4", default-features = false, features = ["std", "derive", "env"] } # Networking / transport -compio = { version = "0.19.0", default-features = false } +compio = { version = "0.19", default-features = false } flume = { version = "0.12", default-features = false } hostaddr = { version = "0.2", default-features = false } lochan = "0.1" diff --git a/serf-proto/Cargo.toml b/serf-proto/Cargo.toml index 3a335caa..547c59f1 100644 --- a/serf-proto/Cargo.toml +++ b/serf-proto/Cargo.toml @@ -50,13 +50,13 @@ derive_more = { workspace = true } rand = { workspace = true } thiserror.workspace = true bitflags = "2" -buffa = { version = "0.7", default-features = false } +buffa = { version = "0.8", default-features = false } smol_str = { workspace = true } zeroize = { version = "1", optional = true } regex = { version = "1", optional = true } [build-dependencies] -buffa-build = "0.7" +buffa-build = "0.8" [dev-dependencies] # The QUIC super-machine loopback test builds a quinn-proto config bundle with a From f499bda7a3d953c135cdd99c39df80544ac30dff Mon Sep 17 00:00:00 2001 From: al8n Date: Wed, 1 Jul 2026 21:47:28 +1200 Subject: [PATCH 025/128] =?UTF-8?q?feat(serf-reactor):=20foundation=20?= =?UTF-8?q?=E2=80=94=20crate,=20command,=20delegate,=20error,=20events,=20?= =?UTF-8?q?options?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The agnostic-generic reactor-driver foundation for serf's Sans-I/O machine (Send/Arc/agnostic R, flume everywhere, no cmd_fairness_budget knob); ports serf-compio's command/delegate/error/events/options into the reactor Send model, mirroring memberlist-reactor's shape. Compiles --no-default-features as the foundation; the tcp/quic-gated driver and transports follow. --- Cargo.toml | 4 + serf-reactor/Cargo.toml | 117 +++++ serf-reactor/build.rs | 16 + serf-reactor/src/command/mod.rs | 368 ++++++++++++++ serf-reactor/src/command/tests.rs | 131 +++++ serf-reactor/src/delegate/mod.rs | 243 +++++++++ serf-reactor/src/delegate/tests.rs | 37 ++ serf-reactor/src/delegate/void.rs | 155 ++++++ serf-reactor/src/driver/mod.rs | 7 + serf-reactor/src/driver/options/mod.rs | 606 +++++++++++++++++++++++ serf-reactor/src/driver/options/tests.rs | 414 ++++++++++++++++ serf-reactor/src/error/mod.rs | 194 ++++++++ serf-reactor/src/error/tests.rs | 86 ++++ serf-reactor/src/events.rs | 66 +++ serf-reactor/src/lib.rs | 126 +++++ 15 files changed, 2570 insertions(+) create mode 100644 serf-reactor/Cargo.toml create mode 100644 serf-reactor/build.rs create mode 100644 serf-reactor/src/command/mod.rs create mode 100644 serf-reactor/src/command/tests.rs create mode 100644 serf-reactor/src/delegate/mod.rs create mode 100644 serf-reactor/src/delegate/tests.rs create mode 100644 serf-reactor/src/delegate/void.rs create mode 100644 serf-reactor/src/driver/mod.rs create mode 100644 serf-reactor/src/driver/options/mod.rs create mode 100644 serf-reactor/src/driver/options/tests.rs create mode 100644 serf-reactor/src/error/mod.rs create mode 100644 serf-reactor/src/error/tests.rs create mode 100644 serf-reactor/src/events.rs create mode 100644 serf-reactor/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 683ee488..20087146 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "serf-proto", "serf-driver", "serf-compio", + "serf-reactor", ] exclude = ["legacy"] resolver = "3" @@ -41,6 +42,9 @@ clap = { version = "4", default-features = false, features = ["std", "derive", " # Networking / transport compio = { version = "0.19", default-features = false } +agnostic = "0.10" +agnostic-lite = { version = "0.6", features = ["time"] } +arc-swap = "1" flume = { version = "0.12", default-features = false } hostaddr = { version = "0.2", default-features = false } lochan = "0.1" diff --git a/serf-reactor/Cargo.toml b/serf-reactor/Cargo.toml new file mode 100644 index 00000000..0c0db280 --- /dev/null +++ b/serf-reactor/Cargo.toml @@ -0,0 +1,117 @@ +[package] +name = "serf-reactor" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true +description = "agnostic-runtime (tokio / smol) async driver for the Sans-I/O serf machine" + +[features] +default = ["tcp", "tag-regex"] +# Plain-TCP reliable coordinator. +tcp = ["serf-proto/tcp", "serf-driver/tcp"] +# TLS-over-TCP reliable coordinator (implies tcp). The record layer comes from +# serf-proto; a crypto-backend feature (tls-rustls-*) supplies the rustls +# provider via memberlist-proto (serf-proto exposes no rustls provider itself). +tls = ["tcp", "serf-proto/tls", "serf-driver/tls"] +tls-rustls-ring = ["tls", "memberlist-proto/tls-rustls-ring"] +tls-rustls-aws-lc-rs = ["tls", "memberlist-proto/tls-rustls-aws-lc-rs"] +# QUIC coordinator. +quic = ["serf-proto/quic", "serf-driver/quic"] +quic-rustls-ring = ["quic", "serf-proto/quic-rustls-ring", "serf-driver/quic-rustls-ring"] +# Vivaldi network coordinate estimation. +coordinates = ["serf-proto/coordinates", "serf-driver/coordinates"] +# Encryption backends — forward to serf-proto (key-management messages) and to +# memberlist-proto (the coordinator's gossip-encryption keyring + encrypt/decrypt). +aes-gcm = ["serf-proto/aes-gcm", "memberlist-proto/aes-gcm"] +chacha20-poly1305 = ["serf-proto/chacha20-poly1305", "memberlist-proto/chacha20-poly1305"] +# Regex-backed tag-filter matching. +tag-regex = ["serf-proto/tag-regex", "serf-driver/tag-regex"] +# Emit `tracing` spans around the public driver operations. +tracing = ["dep:tracing"] +# Runtime conveniences: pull in the concrete agnostic runtime impls. +tokio = ["agnostic/tokio"] +smol = ["agnostic/smol"] +# Optional config layering: `serde` adds Serialize/Deserialize; `clap` adds CLI flags. +serde = ["dep:serde", "dep:humantime-serde"] +clap = ["dep:clap", "dep:humantime"] +# DNS resolver support. +dns = ["dep:hickory-proto", "dep:resolv-conf"] +# Auto-detect the advertise address from the host's own interfaces. +getifs = ["dep:getifs", "dep:iprfc"] + +[dependencies] +serf-proto = { path = "../serf-proto", default-features = false } +serf-driver = { path = "../serf-driver", default-features = false } +# Pinned to the same git+branch as serf-proto to guarantee a single copy in the +# resolver; serf-reactor uses EncryptionError and the SecretKey type under the +# encryption cfg gate. +memberlist-proto = { git = "https://github.com/al8n/memberlist", branch = "main", default-features = false, features = ["std"] } + +rand = { workspace = true, features = ["std", "std_rng", "sys_rng"] } +smol_str.workspace = true +smallvec.workspace = true +bytes = { workspace = true } +thiserror.workspace = true +futures-channel.workspace = true +futures-util = { workspace = true, default-features = false, features = [ + "std", + "async-await-macro", +] } +flume = { workspace = true, default-features = false, features = ["async"] } + +# Agnostic async runtime substrate (tokio / smol) — declared ahead of the driver +# and transport modules that use it, so those can be added without a Cargo.toml +# edit. +agnostic = { workspace = true, features = ["net"] } +agnostic-lite.workspace = true +# Shared-state substrate for the Send/Arc reactor model, used by the driver's +# shared state. +arc-swap.workspace = true +# Address handling used by the transport modules. +hostaddr = { workspace = true, features = ["std", "smol_str"] } + +tracing = { workspace = true, optional = true } +serde = { workspace = true, optional = true } +humantime-serde = { workspace = true, optional = true } +humantime = { workspace = true, optional = true } +clap = { workspace = true, optional = true } +hickory-proto = { workspace = true, optional = true, default-features = false, features = [ + "std", +] } +resolv-conf = { workspace = true, optional = true } +getifs = { workspace = true, optional = true } +iprfc = { workspace = true, optional = true } + +[dev-dependencies] +# The QUIC / TLS loopback smoke tests build a quinn-proto config bundle with a +# self-signed cert and an accept-any verifier. Only the transport-gated test +# modules reference these; they are unconditional dev-deps so the test crate +# always resolves. +quinn-proto = { version = "0.11", default-features = false, features = ["rustls-ring"] } +rustls = { version = "0.23", features = ["ring"] } +rustls-pki-types = "1" +rcgen = "0.14" +ring = "0.17" +# Used in the serde option-layer tests. +serde_json = "1" + +[package.metadata.docs.rs] +# A single coherent crypto-provider set (ring, not aws-lc-rs; tokio, not smol) so +# the doc build documents every gated item without pulling conflicting providers. +features = [ + "tokio", + "tls-rustls-ring", + "quic-rustls-ring", + "coordinates", + "aes-gcm", + "chacha20-poly1305", + "tag-regex", + "tracing", + "serde", + "clap", + "dns", + "getifs", +] +rustdoc-args = ["--cfg", "docsrs"] diff --git a/serf-reactor/build.rs b/serf-reactor/build.rs new file mode 100644 index 00000000..9d8197a0 --- /dev/null +++ b/serf-reactor/build.rs @@ -0,0 +1,16 @@ +//! Emits the aggregate `encryption` cfg when any AEAD cipher backend feature is enabled, +//! so serf-reactor code gates on `#[cfg(encryption)]` instead of repeating the full +//! backend list. + +fn any_feature(names: &[&str]) -> bool { + names + .iter() + .any(|name| std::env::var_os(format!("CARGO_FEATURE_{name}")).is_some()) +} + +fn main() { + println!("cargo::rustc-check-cfg=cfg(encryption)"); + if any_feature(&["AES_GCM", "CHACHA20_POLY1305"]) { + println!("cargo::rustc-cfg=encryption"); + } +} diff --git a/serf-reactor/src/command/mod.rs b/serf-reactor/src/command/mod.rs new file mode 100644 index 00000000..22f34523 --- /dev/null +++ b/serf-reactor/src/command/mod.rs @@ -0,0 +1,368 @@ +//! Internal command queue — the user-facing Serf handle pushes commands; the +//! driver task drains and dispatches them. + +use crate::error::Result; +use bytes::Bytes; +use futures_channel::oneshot::Sender; +use smol_str::SmolStr; + +#[cfg(any(feature = "tcp", feature = "quic"))] +use memberlist_proto::Instant; +#[cfg(encryption)] +use memberlist_proto::SecretKey; +#[cfg(any(feature = "tcp", feature = "quic"))] +use smallvec::SmallVec; +#[cfg(any(feature = "tcp", feature = "quic"))] +use std::net::SocketAddr; + +#[cfg(any(feature = "tcp", feature = "quic"))] +use crate::error::SerfError; +#[cfg(any(feature = "tcp", feature = "quic"))] +use serf_proto::{ + endpoint::{QueryId, QueryParams}, + event::QueryEvent, + typed::Tags, +}; + +/// Address-set reply for [`Command::Join`]. +/// +/// Both join kinds reply through this single type (mirroring how the memberlist +/// driver unifies its join reply into one channel type): `Ok(set)` carries the +/// dispatched set ([`JoinKind::Dispatch`]) or the contacted set +/// ([`JoinKind::WaitForCompletion`] success); `Err((set, err))` is the legacy +/// partial-success tuple, surfacing the reached-so-far set alongside the error. +/// Every error this driver produces — `NotRunning`, `Shutdown`, and +/// `JoinAllFailed` — resolves before any contact is accumulated, so the tuple's +/// set is empty in practice; it is carried for the legacy `join_many` shape. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) type JoinReply = + core::result::Result, (SmallVec<[SocketAddr; 1]>, SerfError)>; + +/// Payload for [`JoinKind::WaitForCompletion`]. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) struct WaitForCompletionArgs { + /// Wall-clock instant past which the driver replies with whatever contacted + /// set it has accumulated (an empty set surfaces as `JoinAllFailed`). + pub(crate) deadline: Instant, +} + +/// Semantic of a [`Command::Join`] dispatch. +/// +/// Both kinds share the same `start_push_pull` fan-out (one outbound exchange +/// per resolved seed); the kind only affects WHEN the reply fires and WHAT it +/// carries: +/// - `Dispatch`: reply immediately with the dispatched seed set +/// (fire-and-forget; the caller does not wait for any exchange to terminate). +/// - `WaitForCompletion`: reply once every dispatched exchange has terminated +/// (an [`Event::ExchangeCompleted`](serf_proto::event::Event) with +/// `kind == ExchangeKind::PushPull` for each) OR the deadline elapses, +/// whichever comes first; the reply carries the contacted set (an empty set +/// surfaces as `JoinAllFailed`). +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) enum JoinKind { + /// Reply immediately with the dispatched seed set. + Dispatch, + /// Reply once every dispatched exchange has terminated OR the deadline expires. + WaitForCompletion(WaitForCompletionArgs), +} + +/// Payload for [`Command::Join`]. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) struct JoinCmd { + /// Pre-resolved socket addresses of the seed peers to contact. The handle + /// resolves every `MaybeResolved` seed through the caller's resolver before + /// sending the command, so the driver only ever sees concrete addresses. + pub(crate) seeds: Vec, + /// Dispatch semantic — see [`JoinKind`]. + pub(crate) kind: JoinKind, + /// When `true`, each seed's join push/pull is started as an `ignore_old` join + /// so the machine records that exchange's `StreamId` and suppresses replay of + /// the seed's pre-join user events. Keyed per-EXCHANGE and one-shot — see + /// `serf_proto::StreamEndpoint::start_join_push_pull`. + pub(crate) ignore_old: bool, + /// One-shot reply channel delivering the address-set result. See [`JoinReply`]. + pub(crate) reply: Sender, +} + +/// Payload for [`Command::Leave`]. +pub(crate) struct LeaveCmd { + /// One-shot reply channel delivering `()` once the graceful leave completes. + pub(crate) reply: Sender>, +} + +/// Payload for [`Command::ForceLeave`]. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) struct ForceLeaveCmd { + /// The node id to force-remove. + pub(crate) id: I, + /// When `true`, the node is pruned immediately rather than after the + /// tombstone timeout — mirrors `Endpoint::force_leave(prune = true)`. + pub(crate) prune: bool, + /// Wall-clock instant passed to the machine's `force_leave` call. + pub(crate) now: Instant, + /// One-shot reply channel for the operation result. + pub(crate) reply: Sender>, +} + +/// Payload for [`Command::UserEvent`]. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) struct UserEventCmd { + /// Event name — must not exceed the configured `max_user_event_size`. + name: SmolStr, + /// Arbitrary application payload bytes. + payload: Bytes, + /// When `true`, the machine deduplicates events with the same name (coalesce mode). + pub(crate) coalesce: bool, + /// One-shot reply channel for the enqueue result. + pub(crate) reply: Sender>, +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +impl UserEventCmd { + /// Construct from event fields and a reply channel. + pub(crate) fn new( + name: SmolStr, + payload: Bytes, + coalesce: bool, + reply: Sender>, + ) -> Self { + Self { + name, + payload, + coalesce, + reply, + } + } + + /// The event name. + pub(crate) fn name(&self) -> &SmolStr { + &self.name + } + + /// The event payload bytes. + pub(crate) const fn payload(&self) -> &Bytes { + &self.payload + } +} + +/// Payload for [`Command::Query`]. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) struct QueryCmd { + /// Query name forwarded to responders. + name: SmolStr, + /// Query payload forwarded to responders. + payload: Bytes, + /// Routing and timing parameters (filters, relay factor, ack, timeout). + pub(crate) params: QueryParams, + /// Wall-clock instant passed to the machine's `query` call. + pub(crate) now: Instant, + /// One-shot reply channel delivering the [`QueryId`] identifying this query. + pub(crate) reply: Sender>, +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +impl QueryCmd { + /// Construct from query fields and a reply channel. + pub(crate) fn new( + name: SmolStr, + payload: Bytes, + params: QueryParams, + now: Instant, + reply: Sender>, + ) -> Self { + Self { + name, + payload, + params, + now, + reply, + } + } + + /// The query name. + pub(crate) fn name(&self) -> &SmolStr { + &self.name + } + + /// The query payload bytes. + pub(crate) const fn payload(&self) -> &Bytes { + &self.payload + } +} + +/// Payload for [`Command::Respond`]. +/// +/// The driver receives a [`serf_proto::event::Event::Query`] carrying a +/// [`QueryEvent`]; the application constructs a `RespondCmd` from that token +/// and its reply bytes, then sends it through the command queue. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) struct RespondCmd { + /// The query token received in `Event::Query` — carries the routing fields + /// (`ltime`, `id`, `from`, `relay_factor`, `deadline`) `Endpoint::respond` + /// needs to dispatch the reply. + pub(crate) token: QueryEvent, + /// The application's response payload bytes. + payload: Bytes, + /// Wall-clock instant at which the driver dispatches the respond call. + pub(crate) now: Instant, + /// One-shot reply channel delivering `Ok(())` on success. + pub(crate) reply: Sender>, +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +impl RespondCmd { + /// Construct from a query token, payload, now instant, and reply channel. + pub(crate) fn new( + token: QueryEvent, + payload: Bytes, + now: Instant, + reply: Sender>, + ) -> Self { + Self { + token, + payload, + now, + reply, + } + } + + /// The response payload bytes. + pub(crate) const fn payload(&self) -> &Bytes { + &self.payload + } +} + +/// Payload for [`Command::SetTags`]. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) struct SetTagsCmd { + /// The new tag map to advertise on the local node. + pub(crate) tags: Tags, + /// One-shot reply channel for the update result. + pub(crate) reply: Sender>, +} + +/// Payload for [`Command::InstallKey`], [`Command::UseKey`], and +/// [`Command::RemoveKey`]. +/// +/// All three carry a single [`SecretKey`] and reply with the [`QueryId`] of the +/// issued cluster-wide key query. +/// +/// Requires the `aes-gcm` or `chacha20-poly1305` feature. +#[cfg(encryption)] +pub(crate) struct KeyCmd { + /// The secret key to install, promote, or remove. + pub(crate) key: SecretKey, + /// Wall-clock instant passed to the machine's key-op call. + #[cfg(any(feature = "tcp", feature = "quic"))] + pub(crate) now: Instant, + /// One-shot reply channel delivering the [`QueryId`] of the issued query. + #[cfg(any(feature = "tcp", feature = "quic"))] + pub(crate) reply: Sender>, +} + +/// Payload for [`Command::ListKeys`]. +/// +/// Requires the `aes-gcm` or `chacha20-poly1305` feature. +#[cfg(encryption)] +pub(crate) struct ListKeysCmd { + /// Wall-clock instant passed to `Endpoint::list_keys`. + #[cfg(any(feature = "tcp", feature = "quic"))] + pub(crate) now: Instant, + /// One-shot reply channel delivering the [`QueryId`] of the issued query. + #[cfg(any(feature = "tcp", feature = "quic"))] + pub(crate) reply: Sender>, +} + +/// Payload for [`Command::Shutdown`]. +pub(crate) struct ShutdownCmd { + /// One-shot reply channel for the shutdown acknowledgement. + pub(crate) reply: Sender>, +} + +/// Commands sent from the public `Serf` handle to the driver task. +/// +/// `I` is the node-id type; `A` is the resolved peer-address type (typically +/// `std::net::SocketAddr`). All variants are newtype-over-payload-struct. +pub(crate) enum Command { + /// Initiate joins to the given resolved seeds. The reply carries the + /// address-set result: a dispatched or contacted set on success, or the + /// legacy partial-success tuple on failure (see [`JoinReply`]). + #[cfg(any(feature = "tcp", feature = "quic"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] + Join(JoinCmd), + + /// Begin a graceful leave from the cluster. + Leave(LeaveCmd), + + /// Force-remove a node from the cluster membership. + #[cfg(any(feature = "tcp", feature = "quic"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] + ForceLeave(ForceLeaveCmd), + + /// Broadcast a user-defined event cluster-wide via gossip. + #[cfg(any(feature = "tcp", feature = "quic"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] + UserEvent(UserEventCmd), + + /// Issue a cluster-wide query and collect responses; reply carries the + /// [`QueryId`] identifying the in-flight query. + #[cfg(any(feature = "tcp", feature = "quic"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] + Query(QueryCmd), + + /// Send a response to an inbound query received via `Event::Query`. + #[cfg(any(feature = "tcp", feature = "quic"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] + Respond(RespondCmd), + + /// Update the local node's advertised tags. + #[cfg(any(feature = "tcp", feature = "quic"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] + SetTags(SetTagsCmd), + + /// Issue a cluster-wide install-key query. + /// + /// Requires the `aes-gcm` or `chacha20-poly1305` feature. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + InstallKey(KeyCmd), + + /// Issue a cluster-wide use-key query to promote a key to primary. + /// + /// Requires the `aes-gcm` or `chacha20-poly1305` feature. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + UseKey(KeyCmd), + + /// Issue a cluster-wide remove-key query. + /// + /// Requires the `aes-gcm` or `chacha20-poly1305` feature. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + RemoveKey(KeyCmd), + + /// Issue a cluster-wide list-keys query to enumerate installed keys. + /// + /// Requires the `aes-gcm` or `chacha20-poly1305` feature. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + ListKeys(ListKeysCmd), + + /// Signal the driver task to shut down gracefully. + Shutdown(ShutdownCmd), +} + +#[cfg(test)] +mod tests; diff --git a/serf-reactor/src/command/tests.rs b/serf-reactor/src/command/tests.rs new file mode 100644 index 00000000..e5df6f9f --- /dev/null +++ b/serf-reactor/src/command/tests.rs @@ -0,0 +1,131 @@ +use super::*; +use smol_str::SmolStr; + +fn addr() -> std::net::SocketAddr { + "127.0.0.1:5000".parse().unwrap() +} + +fn unit_reply() -> Sender> { + futures_channel::oneshot::channel::>().0 +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +#[test] +fn join_cmd_carries_seeds_kind_and_reply() { + use memberlist_proto::Instant; + + let (tx, _rx) = futures_channel::oneshot::channel::(); + let cmd = JoinCmd { + seeds: vec![addr()], + kind: JoinKind::Dispatch, + ignore_old: false, + reply: tx, + }; + assert_eq!(cmd.seeds.len(), 1); + assert_eq!(cmd.seeds[0], addr()); + assert!(matches!(cmd.kind, JoinKind::Dispatch)); + assert!(!cmd.ignore_old); + + // The enum variant wraps the payload transparently; the await kind carries + // its deadline, and the ignore_old flag rides along. + let (tx2, _rx2) = futures_channel::oneshot::channel::(); + let Command::::Join(c) = Command::Join(JoinCmd { + seeds: vec![addr(), addr()], + kind: JoinKind::WaitForCompletion(WaitForCompletionArgs { + deadline: Instant::now(), + }), + ignore_old: true, + reply: tx2, + }) else { + panic!("wrong Command variant"); + }; + assert_eq!(c.seeds.len(), 2); + assert!(matches!(c.kind, JoinKind::WaitForCompletion(_))); + assert!( + c.ignore_old, + "ignore_old must round-trip through the command" + ); +} + +#[test] +fn leave_cmd_constructs() { + let cmd = Command::::Leave(LeaveCmd { + reply: unit_reply(), + }); + assert!(matches!(cmd, Command::Leave(_))); +} + +#[test] +fn shutdown_cmd_constructs() { + let cmd = Command::::Shutdown(ShutdownCmd { + reply: unit_reply(), + }); + assert!(matches!(cmd, Command::Shutdown(_))); +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +mod transport_gated { + use super::*; + use bytes::Bytes; + use memberlist_proto::Instant; + use serf_proto::{endpoint::QueryParams, typed::Tags}; + + #[test] + fn force_leave_cmd_round_trips_fields() { + let cmd = Command::::ForceLeave(ForceLeaveCmd { + id: SmolStr::new("node-1"), + prune: true, + now: Instant::now(), + reply: unit_reply(), + }); + assert!(matches!(cmd, Command::ForceLeave(_))); + if let Command::ForceLeave(c) = cmd { + assert_eq!(c.id.as_str(), "node-1"); + assert!(c.prune); + } + } + + #[test] + fn user_event_cmd_round_trips_fields() { + let cmd = UserEventCmd::new( + SmolStr::new("deploy"), + Bytes::from_static(b"v1.2.3"), + true, + unit_reply(), + ); + assert_eq!(cmd.name().as_str(), "deploy"); + assert_eq!(cmd.payload(), &Bytes::from_static(b"v1.2.3")); + assert!(cmd.coalesce); + + let wrapped = Command::::UserEvent(UserEventCmd::new( + SmolStr::new("x"), + Bytes::from_static(b"y"), + false, + unit_reply(), + )); + assert!(matches!(wrapped, Command::UserEvent(_))); + } + + #[test] + fn query_cmd_round_trips_name_and_payload() { + let (tx, _rx) = futures_channel::oneshot::channel::>(); + let cmd: QueryCmd = QueryCmd::new( + SmolStr::new("health"), + Bytes::from_static(b"ping"), + QueryParams::default(), + Instant::now(), + tx, + ); + assert_eq!(cmd.name().as_str(), "health"); + assert_eq!(cmd.payload(), &Bytes::from_static(b"ping")); + } + + #[test] + fn set_tags_cmd_constructs() { + let cmd = Command::::SetTags(SetTagsCmd { + tags: Tags::default(), + reply: unit_reply(), + }); + assert!(matches!(cmd, Command::SetTags(_))); + } +} diff --git a/serf-reactor/src/delegate/mod.rs b/serf-reactor/src/delegate/mod.rs new file mode 100644 index 00000000..85e6d612 --- /dev/null +++ b/serf-reactor/src/delegate/mod.rs @@ -0,0 +1,243 @@ +//! `Delegate` composite — the reactor driver's per-driver observation hook +//! surface for serf. +//! +//! Composes four observation sub-traits (`MemberDelegate` / `UserEventDelegate` +//! / `QueryDelegate` / `KeyringDelegate`) and a join-admission veto +//! (`MergeDelegate`). Every observation hook returns a `Send` future +//! (`-> impl Future + Send`, not `async fn`) so the observation +//! task can run on a multi-threaded agnostic runtime; the delegate as a whole is +//! `Send + Sync + 'static` and is held behind an `Arc`. +//! +//! `KeyringDelegate` and `MergeDelegate` are separate from the observation +//! `Delegate` composite: `KeyringDelegate` is sync (keyring ops must not +//! block), and `MergeDelegate` is an async admission veto supplied at +//! construction rather than an observation hook. + +mod void; + +pub use void::{NoopMergeDelegate, VoidDelegate}; + +#[cfg(encryption)] +pub use void::VoidKeyringDelegate; + +#[cfg(any(feature = "tcp", feature = "quic"))] +use std::{future::Future, sync::Arc}; + +#[cfg(any(feature = "tcp", feature = "quic"))] +use serf_proto::{event::QueryEvent, members::Member, typed::UserEventMessage}; + +#[cfg(encryption)] +use memberlist_proto::SecretKey; +#[cfg(encryption)] +use serf_proto::KeyResponseArgs; + +/// Async observation hooks for serf membership events. +/// +/// Each method corresponds to one [`serf_proto::event::MemberEventKind`]. +/// Default impls are no-ops; override what the application cares about. +/// +/// Every hook returns `-> impl Future + Send + '_` so the +/// observation task can drive it on a multi-threaded runtime; the delegate is +/// `Send + Sync + 'static`. +/// +/// Requires a stream or QUIC transport feature (`tcp` or `quic`). +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub trait MemberDelegate: Send + Sync + 'static { + /// Node identifier type. + type Id; + /// Address type. + type Address; + + /// Called when a member joins the cluster. + fn notify_join( + &self, + member: Arc>, + ) -> impl Future + Send + '_ { + let _ = member; // Unused: default no-op; override to handle. + async {} + } + + /// Called when a member gracefully leaves or is reaped as dead. + fn notify_leave( + &self, + member: Arc>, + ) -> impl Future + Send + '_ { + let _ = member; // Unused: default no-op; override to handle. + async {} + } + + /// Called when a member is detected as failed (no graceful leave observed). + fn notify_failed( + &self, + member: Arc>, + ) -> impl Future + Send + '_ { + let _ = member; // Unused: default no-op; override to handle. + async {} + } + + /// Called when a member's tags or metadata are updated. + fn notify_update( + &self, + member: Arc>, + ) -> impl Future + Send + '_ { + let _ = member; // Unused: default no-op; override to handle. + async {} + } + + /// Called when a member is reaped from the membership store (tombstone + /// expired). + fn notify_reap( + &self, + member: Arc>, + ) -> impl Future + Send + '_ { + let _ = member; // Unused: default no-op; override to handle. + async {} + } +} + +/// Async observation hook for cluster-wide user-event broadcasts. +/// +/// Returns a `Send` future so the observation task can drive it on a +/// multi-threaded runtime. +/// +/// Requires a stream or QUIC transport feature (`tcp` or `quic`). +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub trait UserEventDelegate: Send + Sync + 'static { + /// Called when a user-event broadcast is received from the cluster. + fn notify_user_event(&self, event: &UserEventMessage) -> impl Future + Send + '_ { + let _ = event; // Unused: default no-op; override to handle. + async {} + } +} + +/// Async observation hook for inbound queries. +/// +/// The driver calls `notify_query` when it receives `Event::Query`. The +/// application may respond through the `Serf` handle's `respond` method; +/// the delegate itself does not hold the respond path. +/// +/// Returns a `Send` future so the observation task can drive it on a +/// multi-threaded runtime. +/// +/// Requires a stream or QUIC transport feature (`tcp` or `quic`). +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub trait QueryDelegate: Send + Sync + 'static { + /// Node identifier type. + type Id; + /// Address type. + type Address; + + /// Called when an inbound query arrives that the application may respond to. + fn notify_query( + &self, + event: &QueryEvent, + ) -> impl Future + Send + '_ { + let _ = event; // Unused: default no-op; override to handle. + async {} + } +} + +/// The reactor driver's per-driver observation hook surface for serf. +/// +/// A type that satisfies `Delegate` implements all three observation sub-traits +/// (`MemberDelegate`, `UserEventDelegate`, `QueryDelegate`) with matching +/// associated types. `Send + Sync + 'static` (inherited from the sub-traits): +/// the driver holds it behind an `Arc` and the observation task fires the hooks +/// on the runtime's worker threads. +/// +/// The keyring delegate (`KeyringDelegate`) and join-admission veto +/// (`MergeDelegate`) are NOT part of this composite — they are supplied +/// separately to the driver constructor. +/// +/// Requires a stream or QUIC transport feature (`tcp` or `quic`). +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub trait Delegate: + MemberDelegate::Id, Address = ::Address> + + UserEventDelegate + + QueryDelegate::Id, Address = ::Address> +{ + /// Node identifier type. + type Id; + /// Address type — always `SocketAddr` in the reactor driver. + type Address; +} + +/// Synchronous delegate the driver implements to apply key-management operations +/// to its local keyring. +/// +/// The driver's event loop calls the matching method when it receives a +/// [`serf_proto::event::Event::KeyRequest`] event, then forwards the result to +/// `StreamEndpoint::respond_key` / `QuicEndpoint::respond_key` so the response +/// is routed back to the originating node. +/// +/// All methods are **synchronous and non-blocking**: key storage is the +/// driver's concern. If a key store requires async I/O, shadow the result +/// through pre-computed state the synchronous method can read. `Send + Sync + +/// 'static` because the driver holds it behind an `Arc` shared across worker +/// threads. +/// +/// Requires the `aes-gcm` or `chacha20-poly1305` feature. +#[cfg(encryption)] +#[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) +)] +pub trait KeyringDelegate: Send + Sync + 'static { + /// Install a new key into the keyring. Returns the response args passed to + /// `respond_key`. + fn install(&self, key: SecretKey) -> KeyResponseArgs; + + /// Promote a key to the primary encryption key. Returns the response args + /// passed to `respond_key`. + fn use_key(&self, key: SecretKey) -> KeyResponseArgs; + + /// Remove a key from the keyring. Returns the response args passed to + /// `respond_key`. + fn remove(&self, key: SecretKey) -> KeyResponseArgs; + + /// List all installed keys and the current primary. Returns the response args + /// (with `keys` populated) passed to `respond_key`. + fn list(&self) -> KeyResponseArgs; +} + +/// Async veto hook invoked by the driver on the join path before accepting +/// remote member state from a push-pull exchange. +/// +/// `Ok(())` permits the merge; `Err(Self::Error)` cancels it. The driver wraps +/// the concrete error into [`SerfError`](crate::SerfError) before forwarding it +/// to the join caller. +/// +/// The hook is **async and driver-side** deliberately: the application may need +/// to consult an ACL service or other async resource before deciding whether to +/// accept a batch of remote peers. A synchronous (Sans-I/O) filter would +/// preclude that. `notify_merge` returns a `Send` future so the driver can drive +/// it on a multi-threaded runtime; the delegate is `Send + Sync + 'static`. +/// +/// Requires a stream or QUIC transport feature (`tcp` or `quic`). +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub trait MergeDelegate: Send + Sync + 'static { + /// The veto/error type this delegate reports when a merge is cancelled. + type Error; + + /// Called before the driver accepts inbound push-pull peer state. + /// + /// `peers` is the slice of remote [`Member`]s the cluster is about to merge. + /// Return `Ok(())` to proceed, or `Err(e)` to cancel the merge. + /// + /// The default implementation always permits the merge. + fn notify_merge( + &self, + peers: &[Arc>], + ) -> impl Future> + Send + '_ { + let _ = peers; // Unused in the default permit-all impl; an overriding delegate inspects it. + async { Ok(()) } + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-reactor/src/delegate/tests.rs b/serf-reactor/src/delegate/tests.rs new file mode 100644 index 00000000..2a9a6e27 --- /dev/null +++ b/serf-reactor/src/delegate/tests.rs @@ -0,0 +1,37 @@ +use super::*; +use smol_str::SmolStr; +use std::net::SocketAddr; + +/// `VoidDelegate` satisfies the observation [`Delegate`] composite with the +/// canonical `(SmolStr, SocketAddr)` id/address pair. +#[test] +fn void_delegate_satisfies_observation_composite() { + fn assert_delegate(_d: &D) + where + D: Delegate, + { + } + let v: VoidDelegate = VoidDelegate::default(); + assert_delegate(&v); +} + +/// `NoopMergeDelegate` satisfies `MergeDelegate` with `Error = Infallible` — a +/// type-level check; no I/O needed. +#[test] +fn noop_merge_delegate_satisfies_trait() { + fn assert_merge(_: &T) + where + T: MergeDelegate, + { + } + assert_merge(&NoopMergeDelegate); +} + +/// The reactor delegate surface is `Send + Sync + 'static`: the driver holds it +/// behind an `Arc` shared across the runtime's worker threads. +#[test] +fn void_delegate_and_noop_merge_are_send_sync() { + fn assert_send_sync() {} + assert_send_sync::>(); + assert_send_sync::(); +} diff --git a/serf-reactor/src/delegate/void.rs b/serf-reactor/src/delegate/void.rs new file mode 100644 index 00000000..ed4ff7bb --- /dev/null +++ b/serf-reactor/src/delegate/void.rs @@ -0,0 +1,155 @@ +//! Zero-cost default delegates — no-op impls of all observation and admission +//! hooks so drivers that do not need them can construct a node without +//! boilerplate. + +use core::marker::PhantomData; + +#[cfg(any(feature = "tcp", feature = "quic"))] +use super::{Delegate, MemberDelegate, MergeDelegate, QueryDelegate, UserEventDelegate}; + +#[cfg(encryption)] +use super::KeyringDelegate; +#[cfg(encryption)] +use serf_proto::KeyResponseArgs; + +/// Zero-cost default observation delegate. Every hook is a no-op. +/// +/// Use when the application does not need to observe membership, user-event, +/// or query notifications. `VoidDelegate` satisfies the +/// [`Delegate`](super::Delegate) composite. +/// +/// Requires a stream or QUIC transport feature (`tcp` or `quic`). +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub struct VoidDelegate { + _phantom: PhantomData, +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +impl VoidDelegate { + /// Construct a `VoidDelegate`. + #[inline] + pub const fn new() -> Self { + Self { + _phantom: PhantomData, + } + } +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +impl Default for VoidDelegate { + fn default() -> Self { + Self::new() + } +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +impl MemberDelegate for VoidDelegate +where + I: Send + Sync + 'static, + A: Send + Sync + 'static, +{ + type Id = I; + type Address = A; +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +impl UserEventDelegate for VoidDelegate +where + I: Send + Sync + 'static, + A: Send + Sync + 'static, +{ +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +impl QueryDelegate for VoidDelegate +where + I: Send + Sync + 'static, + A: Send + Sync + 'static, +{ + type Id = I; + type Address = A; +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +impl Delegate for VoidDelegate +where + I: Send + Sync + 'static, + A: Send + Sync + 'static, +{ + type Id = I; + type Address = A; +} + +/// A merge delegate that always permits merges. +/// +/// The default delegate for drivers that do not need join admission control. +/// Its associated error type is [`core::convert::Infallible`], reflecting that +/// `notify_merge` can never fail. +/// +/// Requires a stream or QUIC transport feature (`tcp` or `quic`). +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub struct NoopMergeDelegate; + +#[cfg(any(feature = "tcp", feature = "quic"))] +impl MergeDelegate for NoopMergeDelegate +where + I: Send + Sync + 'static, + A: Send + Sync + 'static, +{ + type Error = core::convert::Infallible; +} + +/// A keyring delegate that manages no key material. +/// +/// The default for nodes that do not participate in key management: every +/// operation reports `result = false` with an explanatory message and an empty +/// key list, so a cluster-wide key query records this node as not key-managed +/// rather than silently timing out. A node that DOES manage encryption keys +/// supplies its own [`KeyringDelegate`](super::KeyringDelegate) bound to the +/// keyring it shares with the transport's encryption configuration. +/// +/// Requires the `aes-gcm` or `chacha20-poly1305` feature. +#[cfg(encryption)] +#[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) +)] +pub struct VoidKeyringDelegate; + +#[cfg(encryption)] +impl VoidKeyringDelegate { + /// The response returned by every operation: not key-managed. + fn unmanaged() -> KeyResponseArgs { + KeyResponseArgs { + result: false, + message: "node has no keyring delegate configured".into(), + keys: Vec::new(), + primary_key: None, + } + } +} + +#[cfg(encryption)] +impl KeyringDelegate for VoidKeyringDelegate { + #[inline] + fn install(&self, _key: memberlist_proto::SecretKey) -> KeyResponseArgs { + Self::unmanaged() + } + + #[inline] + fn use_key(&self, _key: memberlist_proto::SecretKey) -> KeyResponseArgs { + Self::unmanaged() + } + + #[inline] + fn remove(&self, _key: memberlist_proto::SecretKey) -> KeyResponseArgs { + Self::unmanaged() + } + + #[inline] + fn list(&self) -> KeyResponseArgs { + Self::unmanaged() + } +} diff --git a/serf-reactor/src/driver/mod.rs b/serf-reactor/src/driver/mod.rs new file mode 100644 index 00000000..0b752c49 --- /dev/null +++ b/serf-reactor/src/driver/mod.rs @@ -0,0 +1,7 @@ +//! The driver layer: the generic-free tuning knobs. +//! +//! [`options`] holds the generic-free driver tuning knobs shared by every +//! transport backend. The reactor substrate (the shared `Arc` state) and +//! the per-backend driver loops live behind the transport features. + +pub(crate) mod options; diff --git a/serf-reactor/src/driver/options/mod.rs b/serf-reactor/src/driver/options/mod.rs new file mode 100644 index 00000000..3b2d291c --- /dev/null +++ b/serf-reactor/src/driver/options/mod.rs @@ -0,0 +1,606 @@ +//! Per-`Serf` runtime tuning knobs. +//! +//! [`RuntimeOptions`] carries the generic-free runtime knobs; per-backend knobs +//! live on each backend's `*TransportOptions` struct. +//! +//! Each `DEFAULT_*` constant is `pub` so callers can derive new values from the +//! default (e.g. `DEFAULT_LEAVE_TIMEOUT * 3`). + +use core::time::Duration; + +#[cfg(any(feature = "tcp", feature = "quic"))] +use crate::error::{InvalidOption, SerfError}; + +#[cfg(feature = "clap")] +use humantime::parse_duration; + +/// Default per-call deadline for the await-result +/// [`Serf::join`](crate::Serf::join) / [`Serf::join_many`](crate::Serf::join_many). +/// +/// Past this deadline the driver replies with whatever seeds it has contacted so +/// far (an empty contact set surfaces as +/// [`SerfError::JoinAllFailed`](crate::SerfError::JoinAllFailed)). Mirrors the +/// memberlist driver's `DEFAULT_JOIN_DEADLINE`. +pub const DEFAULT_JOIN_DEADLINE: Duration = Duration::from_secs(10); + +/// Default per-call deadline for [`Serf::leave`](crate::Serf::leave). +pub const DEFAULT_LEAVE_TIMEOUT: Duration = Duration::from_secs(5); + +/// Default fallback driver-loop sleep when the coordinator's `poll_timeout` +/// returns `None`. +pub const DEFAULT_IDLE_WAKE_INTERVAL: Duration = Duration::from_secs(60); + +/// Default per-iteration drain cap for inbound surfaces. +pub const DEFAULT_ITER_DRAIN_CAP: usize = 256; + +/// Default events-channel capacity. +pub const DEFAULT_EVENT_QUEUE_CAP: usize = 1024; + +/// Default outbound dial budget for the stream-transport driver. +pub const DEFAULT_DIAL_TIMEOUT: Duration = Duration::from_secs(5); + +/// Default bridge-inbound channel capacity for the stream-transport driver. +pub const DEFAULT_BRIDGE_INBOUND_CAP: usize = 1024; + +/// Default per-bridge TCP read buffer size for the stream-transport driver. +pub const DEFAULT_BRIDGE_RECV_BUF_LEN: usize = 16 * 1024; + +/// Default bound on a per-bridge graceful-drain write for the stream-transport +/// driver: 10 seconds. +pub const DEFAULT_CLOSE_TIMEOUT: Duration = Duration::from_secs(10); + +/// How the per-driver delegate **observation channel** is bounded. +/// +/// The driver hands every machine `Event` to a separate observation task that +/// runs the user [`Delegate`](crate::Delegate) hooks, decoupled from the +/// protocol loop so a slow hook never stalls the FSM. +/// +/// - [`Unbounded`](Self::Unbounded): the hand-off never drops, so the delegate +/// observes every event in order — but a delegate that persistently runs +/// slower than inbound traffic grows the channel without limit. +/// - [`Bounded`](Self::Bounded): caps the channel at `n` queued events. When +/// full, the driver drops the newest event rather than blocking and counts +/// the drop in `observation_dropped`. +/// +/// `Bounded(0)` is rejected at construction: a zero-capacity channel is a +/// rendezvous the driver's non-blocking send can never deposit into. +/// +/// As a config value (serde / CLI) it is open-vocabulary: `Unbounded` is the +/// bare string `"unbounded"`, `Bounded(n)` is `{"bounded": n}` under serde and +/// `bounded:n` on the CLI (via [`FromStr`](core::str::FromStr) / +/// [`Display`](core::fmt::Display)). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(rename_all = "snake_case"))] +pub enum Channel { + /// Never drop; the channel grows without bound. + Unbounded, + /// Cap the channel at this many queued events; drop-newest and count in + /// `observation_dropped` when full. + Bounded(usize), +} + +impl core::fmt::Display for Channel { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::Unbounded => f.write_str("unbounded"), + Self::Bounded(n) => write!(f, "bounded:{n}"), + } + } +} + +/// Parse a [`Channel`] from its string form — `"unbounded"` or `"bounded:"`. +/// The inverse of [`Channel`]'s `Display`. +impl core::str::FromStr for Channel { + type Err = ParseChannelError; + + fn from_str(s: &str) -> Result { + if s.eq_ignore_ascii_case("unbounded") { + return Ok(Self::Unbounded); + } + let cap = s + .strip_prefix("bounded:") + .or_else(|| s.strip_prefix("bounded=")) + .ok_or(ParseChannelError(()))?; + let n = cap.parse::().map_err(|_| ParseChannelError(()))?; + Ok(Self::Bounded(n)) + } +} + +/// The error from [`Channel::from_str`]: the input was neither `"unbounded"` +/// nor a `"bounded:"` with a valid capacity. +/// +/// Opaque — the private unit field seals construction to this module. +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[error("invalid observation channel (expected `unbounded` or `bounded:`)")] +pub struct ParseChannelError(()); + +/// Default delegate observation channel: [`Channel::Bounded`] at 1024 events. +pub const DEFAULT_OBSERVATION_CHANNEL: Channel = Channel::Bounded(1024); + +/// Per-`Serf` runtime tuning knobs. Generic-free; per-backend knobs live on +/// each backend's `*TransportOptions` struct. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[cfg_attr(feature = "serde", serde(default, deny_unknown_fields))] +pub struct RuntimeOptions { + #[cfg_attr(feature = "serde", serde(with = "humantime_serde"))] + join_deadline: Duration, + #[cfg_attr(feature = "serde", serde(with = "humantime_serde"))] + leave_timeout: Duration, + #[cfg_attr(feature = "serde", serde(with = "humantime_serde"))] + idle_wake_interval: Duration, + iter_drain_cap: usize, + event_queue_cap: usize, + observation_channel: Channel, +} + +// `clap::Args` is NOT derived on `RuntimeOptions`. The derived +// `update_from_arg_matches` treats every `default_value` / `default_value_t` +// arg as present even when the operator did not pass it, so a partial +// `try_update_from` carrying one unrelated flag would reset every other +// defaulted knob to its default. A private mirror carries the `#[arg(...)]` +// attributes and derives `Args`; `RuntimeOptions` delegates its `Args` / +// `FromArgMatches` to the mirror and, on update, applies ONLY the args whose +// value came from the command line or an env var. +#[cfg(feature = "clap")] +const _: () = { + use clap::{ArgMatches, Args, Command, Error, FromArgMatches, parser::ValueSource}; + + #[derive(Args)] + struct RuntimeOptionsCli { + #[arg( + id = "runtime-join-deadline", + long = "runtime-join-deadline", + env = "SERF_RUNTIME_JOIN_DEADLINE", + value_parser = parse_duration, + // The humantime spelling of DEFAULT_JOIN_DEADLINE. + default_value = "10s", + )] + join_deadline: Duration, + #[arg( + id = "runtime-leave-timeout", + long = "runtime-leave-timeout", + env = "SERF_RUNTIME_LEAVE_TIMEOUT", + value_parser = parse_duration, + // The humantime spelling of DEFAULT_LEAVE_TIMEOUT. + default_value = "5s", + )] + leave_timeout: Duration, + #[arg( + id = "runtime-idle-wake-interval", + long = "runtime-idle-wake-interval", + env = "SERF_RUNTIME_IDLE_WAKE_INTERVAL", + value_parser = parse_duration, + // The humantime spelling of DEFAULT_IDLE_WAKE_INTERVAL. + default_value = "60s", + )] + idle_wake_interval: Duration, + #[arg( + id = "runtime-iter-drain-cap", + long = "runtime-iter-drain-cap", + env = "SERF_RUNTIME_ITER_DRAIN_CAP", + default_value_t = DEFAULT_ITER_DRAIN_CAP, + )] + iter_drain_cap: usize, + #[arg( + id = "runtime-event-queue-cap", + long = "runtime-event-queue-cap", + env = "SERF_RUNTIME_EVENT_QUEUE_CAP", + default_value_t = DEFAULT_EVENT_QUEUE_CAP, + )] + event_queue_cap: usize, + #[arg( + id = "runtime-observation-channel", + long = "runtime-observation-channel", + env = "SERF_RUNTIME_OBSERVATION_CHANNEL", + default_value_t = DEFAULT_OBSERVATION_CHANNEL, + )] + observation_channel: Channel, + } + + impl From for RuntimeOptions { + fn from(c: RuntimeOptionsCli) -> Self { + Self { + join_deadline: c.join_deadline, + leave_timeout: c.leave_timeout, + idle_wake_interval: c.idle_wake_interval, + iter_drain_cap: c.iter_drain_cap, + event_queue_cap: c.event_queue_cap, + observation_channel: c.observation_channel, + } + } + } + + impl Args for RuntimeOptions { + fn augment_args(cmd: Command) -> Command { + RuntimeOptionsCli::augment_args(cmd) + } + + fn augment_args_for_update(cmd: Command) -> Command { + RuntimeOptionsCli::augment_args_for_update(cmd) + } + } + + impl FromArgMatches for RuntimeOptions { + fn from_arg_matches(m: &ArgMatches) -> Result { + RuntimeOptionsCli::from_arg_matches(m).map(Into::into) + } + + fn update_from_arg_matches(&mut self, m: &ArgMatches) -> Result<(), Error> { + // Apply ONLY operator-supplied overrides — args whose value came from the + // command line or an env var, not a clap default. A bare derived update + // treats every `default_value` arg as present and would reset unset fields. + macro_rules! take { + ($id:literal, $field:ident, $ty:ty) => { + if matches!( + m.value_source($id), + Some(ValueSource::CommandLine) | Some(ValueSource::EnvVariable) + ) { + if let Some(v) = m.get_one::<$ty>($id) { + self.$field = *v; + } + } + }; + } + take!("runtime-join-deadline", join_deadline, Duration); + take!("runtime-leave-timeout", leave_timeout, Duration); + take!("runtime-idle-wake-interval", idle_wake_interval, Duration); + take!("runtime-iter-drain-cap", iter_drain_cap, usize); + take!("runtime-event-queue-cap", event_queue_cap, usize); + take!("runtime-observation-channel", observation_channel, Channel); + Ok(()) + } + } +}; + +impl RuntimeOptions { + /// Construct from the canonical base defaults. + #[inline] + pub const fn new() -> Self { + Self { + join_deadline: DEFAULT_JOIN_DEADLINE, + leave_timeout: DEFAULT_LEAVE_TIMEOUT, + idle_wake_interval: DEFAULT_IDLE_WAKE_INTERVAL, + iter_drain_cap: DEFAULT_ITER_DRAIN_CAP, + event_queue_cap: DEFAULT_EVENT_QUEUE_CAP, + observation_channel: DEFAULT_OBSERVATION_CHANNEL, + } + } + + /// Builder: per-call deadline for the await-result + /// [`Serf::join`](crate::Serf::join) / [`join_many`](crate::Serf::join_many). + #[must_use] + #[inline] + pub const fn with_join_deadline(mut self, d: Duration) -> Self { + self.join_deadline = d; + self + } + + /// Builder: per-call deadline for [`Serf::leave`](crate::Serf::leave). + #[must_use] + #[inline] + pub const fn with_leave_timeout(mut self, d: Duration) -> Self { + self.leave_timeout = d; + self + } + + /// Builder: fallback driver-loop sleep when the coordinator has no pending + /// deadline. + #[must_use] + #[inline] + pub const fn with_idle_wake_interval(mut self, d: Duration) -> Self { + self.idle_wake_interval = d; + self + } + + /// Builder: per-iteration drain cap. + #[must_use] + #[inline] + pub const fn with_iter_drain_cap(mut self, n: usize) -> Self { + self.iter_drain_cap = n; + self + } + + /// Builder: events-channel capacity. + #[must_use] + #[inline] + pub const fn with_event_queue_cap(mut self, n: usize) -> Self { + self.event_queue_cap = n; + self + } + + /// Builder: how the delegate observation channel is bounded. + #[must_use] + #[inline] + pub const fn with_observation_channel(mut self, c: Channel) -> Self { + self.observation_channel = c; + self + } + + /// Per-call deadline for the await-result [`Serf::join`](crate::Serf::join) / + /// [`join_many`](crate::Serf::join_many). + #[inline] + pub const fn join_deadline(&self) -> Duration { + self.join_deadline + } + + /// Per-call deadline for [`Serf::leave`](crate::Serf::leave). + #[inline] + pub const fn leave_timeout(&self) -> Duration { + self.leave_timeout + } + + /// Fallback driver-loop sleep. + #[inline] + pub const fn idle_wake_interval(&self) -> Duration { + self.idle_wake_interval + } + + /// Per-iteration drain cap. + #[inline] + pub const fn iter_drain_cap(&self) -> usize { + self.iter_drain_cap + } + + /// Events-channel capacity. + #[inline] + pub const fn event_queue_cap(&self) -> usize { + self.event_queue_cap + } + + /// How the delegate observation channel is bounded. + #[inline] + pub const fn observation_channel(&self) -> Channel { + self.observation_channel + } + + /// Validate the generic-free runtime knobs whose value would DETERMINISTICALLY + /// break (not merely degrade) the driver loop, so the misconfiguration + /// surfaces from [`Serf::new`](crate::Serf::new) — before any socket is bound + /// or the detached driver task is spawned — rather than as a panic, a + /// busy-spin, or a silently dead surface inside that task. Each is rejected + /// (not clamped) so the operator learns and fixes the value; every backend + /// (TCP/TLS/QUIC) routes through the one `Serf::new` path, so this single call + /// covers all three. + /// + /// - `join_deadline == 0`: the await-result join parks a waiter until either + /// every dispatched push/pull terminates or this deadline elapses. A zero + /// deadline is past-due the instant the waiter is parked, so the next reap + /// sweep replies before any `ExchangeCompleted` can be observed — every + /// await-result join would spuriously surface `JoinAllFailed` even against a + /// reachable seed. + /// - `idle_wake_interval == 0`: the driver loop's fallback sleep when the + /// coordinator's `poll_timeout` has no nearer deadline + /// (`poll_timeout().unwrap_or(now + idle_wake_interval)`). Zero makes a + /// quiescent endpoint re-arm a zero-duration timer every pass — a busy-spin + /// that pegs a CPU core. + /// - `event_queue_cap == 0`: the observation task forwards every event to the + /// bounded event-stream channel with a non-blocking `try_send`, and a + /// zero-capacity channel is a rendezvous a non-blocking send can never + /// deposit into, so every event would be dropped and the event-observation + /// surface would be non-functional. + /// - `observation_channel == Channel::Bounded(0)`: each driver builds the + /// delegate observation channel as a bounded flume channel; a zero-capacity + /// `Bounded(0)` is a rendezvous the driver's non-blocking `try_send` can + /// never deposit into, so every observed event would be dropped and the + /// delegate would never fire. + /// + /// `iter_drain_cap == 0` and `leave_timeout == 0` are NOT rejected: the former + /// only caps the per-iteration batch drain (the pump still makes one-per-pass + /// forward progress on its readiness arms and the uncapped timeout drain), and + /// the latter is a loud immediate [`SerfError::LeaveTimeout`] rather than a + /// silent break. + /// + /// There is no command-fairness knob to validate: the reactor's command queue + /// is a `Mutex>` woken by a stored `Waker`, drained fully + /// each pass, so commands never starve behind the network arms. + #[cfg(any(feature = "tcp", feature = "quic"))] + pub(crate) fn validate(&self) -> Result<(), SerfError> { + if self.join_deadline.is_zero() { + return Err(SerfError::InvalidOption(InvalidOption::new( + "join_deadline", + "the await-result join deadline must be nonzero: a parked join waiter is past-due the \ + instant it is parked, so the next reap sweep replies before any push/pull \ + ExchangeCompleted can be observed — every await-result join would spuriously surface \ + JoinAllFailed even against a reachable seed" + .to_string(), + ))); + } + if self.idle_wake_interval.is_zero() { + return Err(SerfError::InvalidOption(InvalidOption::new( + "idle_wake_interval", + "the driver-loop fallback sleep must be nonzero: the loop sleeps until \ + `poll_timeout().unwrap_or(now + idle_wake_interval)`, so a zero idle_wake_interval \ + makes a quiescent endpoint re-arm a zero-duration timer every pass — a busy-spin \ + that pegs a CPU core" + .to_string(), + ))); + } + if self.event_queue_cap == 0 { + return Err(SerfError::InvalidOption(InvalidOption::new( + "event_queue_cap", + "the event-stream queue capacity must be nonzero: the observation task forwards every \ + event to this bounded channel with a non-blocking `try_send`, and a zero-capacity \ + channel is a rendezvous a non-blocking send can never deposit into, so every event \ + would be dropped and the event-observation surface would be non-functional" + .to_string(), + ))); + } + if matches!(self.observation_channel, Channel::Bounded(0)) { + return Err(SerfError::InvalidOption(InvalidOption::new( + "observation_channel", + "the delegate observation channel must have a nonzero bounded capacity: each driver \ + builds it as a bounded flume channel, and a `Bounded(0)` rendezvous is one the \ + driver's non-blocking `try_send` can never deposit into, so every observed event \ + would be dropped and the delegate would never fire" + .to_string(), + ))); + } + Ok(()) + } +} + +impl Default for RuntimeOptions { + #[inline] + fn default() -> Self { + Self::new() + } +} + +/// Stream-transport-specific tuning knobs. +/// +/// Apply to the stream-backed (TCP) [`Serf`](crate::Serf) driver. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct StreamTransportOptions { + dial_timeout: Duration, + close_timeout: Duration, + bridge_inbound_cap: usize, + bridge_recv_buf_len: usize, +} + +impl StreamTransportOptions { + /// Construct with the canonical stream-transport defaults. + #[inline] + pub const fn new() -> Self { + Self { + dial_timeout: DEFAULT_DIAL_TIMEOUT, + close_timeout: DEFAULT_CLOSE_TIMEOUT, + bridge_inbound_cap: DEFAULT_BRIDGE_INBOUND_CAP, + bridge_recv_buf_len: DEFAULT_BRIDGE_RECV_BUF_LEN, + } + } + + /// Builder: outbound `TcpStream::connect` budget. + #[must_use] + #[inline] + pub const fn with_dial_timeout(mut self, d: Duration) -> Self { + self.dial_timeout = d; + self + } + + /// Builder: bound on a per-bridge graceful-drain write. + #[must_use] + #[inline] + pub const fn with_close_timeout(mut self, d: Duration) -> Self { + self.close_timeout = d; + self + } + + /// Builder: bridge-inbound channel capacity. + #[must_use] + #[inline] + pub const fn with_bridge_inbound_cap(mut self, n: usize) -> Self { + self.bridge_inbound_cap = n; + self + } + + /// Builder: per-bridge TCP read buffer size. + #[must_use] + #[inline] + pub const fn with_bridge_recv_buf_len(mut self, n: usize) -> Self { + self.bridge_recv_buf_len = n; + self + } + + /// Outbound `TcpStream::connect` budget. + #[inline] + pub const fn dial_timeout(&self) -> Duration { + self.dial_timeout + } + + /// Bound on a per-bridge graceful-drain write. + #[inline] + pub const fn close_timeout(&self) -> Duration { + self.close_timeout + } + + /// Bridge-inbound channel capacity. + #[inline] + pub const fn bridge_inbound_cap(&self) -> usize { + self.bridge_inbound_cap + } + + /// Per-bridge TCP read buffer size. + #[inline] + pub const fn bridge_recv_buf_len(&self) -> usize { + self.bridge_recv_buf_len + } + + /// Validate the stream-transport knobs whose value would DETERMINISTICALLY + /// break (not merely degrade) a stream backend, rejected fail-fast at the + /// transport's `new` (before any socket is bound) rather than constructing `Ok` + /// over a silently-broken backend. Called at the top of both stream backends' + /// `Transport::new` (TCP and TLS); QUIC has no bridges and so no counterpart. + /// + /// - `dial_timeout == 0`: every outbound `StreamAction::Connect` races the + /// runtime's `TcpStream::connect` against `R::sleep(dial_timeout)` in a + /// biased select. A zero timeout makes the sleep ready immediately, so it + /// wins the biased select ahead of the connect and EVERY outbound dial + /// resolves as a spurious dial-timeout failure. Because a serf `join` is + /// dispatch-only — it returns the seed count, not a per-exchange outcome — + /// the node returns `Ok` from `join` while no reliable push/pull exchange + /// ever completes, so it silently never joins. + /// - `bridge_recv_buf_len == 0`: the per-bridge byte-mover reads into a + /// `vec![0u8; bridge_recv_buf_len]`, and a zero-length read returns + /// `Ok(0)`, which the bridge treats as peer EOF — so every reliable bridge + /// would report EOF instead of reading frames. + /// - `close_timeout == 0`: the post-`Close` graceful drain fires immediately, + /// so every graceful close abandons (RSTs) its queued response bytes. + /// - `bridge_inbound_cap == 0`: the per-bridge inbound channel at zero + /// capacity has no buffered slot, so the bridge task's non-blocking hand-off + /// to the driver can never deposit a frame and no reliable-stream frame + /// reaches the driver. + #[cfg(feature = "tcp")] + pub(crate) fn validate(&self) -> Result<(), SerfError> { + if self.dial_timeout.is_zero() { + return Err(SerfError::InvalidOption(InvalidOption::new( + "dial_timeout", + "the outbound dial budget must be nonzero: every Connect races TcpStream::connect \ + against a sleep(dial_timeout) in a biased select, so a zero timeout is ready \ + immediately and wins ahead of the connect, failing every outbound push/pull dial \ + as a spurious timeout — and since serf's join is dispatch-only, the node returns Ok \ + from join while never completing a single reliable exchange" + .to_string(), + ))); + } + if self.bridge_recv_buf_len == 0 { + return Err(SerfError::InvalidOption(InvalidOption::new( + "bridge_recv_buf_len", + "the per-bridge reliable-stream read buffer must be nonzero: a zero-length read \ + returns Ok(0), which the bridge treats as peer EOF, so every reliable \ + push-pull stream exchange would break" + .to_string(), + ))); + } + if self.close_timeout.is_zero() { + return Err(SerfError::InvalidOption(InvalidOption::new( + "close_timeout", + "the reliable graceful-close drain timeout must be nonzero: a zero timeout fires \ + immediately, so a graceful close abandons (RSTs) queued push/pull response bytes \ + instead of draining them, truncating reliable exchanges" + .to_string(), + ))); + } + if self.bridge_inbound_cap == 0 { + return Err(SerfError::InvalidOption(InvalidOption::new( + "bridge_inbound_cap", + "the per-bridge inbound queue capacity must be nonzero: a zero-capacity channel has no \ + buffered slot, so the bridge task's non-blocking hand-off to the driver can never \ + deposit a frame and no reliable-stream frame reaches the driver" + .to_string(), + ))); + } + Ok(()) + } +} + +impl Default for StreamTransportOptions { + #[inline] + fn default() -> Self { + Self::new() + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-reactor/src/driver/options/tests.rs b/serf-reactor/src/driver/options/tests.rs new file mode 100644 index 00000000..2b8535a8 --- /dev/null +++ b/serf-reactor/src/driver/options/tests.rs @@ -0,0 +1,414 @@ +use super::*; +#[cfg(any(feature = "tcp", feature = "quic", feature = "serde", feature = "clap"))] +use core::time::Duration; + +#[test] +fn runtime_options_defaults_are_accessible() { + let opts = RuntimeOptions::new(); + assert_eq!(opts.join_deadline(), DEFAULT_JOIN_DEADLINE); + assert_eq!(opts.leave_timeout(), DEFAULT_LEAVE_TIMEOUT); + assert_eq!(opts.idle_wake_interval(), DEFAULT_IDLE_WAKE_INTERVAL); + assert_eq!(opts.iter_drain_cap(), DEFAULT_ITER_DRAIN_CAP); + assert_eq!(opts.event_queue_cap(), DEFAULT_EVENT_QUEUE_CAP); + assert_eq!(opts.observation_channel(), DEFAULT_OBSERVATION_CHANNEL); +} + +#[test] +fn stream_transport_options_defaults_are_accessible() { + let opts = StreamTransportOptions::new(); + assert_eq!(opts.dial_timeout(), DEFAULT_DIAL_TIMEOUT); + assert_eq!(opts.close_timeout(), DEFAULT_CLOSE_TIMEOUT); + assert_eq!(opts.bridge_inbound_cap(), DEFAULT_BRIDGE_INBOUND_CAP); + assert_eq!(opts.bridge_recv_buf_len(), DEFAULT_BRIDGE_RECV_BUF_LEN); +} + +#[test] +fn channel_display_and_from_str_round_trip() { + assert_eq!(Channel::Unbounded.to_string(), "unbounded"); + assert_eq!(Channel::Bounded(42).to_string(), "bounded:42"); + assert_eq!("unbounded".parse::().unwrap(), Channel::Unbounded); + assert_eq!( + "bounded:42".parse::().unwrap(), + Channel::Bounded(42) + ); + assert!("nonsense".parse::().is_err()); +} + +#[cfg(feature = "tcp")] +#[test] +fn stream_transport_validate_rejects_zero_recv_buf() { + let opts = StreamTransportOptions::new().with_bridge_recv_buf_len(0); + assert!(opts.validate().is_err()); +} + +#[cfg(feature = "tcp")] +#[test] +fn stream_transport_validate_rejects_zero_close_timeout() { + let opts = StreamTransportOptions::new().with_close_timeout(Duration::ZERO); + assert!(opts.validate().is_err()); +} + +#[cfg(feature = "tcp")] +#[test] +fn stream_transport_validate_rejects_zero_bridge_inbound_cap() { + let opts = StreamTransportOptions::new().with_bridge_inbound_cap(0); + assert!(opts.validate().is_err()); +} + +// A `Bounded(0)` observation channel is a flume rendezvous the driver's +// non-blocking `try_send` can never deposit into; `validate` rejects it at +// construction instead. +#[cfg(any(feature = "tcp", feature = "quic"))] +#[test] +fn runtime_validate_rejects_zero_observation_channel() { + let opts = RuntimeOptions::new().with_observation_channel(Channel::Bounded(0)); + assert!(matches!( + opts.validate(), + Err(crate::SerfError::InvalidOption(_)) + )); +} + +// A zero `event_queue_cap` makes the `flume::bounded` event channel a rendezvous +// the non-blocking forward can never deposit into; `validate` rejects it. +#[cfg(any(feature = "tcp", feature = "quic"))] +#[test] +fn runtime_validate_rejects_zero_event_queue_cap() { + let opts = RuntimeOptions::new().with_event_queue_cap(0); + assert!(matches!( + opts.validate(), + Err(crate::SerfError::InvalidOption(_)) + )); +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +#[test] +fn runtime_validate_accepts_unbounded_and_nonzero_caps() { + assert!(RuntimeOptions::new().validate().is_ok()); + assert!( + RuntimeOptions::new() + .with_observation_channel(Channel::Unbounded) + .validate() + .is_ok() + ); + assert!( + RuntimeOptions::new() + .with_observation_channel(Channel::Bounded(1)) + .with_event_queue_cap(1) + .validate() + .is_ok() + ); +} + +// A zero `idle_wake_interval` makes a quiescent endpoint re-arm a zero-duration +// timer every pass — a busy-spin that pegs a CPU core; `validate` rejects it. +#[cfg(any(feature = "tcp", feature = "quic"))] +#[test] +fn runtime_validate_rejects_zero_idle_wake_interval() { + let opts = RuntimeOptions::new().with_idle_wake_interval(Duration::ZERO); + assert!(matches!( + opts.validate(), + Err(crate::SerfError::InvalidOption(_)) + )); +} + +// A zero `join_deadline` makes every parked await-join waiter past-due on insert, +// so the reaper replies `JoinAllFailed` before any push/pull can complete; +// `validate` rejects it at construction. +#[cfg(any(feature = "tcp", feature = "quic"))] +#[test] +fn runtime_validate_rejects_zero_join_deadline() { + let opts = RuntimeOptions::new().with_join_deadline(Duration::ZERO); + assert!(matches!( + opts.validate(), + Err(crate::SerfError::InvalidOption(_)) + )); +} + +// `iter_drain_cap == 0` (the per-iteration batch cap; the pump still makes +// one-per-pass forward progress) and `leave_timeout == 0` (a loud immediate +// `LeaveTimeout`) degrade-but-function, so they are accepted rather than +// rejected. +#[cfg(any(feature = "tcp", feature = "quic"))] +#[test] +fn runtime_validate_accepts_degrade_but_function_knobs() { + assert!( + RuntimeOptions::new() + .with_iter_drain_cap(0) + .validate() + .is_ok() + ); + assert!( + RuntimeOptions::new() + .with_leave_timeout(Duration::ZERO) + .validate() + .is_ok() + ); +} + +// A zero `dial_timeout` makes every outbound dial resolve as an immediate +// biased-select timeout, so the stream transport rejects it at `validate` +// (before any socket bind). +#[cfg(feature = "tcp")] +#[test] +fn stream_transport_validate_rejects_zero_dial_timeout() { + let opts = StreamTransportOptions::new().with_dial_timeout(Duration::ZERO); + assert!(matches!( + opts.validate(), + Err(crate::SerfError::InvalidOption(_)) + )); +} + +// A zero capacity sourced via serde — both the `bounded:0` `FromStr` form and the +// `{"bounded":0}` deserialize form — parses, then fails validation rather than +// reaching a driver-task panic. +#[cfg(all(feature = "serde", any(feature = "tcp", feature = "quic")))] +#[test] +fn runtime_validate_rejects_zero_observation_channel_from_serde() { + let from_str: Channel = "bounded:0".parse().expect("bounded:0 parses"); + assert_eq!(from_str, Channel::Bounded(0)); + let opts: RuntimeOptions = + serde_json::from_str(r#"{"observation_channel":{"bounded":0}}"#).expect("deserialize"); + assert_eq!(opts.observation_channel(), Channel::Bounded(0)); + assert!(matches!( + opts.validate(), + Err(crate::SerfError::InvalidOption(_)) + )); +} + +#[cfg(all(feature = "serde", any(feature = "tcp", feature = "quic")))] +#[test] +fn runtime_validate_rejects_zero_event_queue_cap_from_serde() { + let opts: RuntimeOptions = serde_json::from_str(r#"{"event_queue_cap":0}"#).expect("deserialize"); + assert_eq!(opts.event_queue_cap(), 0); + assert!(matches!( + opts.validate(), + Err(crate::SerfError::InvalidOption(_)) + )); +} + +// The newly-validated runtime knobs are rejected the same way when sourced via +// serde — the humantime `idle_wake_interval` duration parses, then fails +// validation. +#[cfg(all(feature = "serde", any(feature = "tcp", feature = "quic")))] +#[test] +fn runtime_validate_rejects_zero_idle_wake_interval_from_serde() { + let opts: RuntimeOptions = + serde_json::from_str(r#"{"idle_wake_interval":"0s"}"#).expect("deserialize"); + assert_eq!(opts.idle_wake_interval(), Duration::ZERO); + assert!(matches!( + opts.validate(), + Err(crate::SerfError::InvalidOption(_)) + )); +} + +#[cfg(all(feature = "serde", any(feature = "tcp", feature = "quic")))] +#[test] +fn runtime_validate_rejects_zero_join_deadline_from_serde() { + let opts: RuntimeOptions = + serde_json::from_str(r#"{"join_deadline":"0s"}"#).expect("deserialize"); + assert_eq!(opts.join_deadline(), Duration::ZERO); + assert!(matches!( + opts.validate(), + Err(crate::SerfError::InvalidOption(_)) + )); +} + +// A zero capacity sourced via a clap-parsed flag is rejected the same way. +#[cfg(all(feature = "clap", any(feature = "tcp", feature = "quic")))] +#[test] +fn runtime_validate_rejects_zero_observation_channel_from_clap() { + use clap::Parser; + + #[derive(Parser)] + struct Cli { + #[command(flatten)] + runtime: RuntimeOptions, + } + + let cli = Cli::try_parse_from(["app", "--runtime-observation-channel", "bounded:0"]) + .expect("clap parses bounded:0"); + assert_eq!(cli.runtime.observation_channel(), Channel::Bounded(0)); + assert!(matches!( + cli.runtime.validate(), + Err(crate::SerfError::InvalidOption(_)) + )); +} + +// The newly-validated runtime knobs are rejected the same way when sourced via a +// clap-parsed flag. +#[cfg(all(feature = "clap", any(feature = "tcp", feature = "quic")))] +#[test] +fn runtime_validate_rejects_zero_idle_wake_interval_from_clap() { + use clap::Parser; + + #[derive(Parser)] + struct Cli { + #[command(flatten)] + runtime: RuntimeOptions, + } + + let cli = Cli::try_parse_from(["app", "--runtime-idle-wake-interval", "0s"]) + .expect("clap parses idle-wake-interval 0s"); + assert_eq!(cli.runtime.idle_wake_interval(), Duration::ZERO); + assert!(matches!( + cli.runtime.validate(), + Err(crate::SerfError::InvalidOption(_)) + )); +} + +#[cfg(all(feature = "clap", any(feature = "tcp", feature = "quic")))] +#[test] +fn runtime_validate_rejects_zero_join_deadline_from_clap() { + use clap::Parser; + + #[derive(Parser)] + struct Cli { + #[command(flatten)] + runtime: RuntimeOptions, + } + + let cli = Cli::try_parse_from(["app", "--runtime-join-deadline", "0s"]) + .expect("clap parses join-deadline 0s"); + assert_eq!(cli.runtime.join_deadline(), Duration::ZERO); + assert!(matches!( + cli.runtime.validate(), + Err(crate::SerfError::InvalidOption(_)) + )); +} + +#[cfg(feature = "serde")] +#[test] +fn runtime_options_serde_round_trip_and_partial() { + // An empty config deserializes to the full default. + assert_eq!( + serde_json::from_str::("{}").unwrap(), + RuntimeOptions::new() + ); + // A full round-trip preserves every knob, including humantime durations and + // the tagged observation channel. + let opts = RuntimeOptions::new() + .with_leave_timeout(Duration::from_secs(33)) + .with_iter_drain_cap(99) + .with_observation_channel(Channel::Unbounded); + let json = serde_json::to_string(&opts).unwrap(); + assert_eq!(serde_json::from_str::(&json).unwrap(), opts); + // A partial config overrides one field and defaults the rest. + let partial: RuntimeOptions = serde_json::from_str(r#"{"iter_drain_cap": 7}"#).unwrap(); + assert_eq!(partial.iter_drain_cap(), 7); + assert_eq!(partial.leave_timeout(), DEFAULT_LEAVE_TIMEOUT); + assert_eq!(partial.observation_channel(), DEFAULT_OBSERVATION_CHANNEL); + // The bounded channel serializes as its snake_case tagged form. + let bounded = RuntimeOptions::new().with_observation_channel(Channel::Bounded(8)); + let bjson = serde_json::to_string(&bounded).unwrap(); + assert!(bjson.contains("bounded"), "json = {bjson}"); + assert_eq!( + serde_json::from_str::(&bjson).unwrap(), + bounded + ); +} + +#[cfg(feature = "serde")] +#[test] +fn runtime_options_serde_rejects_unknown_field() { + // A misspelled field must be rejected, not silently dropped. + assert!(serde_json::from_str::(r#"{"iter_drain_capp": 7}"#).is_err()); +} + +#[cfg(feature = "clap")] +#[test] +fn runtime_options_clap_parses_flags_and_wires_env() { + use clap::{CommandFactory, Parser}; + + #[derive(Parser)] + struct Cli { + #[command(flatten)] + runtime: RuntimeOptions, + } + + // Flags parse: a humantime duration, a usize, and the channel via FromStr. + let cli = Cli::try_parse_from([ + "app", + "--runtime-leave-timeout", + "30s", + "--runtime-iter-drain-cap", + "12", + "--runtime-observation-channel", + "bounded:64", + ]) + .unwrap(); + assert_eq!(cli.runtime.leave_timeout(), Duration::from_secs(30)); + assert_eq!(cli.runtime.iter_drain_cap(), 12); + assert_eq!(cli.runtime.observation_channel(), Channel::Bounded(64)); + // Unspecified flags stay at the defaults. + let dflt = Cli::try_parse_from(["app"]).unwrap(); + assert_eq!(dflt.runtime, RuntimeOptions::new()); + // The env var is wired — assert via command introspection, never `set_var`. + let cmd = Cli::command(); + let arg = cmd + .get_arguments() + .find(|a| a.get_id().as_str() == "runtime-idle-wake-interval") + .expect("runtime-idle-wake-interval arg is registered"); + assert_eq!( + arg.get_env().and_then(|e| e.to_str()), + Some("SERF_RUNTIME_IDLE_WAKE_INTERVAL") + ); +} + +// A partial `try_update_from` carrying one unrelated flag must NOT reset the +// other defaulted knobs. clap's `default_value` / `default_value_t` makes an +// unset arg look "present" in update mode, so the value-source gate in the +// manual `update_from_arg_matches` is what keeps a seeded non-default value +// alive across an update. +#[cfg(feature = "clap")] +#[test] +fn runtime_options_partial_update_preserves_unset_fields() { + use clap::Parser; + + #[derive(Parser)] + struct Cli { + #[command(flatten)] + o: RuntimeOptions, + } + + // Seed NON-default values on several fields, then run a partial update that + // supplies ONE unrelated flag. + let mut cli = Cli { + o: RuntimeOptions::new() + .with_leave_timeout(Duration::from_secs(42)) + .with_event_queue_cap(7) + .with_observation_channel(Channel::Unbounded), + }; + + cli + .try_update_from(["app", "--runtime-iter-drain-cap", "13"]) + .expect("partial update parses"); + + // The supplied flag is applied. + assert_eq!(cli.o.iter_drain_cap(), 13); + // Every seeded non-default field SURVIVES the partial update. + assert_eq!(cli.o.leave_timeout(), Duration::from_secs(42)); + assert_eq!(cli.o.event_queue_cap(), 7); + assert_eq!(cli.o.observation_channel(), Channel::Unbounded); +} + +// An explicit override on update IS applied (the value-source gate lets a +// command-line value through). +#[cfg(feature = "clap")] +#[test] +fn runtime_options_update_applies_explicit_override() { + use clap::Parser; + + #[derive(Parser)] + struct Cli { + #[command(flatten)] + o: RuntimeOptions, + } + + let mut cli = Cli { + o: RuntimeOptions::new().with_leave_timeout(Duration::from_secs(99)), + }; + cli + .try_update_from(["app", "--runtime-leave-timeout", "3s"]) + .expect("explicit override parses"); + assert_eq!(cli.o.leave_timeout(), Duration::from_secs(3)); +} diff --git a/serf-reactor/src/error/mod.rs b/serf-reactor/src/error/mod.rs new file mode 100644 index 00000000..2f832d08 --- /dev/null +++ b/serf-reactor/src/error/mod.rs @@ -0,0 +1,194 @@ +//! Error types for serf-reactor. + +use core::fmt; +use std::{io, net::SocketAddr}; + +pub use serf_driver::error::{GossipMtuTooSmall, InvalidOption, JoinFailed}; + +/// Payload for [`SerfError::InvalidGossipMtu`]: the configured `gossip_mtu` +/// exceeds the largest plaintext gossip payload that can fit a single UDP +/// datagram once any encryption wrapper is added. Carries the configured value +/// and the effective ceiling. +#[derive(Debug)] +pub struct InvalidGossipMtu { + configured: usize, + ceiling: usize, +} + +impl InvalidGossipMtu { + /// Build a new payload from the configured `gossip_mtu` and the ceiling. + #[inline] + pub fn new(configured: usize, ceiling: usize) -> Self { + Self { + configured, + ceiling, + } + } + + /// The configured `gossip_mtu` that was rejected. + #[inline] + pub fn configured(&self) -> usize { + self.configured + } + + /// The effective ceiling — the largest plaintext `gossip_mtu` whose wire + /// datagram still fits a single UDP packet after any encryption wrapper. + #[inline] + pub fn ceiling(&self) -> usize { + self.ceiling + } +} + +impl fmt::Display for InvalidGossipMtu { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "gossip_mtu {} exceeds the maximum sendable plaintext gossip payload of {} bytes \ + (a gossip packet is one UDP datagram capped after the encryption wrapper); \ + a larger gossip_mtu would make near-MTU gossip packets deterministically unsendable", + self.configured, self.ceiling, + ) + } +} + +/// Payload for [`SerfError::InvalidAdvertiseAddr`]: the resolved advertise +/// address cannot serve as the local node's reachable contact identity. Two +/// independent classes are rejected: +/// +/// - NOT A USABLE UNICAST CONTACT — an unspecified IP, a multicast IP, an IPv4 +/// broadcast IP, or a zero port. Such an address is undialable. +/// - NOT REPRESENTABLE ON THE WIRE — a scoped/flow-labelled IPv6 `SocketAddr` +/// with a nonzero `scope_id` or `flowinfo` that the compact wire layout +/// (`[16B IP][2B port]`) cannot carry. +#[derive(Debug)] +pub struct InvalidAdvertiseAddr { + addr: SocketAddr, + reason: String, +} + +impl InvalidAdvertiseAddr { + /// Build a new payload from the rejected advertise address and the reason. + #[inline] + pub fn new(addr: SocketAddr, reason: String) -> Self { + Self { addr, reason } + } + + /// The advertise address that was rejected. + #[inline] + pub fn addr(&self) -> SocketAddr { + self.addr + } + + /// The reason the address cannot serve as the local node's contact identity. + #[inline] + pub fn reason(&self) -> &str { + &self.reason + } +} + +impl fmt::Display for InvalidAdvertiseAddr { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "advertise address {} cannot serve as this node's reachable contact identity, \ + so peers that learn it could not route serf traffic to this node: {}", + self.addr, self.reason, + ) + } +} + +/// Errors returned by [`Serf`](crate::Serf) operations. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum SerfError { + /// I/O error from the OS, socket, or async runtime. + #[error(transparent)] + Io(#[from] io::Error), + + /// The OS entropy source failed while seeding the gossip RNG. Surfaced in + /// the node constructor (which returns this `Result`) before the driver task + /// is spawned, so a failure is surfaced here rather than panicking in the + /// spawned task. + #[error("OS entropy source failed while seeding the gossip RNG")] + Entropy(#[source] io::Error), + + /// Address resolution failed (DNS error, etc.). + #[error("address resolution: {0}")] + Resolve(io::Error), + + /// A serf endpoint operation (join, leave, set_tags, user_event, query, …) + /// returned a machine-level error. Carries the typed + /// [`serf_proto::endpoint::Error`] so callers can dispatch on the specific + /// cause. + #[cfg(any(feature = "tcp", feature = "quic"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] + #[error(transparent)] + Proto(#[from] serf_proto::endpoint::Error), + + /// Encryption codec error from memberlist-wire. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + #[error(transparent)] + Encryption(#[from] memberlist_proto::EncryptionError), + + /// A graceful [`leave`](crate::Serf::leave) did not complete within the + /// driver's configured leave timeout. The leave was initiated but the driver + /// cannot confirm peers were notified. + #[error("leave did not complete within the configured leave timeout")] + LeaveTimeout, + + /// The driver task has shut down and is no longer accepting commands. + #[error("driver shut down")] + Shutdown, + + /// The local node has left the cluster; the operation requires an active node. + #[error("the local node has left the cluster; the operation requires a running node")] + NotRunning, + + /// An await-result [`join`](crate::Serf::join) dispatched a push/pull to one + /// or more resolved seeds but none was contacted before the call resolved — + /// the typical "all seeds unreachable" cluster-bootstrap failure. A non-empty + /// seed input that resolved to zero addresses surfaces here too, rather than + /// collapsing to a silent success. + #[error(transparent)] + JoinAllFailed(JoinFailed), + + /// The configured `gossip_mtu` exceeds the ceiling after the encryption + /// wrapper is applied. Returned at construction (fail-fast, before any socket + /// is bound) so the misconfiguration is surfaced rather than producing + /// deterministically dropped gossip. + #[error("{0}")] + InvalidGossipMtu(InvalidGossipMtu), + + /// The configured `gossip_mtu` is below the floor needed to carry the + /// mandatory single-datagram control packets the SWIM protocol always emits. + #[error("{0}")] + GossipMtuTooSmall(GossipMtuTooSmall), + + /// The resolved advertise address cannot serve as the local node's reachable + /// contact identity. + #[error("{0}")] + InvalidAdvertiseAddr(InvalidAdvertiseAddr), + + /// An operator-set driver tuning knob was given a value that would + /// deterministically break the node rather than merely degrade it. + #[error("{0}")] + InvalidOption(InvalidOption), + + /// Sending a command to the driver failed because the channel is closed. + #[error("send to driver failed (channel closed)")] + CommandSend, + + /// The driver's reply channel was dropped before a reply arrived. + #[error("driver reply channel closed")] + ReplyClosed, +} + +/// Convenience [`Result`] for [`SerfError`]. +pub type Result = core::result::Result; + +#[cfg(test)] +mod tests; diff --git a/serf-reactor/src/error/tests.rs b/serf-reactor/src/error/tests.rs new file mode 100644 index 00000000..2ee94e47 --- /dev/null +++ b/serf-reactor/src/error/tests.rs @@ -0,0 +1,86 @@ +use super::*; + +#[test] +fn invalid_gossip_mtu_accessors_and_display() { + let payload = InvalidGossipMtu::new(70_000, 65_467); + assert_eq!(payload.configured(), 70_000); + assert_eq!(payload.ceiling(), 65_467); + let shown = format!("{payload}"); + assert!(!shown.is_empty()); + assert!(shown.contains("70000")); + assert!(!format!("{payload:?}").is_empty()); +} + +#[test] +fn gossip_mtu_too_small_accessors_and_display() { + let payload = GossipMtuTooSmall::new(64, 512); + assert_eq!(payload.configured(), 64); + assert_eq!(payload.minimum(), 512); + let shown = format!("{payload}"); + assert!(!shown.is_empty()); + assert!(shown.contains("512")); + assert!(!format!("{payload:?}").is_empty()); +} + +#[test] +fn invalid_advertise_addr_accessors_and_display() { + let addr: std::net::SocketAddr = "0.0.0.0:5000".parse().unwrap(); + let payload = InvalidAdvertiseAddr::new(addr, "wildcard bind".to_string()); + assert_eq!(payload.addr(), addr); + assert_eq!(payload.reason(), "wildcard bind"); + let shown = format!("{payload}"); + assert!(!shown.is_empty()); + assert!(shown.contains("wildcard bind")); + assert!(!format!("{payload:?}").is_empty()); +} + +#[test] +fn invalid_option_accessors_and_display() { + let payload = InvalidOption::new("idle_wake_interval", "must be nonzero".to_string()); + assert_eq!(payload.option(), "idle_wake_interval"); + assert_eq!(payload.reason(), "must be nonzero"); + let shown = format!("{payload}"); + assert!(shown.contains("idle_wake_interval")); + assert!(shown.contains("must be nonzero")); + assert!(!format!("{payload:?}").is_empty()); +} + +#[test] +fn every_variant_displays_and_debugs() { + let variants: &[SerfError] = &[ + SerfError::Io(io::Error::other("disk")), + SerfError::Entropy(io::Error::other("entropy")), + SerfError::Resolve(io::Error::other("dns")), + SerfError::LeaveTimeout, + SerfError::Shutdown, + SerfError::NotRunning, + SerfError::JoinAllFailed(JoinFailed::new(3, 0)), + SerfError::InvalidGossipMtu(InvalidGossipMtu::new(70_000, 65_467)), + SerfError::GossipMtuTooSmall(GossipMtuTooSmall::new(64, 512)), + SerfError::InvalidAdvertiseAddr(InvalidAdvertiseAddr::new( + "0.0.0.0:5000".parse().unwrap(), + "wildcard".to_string(), + )), + SerfError::InvalidOption(InvalidOption::new( + "idle_wake_interval", + "nonzero".to_string(), + )), + SerfError::CommandSend, + SerfError::ReplyClosed, + ]; + + for err in variants { + assert!( + !format!("{err}").is_empty(), + "Display non-empty for {err:?}" + ); + assert!(!format!("{err:?}").is_empty(), "Debug non-empty"); + } +} + +#[test] +fn from_io_conversion() { + let err: SerfError = io::Error::other("boom").into(); + assert!(matches!(err, SerfError::Io(_))); + assert_eq!(err.to_string(), io::Error::other("boom").to_string()); +} diff --git a/serf-reactor/src/events.rs b/serf-reactor/src/events.rs new file mode 100644 index 00000000..8b232260 --- /dev/null +++ b/serf-reactor/src/events.rs @@ -0,0 +1,66 @@ +//! Event stream — the user-facing observation channel for serf events. + +#[cfg(any(feature = "tcp", feature = "quic"))] +use flume::r#async::RecvStream; +#[cfg(any(feature = "tcp", feature = "quic"))] +use futures_util::Stream; +#[cfg(any(feature = "tcp", feature = "quic"))] +use serf_proto::event::Event; +#[cfg(any(feature = "tcp", feature = "quic"))] +use std::{ + pin::Pin, + task::{Context, Poll}, +}; + +/// Stream of serf events. Constructed via [`Serf::events`](crate::Serf::events). +/// +/// Generic over the wire id / address types ``. +/// +/// **Concurrency model:** flume MPMC — multiple `events()` calls each return +/// an independent `EventStream`, but events ROUND-ROBIN between subscribers +/// (NOT broadcast). For single-consumer use this is the right shape. +/// +/// **Lossy under backpressure:** the events channel is bounded. When the queue +/// is full the driver drops the newest event rather than block — a slow +/// subscriber must not stall the membership FSM. +/// +/// Requires a stream or QUIC transport feature (`tcp` or `quic`). +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub struct EventStream +where + I: 'static, + A: 'static, +{ + inner: RecvStream<'static, Event>, +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +impl EventStream +where + I: 'static, + A: 'static, +{ + /// Wrap a flume receiver into an `EventStream`. + /// + /// Consumes the receiver: the resulting stream lives `'static` and owns the + /// queue handle. + pub fn new(rx: flume::Receiver>) -> Self { + Self { + inner: rx.into_stream(), + } + } +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +impl Stream for EventStream +where + I: 'static, + A: 'static, +{ + type Item = Event; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_next(cx) + } +} diff --git a/serf-reactor/src/lib.rs b/serf-reactor/src/lib.rs new file mode 100644 index 00000000..1e33c6cc --- /dev/null +++ b/serf-reactor/src/lib.rs @@ -0,0 +1,126 @@ +//! agnostic-runtime async driver for the Sans-I/O serf machine. +//! +//! `serf-reactor` binds the pure [`serf-proto`] Sans-I/O serf machine to a +//! readiness-based async runtime (tokio or smol, selected through the +//! [`agnostic`](https://docs.rs/agnostic) runtime abstraction). It owns the +//! driver task, the command queue, the observation delegate dispatch, and the +//! transport plumbing — everything that connects the stateless machine to real +//! I/O. The machine itself (`serf-proto`) carries no runtime reference and runs +//! identically regardless of which driver crate wraps it. +//! +//! It is the `Send`/`Arc`/`agnostic`-generic sibling of the `!Send` +//! [`serf-compio`](https://docs.rs/serf-compio) driver: every shared value is an +//! `Arc` with atomic / `arc_swap` interior mutability, the delegate hooks return +//! `Send` futures, and the driver is generic over `R: agnostic::Runtime`. +//! +//! # Feature flags +//! +//! Transport (at least one required): +//! - `tcp` (default) — plain-TCP reliable coordinator. +//! - `tls` — TLS-over-TCP reliable coordinator (implies `tcp`). +//! - `tls-rustls-ring` / `tls-rustls-aws-lc-rs` — TLS with a rustls provider. +//! - `quic` — QUIC coordinator. +//! - `quic-rustls-ring` — QUIC with the rustls+ring TLS backend (implies `quic`). +//! +//! Runtime selection (pick one for a runnable node): +//! - `tokio` — pull in the tokio agnostic runtime impl. +//! - `smol` — pull in the smol agnostic runtime impl. +//! +//! Optional protocol extensions: +//! - `coordinates` — Vivaldi network coordinate estimation. +//! - `aes-gcm` — AES-256-GCM encryption backend. +//! - `chacha20-poly1305` — ChaCha20-Poly1305 encryption backend. +//! - `tag-regex` (default) — regex-backed tag-filter matching. +//! +//! Optional config layering: +//! - `serde` — `Serialize`/`Deserialize` on `*Options` types. +//! - `clap` — `clap::Args` on `*Options` types (CLI flags + env vars). +//! - `tracing` — emit `tracing` spans around public driver operations. +//! +//! Optional resolver support: +//! - `dns` — `DnsResolver` backed by `hickory-proto`. +//! - `getifs` — auto-detect the advertise address from host interfaces. + +#![deny(missing_docs)] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![forbid(unsafe_code)] + +#[cfg(any(feature = "tcp", feature = "quic"))] +mod command; +#[cfg(any(feature = "tcp", feature = "quic"))] +mod delegate; +mod driver; +mod error; +mod events; + +use rand::{ + SeedableRng, + rngs::{StdRng, SysRng}, +}; + +/// A fresh [`StdRng`] seeded directly from the OS entropy source ([`SysRng`], +/// i.e. `getrandom`) — never from a thread-local generator, so a process that +/// forks after building a node cannot inherit a parent's RNG state and derive +/// the same gossip schedule. +/// +/// Drawn before the driver task is spawned (the result is passed to the node +/// constructor, which returns a `Result`), so an OS entropy failure surfaces as +/// [`SerfError::Entropy`] rather than panicking in the spawned task after the +/// handle was already returned. +pub fn gossip_rng() -> crate::Result { + os_seeded_std_rng() +} + +/// Draw a fresh OS-seeded [`StdRng`] — the shared seed source behind both +/// [`gossip_rng`] (the memberlist gossip schedule) and each transport's +/// independent serf-core RNG seed. +/// +/// Every call draws fresh OS entropy, so two machines built in one process get +/// mutually-independent RNG streams; the serf core's RNG (which picks query IDs +/// and relay targets) is therefore never correlated with the gossip RNG or with +/// another node's, ruling out the colliding `(ltime, id)` a shared/zero seed +/// would produce. +pub(crate) fn os_seeded_std_rng() -> crate::Result { + StdRng::try_from_rng(&mut SysRng).map_err(|e| crate::SerfError::Entropy(std::io::Error::other(e))) +} + +pub use error::{ + GossipMtuTooSmall, InvalidAdvertiseAddr, InvalidGossipMtu, InvalidOption, Result, SerfError, +}; + +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub use delegate::{ + Delegate, MemberDelegate, MergeDelegate, NoopMergeDelegate, QueryDelegate, UserEventDelegate, + VoidDelegate, +}; + +#[cfg(encryption)] +#[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) +)] +pub use delegate::{KeyringDelegate, VoidKeyringDelegate}; + +/// Gossip-encryption config types re-exported from `memberlist-proto`, so a +/// caller can build a transport's `with_encryption` keyring without naming +/// `memberlist-proto` directly. `EncryptionOptions` carries an optional +/// `Keyring` (primary + secondary `SecretKey`s); attaching one enables +/// encryption, leaving it absent keeps every plane plaintext. +#[cfg(encryption)] +#[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) +)] +pub use memberlist_proto::{EncryptionOptions, Keyring, SecretKey}; + +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub use events::EventStream; + +pub use driver::options::{ + Channel, DEFAULT_BRIDGE_INBOUND_CAP, DEFAULT_BRIDGE_RECV_BUF_LEN, DEFAULT_CLOSE_TIMEOUT, + DEFAULT_DIAL_TIMEOUT, DEFAULT_EVENT_QUEUE_CAP, DEFAULT_IDLE_WAKE_INTERVAL, + DEFAULT_ITER_DRAIN_CAP, DEFAULT_LEAVE_TIMEOUT, DEFAULT_OBSERVATION_CHANNEL, ParseChannelError, + RuntimeOptions, StreamTransportOptions, +}; From 122bed5de6dc4e0712aac66dde965f0fed10aec1 Mon Sep 17 00:00:00 2001 From: al8n Date: Wed, 1 Jul 2026 22:36:07 +1200 Subject: [PATCH 026/128] feat(serf-reactor): reactor substrate + stream/TCP driver Agnostic-generic reactor substrate: Arc with a Mutex> + stored Waker command queue (no channel), ArcSwap snapshot, drop counters, and teardown latch; a Send Transport trait over R plus agnostic TCP (bind + ephemeral-port retry) and the resolver infra. The stream pump is an impl Future::poll driving serf-proto's StreamEndpoint (readiness recv-loop -> drain_surfaces -> single inline handle_timeout; NO poll_with(ZERO) drain) with the readiness bridge task; ports serf-compio's serf-logic (PendingJoin await-result join, per-exchange ignore_old, events->observation) into the reactor model mirroring memberlist-reactor. The handle carries construction + command dispatch; ergonomic constructors and tests follow. --- serf-reactor/src/bridge/mod.rs | 281 ++++ serf-reactor/src/driver/mod.rs | 13 +- serf-reactor/src/driver/shared.rs | 63 + serf-reactor/src/driver/stream/mod.rs | 1748 ++++++++++++++++++++ serf-reactor/src/lib.rs | 33 + serf-reactor/src/resolver/advertise/mod.rs | 87 + serf-reactor/src/resolver/mod.rs | 54 + serf-reactor/src/resolver/os.rs | 58 + serf-reactor/src/resolver/socket_addr.rs | 27 + serf-reactor/src/serf/mod.rs | 555 +++++++ serf-reactor/src/shared/mod.rs | 204 +++ serf-reactor/src/tcp/mod.rs | 350 ++++ serf-reactor/src/transport/mod.rs | 172 ++ serf-reactor/src/transport/runtime.rs | 67 + 14 files changed, 3709 insertions(+), 3 deletions(-) create mode 100644 serf-reactor/src/bridge/mod.rs create mode 100644 serf-reactor/src/driver/shared.rs create mode 100644 serf-reactor/src/driver/stream/mod.rs create mode 100644 serf-reactor/src/resolver/advertise/mod.rs create mode 100644 serf-reactor/src/resolver/mod.rs create mode 100644 serf-reactor/src/resolver/os.rs create mode 100644 serf-reactor/src/resolver/socket_addr.rs create mode 100644 serf-reactor/src/serf/mod.rs create mode 100644 serf-reactor/src/shared/mod.rs create mode 100644 serf-reactor/src/tcp/mod.rs create mode 100644 serf-reactor/src/transport/mod.rs create mode 100644 serf-reactor/src/transport/runtime.rs diff --git a/serf-reactor/src/bridge/mod.rs b/serf-reactor/src/bridge/mod.rs new file mode 100644 index 00000000..26b2cf33 --- /dev/null +++ b/serf-reactor/src/bridge/mod.rs @@ -0,0 +1,281 @@ +//! Per-bridge byte-mover task — moves bytes between one reliable exchange's TCP +//! stream and the pump, waking the pump after each inbound enqueue. +//! +//! The `Send`/`agnostic` sibling of serf-compio's `!Send` compio bridge, built to +//! memberlist-reactor's readiness `bridge_task` template: `into_split`, a +//! `select_biased` of cancel-vs-`out_rx`-vs-read, an out-of-band oneshot cancel, +//! and an `R::sleep(close_timeout)` no-progress drain backstop. +//! +//! ## Push/pull half-close lifecycle +//! +//! The reliable exchange is a one-shot request-response. A push/pull peer +//! half-closes (FINs) after sending its half, then reads the reply. So a read EOF +//! retires only the read side; the bridge stays alive to write the reply over the +//! half-open connection and tears down only when the pump drops the handle +//! (disconnecting `out_rx` and `cancel_rx`). +//! +//! ## Graceful close vs hard abort +//! +//! - A graceful `StreamAction::Close` drops the [`BridgeHandle`], disconnecting +//! `out_rx` and `cancel_rx`. The bridge first drains the `BridgeOut::Data` +//! already queued (flushing the exchange's final response), then exits on the +//! `out_rx` disconnect. The `cancel_rx` disconnect does NOT preempt that drain — +//! it is mapped to a never-resolving future below. +//! - A failed `StreamAction::Abort` (and the shutdown freeze) sends `()` on +//! `cancel_rx` before dropping the handle. That resolves `cancel_fut`, the +//! biased-FIRST arm of the both-halves-live read select, so it preempts the +//! bridge ahead of a racing read — a peer-FIN readable just after the signal is +//! NOT read and folded into a fabricated EOF — and ahead of a write stalled on +//! an unresponsive peer, discarding the queued bytes. A cancel-break emits no +//! EOF; only a real `read == 0` does. An `inbound_tx` send already in flight is +//! awaited in an arm body, not the select, so it still completes first. +//! +//! ## Graceful-drain backstop (`close_timeout`) +//! +//! A graceful Close has NO remaining cancel path (the handle is gone), so a peer +//! that sent its request+FIN and then STOPPED reading would wedge the post-Close +//! drain forever — leaking this detached task and its socket. The drain is +//! therefore a chunked write loop, and EACH partial write is bounded by a fresh +//! `close_timeout`. Because progress resets the deadline, this is a NO-PROGRESS +//! (idle) timeout, not a cap on total drain duration: a peer that keeps reading — +//! even slowly, so a large frame outlasts `close_timeout` overall — advances on +//! every chunk and never trips it. It fires only when a single partial write +//! makes NO progress for the full `close_timeout`; the bridge is then torn down, +//! dropping the write half so the OS RSTs the stuck stream. + +use std::{future::Future, net::Shutdown, sync::Arc, time::Duration}; + +use agnostic::{Runtime, net::TcpStream}; +use flume::{Receiver, Sender}; +use futures_channel::oneshot; +use futures_util::{ + AsyncReadExt, AsyncWriteExt, FutureExt, + future::{FusedFuture, pending}, + pin_mut, select_biased, +}; +use memberlist_proto::Instant; + +use crate::{ + driver::{ + shared::ExchangeId, + stream::{BridgeData, BridgeEof, BridgeInbound, BridgeOut}, + }, + shared::Shared, +}; + +/// Moves bytes between one exchange's TCP stream and the pump, waking it after +/// each inbound enqueue. Reads forward to `inbound_tx`; `out_rx` drives writes +/// and the write half-close. +#[allow(clippy::too_many_arguments)] +pub(crate) async fn bridge_task( + stream: S, + eid: ExchangeId, + out_rx: Receiver, + cancel_rx: oneshot::Receiver<()>, + inbound_tx: Sender, + shared: Arc>, + recv_buf_len: usize, + close_timeout: Duration, +) where + I: Send + Sync + 'static, + R: Runtime, + S: TcpStream, +{ + let (mut read_half, mut write_half) = stream.into_split(); + let mut buf = vec![0u8; recv_buf_len.max(1)]; + let mut read_eof = false; + let mut write_closed = false; + // Hoist a single cancel future that resolves ONLY on an explicit abort. A + // graceful Close drops `cancel_tx` (Err(Canceled)); that is NOT an abort, so + // map it to a future that never resolves — queued writes then complete and the + // bridge tears down via the `out_rx` disconnect after draining. Because the + // cancellation maps to `pending()`, `cancel_fut` never wins the write race on a + // graceful close, so a write is never dropped mid-flight (no partial-write / + // duplication hazard); an explicit abort still resolves and preempts. + let cancel_fut = async { + match cancel_rx.await { + Ok(()) => (), + Err(_) => pending::<()>().await, + } + } + .fuse(); + pin_mut!(cancel_fut); + loop { + // A push/pull peer half-closes (FINs) after sending its half, then reads the + // reply. So a read EOF retires only the read side; the bridge stays alive to + // write the reply over the half-open connection and tears down only when the + // pump drops the handle (disconnecting out_rx and cancel_rx). + if read_eof { + match out_rx.recv_async().await { + Ok(BridgeOut::Data(bytes)) => { + // Tear down on either teardown signal: an explicit abort, OR a drain + // that makes NO progress for `close_timeout` on a non-reading peer (a + // graceful Close has no cancel path, so the idle timeout is its only + // backstop). A slow-but-reading peer resets the deadline each chunk. + if !write_closed + && write_cancellable::<_, R, _>(&mut write_half, &bytes, &mut cancel_fut, close_timeout) + .await + { + break; + } + } + Ok(BridgeOut::ShutdownWrite) => { + // Ignoring Err: half-closing a gone peer is moot. + let _ = write_half.close().await; + write_closed = true; + } + Err(_) => break, + } + continue; + } + // Both halves live. Bias the shutdown/abort cancel AHEAD of the read so a + // freeze (`cancel_tx.send(())` then handle drop) stops this bridge's reads + // before a racing peer-FIN can be read and folded as a completed EOF: a FIN + // unread at the freeze instant is genuinely in-flight and must be ABSENT from + // the shutdown reached set. The `out` arm is also ahead of the read, so a + // graceful Close (handle drop, no cancel) tears down on the out-channel + // disconnect rather than a racing late read. An already-read EOF is still + // preserved: its `inbound_tx` send is awaited in the read arm BODY, not in + // this select, so once the bridge is parked on that send the cancel cannot + // preempt it — cancel only wins when the loop comes back to a READ. + select_biased! { + // Cancel first: ONLY an explicit abort/freeze (`cancel_tx.send(())`) + // resolves this future — a graceful-Close handle-drop is mapped to + // `pending()`. It breaks at once WITHOUT emitting any EOF; only a real + // `read == 0` below emits one, so a cancel-break never fabricates a seed. + () = &mut cancel_fut => break, + out = out_rx.recv_async().fuse() => match out { + Ok(BridgeOut::Data(bytes)) => { + // Tear down on an explicit abort OR a drain that makes NO progress for + // `close_timeout` on a non-reading peer (the graceful-Close backstop). + // A slow-but-reading peer resets the deadline each chunk and is not + // timed out. + if !write_closed + && write_cancellable::<_, R, _>( + &mut write_half, + &bytes, + &mut cancel_fut, + close_timeout, + ) + .await + { + break; + } + } + Ok(BridgeOut::ShutdownWrite) => { + // Ignoring Err: half-closing a gone peer is moot. + let _ = write_half.close().await; + write_closed = true; + } + // The pump dropped the handle (Close / shutdown): tear down. + Err(_) => break, + }, + read = read_half.read(&mut buf).fuse() => match read { + // A clean `read == 0` (peer half-closed) is a benign EOF anchor; a read + // ERROR is a transport failure and must NOT take the benign-EOF path (it + // would falsely complete a one-way UserMessage as success). Both stop + // this task's reads. + Ok(0) | Err(_) => { + // Timestamp at read completion, before any send backpressure. + let payload = BridgeEof { eid, received_at: Instant::now() }; + let msg = if read.is_err() { + BridgeInbound::Error(payload) + } else { + BridgeInbound::Eof(payload) + }; + // Bounded channel: await space (backpressure), then wake the pump. + if inbound_tx.send_async(msg).await.is_err() { + break; + } + shared.wake_driver(); + read_eof = true; + } + Ok(n) => { + let msg = BridgeInbound::Data(BridgeData { + eid, + bytes: buf[..n].to_vec(), + received_at: Instant::now(), + }); + if inbound_tx.send_async(msg).await.is_err() { + break; + } + shared.wake_driver(); + } + }, + } + } + // Drop the inbound sender BEFORE waking the driver. The shutdown drain reads + // `inbound_rx` to all-senders-gone via `try_recv` and deliberately does NOT + // hold a persistent flume `recv_async` waker (a per-poll temporary registers + // then deregisters on drop), so flume's own last-sender disconnect wakes + // nothing — the driver future would never be re-polled and `shutdown().await` + // would hang. Dropping first, then waking, guarantees the re-polled driver + // observes the disconnect (the drop is release-ordered ahead of the wake). This + // fires on EVERY exit path — a frozen bridge's out-channel disconnect, an + // explicit cancel/abort, an inbound-send error, or a read/write error — which + // is what lets the drain reach Disconnected without awaiting any bridge join. + drop(inbound_tx); + shared.wake_driver(); + // Best-effort: ensure the OS socket is fully closed once the bridge exits. + let _ = ::reunite(read_half, write_half).map(|s| s.shutdown(Shutdown::Both)); +} + +/// Writes `bytes` fully via a chunked loop, racing EACH partial write against TWO +/// backstops. +/// +/// 1. The bridge's hoisted `cancel_fut` — resolves ONLY on an explicit +/// `StreamAction::Abort` (a graceful Close maps its handle-drop disconnect to a +/// never-resolving future). Listed FIRST in every iteration so it preempts +/// immediately, even a write stalled mid-frame on an unresponsive peer; on a +/// graceful close the write always wins this arm and the queued bytes flush. +/// 2. An `R::sleep(close_timeout)` re-armed FRESH on every iteration — the +/// backstop for a post-Close drain that has NO remaining cancel path. Because +/// the deadline resets on each partial write, it is a NO-PROGRESS (idle) +/// timeout, not a cap on total write duration: a peer that keeps reading — +/// even slowly — advances on every chunk and never trips it. It fires only +/// when a single partial write makes NO progress for the full `close_timeout`. +/// +/// Returns true if the bridge should tear down (drop the write half → RST): +/// aborted, the drain made no progress for `close_timeout`, or a write error / +/// write-zero. Returns false only on a fully-written frame. +async fn write_cancellable( + write_half: &mut W, + bytes: &[u8], + mut cancel_fut: &mut C, + close_timeout: Duration, +) -> bool +where + W: futures_util::io::AsyncWrite + Unpin, + R: Runtime, + C: Future + FusedFuture + Unpin, +{ + let mut written = 0; + while written < bytes.len() { + // Re-arm the deadline FRESH each iteration: progress (a non-empty partial + // write) resets the clock, so this is an idle timeout, not a total-duration + // cap. A slow-but-reading peer advances every chunk and never trips it. + let timeout_fut = R::sleep(close_timeout).fuse(); + pin_mut!(timeout_fut); + let res = select_biased! { + // Explicit abort only (a disconnect was mapped to `pending()`). Listed + // first so it can preempt even a write blocked mid-frame on an unresponsive + // peer, ahead of the timeout backstop. + _ = cancel_fut => return true, + // Backstop: no progress on this partial write for the full `close_timeout` + // (a non-reading peer) → tear down (RST on teardown). + _ = timeout_fut => return true, + // Write the unwritten tail; a partial write returns its byte count. + res = write_half.write(&bytes[written..]).fuse() => res, + }; + match res { + // Progress: advance and loop with a fresh deadline. + Ok(n) if n > 0 => written += n, + // A zero-byte write makes no progress, and a write error ends the exchange: + // tear down rather than spin or write further. + // Ignoring Err: the inbound side observes the same broken socket on its next + // read; surfacing it here would race the pump's teardown. + _ => return true, + } + } + false +} diff --git a/serf-reactor/src/driver/mod.rs b/serf-reactor/src/driver/mod.rs index 0b752c49..34f7f83b 100644 --- a/serf-reactor/src/driver/mod.rs +++ b/serf-reactor/src/driver/mod.rs @@ -1,7 +1,14 @@ -//! The driver layer: the generic-free tuning knobs. +//! The driver layer: the generic-free tuning knobs and the per-backend pumps. //! //! [`options`] holds the generic-free driver tuning knobs shared by every -//! transport backend. The reactor substrate (the shared `Arc` state) and -//! the per-backend driver loops live behind the transport features. +//! transport backend. [`shared`] holds the driver-side observation helpers. The +//! per-backend driver loops (behind the transport features) each own a serf +//! `StreamEndpoint` / `QuicEndpoint` and pump it as a quinn-style `Future::poll`. pub(crate) mod options; + +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) mod shared; + +#[cfg(feature = "tcp")] +pub(crate) mod stream; diff --git a/serf-reactor/src/driver/shared.rs b/serf-reactor/src/driver/shared.rs new file mode 100644 index 00000000..dd42e6dc --- /dev/null +++ b/serf-reactor/src/driver/shared.rs @@ -0,0 +1,63 @@ +//! Driver helpers shared by the reactor's backend driver pump. +//! +//! The observation / event hand-off helpers independent of the reliable plane: +//! the [`Delegate`](crate::delegate::Delegate) hook dispatcher, the +//! coordinator-allocated exchange-id alias, and the observation byte-backstop +//! weight. Unlike serf-compio's `driver/shared`, there is **no** `yield_once` +//! and **no** `drain_past_due_udp`: the reactor pump is readiness-based, so it +//! recv-loops the gossip socket to kernel-empty and fires `handle_timeout` inline +//! — there is no completion-backend past-due drain to build. + +/// Coordinator-allocated handle for one in-flight reliable exchange. +/// +/// Shared by the stream driver and the per-bridge task so they agree on the same +/// opaque id without the rest of the crate naming the machine's streams module. +#[cfg(feature = "tcp")] +pub(crate) type ExchangeId = memberlist_proto::event::ExchangeId; + +/// Byte-backstop weight of a serf event. Delegates to +/// [`serf_driver::observation_payload_bytes`]. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) use serf_driver::observation_payload_bytes; + +/// Dispatch the matching [`Delegate`](crate::delegate::Delegate) hook for one +/// drained serf [`Event`](serf_proto::event::Event). +/// +/// Member hooks run once per affected member in the batch; user-event and query +/// hooks run once per event. The observation delegate observes transitions the +/// FSM has already applied — it is NOT an admission gate. Returns a `Send` future +/// (the delegate hooks are `Send`) so the observation task can drive it on a +/// multi-threaded runtime. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) async fn dispatch_event_delegate( + delegate: &D, + ev: &serf_proto::event::Event, +) where + D: crate::delegate::Delegate, + I: Clone, + A: Clone, +{ + use serf_proto::event::{Event, MemberEventKind}; + use std::sync::Arc; + + match ev { + Event::Member(me) => { + for m in me.members() { + let arc = Arc::new(m.clone()); + match me.kind() { + MemberEventKind::Join => delegate.notify_join(arc).await, + MemberEventKind::Leave => delegate.notify_leave(arc).await, + MemberEventKind::Failed => delegate.notify_failed(arc).await, + MemberEventKind::Update => delegate.notify_update(arc).await, + MemberEventKind::Reap => delegate.notify_reap(arc).await, + } + } + } + Event::User(msg) => delegate.notify_user_event(msg).await, + Event::Query(ev) => delegate.notify_query(ev).await, + // Other variants (QueryResponse, QueryAck, Shutdown, LeftCluster, + // RelayDropped, DialRequested, KeyResponse, KeyRequest) carry no observation + // hook — the driver surfaces them through the EventStream. + _ => {} + } +} diff --git a/serf-reactor/src/driver/stream/mod.rs b/serf-reactor/src/driver/stream/mod.rs new file mode 100644 index 00000000..46819feb --- /dev/null +++ b/serf-reactor/src/driver/stream/mod.rs @@ -0,0 +1,1748 @@ +//! The stream-plane driver pump: a quinn-style `Future::poll` that owns the serf +//! [`StreamEndpoint`], its UDP gossip socket, the TCP reliable listener's accept +//! task, and the per-bridge handle table. Per-exchange reliable TCP I/O runs in +//! spawned bridge tasks (one per exchange) wired to the pump by `flume` channels; +//! inbound connections are accepted by a dedicated task (the listener's `accept` +//! is async-only). +//! +//! This is the `Send`/`Arc`/`agnostic` sibling of serf-compio's `!Send` +//! `stream_driver_loop`, restructured onto memberlist-reactor's readiness pump: +//! there is NO top-level `select!` and NO completion-backend drain. Each poll +//! drains queued [`Command`]s, recv-loops the gossip socket to kernel-empty +//! (`poll_recv_from` → `Poll::Pending`), services accept / dial / bridge-inbound +//! channels, runs [`drain_surfaces`](StreamDriver::drain_surfaces) (decode +//! buffered ingress, route transport/gossip egress, emit events), and fires +//! `handle_timeout` INLINE at exactly one site. serf-compio's `poll_with(ZERO)` +//! reap / `drain_past_due_udp` / `fire_timeout_with_drain` are deleted — those +//! exist only because io_uring is completion-based; the reactor is readiness-based +//! and `Poll::Pending` from the socket IS the kernel-empty signal. + +use std::{ + collections::{HashMap, HashSet}, + future::Future, + net::SocketAddr, + pin::Pin, + sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + }, + task::{Context, Poll}, + time::Duration, +}; + +use agnostic::{ + AsyncSpawner, Runtime, + net::{Net, TcpListener, TcpStream, UdpSocket}, +}; +use bytes::Bytes; +use flume::{Receiver, Sender, TrySendError}; +use futures_channel::oneshot; +use futures_util::{FutureExt, select}; +use memberlist_proto::{ + Instant, SeedableRng, StreamId, Transmit, + codec::{ + DecodeOptions, EncodeOptions, decode_incoming, encode_outgoing, encode_outgoing_compound, + parse_messages, + }, + streams::{StreamAction, StreamTransport}, +}; +use serf_driver::SerfSnapshot; +use serf_proto::{ + ExchangeKind, ExchangeStatus, LamportTime, StreamEndpoint, event::Event, members::SerfState, +}; +use smallvec::SmallVec; + +#[cfg(encryption)] +use crate::command::{KeyCmd, ListKeysCmd}; +#[cfg(encryption)] +use crate::delegate::KeyringDelegate; +use crate::{ + Channel, + command::{ + Command, ForceLeaveCmd, JoinCmd, JoinKind, JoinReply, LeaveCmd, QueryCmd, RespondCmd, + SetTagsCmd, ShutdownCmd, WaitForCompletionArgs, + }, + delegate::Delegate, + driver::{ + options::{RuntimeOptions, StreamTransportOptions}, + shared::{ExchangeId, dispatch_event_delegate, observation_payload_bytes}, + }, + error::{JoinFailed, Result, SerfError}, + shared::Shared, +}; +#[cfg(encryption)] +use serf_proto::{KeyRequestOperation, KeyResponseArgs, event::KeyRequest}; + +/// Hard ceiling on the per-recv UDP buffer — UDP's wire payload is capped at +/// 65507 bytes once the IP/UDP headers are deducted, so a larger buffer just +/// wastes an allocation. +const GOSSIP_RECV_BUF_MAX: usize = 65507; + +/// The largest the encrypted wrapper can inflate a gossip datagram, or `0` when no +/// encryption backend is built in — so an encrypted datagram is not silently +/// truncated by the kernel. +#[cfg(encryption)] +const ENCRYPTED_WRAPPER_OVERHEAD: usize = memberlist_proto::ENCRYPTED_WRAPPER_OVERHEAD; +#[cfg(not(encryption))] +const ENCRYPTED_WRAPPER_OVERHEAD: usize = 0; + +/// Capacity of the accepted-connection channel; the accept task backpressures +/// once the pump is this many connections behind. +pub(crate) const ACCEPT_CAP: usize = 256; + +/// Cap on the count of application-data events retained after a full observation +/// channel (the payload byte budget bounds their bytes; this bounds their count). +const OBS_OVERFLOW_MAX: usize = 1024; + +/// A message from the pump to a bridge's TCP write side. Teardown is signalled out +/// of band by dropping the [`BridgeHandle`], not by a variant here, so it can +/// preempt even a write stalled on an unresponsive peer. +pub(crate) enum BridgeOut { + /// Plaintext transport bytes to write to the peer. + Data(Bytes), + /// Half-close the write side (FIN) after the send half retired. + ShutdownWrite, +} + +/// Payload of [`BridgeInbound::Data`]. +pub(crate) struct BridgeData { + pub(crate) eid: ExchangeId, + pub(crate) bytes: Vec, + /// Wall-clock instant the bridge read these bytes; forwarded as the machine's + /// observation time so a response that arrived BEFORE the exchange deadline is + /// not retroactively timed out by the pump's later `Instant::now()` sample. + pub(crate) received_at: Instant, +} + +/// Payload of [`BridgeInbound::Eof`] and [`BridgeInbound::Error`]. +pub(crate) struct BridgeEof { + pub(crate) eid: ExchangeId, + pub(crate) received_at: Instant, +} + +/// Inbound transport bytes / EOF / error from a bridge's TCP read side to the +/// pump. +pub(crate) enum BridgeInbound { + /// Bytes read from the peer for an exchange. + Data(BridgeData), + /// The peer cleanly closed its write side (transport `read == 0`). + Eof(BridgeEof), + /// A transport READ/WRITE error — routed to `handle_transport_error` so a + /// one-way UserMessage is NOT falsely completed as success by the benign-EOF + /// path. + Error(BridgeEof), +} + +/// The pump's end of a live bridge: the write channel plus an explicit-abort +/// channel. A graceful `StreamAction::Close` drops the whole handle (`out_tx` +/// disconnects, the bridge drains queued `Data` then exits); a failed +/// `StreamAction::Abort` sends `()` on `cancel_tx` first, preempting even a +/// stalled write and discarding queued bytes. +struct BridgeHandle { + out_tx: Sender, + cancel_tx: oneshot::Sender<()>, +} + +/// The result a dial task reports back to the pump. +enum DialStatus +where + R: Runtime, +{ + /// The connection succeeded; hand the stream and its channels back. + Connected(DialConnected), + /// The connection failed or timed out; the pump fails the exchange. + Failed(DialFailed), +} + +/// Payload of [`DialStatus::Connected`]. +struct DialConnected +where + R: Runtime, +{ + eid: ExchangeId, + stream: ::TcpStream, + out_rx: Receiver, + cancel_rx: oneshot::Receiver<()>, +} + +/// Payload of [`DialStatus::Failed`]. +struct DialFailed { + eid: ExchangeId, + /// Instant the dial task observed the failure, so a pre-deadline dial failure + /// terminalizes cleanly rather than being read as a timeout. + received_at: Instant, +} + +/// Driver-side state for one outstanding await-result join call. +/// +/// A [`Command::Join`] carrying [`JoinKind::WaitForCompletion`] dispatches one +/// push/pull per resolved seed and parks the per-call state here. Contact +/// accounting is strictly per-OUTBOUND-EXCHANGE, observed via the machine's +/// [`Event::ExchangeCompleted`] filtered to [`ExchangeKind::PushPull`]. Reply +/// resolution and ignore-stream cleanup are SEPARATE terminal states: the reply +/// resolves on all-exchanges-done OR `deadline` (whichever first); the ignore +/// streams are cleared only once every dispatched exchange has completed +/// (`pending` empty), so a `StreamId` recorded for a still-live exchange stays in +/// the machine's ignore set and a late merge still suppresses the peer's pre-join +/// user events. +struct PendingJoin { + /// Outbound exchange ids this waiter dispatched and is still awaiting a terminal + /// `ExchangeCompleted` for. + pending: HashSet, + /// Peer addresses of the dispatched exchanges that terminated `Succeeded`. + /// Duplicate seeds contribute one entry per successful exchange. + contacted: SmallVec<[SocketAddr; 1]>, + /// The `StreamId`s this join recorded in the machine's per-exchange ignore set + /// (non-empty only for an `ignore_old` join). Cleared via + /// `clear_ignore_join_stream` once every dispatched exchange has completed. + ignore_streams: SmallVec<[StreamId; 1]>, + /// Total outbound-exchange count this call dispatched — the `JoinAllFailed` + /// denominator on a zero-contact resolution. + requested: usize, + /// Wall-clock instant past which the driver replies with whatever `contacted` + /// set it has accumulated even if `pending` is non-empty. + deadline: Instant, + /// One-shot reply channel back to the caller, taken when the reply resolves. + /// `None` once resolved; the waiter then lingers — only to drive ignore-stream + /// cleanup — until `pending` empties. + reply: Option>, +} + +impl PendingJoin { + /// Resolve the caller's reply once, from the current `contacted` set. + /// Idempotent: after the first call `reply` is `None` and this is a no-op, so + /// the deadline path and the all-exchanges-done path never double-send. + fn resolve_reply(&mut self) { + if let Some(reply) = self.reply.take() { + let result = if self.contacted.is_empty() { + Err(( + SmallVec::new(), + SerfError::JoinAllFailed(JoinFailed::new(self.requested, 0)), + )) + } else { + Ok(self.contacted.clone()) + }; + // Ignoring Err: caller dropped the reply receiver (the join future was + // cancelled). + let _ = reply.send(result); + } + } + + /// This waiter has reached both terminal states — its reply resolved AND every + /// dispatched exchange completed — so it can be removed and its ignore-stream + /// cleanup run. + fn is_done(&self) -> bool { + self.reply.is_none() && self.pending.is_empty() + } +} + +/// Driver-side state for the single in-flight graceful-leave operation. +/// +/// A [`Command::Leave`] that finds the endpoint `Alive` initiates the machine's +/// `leave()`, which withholds [`Event::LeftCluster`] until the leave notices have +/// drained. The pump parks this and replies only once that `LeftCluster` arrives +/// (success) or `deadline` elapses ([`SerfError::LeaveTimeout`]). Leave is SHARED: +/// a second `Command::Leave` racing an in-flight one joins it by pushing its reply +/// onto `repliers`. +struct PendingLeave { + /// Reply channels of every `leave()` caller that joined this in-flight leave. + repliers: Vec>>, + /// Wall-clock instant past which the pump replies [`SerfError::LeaveTimeout`] to + /// every replier even if `LeftCluster` has not yet fired. + deadline: Instant, +} + +impl PendingLeave { + /// Reply to every joined `leave()` caller with a fresh `Result<()>` from + /// `make_result`. A constructor closure (rather than a cloned value) sidesteps + /// `SerfError` not being `Clone` — every terminal outcome here (`Ok(())`, + /// `LeaveTimeout`, `Shutdown`) is trivially reconstructible. + fn resolve_all(self, mut make_result: impl FnMut() -> Result<()>) { + for replier in self.repliers { + // Ignoring Err: a `leave()` caller dropped its reply receiver. + let _ = replier.send(make_result()); + } + } +} + +/// The single-owner stream driver future. Runs until shutdown (a `Shutdown` +/// command, a lost id-conflict `Event::Shutdown`, or the last handle dropped). +pub(crate) struct StreamDriver +where + // Structurally required: `endpoint` names `StreamEndpoint`, whose struct declares `I: Eq + Hash` and `where T: StreamTransport`. + I: core::hash::Hash + Eq, + R: Runtime, + T: StreamTransport, +{ + endpoint: StreamEndpoint, + /// Unreliable gossip datagrams. `Option` so the shutdown branch can drop it + /// (releasing the bound UDP port) BEFORE acking; `Some` for the running + /// lifetime, taken only during teardown. + socket: Option<::UdpSocket>, + shared: Arc>, + /// Hand-off to the observation task (delegate dispatch + event-stream fan-out). + obs_tx: Sender>, + /// Bytes of payload-bearing events queued in `obs_tx` — the byte backstop's + /// counter (added on enqueue, subtracted by the obs task on dequeue). + obs_payload_bytes: Arc, + /// Queued-payload byte budget on a bounded obs channel, `None` if unbounded. + obs_payload_budget: Option, + /// Application-data events retained after a full obs channel, retried later. + obs_overflow: std::collections::VecDeque>, + /// Cluster label threaded into the gossip codec (outbound stamp + inbound + /// verify). + label: Option, + /// Outstanding await-result join waiters. + pending_joins: Vec, + /// The in-flight graceful leave, resolved on `LeftCluster`. + pending_leave: Option, + /// Parked `Shutdown` replies — acked only after the bind sockets drop, so a + /// caller resuming from `shutdown().await` can rebind the same address. A `Vec` + /// because several callers can race `shutdown()`. + shutdown_reply: Vec>>, + /// Each live exchange's bridge: its write channel and teardown handle. + bridges: HashMap, + /// Inbound connections from the accept task. + accepted_rx: Receiver<(::TcpStream, SocketAddr)>, + /// Held only to be dropped on driver exit; closing it cancels the accept task's + /// pending `accept()` so the listener is released. `Option` so the shutdown + /// branch can drop it before acking. + accept_shutdown_tx: Option>, + /// Join handle of the accept task; awaited on shutdown before acking so the + /// listener FD is released (not merely signalled). + accept_join: Option<::JoinHandle<()>>, + /// Inbound transport bytes/EOF from the bridge read tasks. `Some` through the + /// running lifetime and the shutdown drain; taken once the drain disconnects, + /// which doubles as the one-time reap guard. + inbound_rx: Option>, + /// Template inbound sender cloned into each bridge. `Option` so the shutdown + /// freeze can drop it: with it gone, the channel reaches Disconnected exactly + /// when the last frozen bridge exits — the drain's terminating condition. + inbound_tx: Option>, + /// Dial completions from the dial tasks. + dial_rx: Receiver>, + /// Cloned into each dial task to report its outcome. + dial_tx: Sender>, + recv_buf: Vec, + /// Per-poll cap on each drained surface / recv batch. + iter_drain_cap: usize, + timer: Option>>, + timer_deadline: Option, + idle_wake: Duration, + leave_timeout: Duration, + close_timeout: Duration, + dial_timeout: Duration, + bridge_recv_buf_len: usize, + /// The driver's keyring delegate: applies inbound key-management ops and + /// produces the `respond_key` answer. Present only under an encryption backend. + #[cfg(encryption)] + keyring: Arc, +} + +impl StreamDriver +where + I: memberlist_proto::Id + Clone, + R: Runtime, + T: StreamTransport, + G: rand::Rng, + SR: rand::Rng + SeedableRng, +{ + /// Build the driver from the endpoint, its bound gossip socket, the shared + /// state, the observation hand-off, and the accept task's channels/handle. + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + endpoint: StreamEndpoint, + socket: ::UdpSocket, + shared: Arc>, + obs_tx: Sender>, + obs_payload_bytes: Arc, + obs_payload_budget: Option, + accepted_rx: Receiver<(::TcpStream, SocketAddr)>, + accept_shutdown_tx: Sender<()>, + accept_join: ::JoinHandle<()>, + driver_opts: RuntimeOptions, + stream_opts: StreamTransportOptions, + label: Option, + #[cfg(encryption)] keyring: Arc, + ) -> Self { + let buf_len = endpoint + .gossip_mtu() + .saturating_add(ENCRYPTED_WRAPPER_OVERHEAD) + .min(GOSSIP_RECV_BUF_MAX); + let (inbound_tx, inbound_rx) = flume::bounded(stream_opts.bridge_inbound_cap().max(1)); + let (dial_tx, dial_rx) = flume::unbounded(); + Self { + endpoint, + socket: Some(socket), + shared, + obs_tx, + obs_payload_bytes, + obs_payload_budget, + obs_overflow: std::collections::VecDeque::new(), + label, + pending_joins: Vec::new(), + pending_leave: None, + shutdown_reply: Vec::new(), + bridges: HashMap::new(), + accepted_rx, + accept_shutdown_tx: Some(accept_shutdown_tx), + accept_join: Some(accept_join), + inbound_rx: Some(inbound_rx), + inbound_tx: Some(inbound_tx), + dial_rx, + dial_tx, + recv_buf: vec![0u8; buf_len.max(1)], + iter_drain_cap: driver_opts.iter_drain_cap().max(1), + timer: None, + timer_deadline: None, + idle_wake: driver_opts.idle_wake_interval(), + leave_timeout: driver_opts.leave_timeout(), + close_timeout: stream_opts.close_timeout(), + dial_timeout: stream_opts.dial_timeout(), + bridge_recv_buf_len: stream_opts.bridge_recv_buf_len(), + #[cfg(encryption)] + keyring, + } + } + + /// Spawn the dial task that connects to `peer` for outbound exchange `eid`. + fn spawn_dial( + &self, + eid: ExchangeId, + peer: SocketAddr, + out_rx: Receiver, + cancel_rx: oneshot::Receiver<()>, + ) { + R::spawn_detach(dial_task::( + eid, + peer, + self.dial_timeout, + out_rx, + cancel_rx, + self.dial_tx.clone(), + self.shared.clone(), + )); + } + + /// Spawn the per-exchange bridge byte-mover for `eid`. The caller has already + /// inserted the matching [`BridgeHandle`] so bytes queued before the bridge + /// spawned reach the wire via the `out_rx` handed in here. + fn spawn_bridge( + &self, + eid: ExchangeId, + stream: ::TcpStream, + out_rx: Receiver, + cancel_rx: oneshot::Receiver<()>, + ) where + I: Send + Sync + 'static, + { + R::spawn_detach(crate::bridge::bridge_task::::TcpStream>( + stream, + eid, + out_rx, + cancel_rx, + self + .inbound_tx + .as_ref() + .expect("a bridge is only spawned while running, before the shutdown freeze drops the template inbound sender") + .clone(), + self.shared.clone(), + self.bridge_recv_buf_len, + self.close_timeout, + )); + } + + /// Applies one handle command to the machine. + fn dispatch(&mut self, cmd: Command, now: Instant) + where + I: Send + Sync + 'static, + { + let running = self.endpoint.state() == SerfState::Alive; + match cmd { + Command::Join(JoinCmd { + seeds, + kind, + ignore_old, + reply, + }) => { + // Gate on a running node: `leave()` stops the periodic schedulers, so a + // join after leave would leave the node non-participating. + if !running { + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(Err((SmallVec::new(), SerfError::NotRunning))); + return; + } + // Announce the serf-level join intent so peers learn the local join ltime + // without waiting for the next anti-entropy round. + if let Err(e) = self.endpoint.join() { + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(Err((SmallVec::new(), SerfError::from(e)))); + return; + } + match kind { + JoinKind::Dispatch => { + let mut dispatched: SmallVec<[SocketAddr; 1]> = SmallVec::new(); + for seed in seeds { + // Ignoring StreamId: the Dispatch arm tracks no per-exchange waiter + // state — completion / failure surfaces through `poll_event`. + let _sid = self.endpoint.start_join_push_pull(seed, ignore_old, now); + while let Some(action) = self.endpoint.poll_action() { + self.handle_stream_action(action, None); + } + dispatched.push(seed); + } + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(Ok(dispatched)); + } + JoinKind::WaitForCompletion(WaitForCompletionArgs { deadline }) => { + // Capture the resolved seed count BEFORE the loop consumes `seeds`: + // this is the `JoinAllFailed` denominator. A seed that retires before + // producing a `Connect` never enters `exchange_ids`, so deriving + // `requested` from the captured-exchange count would undercount. + let requested = seeds.len(); + let mut exchange_ids: HashSet = HashSet::with_capacity(requested); + // The `StreamId`s this join's `start_join_push_pull` calls returned; + // the Connect capture keys on this set (not the peer) so a same-peer + // dial flushed for another subsystem is never misattributed here. + let mut started: HashSet = HashSet::with_capacity(requested); + for seed in seeds { + let sid = self.endpoint.start_join_push_pull(seed, ignore_old, now); + started.insert(sid); + while let Some(action) = self.endpoint.poll_action() { + self.handle_stream_action(action, Some((&started, &mut exchange_ids))); + } + } + // An `ignore_old` join recorded every seed's `StreamId` in the machine; + // the driver owns clearing any that fail to merge. A plain join + // recorded nothing, so this stays empty. + let ignore_streams: SmallVec<[StreamId; 1]> = if ignore_old { + started.iter().copied().collect() + } else { + SmallVec::new() + }; + if exchange_ids.is_empty() { + // Every seed retired before a `Connect`: no exchange will ever + // surface a terminal completion (nor a merge). Resolve now with the + // all-failed outcome, clearing recorded ignore streams that never + // merge. + for s in &ignore_streams { + self.endpoint.clear_ignore_join_stream(*s); + } + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(Err(( + SmallVec::new(), + SerfError::JoinAllFailed(JoinFailed::new(requested, 0)), + ))); + } else { + self.pending_joins.push(PendingJoin { + pending: exchange_ids, + contacted: SmallVec::new(), + ignore_streams, + requested, + deadline, + reply: Some(reply), + }); + } + } + } + } + Command::Leave(LeaveCmd { reply }) => { + // Leave is a SHARED in-flight operation. If one is in flight, JOIN it (do + // not re-invoke `leave()`, a terminal no-op once `Leaving`/`Left` that + // emits no second `LeftCluster`). Otherwise INITIATE: snapshot `Alive` + // before the call, then park (was Alive) or reply immediately (no-op / + // error). + if let Some(pl) = self.pending_leave.as_mut() { + pl.repliers.push(reply); + } else { + let was_alive = running; + let leave_timeout = self.leave_timeout; + let res: Result<()> = self.endpoint.leave(now).map_err(SerfError::from); + match res { + Ok(()) if was_alive => { + self.pending_leave = Some(PendingLeave { + repliers: vec![reply], + deadline: now + leave_timeout, + }); + } + other => { + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(other); + } + } + } + } + Command::ForceLeave(ForceLeaveCmd { + id, + prune, + now: at, + reply, + }) => { + let res = if running { + self + .endpoint + .force_leave(id, prune, at) + .map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(res); + } + Command::UserEvent(cmd) => { + let res = if running { + let name = cmd.name().clone(); + let payload = cmd.payload().clone(); + self + .endpoint + .user_event(name, payload, cmd.coalesce) + .map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = cmd.reply.send(res); + } + Command::Query(cmd) => { + let res = if running { + let name = cmd.name().clone(); + let payload = cmd.payload().clone(); + let QueryCmd { + params, now: at, .. + } = &cmd; + self + .endpoint + .query(name, payload, params.clone(), *at) + .map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = cmd.reply.send(res); + } + Command::Respond(cmd) => { + let res = if running { + let payload = cmd.payload().clone(); + self + .endpoint + .respond(&cmd.token, payload, cmd.now) + .map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = cmd.reply.send(res); + } + Command::SetTags(SetTagsCmd { tags, reply }) => { + let res = if running { + self.endpoint.set_tags(tags).map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(res); + } + #[cfg(encryption)] + Command::InstallKey(KeyCmd { + key, + now: at, + reply, + }) => { + let res = if running { + self.endpoint.install_key(key, at).map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(res); + } + #[cfg(encryption)] + Command::UseKey(KeyCmd { + key, + now: at, + reply, + }) => { + let res = if running { + self.endpoint.use_key(key, at).map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(res); + } + #[cfg(encryption)] + Command::RemoveKey(KeyCmd { + key, + now: at, + reply, + }) => { + let res = if running { + self.endpoint.remove_key(key, at).map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(res); + } + #[cfg(encryption)] + Command::ListKeys(ListKeysCmd { now: at, reply }) => { + let res = if running { + self.endpoint.list_keys(at).map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(res); + } + Command::Shutdown(ShutdownCmd { reply }) => { + // Do NOT ack inline: the gossip socket and TCP listener are still bound. + // Flag shutdown and park the reply; the teardown branch acks every parked + // caller only AFTER it drops both, so an immediate rebind on the same + // address after `shutdown().await` succeeds. + self.shared.begin_shutdown(); + self.shutdown_reply.push(reply); + } + } + } + + /// Applies one stream action: open a dial, half-close a bridge's write, or tear + /// a bridge down. On the await-result join path `capture` records each + /// exchange's `Connect` id synchronously so the completion accounting can bind + /// it to that join alone. + fn handle_stream_action( + &mut self, + action: StreamAction, + capture: Option<(&HashSet, &mut HashSet)>, + ) where + I: Send + Sync + 'static, + { + match action { + StreamAction::Connect(info) => { + let eid = info.id(); + let peer = info.peer(); + if let Some((started, pending_exchanges)) = capture + && started.contains(&info.stream_id()) + { + pending_exchanges.insert(eid); + } + let (out_tx, out_rx) = flume::unbounded(); + let (cancel_tx, cancel_rx) = oneshot::channel(); + // No bridge task yet — the dial owns the connecting FD until it completes, + // at which point `DialStatus::Connected` spawns the bridge. + self.bridges.insert(eid, BridgeHandle { out_tx, cancel_tx }); + self.spawn_dial(eid, peer, out_rx, cancel_rx); + } + StreamAction::Shutdown(eref) => { + if let Some(handle) = self.bridges.get(&eref.id()) { + // Ignoring Err: the bridge task exited; its socket is already gone. + let _ = handle.out_tx.try_send(BridgeOut::ShutdownWrite); + } + } + StreamAction::Close(eref) => { + // Graceful close. Dropping the handle disconnects `out_tx` and + // `cancel_tx`: the bridge drains the `BridgeOut::Data` it already queued + // (writing the exchange's final response), then exits on the `out_rx` + // disconnect. The `cancel_tx` disconnect is mapped to a never-resolving + // future in the bridge, so it does NOT preempt that drain. + self.bridges.remove(&eref.id()); + } + StreamAction::Abort(eref) => { + // A FAILED exchange. Send the explicit cancel BEFORE dropping the handle: + // it preempts the bridge — even a write stalled on an unresponsive peer — + // and discards any queued bytes. + if let Some(handle) = self.bridges.remove(&eref.id()) { + // Ignoring Err: the bridge already exited (cancel receiver gone). + let _ = handle.cancel_tx.send(()); + } + } + } + } + + /// Route one accepted inbound connection: allocate the exchange, register the + /// bridge handle, and spawn the byte mover. Returns `true` iff an accept was + /// processed (a state-affecting event the caller treats as progress). + fn handle_accepted( + &mut self, + stream: ::TcpStream, + peer: SocketAddr, + now: Instant, + ) -> bool + where + I: Send + Sync + 'static, + { + let Some(eid) = self.endpoint.accept_connection(peer, now) else { + // Not admitted (leaving, the inbound-stream cap is reached, or a + // record-layer config error): drop the accepted stream rather than spawn a + // byte mover for a connection the machine will never feed. + drop(stream); + return true; + }; + let (out_tx, out_rx) = flume::unbounded(); + let (cancel_tx, cancel_rx) = oneshot::channel(); + self.bridges.insert(eid, BridgeHandle { out_tx, cancel_tx }); + self.spawn_bridge(eid, stream, out_rx, cancel_rx); + true + } + + /// Route one dial completion: bridge a connected stream, or fail the exchange. + fn handle_dial_status(&mut self, status: DialStatus) + where + I: Send + Sync + 'static, + { + match status { + DialStatus::Connected(DialConnected { + eid, + stream, + out_rx, + cancel_rx, + }) => { + // If the exchange was reaped while dialing, its handle is gone; drop the + // stream and channels rather than bridge a dead exchange. + if self.bridges.contains_key(&eid) { + self.spawn_bridge(eid, stream, out_rx, cancel_rx); + } + } + DialStatus::Failed(DialFailed { eid, received_at }) => { + // Drive the exchange to a DIAL FAILURE — NOT a benign EOF feed. A connect + // that never established has no wire, and a one-way `UserMessage` maps a + // clean EOF to a SUCCESSFUL completion, which would falsely report a + // reliable send as succeeding on an unreachable peer. + self.bridges.remove(&eid); + self.endpoint.handle_dial_failed(eid, received_at); + } + } + } + + /// Route one bridge inbound message into the coordinator, forwarding each + /// message's own `received_at` so the stream FSM's deadline gate compares + /// against the true arrival time. + fn dispatch_bridge_inbound(&mut self, inbound: BridgeInbound) { + match inbound { + BridgeInbound::Data(BridgeData { + eid, + bytes, + received_at, + }) => { + self + .endpoint + .handle_transport_data(eid, &bytes, false, received_at); + } + BridgeInbound::Eof(BridgeEof { eid, received_at }) => { + // Feed the read-half EOF anchor. Do NOT remove the `BridgeHandle` — for an + // inbound (server-side) push/pull bridge the read EOF arrives BEFORE the + // response is generated; the bridge entry stays until the matching + // `StreamAction::Close`. + self + .endpoint + .handle_transport_data(eid, &[], true, received_at); + } + BridgeInbound::Error(BridgeEof { eid, received_at }) => { + // A transport ERROR is NOT a clean EOF: route it to `handle_transport_error` + // so the bridge fails rather than taking the benign-EOF path. + self.endpoint.handle_transport_error(eid, received_at); + } + } + } + + /// Drains each machine surface up to `iter_drain_cap` items in one pass. + /// Returns `(worked, more)`: whether any surface produced work, and whether any + /// surface hit its cap with work left (self-wake). + fn drain_surfaces(&mut self, cx: &mut Context<'_>) -> (bool, bool) + where + I: Send + Sync + 'static, + { + let now = Instant::now(); + let budget = self.iter_drain_cap; + let mut worked = false; + let mut more = false; + + // Inbound gossip: decrypt + strip-label + parse, inline on the pump. The + // parsed messages reach the single-owner machine HERE, with their arrival-time + // `now`. + let decode_opts = DecodeOptions::new(self.label.clone()); + let mut ingress = 0; + while ingress < budget { + let Some((from, raw)) = self.endpoint.poll_memberlist_ingress() else { + break; + }; + ingress += 1; + // Reverse the wire transform stack: with an encryption backend built in, + // `decrypt_gossip` strips (and authenticates) the encryption wrapper; with + // none the serf gossip plane carries no transforms so the raw bytes are the + // plain label frame. A dropped datagram self-heals on the next gossip round. + #[cfg(encryption)] + let plain = match self.endpoint.decrypt_gossip(&raw) { + Ok(p) => Bytes::from(p), + Err(_) => continue, + }; + #[cfg(not(encryption))] + let plain = raw; + let inner = match decode_incoming(plain, &decode_opts) { + Ok(b) => b, + Err(_) => continue, + }; + let msgs = match parse_messages::(inner) { + Ok(m) => m, + Err(_) => continue, + }; + for msg in msgs { + self.endpoint.handle_message(from, msg, now); + } + } + worked |= ingress > 0; + more |= ingress == budget; + + // Stream actions: open dials, half-close, or tear down reliable exchanges. + let mut actions = 0; + while actions < budget { + let Some(action) = self.endpoint.poll_action() else { + break; + }; + actions += 1; + self.handle_stream_action(action, None); + } + worked |= actions > 0; + more |= actions == budget; + + // Outbound transport bytes: route each exchange's plaintext to its bridge. + let mut tx = 0; + while tx < budget { + let Some((eid, _peer, bytes)) = self.endpoint.poll_transport_transmit() else { + break; + }; + tx += 1; + if let Some(handle) = self.bridges.get(&eid) { + // Ignoring Err: the bridge task exited; the exchange will time out. + let _ = handle.out_tx.try_send(BridgeOut::Data(bytes)); + } + } + worked |= tx > 0; + more |= tx == budget; + + // Outbound gossip: encode (plain or compound) + encrypt, then send. Popping + // the last transmit is the endpoint's leave-completion fence (it emits + // `LeftCluster`), so the leave/shutdown datagrams reach the socket before that + // fence fires. + let encode_opts = EncodeOptions::new(self.label.clone()); + let mut sent = 0; + while sent < budget { + let Some(transmit) = self.endpoint.poll_memberlist_transmit() else { + break; + }; + sent += 1; + let (peer, plain): (SocketAddr, Bytes) = match transmit { + Transmit::Packet(pkt) => { + let (to, msg) = pkt.into_parts(); + match encode_outgoing(&msg, &encode_opts) { + Ok(b) => (to, b), + Err(_) => continue, + } + } + Transmit::Compound(cmp) => { + let (to, msgs) = cmp.into_parts(); + match encode_outgoing_compound(&msgs, &encode_opts) { + Ok(b) => (to, b), + Err(_) => continue, + } + } + }; + #[allow(unused_mut)] + let mut on_wire: Vec = plain.to_vec(); + #[cfg(encryption)] + { + on_wire = match self.endpoint.encrypt_gossip(&on_wire) { + Ok(bytes) => bytes, + Err(_) => continue, + }; + } + if let Some(socket) = self.socket.as_ref() { + // Ignoring Poll: gossip is best-effort — a full or errored UDP send drops + // the datagram and SWIM recovers on the next round. + let _ = socket.poll_send_to(cx, &on_wire, peer); + } + } + worked |= sent > 0; + more |= sent == budget; + + // Observation events: retry the overflow first, then drain up to the budget. + self.flush_obs_overflow(); + let mut events = 0; + while events < budget { + let Some(ev) = self.endpoint.poll_event() else { + break; + }; + events += 1; + self.send_observation(ev); + } + worked |= events > 0; + more |= events == budget; + + (worked, more) + } + + /// Retries retained overflow events into the obs channel, stopping at the first + /// `Full`. + fn flush_obs_overflow(&mut self) { + while let Some(ev) = self.obs_overflow.pop_front() { + match self.obs_tx.try_send(ev) { + Ok(()) => {} + Err(TrySendError::Full(ev)) => { + self.obs_overflow.push_front(ev); + break; + } + Err(TrySendError::Disconnected(ev)) => { + // The obs task is gone: reclaim this event's reserved payload bytes. + if let Some(bytes) = observation_payload_bytes(&ev) { + self.obs_payload_bytes.fetch_sub(bytes, Ordering::Relaxed); + } + } + } + } + } + + /// Hands one event to the obs task. Applies the synchronous protocol accounting + /// first (join/leave/conflict/key). A full channel retains application data for + /// retry (bounded by the payload byte budget and `OBS_OVERFLOW_MAX`) and drops + /// recoverable membership/control events, counting them. + fn send_observation(&mut self, ev: Event) { + self.account_event(&ev); + let payload = observation_payload_bytes(&ev); + // Byte backstop: refuse a payload event if enqueuing it would push the queued + // payload bytes over budget. + if let (Some(budget), Some(bytes)) = (self.obs_payload_budget, payload) + && self + .obs_payload_bytes + .load(Ordering::Relaxed) + .saturating_add(bytes) + > budget + { + self.shared.add_observation_dropped(1); + return; + } + // Reserve the payload bytes before the event becomes visible to the obs task, + // so its release (subtract on receive) can never run ahead of the reservation. + if let Some(bytes) = payload { + self.obs_payload_bytes.fetch_add(bytes, Ordering::Relaxed); + } + match self.obs_tx.try_send(ev) { + Ok(()) => {} + Err(TrySendError::Full(ev)) => match payload { + // Application data the event stream cannot reconstruct: retain (still + // reserved) for a retry. + Some(_) if self.obs_overflow.len() < OBS_OVERFLOW_MAX => { + self.obs_overflow.push_back(ev); + } + // Recoverable membership/control, or the overflow is full: drop, count, + // and roll back any reservation. + _ => { + if let Some(bytes) = payload { + self.obs_payload_bytes.fetch_sub(bytes, Ordering::Relaxed); + } + self.shared.add_observation_dropped(1); + } + }, + // The obs task is gone: roll back the reservation. + Err(TrySendError::Disconnected(_)) => { + if let Some(bytes) = payload { + self.obs_payload_bytes.fetch_sub(bytes, Ordering::Relaxed); + } + } + } + } + + /// Synchronous protocol accounting for a surfaced event: reduce the matching + /// await-result join on a push/pull `ExchangeCompleted`, resolve a parked leave + /// on `LeftCluster`, begin teardown on a lost id-conflict `Shutdown`, and answer + /// an inbound key-management request. + fn account_event(&mut self, ev: &Event) { + if let Event::ExchangeCompleted(c) = ev + && c.kind() == ExchangeKind::PushPull + { + let Self { + endpoint, + pending_joins, + .. + } = self; + complete_join_exchange( + endpoint, + pending_joins, + c.eid(), + *c.peer(), + matches!(c.outcome(), ExchangeStatus::Succeeded), + ); + } + if matches!(ev, Event::LeftCluster) + && let Some(pl) = self.pending_leave.take() + { + pl.resolve_all(|| Ok(())); + } + // A lost id-conflict vote means the local node MUST stop, exactly as for a + // `Command::Shutdown`. Flag shutdown; the pump self-wakes into the teardown + // branch. The event still reaches subscribers through the obs hand-off. + if matches!(ev, Event::Shutdown) { + self.shared.begin_shutdown(); + } + #[cfg(encryption)] + if let Event::KeyRequest(req) = ev { + let resp = apply_key_request(&*self.keyring, req); + // Ignoring Err: `respond_key` fails only when the response cannot be routed; + // the key op has already applied locally. + let _ = self.endpoint.respond_key(req, resp, Instant::now()); + } + } + + /// Reap await-result join waiters on the deadline timer (the reply terminal), + /// then remove any waiter that has reached BOTH terminals (reply resolved AND + /// `pending` empty), clearing its still-recorded ignore-join streams. + fn reap_pending_joins(&mut self, now: Instant) { + let Self { + endpoint, + pending_joins, + .. + } = self; + let mut i = 0; + while i < pending_joins.len() { + if now >= pending_joins[i].deadline { + pending_joins[i].resolve_reply(); + } + if pending_joins[i].is_done() { + let pj = pending_joins.swap_remove(i); + for s in &pj.ignore_streams { + endpoint.clear_ignore_join_stream(*s); + } + } else { + i += 1; + } + } + } + + /// Reap a deadline-expired graceful-leave waiter. + fn reap_pending_leave(&mut self, now: Instant) { + if let Some(pl) = self.pending_leave.as_ref() + && now >= pl.deadline + { + let pl = self.pending_leave.take().expect("checked Some above"); + pl.resolve_all(|| Err(SerfError::LeaveTimeout)); + } + } + + /// Earliest pending-join deadline of a still-unreplied waiter, folded into the + /// per-poll timer target so it fires by the first expiring join. + fn min_pending_join_deadline(&self) -> Option { + self + .pending_joins + .iter() + .filter(|pj| pj.reply.is_some()) + .map(|pj| pj.deadline) + .min() + } + + /// Earliest pending-leave deadline, folded into the per-poll timer target. + fn min_pending_leave_deadline(&self) -> Option { + self.pending_leave.as_ref().map(|pl| pl.deadline) + } + + /// Publish a fresh [`SerfSnapshot`] of the endpoint's observable membership. + /// Skips the publish when the local node is not yet present in the membership + /// store (the local `NodeJoined` sieve has not fired), so `SerfSnapshot::new` + /// (which requires the local node) is never called with it absent. + fn refresh_snapshot(&self) { + let members = self.endpoint.members_snapshot(); + let local_id = self.endpoint.local_id(); + if !members.iter().any(|m| m.node().id_ref() == local_id) { + return; + } + let snap = SerfSnapshot::new( + members, + local_id, + self.endpoint.state(), + LamportTime::from(self.endpoint.member_time()), + LamportTime::from(self.endpoint.event_time()), + LamportTime::from(self.endpoint.query_time()), + ); + self.shared.publish(snap); + } + + /// (Re)arms the wakeup timer for `target` if it is not already armed for it. + fn arm_timer(&mut self, target: Instant, now: Instant) { + if self.timer_deadline != Some(target) { + self.timer = Some(Box::pin(R::sleep(target.saturating_duration_since(now)))); + self.timer_deadline = Some(target); + } + } + + /// Reply `Err(Shutdown)` to a command drained during teardown. + fn reply_shutdown(cmd: Command) { + match cmd { + Command::Join(JoinCmd { reply, .. }) => { + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(Err((SmallVec::new(), SerfError::Shutdown))); + } + Command::Leave(LeaveCmd { reply }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + Command::ForceLeave(ForceLeaveCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + Command::UserEvent(cmd) => { + let _ = cmd.reply.send(Err(SerfError::Shutdown)); + } + Command::Query(QueryCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + Command::Respond(RespondCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + Command::SetTags(SetTagsCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + #[cfg(encryption)] + Command::InstallKey(KeyCmd { reply, .. }) + | Command::UseKey(KeyCmd { reply, .. }) + | Command::RemoveKey(KeyCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + #[cfg(encryption)] + Command::ListKeys(ListKeysCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + Command::Shutdown(ShutdownCmd { reply }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + } + } +} + +impl Future for StreamDriver +where + I: memberlist_proto::Id + Clone + Send + Sync + Unpin + 'static, + R: Runtime, + T: StreamTransport + Unpin, + T::Options: Unpin, + G: rand::Rng + Unpin, + SR: rand::Rng + SeedableRng + Unpin, + StreamEndpoint: Unpin, +{ + type Output = (); + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { + let this = self.get_mut(); + let now = Instant::now(); + let mut progress = false; + let mut more = false; + + // Drain queued commands (parks the waker for the next push). + for cmd in this.shared.drain_commands(cx.waker()) { + this.dispatch(cmd, now); + progress = true; + } + + // Shutdown, in ordered phases: (1) FREEZE — best-effort leave, cancel every + // bridge's reads, drop the template inbound sender, and release the bind + // sockets; (2) DRAIN the bridge inbound channel to all-senders-gone, folding + // every already-read completion into its join's contacted set; (3) REAP the + // parked joins/leave and fail the queued commands; (4) await ONLY the accept + // task's exit before acking. The completion latch promises the bind address + // is free (the UDP gossip socket + the TCP listener), not that every connected + // stream FD has closed. + if this.shared.is_shutdown() { + if this.accept_shutdown_tx.is_some() { + // FREEZE (one-time). Best-effort leave, freeze every live bridge, drop the + // template inbound sender so the channel can reach all-senders-gone, and + // release the bind sockets. + // Ignoring Err: best-effort leave during shutdown. + let _ = this.endpoint.leave(Instant::now()); + for (_, handle) in this.bridges.drain() { + // Ignoring Err: the bridge may have already exited (cancel receiver + // gone); the freeze is best-effort. + let _ = handle.cancel_tx.send(()); + } + this.inbound_tx = None; + // Dropping `accept_shutdown_tx` cancels the accept task's pending + // `accept()`; its `listener` local is released when the task is next + // scheduled (awaited below). Dropping the gossip socket closes its UDP FD + // synchronously. + drop(this.accept_shutdown_tx.take()); + drop(this.socket.take()); + } + + // DRAIN-TO-DISCONNECTED (re-entrant), then REAP (once). Fold every + // already-read completion into its join's contacted set BEFORE reaping. The + // drain reads `inbound_rx` to all-senders-gone via `try_recv`; each frozen + // bridge calls `wake_driver()` on exit, so the last exit re-polls us to + // observe Disconnected. `inbound_rx` is taken once it disconnects, doubling + // as the one-time reap guard. + if this.inbound_rx.is_some() { + let drained_to_disconnect = loop { + let mut channel_work = false; + let mut hit_disconnect = false; + if let Some(inbound_rx) = this.inbound_rx.as_ref() { + loop { + match inbound_rx.try_recv() { + // Inline the machine feed (rather than `dispatch_bridge_inbound`, + // which borrows all of `*this`) so the `&this.inbound_rx` held by + // the outer `if let` and the disjoint `&mut this.endpoint` here do + // not overlap. + Ok(BridgeInbound::Data(BridgeData { + eid, + bytes, + received_at, + })) => { + this + .endpoint + .handle_transport_data(eid, &bytes, false, received_at); + channel_work = true; + } + Ok(BridgeInbound::Eof(BridgeEof { eid, received_at })) => { + this + .endpoint + .handle_transport_data(eid, &[], true, received_at); + channel_work = true; + } + Ok(BridgeInbound::Error(BridgeEof { eid, received_at })) => { + this.endpoint.handle_transport_error(eid, received_at); + channel_work = true; + } + Err(flume::TryRecvError::Empty) => break, + Err(flume::TryRecvError::Disconnected) => { + hit_disconnect = true; + break; + } + } + } + } + // Pull the resulting machine surfaces to QUIESCENCE; account_event folds + // every terminal completion into the matching pending join. + let (_, surf_more) = this.drain_surfaces(cx); + if channel_work || surf_more { + continue; + } + break hit_disconnect; + }; + if !drained_to_disconnect { + return Poll::Pending; + } + drop(this.inbound_rx.take()); + // Close the command queue and fail any still-queued commands. + for cmd in this.shared.close_and_drain() { + if let Command::Shutdown(ShutdownCmd { reply }) = cmd { + // A straggler `Shutdown`: park it too, so it is acked after the socket + // and listener drop like every other caller. + this.shutdown_reply.push(reply); + } else { + Self::reply_shutdown(cmd); + } + } + for mut pj in this.pending_joins.drain(..) { + for s in &pj.ignore_streams { + this.endpoint.clear_ignore_join_stream(*s); + } + if let Some(reply) = pj.reply.take() { + // Ignoring Err: the join caller dropped its reply receiver. Carry the + // addresses reached before shutdown raced this waiter. + let _ = reply.send(Err(( + std::mem::take(&mut pj.contacted), + SerfError::Shutdown, + ))); + } + } + if let Some(pl) = this.pending_leave.take() { + pl.resolve_all(|| Err(SerfError::Shutdown)); + } + } + + // Await the accept task's exit before acking so the TCP listener FD is + // actually released (not merely signalled). + if let Some(join) = this.accept_join.as_mut() { + // Ignoring Ok/Err: only readiness matters — the listener is released once + // the task has exited, regardless of how. + if join.poll_unpin(cx).is_pending() { + return Poll::Pending; + } + this.accept_join = None; + } + // The bind address is now free. Ack the stashed replies and release any late + // `shutdown()` caller parked on the completion latch, then stop. + for reply in this.shutdown_reply.drain(..) { + // Ignoring Err: the caller dropped its reply receiver. + let _ = reply.send(Ok(())); + } + this.shared.mark_shutdown_complete(); + return Poll::Ready(()); + } + + // Receive gossip (bounded; a full batch means more may be waiting). The socket + // is always `Some` here — the shutdown branch above (which takes it) returned + // before reaching this point. `Poll::Pending` from the socket IS the + // kernel-empty signal (the readiness analogue of serf-compio's completion + // reap), so the recv-loop stops and `handle_timeout` fires inline below. + let mut recv_n = 0; + while recv_n < this.iter_drain_cap { + let Some(socket) = this.socket.as_ref() else { + break; + }; + match socket.poll_recv_from(cx, &mut this.recv_buf) { + Poll::Ready(Ok((n, src))) => { + this.endpoint.handle_gossip(src, &this.recv_buf[..n], now); + recv_n += 1; + } + // Ignoring Err: a transient recv error is non-fatal; re-armed next poll. + Poll::Ready(Err(_)) => break, + Poll::Pending => break, + } + } + if recv_n > 0 { + progress = true; + } + if recv_n == this.iter_drain_cap { + more = true; + } + + // Accept inbound connections. Aux tasks wake the driver after enqueueing, so + // `try_recv` (no waker registration) is sufficient. + while let Ok((stream, peer)) = this.accepted_rx.try_recv() { + if this.handle_accepted(stream, peer, now) { + progress = true; + } + } + + // Dial outcomes. + while let Ok(status) = this.dial_rx.try_recv() { + this.handle_dial_status(status); + progress = true; + } + + // Inbound transport bytes/EOF from the bridge read tasks (bounded per poll; a + // full batch self-wakes via `more`). + let mut inbound_n = 0; + while inbound_n < this.iter_drain_cap { + let Some(inbound_rx) = this.inbound_rx.as_ref() else { + break; + }; + let Ok(msg) = inbound_rx.try_recv() else { + break; + }; + inbound_n += 1; + this.dispatch_bridge_inbound(msg); + } + if inbound_n > 0 { + progress = true; + } + if inbound_n == this.iter_drain_cap { + more = true; + } + + // Drain machine surfaces (bounded per surface). + let (drained, drain_more) = this.drain_surfaces(cx); + progress |= drained; + more |= drain_more; + // A conflict `Event::Shutdown` observed during the drain flips the shutdown + // latch; self-wake so the next poll enters the teardown branch. + if this.shared.is_shutdown() { + more = true; + } + + // Timer: fire an overdue deadline inline (the single `handle_timeout` site), + // else arm + poll the sleep. Fold in the earliest pending-join / -leave + // deadline so a parked waiter's timeout fires even when the coordinator has no + // nearer deadline. + let endpoint_deadline = this + .endpoint + .poll_timeout() + .map(|d| d.min(now + this.idle_wake)) + .unwrap_or(now + this.idle_wake); + let target = [ + Some(endpoint_deadline), + this.min_pending_join_deadline(), + this.min_pending_leave_deadline(), + ] + .into_iter() + .flatten() + .min() + .unwrap_or(endpoint_deadline); + if target <= now { + this.endpoint.handle_timeout(now); + progress = true; + more = true; + } else { + this.arm_timer(target, now); + if let Some(timer) = this.timer.as_mut() + && timer.as_mut().poll(cx).is_ready() + { + this.endpoint.handle_timeout(Instant::now()); + this.timer = None; + this.timer_deadline = None; + progress = true; + more = true; + } + } + + // Reap deadline-expired join / leave waiters (a fired `handle_timeout` may have + // completed exchanges; the deadline path resolves the rest). + this.reap_pending_joins(now); + this.reap_pending_leave(now); + + // Republish the snapshot whenever the pump made progress (the serf endpoint + // exposes no cheap version stamp, so — as in serf-compio — a productive poll + // rebuilds and republishes the observable membership). + if progress { + this.refresh_snapshot(); + } + + // Yield to other tasks, but re-poll promptly while work remains. + if more { + cx.waker().wake_by_ref(); + } + Poll::Pending + } +} + +/// Apply one terminal `ExchangeCompleted` to its await-result join waiter (if +/// any): remove `eid` from `pending`, push the peer into `contacted` on success, +/// resolve the caller's reply the instant `pending` empties (ahead of the obs +/// hand-off, so a slow delegate cannot delay it), and once fully done clear the +/// still-recorded ignore-join streams and reap the waiter. +fn complete_join_exchange( + endpoint: &mut StreamEndpoint, + pending_joins: &mut Vec, + eid: ExchangeId, + peer: SocketAddr, + succeeded: bool, +) where + I: memberlist_proto::Id + Clone, + T: StreamTransport, + G: rand::Rng, + SR: rand::Rng + SeedableRng, +{ + let Some(idx) = pending_joins + .iter() + .position(|pj| pj.pending.contains(&eid)) + else { + return; + }; + let pj = &mut pending_joins[idx]; + pj.pending.remove(&eid); + if succeeded { + pj.contacted.push(peer); + } + // Resolve the reply the moment every dispatched exchange has terminated. If the + // deadline already replied, `reply` is `None` and this is a no-op. + if pj.pending.is_empty() { + pj.resolve_reply(); + } + if pending_joins[idx].is_done() { + let pj = pending_joins.swap_remove(idx); + for s in &pj.ignore_streams { + endpoint.clear_ignore_join_stream(*s); + } + } +} + +/// Apply one inbound [`KeyRequest`] to the driver's keyring delegate, producing +/// the [`KeyResponseArgs`] the pump forwards to `respond_key`. +#[cfg(encryption)] +fn apply_key_request( + keyring: &dyn KeyringDelegate, + req: &KeyRequest, +) -> KeyResponseArgs { + match (req.op(), req.key()) { + (KeyRequestOperation::Install, Some(key)) => keyring.install(*key), + (KeyRequestOperation::Use, Some(key)) => keyring.use_key(*key), + (KeyRequestOperation::Remove, Some(key)) => keyring.remove(*key), + (KeyRequestOperation::List, _) => keyring.list(), + (_, None) => KeyResponseArgs { + result: false, + message: "key-management request missing its required key".into(), + keys: Vec::new(), + primary_key: None, + }, + } +} + +/// Accepts inbound TCP connections and forwards each to the pump, waking it after +/// each enqueue. `accept` is async-only, so this cannot fold into the pump's poll; +/// it stops promptly when the driver drops `shutdown_rx`'s sender, which cancels +/// the pending `accept()` so the listener is released. +pub(crate) async fn accept_task( + listener: L, + accepted_tx: Sender<(L::Stream, SocketAddr)>, + shutdown_rx: Receiver<()>, + shared: Arc>, +) where + I: Send + Sync + 'static, + L: TcpListener, +{ + loop { + select! { + conn = listener.accept().fuse() => match conn { + Ok((stream, peer)) => { + // Bounded channel: await space (accept backpressure). Race the send + // against shutdown so a full queue at shutdown cannot wedge the task. + select! { + res = accepted_tx.send_async((stream, peer)).fuse() => { + if res.is_err() { + break; + } + shared.wake_driver(); + } + _ = shutdown_rx.recv_async().fuse() => break, + } + } + // Ignoring Err: a transient accept error (e.g. a reset mid-handshake) is + // non-fatal; keep listening for the next connection. + Err(_) => continue, + }, + // The driver dropped its shutdown sender: stop and release the listener. + _ = shutdown_rx.recv_async().fuse() => break, + } + } +} + +/// Dials `peer` for outbound exchange `eid`, bounded by `dial_timeout`, and +/// reports the outcome to the pump, waking it afterwards. +async fn dial_task( + eid: ExchangeId, + peer: SocketAddr, + dial_timeout: Duration, + out_rx: Receiver, + cancel_rx: oneshot::Receiver<()>, + dial_tx: Sender>, + shared: Arc>, +) where + I: Send + Sync + 'static, + R: Runtime, +{ + let outcome = match ::TcpStream::connect_timeout(&peer, dial_timeout).await { + Ok(stream) => DialStatus::Connected(DialConnected { + eid, + stream, + out_rx, + cancel_rx, + }), + Err(_) => DialStatus::Failed(DialFailed { + eid, + received_at: Instant::now(), + }), + }; + // Ignoring Err: the pump dropped its receiver (driver shut down). + let _ = dial_tx.send(outcome); + shared.wake_driver(); +} + +/// The observation task: drains machine events off the pump, invokes the +/// [`Delegate`] hooks, and forwards every serf event to the +/// [`EventStream`](crate::EventStream). Every serf event is the application's +/// observation surface, so all are forwarded to subscribers; the forward is +/// best-effort (a full queue drops + counts, never blocks). +async fn observation_task( + obs_rx: Receiver>, + delegate: D, + events_tx: Sender>, + shared: Arc>, + obs_payload_bytes: Arc, +) where + I: Clone + Send + Sync + 'static, + D: Delegate, +{ + use std::panic::AssertUnwindSafe; + while let Ok(ev) = obs_rx.recv_async().await { + // Reclaim the byte-backstop budget this event occupied, before the (possibly + // slow) delegate hook, so the pump's enqueue side sees it promptly. + let payload = observation_payload_bytes(&ev); + if let Some(bytes) = payload { + obs_payload_bytes.fetch_sub(bytes, Ordering::Relaxed); + } + // Contain a panicking delegate hook so the task survives and keeps releasing + // the reservations of still-queued events. Ignoring the unwind result: the + // panic is contained and the event is still forwarded to subscribers below. + let _ = AssertUnwindSafe(dispatch_event_delegate(&delegate, &ev)) + .catch_unwind() + .await; + if events_tx + .try_send(ev) + .is_err_and(|e| matches!(e, TrySendError::Full(_))) + { + shared.add_events_dropped(1); + } + } +} + +/// Set up the stream driver: spawn the observation task and the accept task, then +/// build the [`StreamDriver`] future (with the periodic schedulers armed). The +/// caller (`Transport::run`) awaits the returned future. +#[allow(clippy::too_many_arguments)] +pub(crate) fn spawn_stream_driver( + mut endpoint: StreamEndpoint, + gossip_socket: ::UdpSocket, + listener: ::TcpListener, + shared: Arc>, + events_tx: Sender>, + delegate: D, + driver_opts: RuntimeOptions, + stream_opts: StreamTransportOptions, + label: Option, + #[cfg(encryption)] keyring: Arc, +) -> StreamDriver +where + I: memberlist_proto::Id + Clone + Send + Sync + Unpin + 'static, + R: Runtime, + T: StreamTransport, + D: Delegate, + G: rand::Rng, + SR: rand::Rng + SeedableRng, +{ + // Arm the periodic probe / gossip / push-pull schedulers. Without this the + // coordinator's schedulers stay unset, so failure detection, dissemination, and + // anti-entropy never run. + endpoint.start_scheduling(Instant::now()); + + // Observation byte backstop: bound the queued payload bytes a slow delegate can + // pin (the obs-channel count cap alone does not). + let obs_payload_bytes = Arc::new(AtomicU64::new(0)); + let obs_payload_budget: Option = match driver_opts.observation_channel() { + Channel::Bounded(_) => Some((endpoint.max_stream_frame_size() as u64).saturating_mul(4)), + Channel::Unbounded => None, + }; + let (obs_tx, obs_rx) = match driver_opts.observation_channel() { + Channel::Bounded(n) => flume::bounded(n), + Channel::Unbounded => flume::unbounded(), + }; + R::spawn_detach(observation_task::( + obs_rx, + delegate, + events_tx, + shared.clone(), + obs_payload_bytes.clone(), + )); + + // Inbound connections arrive on a dedicated accept task (accept is async); it is + // cancelled when the driver drops `accept_shutdown_tx`. Retain its join handle + // (spawn, not spawn_detach) so the driver AWAITS its exit on shutdown before + // acking — the listener FD lives in the task and is released only when it exits. + let (accepted_tx, accepted_rx) = flume::bounded(ACCEPT_CAP); + let (accept_shutdown_tx, accept_shutdown_rx) = flume::bounded(1); + let accept_join = R::spawn(accept_task::::TcpListener>( + listener, + accepted_tx, + accept_shutdown_rx, + shared.clone(), + )); + + StreamDriver::::new( + endpoint, + gossip_socket, + shared, + obs_tx, + obs_payload_bytes, + obs_payload_budget, + accepted_rx, + accept_shutdown_tx, + accept_join, + driver_opts, + stream_opts, + label, + #[cfg(encryption)] + keyring, + ) +} diff --git a/serf-reactor/src/lib.rs b/serf-reactor/src/lib.rs index 1e33c6cc..7501e303 100644 --- a/serf-reactor/src/lib.rs +++ b/serf-reactor/src/lib.rs @@ -45,6 +45,8 @@ #![cfg_attr(docsrs, feature(doc_cfg))] #![forbid(unsafe_code)] +#[cfg(feature = "tcp")] +mod bridge; #[cfg(any(feature = "tcp", feature = "quic"))] mod command; #[cfg(any(feature = "tcp", feature = "quic"))] @@ -52,6 +54,15 @@ mod delegate; mod driver; mod error; mod events; +mod resolver; +#[cfg(any(feature = "tcp", feature = "quic"))] +mod serf; +#[cfg(any(feature = "tcp", feature = "quic"))] +mod shared; +#[cfg(feature = "tcp")] +mod tcp; +#[cfg(any(feature = "tcp", feature = "quic"))] +mod transport; use rand::{ SeedableRng, @@ -88,6 +99,28 @@ pub use error::{ GossipMtuTooSmall, InvalidAdvertiseAddr, InvalidGossipMtu, InvalidOption, Result, SerfError, }; +/// The seed/advertise address form re-exported from `memberlist-proto`: either an +/// already-`Resolved` wire [`std::net::SocketAddr`] or an `Unresolved` user +/// address the caller's [`Resolver`] resolves at the boundary. +pub use memberlist_proto::MaybeResolved; + +pub use resolver::{ + AdvertiseAddrResolver, AdvertiseResolutionError, FirstAddrResolver, Ipv4PreferringResolver, + Ipv6PreferringResolver, OsResolver, Resolver, SocketAddrResolver, +}; + +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub use serf::Serf; + +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub use transport::{Transport, TransportRuntime}; + +#[cfg(feature = "tcp")] +#[cfg_attr(docsrs, doc(cfg(feature = "tcp")))] +pub use tcp::{TcpTransport, TcpTransportOptions}; + #[cfg(any(feature = "tcp", feature = "quic"))] #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] pub use delegate::{ diff --git a/serf-reactor/src/resolver/advertise/mod.rs b/serf-reactor/src/resolver/advertise/mod.rs new file mode 100644 index 00000000..55ae08cf --- /dev/null +++ b/serf-reactor/src/resolver/advertise/mod.rs @@ -0,0 +1,87 @@ +//! [`AdvertiseAddrResolver`] — picks one [`SocketAddr`] from a candidate set +//! during local-node advertise resolution. Called once at `Transport::new` when +//! the configured advertise address is `MaybeResolved::Unresolved(addr)` and +//! [`Resolver::resolve`](crate::resolver::Resolver::resolve) returns multiple +//! candidates. + +use std::net::SocketAddr; + +/// Picks one [`SocketAddr`] from a candidate set. +/// +/// The driver calls this immediately after +/// [`Resolver::resolve`](crate::resolver::Resolver::resolve) returns more than +/// one result for the configured advertise address. An implementor can express +/// simple policies (prefer IPv4, prefer IPv6, take the first) or consult external +/// state. `Send + Sync + 'static` so it can be held across the multi-threaded +/// construction path. +pub trait AdvertiseAddrResolver: Send + Sync + 'static { + /// Error type returned by [`Self::pick`]. + type Error: core::error::Error + Send + Sync + 'static; + + /// Pick one candidate. Returns an error when `candidates` is empty. + fn pick(&self, candidates: Vec) -> Result; +} + +/// Error variants returned by the built-in [`AdvertiseAddrResolver`] impls. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum AdvertiseResolutionError { + /// The candidate set was empty — resolution returned no addresses. + #[error("advertise resolution: no candidate addresses returned")] + Empty, +} + +/// Default — returns the first candidate address in the set. +#[derive(Debug, Default, Clone, Copy)] +pub struct FirstAddrResolver; + +impl AdvertiseAddrResolver for FirstAddrResolver { + type Error = AdvertiseResolutionError; + + fn pick(&self, candidates: Vec) -> Result { + candidates + .into_iter() + .next() + .ok_or(AdvertiseResolutionError::Empty) + } +} + +/// Prefers the first IPv4 candidate; falls through to the first address of any +/// family if no IPv4 candidates are present. +#[derive(Debug, Default, Clone, Copy)] +pub struct Ipv4PreferringResolver; + +impl AdvertiseAddrResolver for Ipv4PreferringResolver { + type Error = AdvertiseResolutionError; + + fn pick(&self, candidates: Vec) -> Result { + let v4 = candidates.iter().find(|s| s.is_ipv4()).copied(); + if let Some(s) = v4 { + return Ok(s); + } + candidates + .into_iter() + .next() + .ok_or(AdvertiseResolutionError::Empty) + } +} + +/// Prefers the first IPv6 candidate; falls through to the first address of any +/// family if no IPv6 candidates are present. +#[derive(Debug, Default, Clone, Copy)] +pub struct Ipv6PreferringResolver; + +impl AdvertiseAddrResolver for Ipv6PreferringResolver { + type Error = AdvertiseResolutionError; + + fn pick(&self, candidates: Vec) -> Result { + let v6 = candidates.iter().find(|s| s.is_ipv6()).copied(); + if let Some(s) = v6 { + return Ok(s); + } + candidates + .into_iter() + .next() + .ok_or(AdvertiseResolutionError::Empty) + } +} diff --git a/serf-reactor/src/resolver/mod.rs b/serf-reactor/src/resolver/mod.rs new file mode 100644 index 00000000..72adbb20 --- /dev/null +++ b/serf-reactor/src/resolver/mod.rs @@ -0,0 +1,54 @@ +//! Address resolution for serf-reactor. +//! +//! [`Resolver`] is the generic address-to-[`SocketAddr`] conversion trait; +//! [`AdvertiseAddrResolver`] picks the single advertise address from a +//! multi-candidate resolution result. Both are the `Send`/`Sync` siblings of +//! serf-compio's `!Send` resolvers — mirroring memberlist-reactor's +//! `AddressResolver`, `resolve` returns `-> impl Future + Send` (not `async fn`) +//! so resolution can run on a multi-threaded agnostic runtime. +//! +//! Built-in resolvers: +//! - [`OsResolver`]: `getaddrinfo`-backed via the runtime's blocking pool. +//! - [`SocketAddrResolver`]: identity pass-through for already-resolved addrs. +//! +//! The `hickory`-backed `DnsResolver` (`dns` feature) and the getifs +//! `LocalAddrResolver` (`getifs` feature) are added in a later chunk. + +mod advertise; +mod os; +mod socket_addr; + +pub use advertise::{ + AdvertiseAddrResolver, AdvertiseResolutionError, FirstAddrResolver, Ipv4PreferringResolver, + Ipv6PreferringResolver, +}; +pub use os::OsResolver; +pub use socket_addr::SocketAddrResolver; + +use core::future::Future; +use std::net::SocketAddr; + +/// Resolve a user-facing address into one or more concrete [`SocketAddr`]s. +/// +/// The input address type is the implementor's choice: [`SocketAddrResolver`] +/// takes [`SocketAddr`] (identity pass-through); [`OsResolver`] takes +/// [`hostaddr::HostAddr`](hostaddr::HostAddr). Custom resolvers may take +/// any type (e.g. a service-discovery handle). +/// +/// Invoked only at the boundary (bootstrap + `join`). The returned future is +/// `Send` so resolution can run on a multi-threaded runtime; it is written +/// `-> impl Future + Send` rather than `async fn` so the `Send` bound is part of +/// the trait contract. +pub trait Resolver: Send + Sync + 'static { + /// The user-facing address type this resolver consumes. + type Address: Send + Sync + 'static; + + /// The error type returned on resolution failure. + type Error: core::error::Error + Send + Sync + 'static; + + /// Resolve `addr` to its concrete socket addresses. + fn resolve( + &self, + addr: &Self::Address, + ) -> impl Future, Self::Error>> + Send + '_; +} diff --git a/serf-reactor/src/resolver/os.rs b/serf-reactor/src/resolver/os.rs new file mode 100644 index 00000000..007342bb --- /dev/null +++ b/serf-reactor/src/resolver/os.rs @@ -0,0 +1,58 @@ +//! OS-based resolver — delegates to the runtime's `getaddrinfo` equivalent. + +use core::{future::Future, marker::PhantomData}; +use std::{io, net::SocketAddr}; + +use agnostic::net::ToSocketAddrs; +use agnostic_lite::RuntimeLite; +use hostaddr::Host; +use smol_str::SmolStr; + +use crate::resolver::Resolver; + +/// OS-based resolver. Runs `getaddrinfo` on the runtime's blocking pool via the +/// agnostic [`ToSocketAddrs`] abstraction, so it stays runtime-agnostic yet never +/// blocks the async worker. +/// +/// Generic over the runtime `R` (the same `agnostic::Runtime` the node's driver +/// is spawned on). Suitable for typical hostname lookups; large DNS records that +/// may be truncated over UDP are better served by the `dns`-feature resolver. +pub struct OsResolver(PhantomData R>); + +impl OsResolver { + /// Construct an OS resolver for the runtime `R`. + #[inline] + pub const fn new() -> Self { + Self(PhantomData) + } +} + +impl Default for OsResolver { + #[inline] + fn default() -> Self { + Self::new() + } +} + +impl Resolver for OsResolver +where + R: RuntimeLite, +{ + type Address = hostaddr::HostAddr; + type Error = io::Error; + + fn resolve( + &self, + addr: &Self::Address, + ) -> impl Future, Self::Error>> + Send + '_ { + let port = addr.port().unwrap_or(0); + let host = match addr.host() { + Host::Ip(ip) => ip.to_string(), + Host::Domain(name) => name.to_string(), + }; + async move { + let iter = ToSocketAddrs::::to_socket_addrs(&(host, port)).await?; + Ok(iter.collect()) + } + } +} diff --git a/serf-reactor/src/resolver/socket_addr.rs b/serf-reactor/src/resolver/socket_addr.rs new file mode 100644 index 00000000..3b38b253 --- /dev/null +++ b/serf-reactor/src/resolver/socket_addr.rs @@ -0,0 +1,27 @@ +//! Identity pass-through resolver — input is already a [`SocketAddr`]. + +use crate::resolver::Resolver; +use core::future::Future; +use std::{io, net::SocketAddr}; + +/// Identity pass-through resolver. Declares +/// [`Resolver::Address`](crate::resolver::Resolver::Address)`= SocketAddr` and +/// returns the input verbatim. +/// +/// Use when seed addresses are already concrete socket addresses — no DNS lookup +/// or hostname parsing is required. +#[derive(Debug, Default, Clone, Copy)] +pub struct SocketAddrResolver; + +impl Resolver for SocketAddrResolver { + type Address = SocketAddr; + type Error = io::Error; + + fn resolve( + &self, + addr: &SocketAddr, + ) -> impl Future, Self::Error>> + Send + '_ { + let addr = *addr; + async move { Ok(vec![addr]) } + } +} diff --git a/serf-reactor/src/serf/mod.rs b/serf-reactor/src/serf/mod.rs new file mode 100644 index 00000000..54747a71 --- /dev/null +++ b/serf-reactor/src/serf/mod.rs @@ -0,0 +1,555 @@ +//! The [`Serf`] handle — construction, lifecycle, command dispatch, and lock-free +//! reads. +//! +//! [`Serf::new`] binds a [`Transport`], builds the [`Shared`] state + events +//! channel, and spawns the driver pump on the agnostic runtime `R`. Every command +//! method pushes one [`Command`] onto the shared queue and (for the awaited kinds) +//! parks on a one-shot reply the driver resolves. `Clone` / `Drop` reference-count +//! the shared driver; membership reads go through the lock-free +//! [`SerfSnapshot`]. The ergonomic per-backend constructors (`tcp` / `tls` / +//! `quic`), the richer snapshot accessors, and the full test suite are layered on +//! in a later chunk. + +use core::{marker::PhantomData, time::Duration}; +use std::{net::SocketAddr, sync::Arc}; + +use agnostic::Runtime; +use bytes::Bytes; +use futures_channel::oneshot; +use memberlist_proto::{Instant, Node}; +use serf_driver::SerfSnapshot; +use serf_proto::{ + LamportTime, + endpoint::{QueryId, QueryParams}, + event::{Event, QueryEvent}, + members::{Member, MemberStatus, SerfState}, + options::Options as SerfOptions, + typed::Tags, +}; +use smallvec::SmallVec; +use smol_str::SmolStr; + +#[cfg(encryption)] +use crate::command::{KeyCmd, ListKeysCmd}; +#[cfg(encryption)] +use crate::delegate::KeyringDelegate; +use crate::{ + MaybeResolved, + command::{ + Command, ForceLeaveCmd, JoinCmd, JoinKind, JoinReply, LeaveCmd, QueryCmd, RespondCmd, + SetTagsCmd, ShutdownCmd, UserEventCmd, WaitForCompletionArgs, + }, + delegate::Delegate, + driver::options::RuntimeOptions, + error::{JoinFailed, Result, SerfError}, + events::EventStream, + resolver::{AdvertiseAddrResolver, Resolver}, + shared::Shared, + transport::{Transport, TransportRuntime}, +}; +#[cfg(encryption)] +use memberlist_proto::SecretKey; + +/// The initial published snapshot: the local node, `Alive`, with empty tags and +/// zeroed Lamport clocks. Superseded by the driver's first real republish. +fn initial_snapshot(local_id: &I, advertise: SocketAddr) -> SerfSnapshot +where + I: Clone + PartialEq, +{ + let member = Member::new( + Node::new(local_id.clone(), advertise), + Tags::new(), + MemberStatus::Alive, + ); + SerfSnapshot::new( + vec![Arc::new(member)], + local_id, + SerfState::Alive, + LamportTime::from(0u64), + LamportTime::from(0u64), + LamportTime::from(0u64), + ) +} + +/// A handle to a running serf node. +/// +/// Cheap to clone; every clone shares the one backend driver, which runs until the +/// last handle is dropped (or [`shutdown`](Serf::shutdown) is called). Membership +/// reads are lock-free via the published [`SerfSnapshot`]. +/// +/// `Serf` carries the wire id type `I`, the resolver's unresolved address +/// type `A`, and the agnostic runtime `R` its driver was spawned on. `I` flows +/// into the snapshot and events channel (both ``); `A` ties `join`'s +/// seeds to the address domain the node was built with; `R` brands the handle so a +/// tokio-backed node is a distinct type from a smol-backed one. +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub struct Serf { + shared: Arc>, + events_rx: flume::Receiver>, + /// Per-call await-result join deadline offset, cached from the runtime options + /// so each `join` can stamp its absolute `WaitForCompletion` deadline. + join_deadline: Duration, + /// Ties the handle to the resolver's unresolved address type. Not held in any + /// field — `join` enforces seeds resolve in this address domain. + _a: PhantomData, + /// Brands the handle with the agnostic runtime its driver was spawned on. Not + /// held in any field — the driver task is spawned detached. + _r: PhantomData, +} + +impl Clone for Serf { + fn clone(&self) -> Self { + self.shared.handle_cloned(); + Self { + shared: self.shared.clone(), + events_rx: self.events_rx.clone(), + join_deadline: self.join_deadline, + _a: PhantomData, + _r: PhantomData, + } + } +} + +impl Drop for Serf { + fn drop(&mut self) { + if self.shared.handle_dropped() { + self.shared.begin_shutdown(); + self.shared.wake_driver(); + } + } +} + +impl Serf +where + I: memberlist_proto::Id + Clone + Send + Sync + Unpin + 'static, + R: Runtime, +{ + /// Build a node: construct the transport `T`, build the shared state + events + /// channel, and spawn its driver pump on the runtime `R`. Every clone shares the + /// one driver. + /// + /// `resolver` / `advertise_resolver` resolve the advertise address at + /// construction; they are not retained. `gossip_rng` seeds the memberlist gossip + /// schedule (draw it via [`gossip_rng`](crate::gossip_rng)). Under an encryption + /// backend, pass an [`Arc`](crate::KeyringDelegate) + /// (`Arc::new(VoidKeyringDelegate)` for a node that manages no keys). + #[allow(clippy::too_many_arguments)] + pub async fn new( + options: T::Options, + resolver: &RES, + advertise_resolver: &AR, + delegate: D, + runtime_options: RuntimeOptions, + serf_options: SerfOptions, + gossip_rng: G, + #[cfg(encryption)] keyring: Arc, + ) -> core::result::Result + where + T: Transport, + RES: Resolver
, + AR: AdvertiseAddrResolver, + D: Delegate, + G: rand::Rng + Send + Unpin + 'static, + T::Error: From, + { + // Reject runtime knobs a zero capacity would deterministically break BEFORE + // binding any socket or spawning the detached driver. + runtime_options.validate().map_err(T::Error::from)?; + // Cache the join deadline before `runtime_options` moves into the bundle. + let join_deadline = runtime_options.join_deadline(); + + let transport = T::new(options, resolver, advertise_resolver).await?; + let local_id = transport.local_id().clone(); + let advertise = *transport.advertise_address(); + + let (events_tx, events_rx) = + flume::bounded::>(runtime_options.event_queue_cap()); + let shared = Arc::new(Shared::new(initial_snapshot(&local_id, advertise))); + + let runtime = TransportRuntime::::new( + delegate, + shared.clone(), + events_tx, + runtime_options, + serf_options, + #[cfg(encryption)] + keyring, + ); + + // The driver pump owns the transport, the bound sockets, and the endpoint; it + // runs detached until a `Shutdown` command or all handles drop. + R::spawn_detach(transport.run(runtime, gossip_rng)); + + Ok(Self { + shared, + events_rx, + join_deadline, + _a: PhantomData, + _r: PhantomData, + }) + } +} + +// Handle operations that read a cached snapshot or push a command over the queue — +// none touch node identity directly, so they impose no bound and stay callable on +// a `Serf` of any id type. +impl Serf { + /// The latest membership snapshot, read lock-free. + #[must_use] + pub fn snapshot(&self) -> Arc> { + self.shared.load_snapshot() + } + + /// The number of known members in the latest published snapshot. + #[must_use] + pub fn num_members(&self) -> usize { + self.shared.load_snapshot().num_members() + } + + /// Subscribe to the serf event stream (membership transitions, user events, + /// queries, responses). + #[must_use] + pub fn events(&self) -> EventStream + where + I: 'static, + { + EventStream::new(self.events_rx.clone()) + } + + /// The cumulative count of events dropped at the event-stream fan-out (a slow + /// subscriber); these are recoverable from the snapshot. + #[must_use] + pub fn events_dropped(&self) -> u64 { + self.shared.events_dropped() + } + + /// The cumulative count of events dropped at the observation channel (a slow + /// delegate); these may include unrecoverable application data. + #[must_use] + pub fn observation_dropped(&self) -> u64 { + self.shared.observation_dropped() + } + + /// Send `cmd` to the driver, failing fast if the node has shut down. + fn send(&self, cmd: Command) -> Result<()> { + if self.shared.is_shutdown() { + return Err(SerfError::Shutdown); + } + if self.shared.push_command(cmd) { + Ok(()) + } else { + Err(SerfError::CommandSend) + } + } + + /// Join an existing cluster through a single seed, waiting for the seed to be + /// contacted. Returns the resolved [`SocketAddr`] of the seed actually reached. + /// + /// When `ignore_old` is `true`, replay of the seed's pre-join user events is + /// suppressed (the machine records the join exchange's `StreamId` as a one-shot + /// ignore-join target consumed at that exchange's own merge). + pub async fn join( + &self, + resolver: &RES, + node: MaybeResolved, + ignore_old: bool, + ) -> Result + where + RES: Resolver, + { + let addrs = resolve_seeds(resolver, core::slice::from_ref(&node)).await?; + match self.join_await(addrs, 1, ignore_old).await { + // The driver only replies `Ok` with a non-empty contacted set, so `next` is + // the single reached address; the fallback can never fire. + Ok(reached) => reached + .into_iter() + .next() + .ok_or_else(|| SerfError::JoinAllFailed(JoinFailed::new(1, 0))), + Err((_, e)) => Err(e), + } + } + + /// Join through several seeds, waiting for the join to complete. Returns the set + /// of seed addresses actually contacted on success, or the legacy + /// partial-success tuple `(reached_so_far, error)` on failure. An empty + /// `existing` iterator is a trivial `Ok(empty)`. + pub async fn join_many( + &self, + resolver: &RES, + existing: impl Iterator>, + ignore_old: bool, + ) -> core::result::Result, (SmallVec<[SocketAddr; 1]>, SerfError)> + where + RES: Resolver, + { + let seeds: Vec> = existing.collect(); + if seeds.is_empty() { + return Ok(SmallVec::new()); + } + let requested = seeds.len(); + let addrs = match resolve_seeds(resolver, &seeds).await { + Ok(a) => a, + Err(e) => return Err((SmallVec::new(), e)), + }; + self.join_await(addrs, requested, ignore_old).await + } + + /// Fire-and-forget join: resolve `seeds`, dispatch a push/pull against each, and + /// return the dispatched-exchange count immediately without waiting for contact. + /// Actual membership surfaces through [`events`](Self::events) and + /// [`snapshot`](Self::snapshot). + pub async fn dispatch_join( + &self, + resolver: &RES, + seeds: &[MaybeResolved], + ) -> Result + where + RES: Resolver, + { + let addrs = resolve_seeds(resolver, seeds).await?; + let (tx, rx) = oneshot::channel(); + self.send(Command::Join(JoinCmd { + seeds: addrs, + kind: JoinKind::Dispatch, + // Fire-and-forget joins never ignore old events. + ignore_old: false, + reply: tx, + }))?; + match await_join_reply(rx).await { + Ok(dispatched) => Ok(dispatched.len()), + Err((_, e)) => Err(e), + } + } + + /// Drive an await-result join over already-resolved `addrs`, threading + /// `ignore_old` to each seed's join push/pull. `requested_if_empty` is the + /// `JoinAllFailed` denominator used only when `addrs` is empty. + async fn join_await( + &self, + addrs: Vec, + requested_if_empty: usize, + ignore_old: bool, + ) -> JoinReply { + // A non-empty seed input that resolved to zero addresses is NOT a silent + // success — surface `JoinAllFailed` so a bootstrap outage is never reported as + // a healthy zero-contact join. + if addrs.is_empty() { + return Err(( + SmallVec::new(), + SerfError::JoinAllFailed(JoinFailed::new(requested_if_empty, 0)), + )); + } + let deadline = Instant::now() + self.join_deadline; + let (tx, rx) = oneshot::channel(); + match self.send(Command::Join(JoinCmd { + seeds: addrs, + kind: JoinKind::WaitForCompletion(WaitForCompletionArgs { deadline }), + ignore_old, + reply: tx, + })) { + Ok(()) => await_join_reply(rx).await, + Err(e) => Err((SmallVec::new(), e)), + } + } + + /// Gracefully leave the cluster. Resolves once peers have been notified or the + /// configured leave timeout elapses. + pub async fn leave(&self) -> Result<()> { + let (tx, rx) = oneshot::channel(); + self.send(Command::Leave(LeaveCmd { reply: tx }))?; + await_reply(rx).await + } + + /// Force-remove `id` from the membership. With `prune`, the node is removed + /// immediately rather than after the tombstone timeout. + pub async fn force_leave(&self, id: I, prune: bool) -> Result<()> { + let (tx, rx) = oneshot::channel(); + self.send(Command::ForceLeave(ForceLeaveCmd { + id, + prune, + now: Instant::now(), + reply: tx, + }))?; + await_reply(rx).await + } + + /// Broadcast a user-defined event cluster-wide. + pub async fn user_event( + &self, + name: impl Into, + payload: Bytes, + coalesce: bool, + ) -> Result<()> { + let (tx, rx) = oneshot::channel(); + self.send(Command::UserEvent(UserEventCmd::new( + name.into(), + payload, + coalesce, + tx, + )))?; + await_reply(rx).await + } + + /// Issue a cluster-wide query; returns the [`QueryId`] identifying it. + pub async fn query( + &self, + name: impl Into, + payload: Bytes, + params: QueryParams, + ) -> Result { + let (tx, rx) = oneshot::channel(); + self.send(Command::Query(QueryCmd::new( + name.into(), + payload, + params, + Instant::now(), + tx, + )))?; + await_reply(rx).await + } + + /// Respond to an inbound query received via [`Event::Query`]. + pub async fn respond(&self, token: QueryEvent, payload: Bytes) -> Result<()> { + let (tx, rx) = oneshot::channel(); + self.send(Command::Respond(RespondCmd::new( + token, + payload, + Instant::now(), + tx, + )))?; + await_reply(rx).await + } + + /// Replace the local node's advertised tags. + pub async fn set_tags(&self, tags: Tags) -> Result<()> { + let (tx, rx) = oneshot::channel(); + self.send(Command::SetTags(SetTagsCmd { tags, reply: tx }))?; + await_reply(rx).await + } + + /// Issue a cluster-wide install-key query; returns the issued [`QueryId`]. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub async fn install_key(&self, key: SecretKey) -> Result { + let (tx, rx) = oneshot::channel(); + self.send(Command::InstallKey(KeyCmd { + key, + now: Instant::now(), + reply: tx, + }))?; + await_reply(rx).await + } + + /// Issue a cluster-wide use-key query to promote `key` to primary. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub async fn use_key(&self, key: SecretKey) -> Result { + let (tx, rx) = oneshot::channel(); + self.send(Command::UseKey(KeyCmd { + key, + now: Instant::now(), + reply: tx, + }))?; + await_reply(rx).await + } + + /// Issue a cluster-wide remove-key query. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub async fn remove_key(&self, key: SecretKey) -> Result { + let (tx, rx) = oneshot::channel(); + self.send(Command::RemoveKey(KeyCmd { + key, + now: Instant::now(), + reply: tx, + }))?; + await_reply(rx).await + } + + /// Issue a cluster-wide list-keys query. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub async fn list_keys(&self) -> Result { + let (tx, rx) = oneshot::channel(); + self.send(Command::ListKeys(ListKeysCmd { + now: Instant::now(), + reply: tx, + }))?; + await_reply(rx).await + } + + /// Stop the driver and release its bound sockets, so an immediate rebind on the + /// same address succeeds with no grace period. Returns once those sockets are + /// released; it aborts in-flight reliable-stream exchanges but does not block on + /// their connection cleanup. + pub async fn shutdown(&self) -> Result<()> { + let (tx, rx) = oneshot::channel(); + if !self + .shared + .push_command(Command::Shutdown(ShutdownCmd { reply: tx })) + { + // The queue is already closed: a shutdown is in flight (or done). The driver + // may still hold its bind sockets, so await teardown completion before + // reporting success rather than returning into a still-bound port. + self.shared.wait_shutdown_complete().await; + return Ok(()); + } + await_reply(rx).await + } +} + +/// Await a driver reply, mapping a dropped reply channel to +/// [`SerfError::ReplyClosed`]. +async fn await_reply(rx: oneshot::Receiver>) -> Result { + match rx.await { + Ok(res) => res, + Err(_) => Err(SerfError::ReplyClosed), + } +} + +/// Await a join reply, mapping a dropped reply channel to the empty-reached, +/// [`SerfError::ReplyClosed`] partial-success tuple. +async fn await_join_reply(rx: oneshot::Receiver) -> JoinReply { + match rx.await { + Ok(res) => res, + Err(_) => Err((SmallVec::new(), SerfError::ReplyClosed)), + } +} + +/// Resolve every `MaybeResolved` seed through `resolver` into concrete +/// [`SocketAddr`]s (an already-`Resolved` seed passes straight through). +async fn resolve_seeds( + resolver: &RES, + seeds: &[MaybeResolved], +) -> Result> +where + RES: Resolver, +{ + let mut addrs: Vec = Vec::new(); + for seed in seeds { + match seed { + MaybeResolved::Resolved(s) => addrs.push(*s), + MaybeResolved::Unresolved(a) => { + let resolved = resolver + .resolve(a) + .await + .map_err(|e| SerfError::Resolve(std::io::Error::other(e.to_string())))?; + addrs.extend(resolved); + } + } + } + Ok(addrs) +} diff --git a/serf-reactor/src/shared/mod.rs b/serf-reactor/src/shared/mod.rs new file mode 100644 index 00000000..8064aab9 --- /dev/null +++ b/serf-reactor/src/shared/mod.rs @@ -0,0 +1,204 @@ +//! [`Shared`] — the uniform handle-to-driver shared state: a command queue + the +//! driver's parked waker, the published membership snapshot, the drop counters, +//! the teardown latch, and the live-handle count. The serf `StreamEndpoint` is +//! owned privately by the driver; handles reach it only by pushing [`Command`]s +//! and reading the snapshot. +//! +//! This is the `Send`/`Arc`/atomic sibling of serf-compio's `!Send` `Rc` +//! plus `Rc>` state, mirroring memberlist-reactor's `shared` module. The +//! command queue is a `Mutex>` woken by a stored [`Waker`] (NOT +//! a channel); the snapshot is an [`ArcSwap`] read lock-free; the counters and +//! latches are atomics. + +use std::{ + collections::VecDeque, + net::SocketAddr, + sync::{ + Arc, Mutex, + atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, + }, + task::Waker, +}; + +use arc_swap::ArcSwap; +use flume::{Receiver, Sender}; + +use crate::command::Command; +use serf_driver::SerfSnapshot; + +/// The lock-guarded part of [`Shared`]: the command queue handles push onto, the +/// driver's parked waker, and the closed latch. +struct Inner { + commands: VecDeque>, + driver_waker: Option, + /// Set once the driver has exited; no further commands are accepted, so a + /// handle never waits on a reply that can never come. + closed: bool, +} + +/// State shared between the `Serf` handle (and its clones) and the backend driver +/// task. +/// +/// Generic over the wire id type `I`; the membership address is always +/// [`SocketAddr`], so the snapshot and command channel are pinned to it. +pub(crate) struct Shared { + inner: Mutex>, + snapshot: ArcSwap>, + /// Recoverable EventStream-forward drops (a slow subscriber). + events_dropped: AtomicU64, + /// Observation-channel drops (a slow delegate; may lose application data). + observation_dropped: AtomicU64, + /// Set once the driver is shutting down; handle command methods observe it. + shutdown: AtomicBool, + /// Count of live `Serf` handles. The last to drop flips `shutdown` and wakes + /// the driver so it exits. + handles: AtomicUsize, + /// The sole sender of the teardown-completion latch, held until the driver has + /// freed its bind sockets (the UDP gossip socket and the TCP listener). The + /// latch fires once those are released — NOT once every connected stream FD has + /// closed. Dropping it disconnects `shutdown_complete_rx`; a late `shutdown()` + /// caller (whose command the closed queue rejected) awaits that disconnect so + /// it never returns into a still-bound port. + shutdown_complete_tx: Mutex>>, + /// The receiving end of the teardown-completion latch. + shutdown_complete_rx: Receiver<()>, +} + +impl Shared { + /// Builds the shared state around an initial published snapshot, with one live + /// handle. + pub(crate) fn new(initial: SerfSnapshot) -> Self { + let (shutdown_complete_tx, shutdown_complete_rx) = flume::bounded(0); + Self { + inner: Mutex::new(Inner:: { + commands: VecDeque::new(), + driver_waker: None, + closed: false, + }), + snapshot: ArcSwap::from_pointee(initial), + events_dropped: AtomicU64::new(0), + observation_dropped: AtomicU64::new(0), + shutdown: AtomicBool::new(false), + handles: AtomicUsize::new(1), + shutdown_complete_tx: Mutex::new(Some(shutdown_complete_tx)), + shutdown_complete_rx, + } + } + + /// Pushes a command and wakes the driver. Returns `false` (dropping `cmd`) if + /// the driver has already exited, so the caller can fail fast instead of + /// awaiting a reply that will never arrive. + pub(crate) fn push_command(&self, cmd: Command) -> bool { + let mut inner = self.inner.lock().unwrap(); + if inner.closed { + return false; + } + inner.commands.push_back(cmd); + if let Some(waker) = inner.driver_waker.take() { + waker.wake(); + } + true + } + + /// Driver side: parks `waker` for the next push and returns all queued + /// commands. + pub(crate) fn drain_commands(&self, waker: &Waker) -> VecDeque> { + let mut inner = self.inner.lock().unwrap(); + match &inner.driver_waker { + Some(w) if w.will_wake(waker) => {} + _ => inner.driver_waker = Some(waker.clone()), + } + core::mem::take(&mut inner.commands) + } + + /// Driver side, on exit: closes the queue (rejecting further pushes) and + /// returns any still-queued commands so the driver can fail their repliers. + pub(crate) fn close_and_drain(&self) -> VecDeque> { + let mut inner = self.inner.lock().unwrap(); + inner.closed = true; + core::mem::take(&mut inner.commands) + } + + /// Publishes a fresh membership snapshot for handles to read lock-free. + pub(crate) fn publish(&self, snap: SerfSnapshot) { + self.snapshot.store(Arc::new(snap)); + } + + /// Loads the latest published snapshot. + pub(crate) fn load_snapshot(&self) -> Arc> { + self.snapshot.load_full() + } + + /// Records `n` recoverable EventStream-forward drops. + pub(crate) fn add_events_dropped(&self, n: u64) { + self.events_dropped.fetch_add(n, Ordering::Relaxed); + } + + /// Records `n` observation-channel drops. + pub(crate) fn add_observation_dropped(&self, n: u64) { + self.observation_dropped.fetch_add(n, Ordering::Relaxed); + } + + /// The cumulative recoverable EventStream-forward drop count. + pub(crate) fn events_dropped(&self) -> u64 { + self.events_dropped.load(Ordering::Relaxed) + } + + /// The cumulative observation-channel drop count. + pub(crate) fn observation_dropped(&self) -> u64 { + self.observation_dropped.load(Ordering::Relaxed) + } + + /// Whether the driver is shutting down. + pub(crate) fn is_shutdown(&self) -> bool { + self.shutdown.load(Ordering::Acquire) + } + + /// Marks the driver as shutting down. + pub(crate) fn begin_shutdown(&self) { + self.shutdown.store(true, Ordering::Release); + } + + /// Registers a freshly cloned handle. + pub(crate) fn handle_cloned(&self) { + self.handles.fetch_add(1, Ordering::Relaxed); + } + + /// Deregisters a dropped handle; returns `true` if it was the last one (the + /// caller should then begin shutdown and wake the driver). + pub(crate) fn handle_dropped(&self) -> bool { + self.handles.fetch_sub(1, Ordering::AcqRel) == 1 + } + + /// Wakes the driver without enqueuing a command (used on last-handle drop and + /// by the accept / dial / bridge aux tasks after they enqueue for the pump). + pub(crate) fn wake_driver(&self) { + if let Some(waker) = self.inner.lock().unwrap().driver_waker.take() { + waker.wake(); + } + } + + /// Driver side, at the very end of teardown — once the bind sockets are free + /// (the driver has dropped its UDP gossip socket and released the accept task's + /// TCP listener): fire the completion latch so any late `shutdown()` caller + /// parked in [`wait_shutdown_complete`](Self::wait_shutdown_complete) returns. + /// Dropping the sole sender disconnects the receiver; idempotent across + /// re-entrant polls. + pub(crate) fn mark_shutdown_complete(&self) { + // Ignoring the take result: dropping the sender fires the latch; a second + // teardown poll finds `None` and is a no-op. + let _ = self.shutdown_complete_tx.lock().unwrap().take(); + } + + /// Handle side: await teardown completion. Returns as soon as the driver has + /// fired the latch via [`mark_shutdown_complete`](Self::mark_shutdown_complete), + /// or immediately if it already has — i.e. once the bind address is free, not + /// once every connected stream FD has closed. A `shutdown()` caller whose + /// command the closed queue rejected waits here for the ports to free rather + /// than returning into a still-bound address. + pub(crate) async fn wait_shutdown_complete(&self) { + // Ignoring the recv result: the latch fires by sender-disconnect, never by a + // sent value, so recv resolves to Err exactly once teardown completes. + let _ = self.shutdown_complete_rx.recv_async().await; + } +} diff --git a/serf-reactor/src/tcp/mod.rs b/serf-reactor/src/tcp/mod.rs new file mode 100644 index 00000000..63e17823 --- /dev/null +++ b/serf-reactor/src/tcp/mod.rs @@ -0,0 +1,350 @@ +//! TCP-backed serf driver over the agnostic runtime. +//! +//! [`TcpTransport`] owns the bound UDP gossip socket and TCP reliable listener. +//! The machine-layer `serf_proto::StreamEndpoint` is +//! built inside [`TcpTransport::run`] from the stored stream knobs and the serf +//! `Options` carried by the [`TransportRuntime`](crate::TransportRuntime). This is +//! the `Send`/`agnostic` sibling of serf-compio's `!Send`, compio-bound +//! `TcpTransport`. + +#![cfg(feature = "tcp")] + +use core::num::NonZeroU8; +use std::{io::ErrorKind, net::SocketAddr}; + +use agnostic::{ + Runtime, + net::{Net, TcpListener, UdpSocket}, +}; +use hostaddr::HostAddr; +use memberlist_proto::{ + CheapClone, Data, Endpoint, EndpointOptions, Id, MaybeResolved, RawRecords, + streams::{LabelOptions, StreamEndpoint as Coordinator}, +}; +use rand::rngs::StdRng; +use smol_str::SmolStr; + +#[cfg(encryption)] +use memberlist_proto::EncryptionOptions; + +use crate::{ + SerfError, + delegate::Delegate, + driver::options::StreamTransportOptions, + resolver::{AdvertiseAddrResolver, Resolver}, + transport::{Transport, TransportRuntime}, +}; + +/// Per-backend TCP-specific transport options. +/// +/// Bundles the local node identifier, the (possibly-unresolved) advertise +/// address, and the stream-transport tuning knobs. The cluster label and +/// inbound-label-check policy are supplied via the serf `Options` block (not +/// here), feeding both planes from a single validated source. +pub struct TcpTransportOptions> { + local_id: Option, + advertise_addr: Option>, + stream: StreamTransportOptions, + /// Gossip-and-reliable encryption policy. The default (no keyring) leaves both + /// planes plaintext; attaching a keyring via [`with_encryption`](Self::with_encryption) + /// makes the coordinator's `encrypt_gossip`/`decrypt_gossip` (and the plain-TCP + /// reliable record layer) AEAD-protect every datagram and stream unit. + #[cfg(encryption)] + encryption: EncryptionOptions, +} + +impl TcpTransportOptions { + /// Construct with defaults. Caller MUST chain + /// [`with_local_id`](Self::with_local_id) and + /// [`with_advertise_addr`](Self::with_advertise_addr) before passing to + /// `TcpTransport::new`. + #[inline] + pub fn new() -> Self { + Self { + local_id: None, + advertise_addr: None, + stream: StreamTransportOptions::new(), + #[cfg(encryption)] + encryption: EncryptionOptions::new(), + } + } + + /// Builder: local node identifier. + #[must_use] + #[inline] + pub fn with_local_id(mut self, id: I) -> Self { + self.local_id = Some(id); + self + } + + /// Builder: advertise address (resolved or unresolved). + #[must_use] + #[inline] + pub fn with_advertise_addr(mut self, addr: MaybeResolved) -> Self { + self.advertise_addr = Some(addr); + self + } + + /// Builder: stream-transport tuning knobs. + #[must_use] + #[inline] + pub fn with_stream(mut self, opts: StreamTransportOptions) -> Self { + self.stream = opts; + self + } + + /// Builder: gossip-and-reliable encryption policy. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + #[must_use] + #[inline] + pub fn with_encryption(mut self, encryption: EncryptionOptions) -> Self { + self.encryption = encryption; + self + } + + /// Local node identifier, if set. + #[inline] + pub const fn local_id(&self) -> Option<&I> { + self.local_id.as_ref() + } + + /// Advertise address, if set. + #[inline] + pub const fn advertise_addr(&self) -> Option<&MaybeResolved> { + self.advertise_addr.as_ref() + } + + /// Stream-transport tuning knobs. + #[inline] + pub const fn stream(&self) -> &StreamTransportOptions { + &self.stream + } + + /// Gossip-and-reliable encryption policy. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + #[inline] + pub const fn encryption(&self) -> &EncryptionOptions { + &self.encryption + } +} + +impl Default for TcpTransportOptions { + #[inline] + fn default() -> Self { + Self::new() + } +} + +/// TCP-backed serf transport. +/// +/// Owns the bound `UdpSocket` (gossip unreliable plane) and `TcpListener` +/// (reliable coordinator). The machine-layer +/// `serf_proto::StreamEndpoint` is built inside +/// [`Transport::run`] from the serf options sourced from +/// [`TransportRuntime`](crate::TransportRuntime). +pub struct TcpTransport +where + R: Runtime, +{ + local_id: I, + local_address: MaybeResolved, + advertise_socket: SocketAddr, + gossip_socket: ::UdpSocket, + tcp_listener: ::TcpListener, + stream_options: StreamTransportOptions, + /// Independent OS-seeded seed for the serf core's RNG, drawn once per node in + /// [`Transport::new`] and consumed when [`Transport::run`] builds the endpoint. + serf_rng: StdRng, + /// Gossip-and-reliable encryption policy applied to the coordinator built in + /// [`Transport::run`]. Absent keyring ⇒ plaintext (the default). + #[cfg(encryption)] + encryption: EncryptionOptions, +} + +impl Transport for TcpTransport +where + R: Runtime, + I: + Id + CheapClone + Clone + core::fmt::Debug + core::fmt::Display + Send + Sync + Unpin + 'static, + A: Data + Clone + Send + Sync + 'static, +{ + type Error = SerfError; + type Id = I; + type Address = A; + type Options = TcpTransportOptions; + + async fn new( + options: Self::Options, + resolver: &RES, + advertise_resolver: &AR, + ) -> Result + where + RES: Resolver
, + AR: AdvertiseAddrResolver, + { + // Validate stream knobs that would deterministically break the backend BEFORE + // binding any socket. + options.stream.validate()?; + + let local_id = options.local_id.ok_or_else(|| { + SerfError::Io(std::io::Error::new( + ErrorKind::InvalidInput, + "local_id required", + )) + })?; + let advertise_input = options.advertise_addr.ok_or_else(|| { + SerfError::Io(std::io::Error::new( + ErrorKind::InvalidInput, + "advertise_addr required", + )) + })?; + + let advertise_socket = match &advertise_input { + MaybeResolved::Resolved(s) => *s, + MaybeResolved::Unresolved(a) => { + let candidates = resolver + .resolve(a) + .await + .map_err(|e| SerfError::Resolve(std::io::Error::other(e.to_string())))?; + advertise_resolver.pick(candidates).map_err(|e| { + SerfError::Resolve(std::io::Error::new( + ErrorKind::AddrNotAvailable, + e.to_string(), + )) + })? + } + }; + + // Bind the TCP listener first to claim an OS-assigned free port, then bind the + // gossip UDP socket to that same port. TCP and UDP port spaces are independent, + // so for an ephemeral (`:0`) advertise we retry the pair on a fresh port when + // the UDP bind fails transiently: AddrInUse from the port-space race, or + // PermissionDenied when the TCP-claimed port falls in a UDP-excluded range. A + // dropped agnostic socket closes its FD synchronously, so an abandoned attempt + // never leaks a bound port. + const EPHEMERAL_BIND_RETRIES: usize = 16; + let ephemeral = advertise_socket.port() == 0; + let (tcp_listener, bound, gossip_socket) = { + let mut attempt = 0usize; + loop { + let tcp_listener = ::TcpListener::bind(advertise_socket) + .await + .map_err(SerfError::Io)?; + let bound = tcp_listener.local_addr().map_err(SerfError::Io)?; + match ::UdpSocket::bind(bound).await { + Ok(gossip_socket) => break (tcp_listener, bound, gossip_socket), + Err(e) + if ephemeral + && matches!(e.kind(), ErrorKind::AddrInUse | ErrorKind::PermissionDenied) + && attempt < EPHEMERAL_BIND_RETRIES => + { + // Release the claimed TCP port (drop closes the FD) and retry a fresh + // ephemeral pair. + drop(tcp_listener); + attempt += 1; + } + Err(e) => return Err(SerfError::Io(e)), + } + } + }; + + // Both sockets are now bound. The readback resolves an ephemeral `:0` to a + // concrete port but keeps an unspecified IP: `post_bind_setup` rejects an + // advertise address peers could not route serf traffic back to, then draws the + // OS-seeded serf-core RNG. Either failure drops BOTH bound sockets (dropping an + // agnostic socket closes its FD) before returning. + let serf_rng = match crate::transport::post_bind_setup(&bound) { + Ok(rng) => rng, + Err(e) => { + drop(tcp_listener); + drop(gossip_socket); + return Err(e); + } + }; + + Ok(Self { + local_id, + local_address: advertise_input, + advertise_socket: bound, + gossip_socket, + tcp_listener, + stream_options: options.stream, + serf_rng, + #[cfg(encryption)] + encryption: options.encryption, + }) + } + + #[inline] + fn local_id(&self) -> &Self::Id { + &self.local_id + } + + #[inline] + fn local_address(&self) -> &MaybeResolved { + &self.local_address + } + + #[inline] + fn advertise_address(&self) -> &SocketAddr { + &self.advertise_socket + } + + async fn run(self, runtime: TransportRuntime, gossip_rng: G) + where + D: Delegate, + G: rand::Rng + Send + Unpin + 'static, + { + // `Serf::new` is generic over `T` and cannot build the record-layer-specific + // endpoint; build it here from `self`'s stored config. Serf ranks its user + // broadcasts on three tiers (intent / event / query → ranks 0 / 1 / 2), so the + // inner memberlist endpoint needs at least three broadcast tiers. + let inner_opts = EndpointOptions::new(self.local_id, self.advertise_socket) + .with_user_broadcast_tiers(NonZeroU8::new(3).expect("3 is nonzero")); + let inner = Endpoint::new(inner_opts, gossip_rng); + // Plain TCP has no SNI (`|_| None`) and a membership address that IS the + // transport socket (`|addr| *addr`). No cluster label at this stage. + #[allow(unused_mut)] + let mut coord = Coordinator::<_, _, RawRecords, G>::new( + inner, + LabelOptions::new_in(None::>, ()), + Box::new(|_: &SocketAddr| -> Option { None }), + Box::new(|addr: &SocketAddr| *addr), + ); + // Install the gossip-encryption keyring. A no-keyring policy is the identity + // transform, so an unencrypted node is unaffected. + #[cfg(encryption)] + coord.set_encryption_options(self.encryption); + // Serf's core RNG is seeded from its own OS-drawn entropy (`self.serf_rng`), + // independent of the coordinator's gossip RNG. + let endpoint = + serf_proto::StreamEndpoint::::new_with_rng( + coord, + runtime.serf_options, + self.serf_rng, + ); + + let driver = crate::driver::stream::spawn_stream_driver::( + endpoint, + self.gossip_socket, + self.tcp_listener, + runtime.shared, + runtime.events_tx, + runtime.delegate, + runtime.driver_options, + self.stream_options, + None, + #[cfg(encryption)] + runtime.keyring, + ); + driver.await; + } +} diff --git a/serf-reactor/src/transport/mod.rs b/serf-reactor/src/transport/mod.rs new file mode 100644 index 00000000..719d4b13 --- /dev/null +++ b/serf-reactor/src/transport/mod.rs @@ -0,0 +1,172 @@ +//! `Transport` — abstracts a per-backend serf driver (TCP/TLS/QUIC) over the +//! agnostic runtime `R`. +//! +//! Concrete impls live in `src/{tcp,tls,quic}.rs`. The Sans-I/O machine endpoint +//! (`serf_proto::StreamEndpoint` for TCP/TLS) is built inside +//! `T::run` from the transport's stored config — a generic `Serf::new` cannot +//! build the backend's private record-layer config + dial closures, so the +//! endpoint never flows through the [`TransportRuntime`] bundle. +//! +//! This is the `Send`/`agnostic` sibling of serf-compio's `!Send`, compio-implicit +//! `Transport`: the trait is generic over `R: agnostic::Runtime`, and `new` / +//! `run` return `Send` futures. + +use core::future::Future; +use std::net::SocketAddr; + +use agnostic::Runtime; +use memberlist_proto::MaybeResolved; + +use crate::{ + delegate::Delegate, + resolver::{AdvertiseAddrResolver, Resolver}, +}; + +pub mod runtime; +pub use runtime::TransportRuntime; + +/// Abstracts a per-backend serf driver over the agnostic runtime `R`. The trait +/// owns construction, resource ownership (bound sockets / TCP listener / quinn +/// endpoint), the Sans-I/O machine endpoint, and — via +/// [`run`](Transport::run) — the spawned driver pump. +/// +/// `Self::Error` is bounded by `From` — convertible from the OS +/// layer. `Resolver` and `AdvertiseAddrResolver` are call-site arguments to +/// `Self::new`, NOT associated types, so users can swap resolvers without changing +/// the `Transport` type. +/// +/// `Self::new` / `Self::run` return `Send` futures so a node can be built and +/// driven on a multi-threaded agnostic runtime. +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub trait Transport: Sized + Send + 'static +where + R: Runtime, +{ + /// Per-backend error type. + type Error: core::error::Error + From + Send + Sync + 'static; + + /// Node identifier type. + type Id; + + /// User-facing unresolved address type (e.g. `hostaddr::HostAddr`). + type Address; + + /// Per-backend transport-knobs block embedded into `Options`. + type Options; + + /// Construct the transport. Resolves `options.advertise_addr` via the + /// caller-supplied resolvers if it is `MaybeResolved::Unresolved(…)`; binds the + /// UDP gossip socket + TCP listener (or the QUIC endpoint) and stores them. + fn new( + options: Self::Options, + resolver: &RES, + advertise_resolver: &AR, + ) -> impl Future> + Send + where + RES: Resolver
, + AR: AdvertiseAddrResolver; + + /// Local node identifier. + fn local_id(&self) -> &Self::Id; + + /// Original advertise input form — `Resolved` if the user supplied a concrete + /// `SocketAddr`; `Unresolved` if a hostname was resolved at construction. + fn local_address(&self) -> &MaybeResolved; + + /// Bound advertise `SocketAddr` — what the local node gossips to peers and what + /// the UDP / QUIC socket is bound to. + fn advertise_address(&self) -> &SocketAddr; + + /// Run the driver pump. Consumes `self` (sockets and listener move into the + /// driver), the [`TransportRuntime`] bundle (shared state, events sender, + /// delegate, tuning knobs), and the gossip RNG the node constructor drew; the + /// body builds the machine endpoint from `self`'s stored config, spawns the + /// observation + accept tasks, and awaits the stream driver. Returns when + /// shutdown is requested. + fn run( + self, + runtime: TransportRuntime, + gossip_rng: G, + ) -> impl Future + Send + where + D: Delegate, + G: rand::Rng + Send + Unpin + 'static; +} + +/// Validate that the resolved advertise address can serve as the local node's +/// reachable, wire-representable contact identity. +/// +/// Each transport reads the advertise `SocketAddr` back from its bound socket +/// after construction and gossips it as the node's published contact. An address +/// that the codec encodes fine but that is undialable — classically the wildcard +/// `0.0.0.0:0` / `[::]:0`, whose `local_addr()` readback keeps the unspecified IP +/// — would let the node join a cluster as a member no peer can route to. +/// +/// Rejected with [`SerfError::InvalidAdvertiseAddr`](crate::SerfError::InvalidAdvertiseAddr) +/// (not clamped) for either class: +/// +/// - NOT A USABLE UNICAST CONTACT — an unspecified IP (`0.0.0.0` / `::`), a +/// multicast IP, an IPv4 broadcast IP (`255.255.255.255`), or a zero port. +/// - NOT REPRESENTABLE ON THE WIRE — a scoped/flow-labelled IPv6 address with a +/// nonzero `scope_id` or `flowinfo`, which the compact `[16B IP][2B port]` wire +/// layout carries neither field of. +/// +/// Loopback, private, and global unicast addresses stay valid. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) fn validate_advertise_addr(advertise_addr: &SocketAddr) -> Result<(), crate::SerfError> { + let reject = |reason: &str| { + Err(crate::SerfError::InvalidAdvertiseAddr( + crate::error::InvalidAdvertiseAddr::new(*advertise_addr, reason.to_string()), + )) + }; + + let ip = advertise_addr.ip(); + if ip.is_unspecified() { + return reject( + "an unspecified IP (0.0.0.0 / ::) is the wildcard-bind address, not a routable contact \ + — peers cannot dial it (set a concrete advertise address, or resolve one from the \ + host's interfaces, when binding the wildcard)", + ); + } + if ip.is_multicast() { + return reject("a multicast IP is a group address, not a single peer's unicast contact"); + } + if let SocketAddr::V4(v4) = advertise_addr + && v4.ip().is_broadcast() + { + return reject("an IPv4 broadcast IP (255.255.255.255) is not a unicast contact"); + } + if advertise_addr.port() == 0 { + return reject( + "a zero port is undialable — the bound socket's local_addr() readback must carry a \ + concrete port", + ); + } + if let SocketAddr::V6(v6) = advertise_addr + && (v6.scope_id() != 0 || v6.flowinfo() != 0) + { + return reject( + "a scoped/flow-labelled IPv6 address (nonzero scope_id or flowinfo) is not representable \ + on the compact `[16B IP][2B port]` wire layout, so peers could not decode a routable \ + contact for this node", + ); + } + Ok(()) +} + +/// Run the post-bind construction steps common to every transport: reject an +/// undialable advertise address, then draw the OS-seeded serf-core RNG (distinct +/// from the coordinator's gossip RNG so serf's query IDs and relay choices are not +/// correlated across nodes). +/// +/// Both are fallible AFTER the transport's socket(s) are already bound, so the +/// caller drops its bound socket(s) before returning the `Err` this produces +/// (dropping an agnostic socket closes its FD synchronously). +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) fn post_bind_setup( + advertise_addr: &SocketAddr, +) -> Result { + validate_advertise_addr(advertise_addr)?; + crate::os_seeded_std_rng() +} diff --git a/serf-reactor/src/transport/runtime.rs b/serf-reactor/src/transport/runtime.rs new file mode 100644 index 00000000..e0d76377 --- /dev/null +++ b/serf-reactor/src/transport/runtime.rs @@ -0,0 +1,67 @@ +//! [`TransportRuntime`] — the bundle handed to `Transport::run(self, +//! runtime, gossip_rng)`. +//! +//! Carries the serf observation delegate, the handle-to-driver [`Shared`] state +//! (the reactor's command queue lives there, NOT in a channel), the events +//! sender, and the driver / serf tuning knobs. The concrete machine endpoint is +//! NOT carried here: a generic `Serf::new` cannot build the backend's +//! record-layer config + dial closures, so each `T::run` body builds its own +//! endpoint from the transport's stored config and then drives the shared stream +//! driver. + +use std::{net::SocketAddr, sync::Arc}; + +use flume::Sender; +use serf_proto::{event::Event, options::Options as SerfOptions}; + +use crate::{driver::options::RuntimeOptions, shared::Shared}; + +#[cfg(encryption)] +use crate::delegate::KeyringDelegate; + +/// Bundle handed to `Transport::run(self, runtime, gossip_rng)`. +/// +/// Generic over the wire id `I` and the observation delegate `D`. The membership +/// address is always [`SocketAddr`], so the events channel and [`Shared`] state +/// are pinned to it. The machine endpoint is built inside `T::run` (it needs the +/// backend's private record-layer config), so it is deliberately absent here. +/// +/// Requires a stream or QUIC transport feature. +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub struct TransportRuntime { + pub(crate) delegate: D, + pub(crate) shared: Arc>, + pub(crate) events_tx: Sender>, + pub(crate) driver_options: RuntimeOptions, + pub(crate) serf_options: SerfOptions, + /// The driver's keyring delegate, applied to inbound key-management requests. + /// Present only under an encryption backend. + #[cfg(encryption)] + pub(crate) keyring: Arc, +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +impl TransportRuntime { + /// Construct the runtime bundle. Called by the `Serf` handle constructor. + #[allow(clippy::too_many_arguments)] + #[inline] + pub(crate) fn new( + delegate: D, + shared: Arc>, + events_tx: Sender>, + driver_options: RuntimeOptions, + serf_options: SerfOptions, + #[cfg(encryption)] keyring: Arc, + ) -> Self { + Self { + delegate, + shared, + events_tx, + driver_options, + serf_options, + #[cfg(encryption)] + keyring, + } + } +} From ae19315e09da0494336532f198e561aa33a292fa Mon Sep 17 00:00:00 2001 From: al8n Date: Wed, 1 Jul 2026 23:04:53 +1200 Subject: [PATCH 027/128] feat(serf-reactor): handle constructors, snapshot accessors, real-node TCP tests Add ergonomic Serf::tcp / tcp_with_rng over the generic Serf::new::, plus src/snapshot.rs and the snapshot public read-forwarders (members/local_member/state/advertise_node/advertise_address/local_id/default_query_*/remove_failed_node*) porting serf-compio's semantics. Add the real-node TCP suite on tokio (2-node join/converge/user_event/query/leave-LeftCluster/shutdown) proving the reactor stream driver works end-to-end; mirrors memberlist-reactor's harness. --- serf-reactor/Cargo.toml | 11 + serf-reactor/src/lib.rs | 6 + serf-reactor/src/serf/mod.rs | 215 +++++++++++++++++++- serf-reactor/src/serf/tests.rs | 243 ++++++++++++++++++++++ serf-reactor/src/snapshot.rs | 16 ++ serf-reactor/tests/tcp.rs | 356 +++++++++++++++++++++++++++++++++ 6 files changed, 842 insertions(+), 5 deletions(-) create mode 100644 serf-reactor/src/serf/tests.rs create mode 100644 serf-reactor/src/snapshot.rs create mode 100644 serf-reactor/tests/tcp.rs diff --git a/serf-reactor/Cargo.toml b/serf-reactor/Cargo.toml index 0c0db280..37cdb841 100644 --- a/serf-reactor/Cargo.toml +++ b/serf-reactor/Cargo.toml @@ -96,6 +96,17 @@ rcgen = "0.14" ring = "0.17" # Used in the serde option-layer tests. serde_json = "1" +# The real-node TCP tests run the reactor stream driver on a concrete runtime; +# `#[tokio::test]` needs the tokio macros + a multi-thread runtime with timers. +# The `TokioRuntime` itself comes from `agnostic` under the crate's `tokio` +# feature (which the test targets require). +tokio = { version = "1", features = ["rt-multi-thread", "macros", "time"] } + +# The real-node TCP test suite (two loopback nodes: join/converge, user event, +# query round-trip, leave/LeftCluster, snapshot forwarders, shutdown). +[[test]] +name = "tcp" +required-features = ["tcp", "tokio"] [package.metadata.docs.rs] # A single coherent crypto-provider set (ring, not aws-lc-rs; tokio, not smol) so diff --git a/serf-reactor/src/lib.rs b/serf-reactor/src/lib.rs index 7501e303..42201c76 100644 --- a/serf-reactor/src/lib.rs +++ b/serf-reactor/src/lib.rs @@ -59,6 +59,7 @@ mod resolver; mod serf; #[cfg(any(feature = "tcp", feature = "quic"))] mod shared; +mod snapshot; #[cfg(feature = "tcp")] mod tcp; #[cfg(any(feature = "tcp", feature = "quic"))] @@ -113,6 +114,11 @@ pub use resolver::{ #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] pub use serf::Serf; +/// The published, lock-free membership snapshot a [`Serf`] handle reads. +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub use snapshot::SerfSnapshot; + #[cfg(any(feature = "tcp", feature = "quic"))] #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] pub use transport::{Transport, TransportRuntime}; diff --git a/serf-reactor/src/serf/mod.rs b/serf-reactor/src/serf/mod.rs index 54747a71..b5ee293d 100644 --- a/serf-reactor/src/serf/mod.rs +++ b/serf-reactor/src/serf/mod.rs @@ -5,10 +5,11 @@ //! channel, and spawns the driver pump on the agnostic runtime `R`. Every command //! method pushes one [`Command`] onto the shared queue and (for the awaited kinds) //! parks on a one-shot reply the driver resolves. `Clone` / `Drop` reference-count -//! the shared driver; membership reads go through the lock-free -//! [`SerfSnapshot`]. The ergonomic per-backend constructors (`tcp` / `tls` / -//! `quic`), the richer snapshot accessors, and the full test suite are layered on -//! in a later chunk. +//! the shared driver; membership reads go through the lock-free [`SerfSnapshot`] +//! and the snapshot read-forwarders (`members` / `local_member` / `state` / +//! `advertise_node` / `default_query_*`). The ergonomic per-backend constructor +//! [`Serf::tcp`] instantiates the transport for the caller; `tls` / `quic` slot in +//! the same way once those backends land. use core::{marker::PhantomData, time::Duration}; use std::{net::SocketAddr, sync::Arc}; @@ -17,7 +18,6 @@ use agnostic::Runtime; use bytes::Bytes; use futures_channel::oneshot; use memberlist_proto::{Instant, Node}; -use serf_driver::SerfSnapshot; use serf_proto::{ LamportTime, endpoint::{QueryId, QueryParams}, @@ -33,6 +33,8 @@ use smol_str::SmolStr; use crate::command::{KeyCmd, ListKeysCmd}; #[cfg(encryption)] use crate::delegate::KeyringDelegate; +#[cfg(feature = "tcp")] +use crate::tcp::{TcpTransport, TcpTransportOptions}; use crate::{ MaybeResolved, command::{ @@ -45,10 +47,13 @@ use crate::{ events::EventStream, resolver::{AdvertiseAddrResolver, Resolver}, shared::Shared, + snapshot::SerfSnapshot, transport::{Transport, TransportRuntime}, }; #[cfg(encryption)] use memberlist_proto::SecretKey; +#[cfg(feature = "tcp")] +use memberlist_proto::{CheapClone, Data}; /// The initial published snapshot: the local node, `Alive`, with empty tags and /// zeroed Lamport clocks. Superseded by the driver's first real republish. @@ -90,6 +95,10 @@ pub struct Serf { /// Per-call await-result join deadline offset, cached from the runtime options /// so each `join` can stamp its absolute `WaitForCompletion` deadline. join_deadline: Duration, + /// Cached `query_timeout_mult` from the serf options, so + /// [`default_query_timeout`](Serf::default_query_timeout) derives the query + /// timeout from the live snapshot member count without a driver round-trip. + query_timeout_mult: usize, /// Ties the handle to the resolver's unresolved address type. Not held in any /// field — `join` enforces seeds resolve in this address domain. _a: PhantomData, @@ -105,6 +114,7 @@ impl Clone for Serf { shared: self.shared.clone(), events_rx: self.events_rx.clone(), join_deadline: self.join_deadline, + query_timeout_mult: self.query_timeout_mult, _a: PhantomData, _r: PhantomData, } @@ -158,6 +168,9 @@ where runtime_options.validate().map_err(T::Error::from)?; // Cache the join deadline before `runtime_options` moves into the bundle. let join_deadline = runtime_options.join_deadline(); + // Cache `query_timeout_mult` before `serf_options` moves into the bundle, so + // the handle can compute `default_query_timeout` without a driver round-trip. + let query_timeout_mult = serf_options.query_timeout_mult(); let transport = T::new(options, resolver, advertise_resolver).await?; let local_id = transport.local_id().clone(); @@ -185,12 +198,107 @@ where shared, events_rx, join_deadline, + query_timeout_mult, _a: PhantomData, _r: PhantomData, }) } } +// Ergonomic per-backend constructors: instantiate the transport for the caller so +// a node can be built without naming the generic `Serf::new::` machinery. +// `tls` / `quic` slot in the same way once those backends land. +#[cfg(feature = "tcp")] +#[cfg_attr(docsrs, doc(cfg(feature = "tcp")))] +impl Serf +where + I: memberlist_proto::Id + + CheapClone + + Clone + + core::fmt::Debug + + core::fmt::Display + + Send + + Sync + + Unpin + + 'static, + A: Data + Clone + Send + Sync + 'static, + R: Runtime, +{ + /// Build a TCP-backed serf node and spawn its driver on the runtime `R`. + /// + /// The ergonomic wrapper over [`Serf::new`] that instantiates the + /// [`TcpTransport`](crate::TcpTransport) for the caller: it binds a UDP gossip + /// socket and a TCP reliable listener on the advertise address (resolved once + /// via `resolver` / `advertise_resolver`), then spawns the stream driver. The + /// gossip RNG is drawn from OS entropy via [`gossip_rng`](crate::gossip_rng); + /// use [`tcp_with_rng`](Self::tcp_with_rng) to supply your own. + /// + /// Under an encryption backend, pass an + /// [`Arc`](crate::KeyringDelegate) + /// (`Arc::new(VoidKeyringDelegate)` for a node that manages no keys). + #[allow(clippy::too_many_arguments)] + pub async fn tcp( + options: TcpTransportOptions, + resolver: &RES, + advertise_resolver: &AR, + delegate: D, + runtime_options: RuntimeOptions, + serf_options: SerfOptions, + #[cfg(encryption)] keyring: Arc, + ) -> Result + where + RES: Resolver
, + AR: AdvertiseAddrResolver, + D: Delegate, + { + Self::tcp_with_rng( + options, + resolver, + advertise_resolver, + delegate, + runtime_options, + serf_options, + crate::gossip_rng()?, + #[cfg(encryption)] + keyring, + ) + .await + } + + /// Like [`tcp`](Self::tcp) but with a caller-supplied gossip RNG `G` — draw it + /// via [`gossip_rng`](crate::gossip_rng) for fork-safe OS entropy. + #[allow(clippy::too_many_arguments)] + pub async fn tcp_with_rng( + options: TcpTransportOptions, + resolver: &RES, + advertise_resolver: &AR, + delegate: D, + runtime_options: RuntimeOptions, + serf_options: SerfOptions, + gossip_rng: G, + #[cfg(encryption)] keyring: Arc, + ) -> Result + where + RES: Resolver
, + AR: AdvertiseAddrResolver, + D: Delegate, + G: rand::Rng + Send + Unpin + 'static, + { + Self::new::, RES, AR, D, G>( + options, + resolver, + advertise_resolver, + delegate, + runtime_options, + serf_options, + gossip_rng, + #[cfg(encryption)] + keyring, + ) + .await + } +} + // Handle operations that read a cached snapshot or push a command over the queue — // none touch node identity directly, so they impose no bound and stay callable on // a `Serf` of any id type. @@ -207,6 +315,86 @@ impl Serf { self.shared.load_snapshot().num_members() } + /// All known cluster members at the latest published snapshot instant — alive, + /// leaving, left, and failed within the reap window. Mirrors serf-compio's + /// `Serf::members`. + #[must_use] + pub fn members(&self) -> Vec>> { + self.snapshot().members().to_vec() + } + + /// The local node's full membership view at the latest published snapshot + /// instant. Returns the same `Arc` that lives at the local index in + /// [`members`](Self::members), so it is always consistent with that view. + /// Mirrors serf-compio's `Serf::local_member`. + #[must_use] + pub fn local_member(&self) -> Arc> { + self.snapshot().local() + } + + /// The lifecycle state of the local serf endpoint, derived from the latest + /// published snapshot. Mirrors serf-compio's `Serf::state`. + #[must_use] + pub fn state(&self) -> SerfState { + self.snapshot().state() + } + + /// The bound advertise address this node gossips to peers, read from the local + /// member of the latest published snapshot. Mirrors serf-compio's + /// `Serf::advertise_address`. + #[must_use] + pub fn advertise_address(&self) -> SocketAddr { + *self.snapshot().local_ref().node().addr_ref() + } + + /// The local node's id, read from the local member of the latest published + /// snapshot. Mirrors serf-compio's `Serf::local_id` (returned owned here, as the + /// snapshot is loaded by value). + #[must_use] + pub fn local_id(&self) -> I + where + I: Clone, + { + self.snapshot().local_ref().node().id_ref().clone() + } + + /// The local node as an `(id, advertise-address)` [`Node`], composed from the + /// local member of the latest published snapshot. Mirrors serf-compio's + /// `Serf::advertise_node`. + #[must_use] + pub fn advertise_node(&self) -> Node + where + I: Clone, + { + self.snapshot().local().node().clone() + } + + /// Default query timeout derived from the current snapshot member count. + /// + /// Computed as `200ms × query_timeout_mult × ⌈log₁₀(N+1)⌉` where N is the + /// snapshot member count, matching the machine's own zero-timeout resolution. + /// Mirrors serf-compio's `Serf::default_query_timeout`. + #[must_use] + pub fn default_query_timeout(&self) -> Duration { + let n = self.num_members(); + let log_factor = ((n as f64 + 1.0).log10().ceil() as u32).max(1); + Duration::from_millis(200) * self.query_timeout_mult as u32 * log_factor + } + + /// Default query parameters derived from the current snapshot: no filters, no + /// relay, no ACK, and a timeout from + /// [`default_query_timeout`](Self::default_query_timeout). Mirrors serf-compio's + /// `Serf::default_query_param`. + #[must_use] + pub fn default_query_param(&self) -> QueryParams { + QueryParams { + filters: Vec::new(), + relay_factor: 0, + request_ack: false, + timeout: self.default_query_timeout(), + } + } + /// Subscribe to the serf event stream (membership transitions, user events, /// queries, responses). #[must_use] @@ -374,6 +562,20 @@ impl Serf { await_reply(rx).await } + /// Force-remove a failed node immediately without pruning the tombstone. Thin + /// alias for `force_leave(id, false)`; serf stops attempting to reconnect. + /// Mirrors serf-compio's `Serf::remove_failed_node`. + pub async fn remove_failed_node(&self, id: I) -> Result<()> { + self.force_leave(id, false).await + } + + /// Force-remove a failed node immediately and prune the tombstone. Thin alias + /// for `force_leave(id, true)`; the node is removed at once rather than after + /// the tombstone timeout. Mirrors serf-compio's `Serf::remove_failed_node_prune`. + pub async fn remove_failed_node_prune(&self, id: I) -> Result<()> { + self.force_leave(id, true).await + } + /// Broadcast a user-defined event cluster-wide. pub async fn user_event( &self, @@ -553,3 +755,6 @@ where } Ok(addrs) } + +#[cfg(all(test, feature = "tcp", feature = "tokio"))] +mod tests; diff --git a/serf-reactor/src/serf/tests.rs b/serf-reactor/src/serf/tests.rs new file mode 100644 index 00000000..b5dc1c48 --- /dev/null +++ b/serf-reactor/src/serf/tests.rs @@ -0,0 +1,243 @@ +//! Handle-level unit tests for the reactor `Serf` on tokio: the ergonomic +//! `Serf::tcp` constructor's option validation, single-node shutdown/rebind, and +//! the fast join-failure paths. The multi-node convergence / event / query / +//! leave behavior is covered by the real-node suite in `tests/tcp.rs`. + +use core::{future::Future, time::Duration}; +use std::net::SocketAddr; + +use agnostic::tokio::TokioRuntime; + +use crate::{ + Channel, FirstAddrResolver, MaybeResolved, Resolver, RuntimeOptions, Serf, SerfError, + SocketAddrResolver, TcpTransportOptions, VoidDelegate, +}; +use serf_proto::options::Options as SerfOptions; +use smol_str::SmolStr; + +/// A tokio-backed reactor TCP node handle. +type Node = Serf; + +/// A loopback address with a port nothing listens on — `connect()` returns +/// `ECONNREFUSED` immediately, so its push/pull exchange fails fast. The port is +/// below the OS ephemeral range, so a `:0` test bind never collides with it. +fn blackhole_addr() -> SocketAddr { + "127.0.0.1:7217".parse().expect("loopback addr") +} + +/// Resolver that always resolves to an empty address list — models a +/// service-discovery resolver that finds no live endpoints under a service key. +struct EmptyResolver; + +impl Resolver for EmptyResolver { + type Address = String; + type Error = std::io::Error; + + // The trait's `resolve` future is bound `+ '_` to the `&self` lifetime; an + // `async fn` would also capture the (unused) `&addr` lifetime and fail to + // satisfy it, so the future is written explicitly, borrowing neither argument. + #[allow(clippy::manual_async_fn)] + fn resolve( + &self, + _addr: &String, + ) -> impl Future, std::io::Error>> + Send + '_ { + async move { Ok(Vec::new()) } + } +} + +/// Build a reactor TCP node bound to a specific advertise address, returning the +/// construction result so the same-address rebind regression can assert a freed +/// port accepts an immediate rebind. +async fn try_spawn_node_at(id: &str, bind: SocketAddr) -> Result { + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(bind)); + Serf::::tcp( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + #[cfg(encryption)] + std::sync::Arc::new(crate::VoidKeyringDelegate), + ) + .await +} + +/// Build and spawn a reactor TCP node bound to an ephemeral loopback port. +async fn spawn_node(id: &str) -> Node { + try_spawn_node_at(id, "127.0.0.1:0".parse().expect("loopback addr")) + .await + .expect("spawn serf node") +} + +/// Build VALID TCP transport options paired with a deliberately invalid +/// `runtime`, and assert `Serf::tcp` rejects it with [`SerfError::InvalidOption`] +/// — before binding a socket or spawning the detached driver — rather than +/// returning `Ok` and later panicking the driver task on a zero-capacity channel. +async fn assert_tcp_new_rejects(runtime: RuntimeOptions) { + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new("bad-opt-node")) + .with_advertise_addr(MaybeResolved::Resolved(bind)); + let res = Serf::::tcp( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + runtime, + SerfOptions::new(), + #[cfg(encryption)] + std::sync::Arc::new(crate::VoidKeyringDelegate), + ) + .await; + match res { + Err(SerfError::InvalidOption(_)) => {} + Err(other) => panic!("expected InvalidOption, got {other:?}"), + Ok(_) => panic!("a zero-capacity channel option must be rejected at construction"), + } +} + +/// A `Bounded(0)` observation channel is rejected by `Serf::tcp` instead of +/// panicking the detached driver task. +#[tokio::test] +async fn tcp_new_rejects_zero_observation_channel() { + assert_tcp_new_rejects(RuntimeOptions::new().with_observation_channel(Channel::Bounded(0))).await; +} + +/// A zero `event_queue_cap` is rejected at construction. +#[tokio::test] +async fn tcp_new_rejects_zero_event_queue_cap() { + assert_tcp_new_rejects(RuntimeOptions::new().with_event_queue_cap(0)).await; +} + +/// A `Bounded(0)` observation channel sourced from a serde config is rejected. +#[cfg(feature = "serde")] +#[tokio::test] +async fn tcp_new_rejects_zero_observation_channel_from_serde() { + let runtime: RuntimeOptions = + serde_json::from_str(r#"{"observation_channel":{"bounded":0}}"#).expect("deserialize"); + assert_tcp_new_rejects(runtime).await; +} + +/// A `bounded:0` observation channel parsed from a clap flag is rejected. +#[cfg(feature = "clap")] +#[tokio::test] +async fn tcp_new_rejects_zero_observation_channel_from_clap() { + use clap::Parser; + + #[derive(Parser)] + struct Cli { + #[command(flatten)] + runtime: RuntimeOptions, + } + + let cli = Cli::try_parse_from(["app", "--runtime-observation-channel", "bounded:0"]) + .expect("clap parses bounded:0"); + assert_tcp_new_rejects(cli.runtime).await; +} + +/// `shutdown().await` must release the bound TCP listener and UDP gossip socket +/// before it resolves: a second node binding the SAME advertise address the +/// instant the first shuts down must construct successfully, not fail with +/// `AddrInUse`. The driver awaits an explicit release of both sockets before +/// acking the shutdown caller. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn tcp_shutdown_releases_bound_address_for_rebind() { + let first = spawn_node("rebind-first").await; + let addr = first.advertise_address(); + first.shutdown().await.expect("first node shuts down"); + + let second = try_spawn_node_at("rebind-second", addr) + .await + .expect("rebinding the freed address must succeed, not AddrInUse"); + assert_eq!( + second.advertise_address(), + addr, + "the second node rebinds the exact freed address" + ); + second.shutdown().await.expect("second node shuts down"); +} + +/// An await-result `join` against an unreachable blackhole seed surfaces +/// `SerfError::JoinAllFailed { requested: 1, contacted: 0 }` once the dial fails +/// fast — well before the join deadline. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn tcp_join_unreachable_seed_surfaces_join_all_failed() { + let a = spawn_node("blackhole-joiner").await; + + let err = a + .join( + &SocketAddrResolver, + MaybeResolved::Resolved(blackhole_addr()), + false, + ) + .await + .expect_err("join against a blackhole must fail"); + + match err { + SerfError::JoinAllFailed(payload) => { + assert_eq!(payload.requested(), 1, "one seed requested"); + assert_eq!(payload.contacted(), 0, "no seed contacted"); + } + other => panic!("expected JoinAllFailed, got {other:?}"), + } + + a.shutdown().await.expect("joiner shuts down"); +} + +/// A non-empty `join` whose resolver returns zero addresses surfaces +/// `JoinAllFailed`, NOT a silent success. An empty `join_many` input is instead a +/// trivial `Ok(empty)` (no command is sent). +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn tcp_join_zero_resolution_surfaces_join_all_failed() { + let a = spawn_node("empty-resolve-joiner").await; + + let err = a + .join( + &EmptyResolver, + MaybeResolved::Unresolved("svc-a".into()), + false, + ) + .await + .expect_err("a seed resolving to zero addresses must fail"); + + match err { + SerfError::JoinAllFailed(payload) => { + assert_eq!(payload.requested(), 1, "one input seed requested"); + assert_eq!(payload.contacted(), 0); + } + other => panic!("expected JoinAllFailed, got {other:?}"), + } + + let empty: Vec> = Vec::new(); + let reached = a + .join_many(&EmptyResolver, empty.into_iter(), false) + .await + .expect("empty input is a trivial success"); + assert!(reached.is_empty(), "empty input contacts nothing"); + + a.shutdown().await.expect("joiner shuts down"); +} + +/// `default_query_timeout` on a fresh single-member node is a positive duration, +/// and `default_query_param` carries that timeout with no filters / relay / ack. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn tcp_default_query_param_defaults() { + let a = spawn_node("dqp-node").await; + + let qt = a.default_query_timeout(); + assert!( + qt > Duration::ZERO, + "default_query_timeout must be positive" + ); + + let qp = a.default_query_param(); + assert_eq!(qp.timeout, qt, "default_query_param timeout matches"); + assert!(qp.filters.is_empty(), "no filters"); + assert!(!qp.request_ack, "no ack"); + assert_eq!(qp.relay_factor, 0, "no relay"); + + a.shutdown().await.expect("node shuts down"); +} diff --git a/serf-reactor/src/snapshot.rs b/serf-reactor/src/snapshot.rs new file mode 100644 index 00000000..7cfc8e14 --- /dev/null +++ b/serf-reactor/src/snapshot.rs @@ -0,0 +1,16 @@ +//! Snapshot of serf membership state, published lock-free for handle reads. +//! +//! [`SerfSnapshot`] is the immutable membership view the driver republishes after +//! every state-affecting tick. The reactor driver stores the current one in the +//! [`Shared`](crate::shared) state's [`arc_swap::ArcSwap`], and a +//! [`Serf`](crate::Serf) handle (plus every clone) reads the latest with a +//! lock-free `load_full`. It is the `Send`/`Arc` analog of serf-compio's +//! single-owner `Rc>>` cell. +//! +//! The handle's snapshot read-forwarders (`members` / `local_member` / `state` / +//! `advertise_address` / `advertise_node` / `local_id` / `default_query_timeout` +//! / `default_query_param`) live on [`Serf`](crate::Serf); they each load the +//! published snapshot and project one view out of it. + +#[cfg(any(feature = "tcp", feature = "quic"))] +pub use serf_driver::SerfSnapshot; diff --git a/serf-reactor/tests/tcp.rs b/serf-reactor/tests/tcp.rs new file mode 100644 index 00000000..5337cb40 --- /dev/null +++ b/serf-reactor/tests/tcp.rs @@ -0,0 +1,356 @@ +//! Real-node TCP serf tests on tokio: two loopback nodes exercising the reactor +//! stream driver end-to-end. Each test spins up ephemeral `127.0.0.1:0` nodes via +//! the ergonomic [`Serf::tcp`] constructor and drives the full pump — join +//! push/pull, coordinator merge, gossip, user events, queries, and graceful +//! leave/shutdown — end-to-end proof the reactor stream driver works over a +//! concrete runtime. +//! +//! Mirrors serf-compio's serf behavior tests and memberlist-reactor's real-node +//! harness (bind loopback, join, poll-until-converged with a timeout, assert +//! membership / events), adapted to the reactor's `Send`/`agnostic` model. + +#![cfg(all(feature = "tcp", feature = "tokio"))] + +use core::time::Duration; +use std::net::SocketAddr; + +use agnostic::tokio::TokioRuntime; +use bytes::Bytes; +use futures_util::{StreamExt, future}; +use serf_proto::{event::Event, members::SerfState, options::Options as SerfOptions}; +use serf_reactor::{ + FirstAddrResolver, MaybeResolved, RuntimeOptions, Serf, SocketAddrResolver, TcpTransportOptions, + VoidDelegate, +}; +use smol_str::SmolStr; + +/// A tokio-backed reactor TCP node handle. +type Node = Serf; + +/// Build and spawn a reactor TCP node on an ephemeral loopback port through the +/// ergonomic `Serf::tcp` constructor. +async fn spawn_node(id: &str) -> Node { + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(bind)); + Serf::::tcp( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + #[cfg(encryption)] + std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), + ) + .await + .expect("spawn serf tcp node") +} + +/// Poll both nodes until each reports the full two-member cluster, or fail on a +/// generous timeout so a convergence regression surfaces as a timeout, not a hang. +async fn converge(a: &Node, b: &Node) { + tokio::time::timeout(Duration::from_secs(20), async { + loop { + if a.num_members() == 2 && b.num_members() == 2 { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("both nodes converge to a 2-member cluster"); +} + +/// Two nodes on loopback: A joins B (await-result), then BOTH converge to a +/// two-member cluster and shut down cleanly. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn two_node_join_converges() { + let b = spawn_node("conv-b").await; + let a = spawn_node("conv-a").await; + let b_addr = b.advertise_address(); + + let reached = a + .join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + assert_eq!(reached, b_addr, "join returns the reached seed address"); + + converge(&a, &b).await; + assert_eq!(a.num_members(), 2, "A sees the 2-member cluster"); + assert_eq!(b.num_members(), 2, "B sees the 2-member cluster"); + + a.shutdown().await.expect("conv-a shuts down"); + b.shutdown().await.expect("conv-b shuts down"); +} + +/// After a two-node join, a user event broadcast by B is delivered to A's event +/// stream carrying the original name and payload. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn user_event_delivered() { + let b = spawn_node("ue-b").await; + let a = spawn_node("ue-a").await; + let b_addr = b.advertise_address(); + + // Subscribe before joining so the user event cannot race the subscription. + let mut a_events = a.events(); + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + converge(&a, &b).await; + + b.user_event("greet", Bytes::from_static(b"hello"), false) + .await + .expect("user event dispatched"); + + let got = tokio::time::timeout(Duration::from_secs(20), async { + loop { + match a_events.next().await { + Some(Event::User(u)) if u.name.as_str() == "greet" => break Some(u.payload.clone()), + Some(_) => {} + None => break None, + } + } + }) + .await + .expect("A observes B's user event within the timeout"); + assert_eq!( + got, + Some(Bytes::from_static(b"hello")), + "A receives B's user-event payload" + ); + + a.shutdown().await.expect("ue-a shuts down"); + b.shutdown().await.expect("ue-b shuts down"); +} + +/// After a two-node join, a query issued by A round-trips: B receives the +/// `Event::Query`, responds, and A surfaces the matching `Event::QueryResponse`. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn query_round_trip() { + let b = spawn_node("q-b").await; + let a = spawn_node("q-a").await; + let b_addr = b.advertise_address(); + + // Subscribe both before the join so neither the query nor its response races + // ahead of a subscription. + let mut b_events = b.events(); + let mut a_events = a.events(); + + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + converge(&a, &b).await; + + let want = Bytes::from_static(b"pong"); + a.query( + "ping", + Bytes::from_static(b"ping-payload"), + a.default_query_param(), + ) + .await + .expect("query issued"); + + // B answers the first "ping" query it sees; A collects the matching response. + let responder = async { + loop { + match b_events.next().await { + Some(Event::Query(qe)) if qe.name() == "ping" => { + b.respond(qe, want.clone()) + .await + .expect("B responds to the query"); + break; + } + Some(_) => {} + None => panic!("B's event stream closed before the query arrived"), + } + } + }; + let collector = async { + loop { + match a_events.next().await { + Some(Event::QueryResponse(qr)) if qr.payload() == &want => break true, + Some(_) => {} + None => break false, + } + } + }; + + let got = tokio::time::timeout(Duration::from_secs(20), async { + let (_, got) = future::join(responder, collector).await; + got + }) + .await + .expect("query round-trip completes within the timeout"); + assert!(got, "A must receive B's query response"); + + a.shutdown().await.expect("q-a shuts down"); + b.shutdown().await.expect("q-b shuts down"); +} + +/// A graceful leave completes the machine's leave chain: `leave()` resolves only +/// once `LeftCluster` fires (the reactor gates the reply on it), that event +/// surfaces on the leaver's own stream, and the local endpoint settles at `Left`. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn leave_emits_left_cluster() { + let b = spawn_node("lv-b").await; + let a = spawn_node("lv-a").await; + let b_addr = b.advertise_address(); + + let mut a_events = a.events(); + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + converge(&a, &b).await; + + // The reactor resolves `leave()` only once the machine's `LeftCluster` fires, so + // a successful return already proves the graceful-leave chain completed. + a.leave().await.expect("A leaves the cluster"); + + // `LeftCluster` is also forwarded to A's own subscribers. + let saw = tokio::time::timeout(Duration::from_secs(20), async { + loop { + match a_events.next().await { + Some(Event::LeftCluster) => break true, + Some(_) => {} + None => break false, + } + } + }) + .await + .expect("A observes LeftCluster within the timeout"); + assert!(saw, "A must surface Event::LeftCluster after leave()"); + + // The local endpoint state settles at `Left` (poll to absorb the snapshot-refresh + // race after the leave chain completes). + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if a.state() == SerfState::Left { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("A's endpoint state becomes Left"); + + a.shutdown().await.expect("lv-a shuts down"); + b.shutdown().await.expect("lv-b shuts down"); +} + +/// After a two-node join, the snapshot read-forwarders on the joined node reflect +/// the two-member cluster: `members` returns both nodes, `local_member` / `local_id` +/// return this node, `state` is `Alive`, `advertise_node` composes id + advertise, +/// and `default_query_*` produce a positive, filter-free query default. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn snapshot_forwarders_reflect_joined_cluster() { + let b = spawn_node("snap-b").await; + let a = spawn_node("snap-a").await; + let b_addr = b.advertise_address(); + let a_id = SmolStr::new("snap-a"); + let b_id = SmolStr::new("snap-b"); + + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + converge(&a, &b).await; + + let members = a.members(); + assert_eq!(members.len(), 2, "members() returns the 2-member cluster"); + let ids: Vec<&SmolStr> = members.iter().map(|m| m.node().id_ref()).collect(); + assert!(ids.contains(&&a_id), "members() includes local node A"); + assert!(ids.contains(&&b_id), "members() includes peer node B"); + + assert_eq!( + a.local_member().node().id_ref(), + &a_id, + "local_member() returns local node A" + ); + assert_eq!(a.local_id(), a_id, "local_id() returns local node A"); + assert_eq!(a.state(), SerfState::Alive, "state() is Alive after join"); + + let anode = a.advertise_node(); + assert_eq!( + anode.id_ref(), + &a_id, + "advertise_node() id matches local_id()" + ); + assert_eq!( + anode.addr_ref(), + &a.advertise_address(), + "advertise_node() addr matches advertise_address()" + ); + + let qt = a.default_query_timeout(); + assert!(qt > Duration::ZERO, "default_query_timeout() is positive"); + let qp = a.default_query_param(); + assert_eq!(qp.timeout, qt, "default_query_param() timeout matches"); + assert!( + qp.filters.is_empty(), + "default_query_param() has no filters" + ); + assert!(!qp.request_ack, "default_query_param() has no ack"); + assert_eq!(qp.relay_factor, 0, "default_query_param() has no relay"); + + a.shutdown().await.expect("snap-a shuts down"); + b.shutdown().await.expect("snap-b shuts down"); +} + +/// `join_many` over two seeds — one reachable (node B), one an unroutable +/// blackhole port — returns only the reached seed's address once both exchanges +/// terminate. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn join_many_returns_only_reached_seeds() { + let b = spawn_node("jm-b").await; + let a = spawn_node("jm-a").await; + let b_addr = b.advertise_address(); + let blackhole: SocketAddr = "127.0.0.1:7219".parse().expect("loopback addr"); + + let reached = a + .join_many( + &SocketAddrResolver, + [ + MaybeResolved::Resolved(b_addr), + MaybeResolved::Resolved(blackhole), + ] + .into_iter(), + false, + ) + .await + .expect("join_many reaches the reachable seed"); + + assert_eq!(reached.len(), 1, "only the reachable seed is contacted"); + assert_eq!( + reached[0], b_addr, + "the reached set carries node B's address" + ); + + a.shutdown().await.expect("jm-a shuts down"); + b.shutdown().await.expect("jm-b shuts down"); +} + +/// `remove_failed_node` / `remove_failed_node_prune` are thin `force_leave` +/// aliases; calling both on a valid joined node-id completes without error. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn remove_failed_node_alias_succeeds() { + let b = spawn_node("rfn-b").await; + let a = spawn_node("rfn-a").await; + let b_addr = b.advertise_address(); + let b_id = SmolStr::new("rfn-b"); + + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + converge(&a, &b).await; + + a.remove_failed_node(b_id.clone()) + .await + .expect("remove_failed_node must not error"); + a.remove_failed_node_prune(b_id) + .await + .expect("remove_failed_node_prune must not error"); + + a.shutdown().await.expect("rfn-a shuts down"); + b.shutdown().await.expect("rfn-b shuts down"); +} From 688790645cd2cc0032fcc10221971ec5da1fc665 Mon Sep 17 00:00:00 2001 From: al8n Date: Wed, 1 Jul 2026 23:42:42 +1200 Subject: [PATCH 028/128] fix(serf-reactor): drain the socket and machine surfaces to quiescence before the timer The readiness pump capped the UDP recv (and bridge-inbound) at iter_drain_cap but still ran the single handle_timeout and the join/leave deadline reaps, so a due timeout could fire before a ready pre-deadline Ack / ExchangeCompleted / LeftCluster queued behind the cap was drained (false suspicion / spurious JoinAllFailed / LeaveTimeout under bursts). It keeps memberlist-reactor's bounded per-poll recv cap but now gates the single handle_timeout site and the deadline reaps on quiescence (!more), so they run only once the socket is drained to Poll::Pending and drain_surfaces has no more ready work. Regression with iter_drain_cap=1. --- serf-reactor/src/driver/stream/mod.rs | 89 ++++--- serf-reactor/src/driver/stream/tests.rs | 295 ++++++++++++++++++++++++ 2 files changed, 348 insertions(+), 36 deletions(-) create mode 100644 serf-reactor/src/driver/stream/tests.rs diff --git a/serf-reactor/src/driver/stream/mod.rs b/serf-reactor/src/driver/stream/mod.rs index 46819feb..deb8fe7f 100644 --- a/serf-reactor/src/driver/stream/mod.rs +++ b/serf-reactor/src/driver/stream/mod.rs @@ -1376,7 +1376,8 @@ where // is always `Some` here — the shutdown branch above (which takes it) returned // before reaching this point. `Poll::Pending` from the socket IS the // kernel-empty signal (the readiness analogue of serf-compio's completion - // reap), so the recv-loop stops and `handle_timeout` fires inline below. + // reap), so the recv-loop stops. A full batch sets `more`, which defers the + // single `handle_timeout` site below to a later, quiescent poll. let mut recv_n = 0; while recv_n < this.iter_drain_cap { let Some(socket) = this.socket.as_ref() else { @@ -1443,45 +1444,58 @@ where more = true; } - // Timer: fire an overdue deadline inline (the single `handle_timeout` site), - // else arm + poll the sleep. Fold in the earliest pending-join / -leave - // deadline so a parked waiter's timeout fires even when the coordinator has no - // nearer deadline. - let endpoint_deadline = this - .endpoint - .poll_timeout() - .map(|d| d.min(now + this.idle_wake)) - .unwrap_or(now + this.idle_wake); - let target = [ - Some(endpoint_deadline), - this.min_pending_join_deadline(), - this.min_pending_leave_deadline(), - ] - .into_iter() - .flatten() - .min() - .unwrap_or(endpoint_deadline); - if target <= now { - this.endpoint.handle_timeout(now); - progress = true; - more = true; - } else { - this.arm_timer(target, now); - if let Some(timer) = this.timer.as_mut() - && timer.as_mut().poll(cx).is_ready() - { - this.endpoint.handle_timeout(Instant::now()); - this.timer = None; - this.timer_deadline = None; + // Timer + deadline reaps, gated on quiescence (`!more`). While `more` — the UDP + // recv loop or the bridge-inbound loop hit its per-poll `iter_drain_cap`, or a + // machine surface still had queued work — a ready pre-deadline datagram or + // completion may sit BEHIND that cap, undrained. Firing `handle_timeout` (or a + // deadline reap) now could time out a probe / await-result join / graceful + // leave whose resolving Ack / ExchangeCompleted / LeftCluster is already + // waiting one poll behind, yielding false suspicion, a spurious `JoinAllFailed`, + // or a `LeaveTimeout`. The `more` self-wake below re-polls and drains that work + // first; the single `handle_timeout` site and the join / leave deadline reaps + // run only once the socket is drained to `Poll::Pending` and `drain_surfaces` + // is quiescent. The kernel buffer is finite and each poll makes `iter_drain_cap` + // progress before re-polling, so this defers the timer without starving it. + if !more { + // Fire an overdue deadline inline (the single `handle_timeout` site), else arm + // + poll the sleep. Fold in the earliest pending-join / -leave deadline so a + // parked waiter's timeout fires even when the coordinator has no nearer one. + let endpoint_deadline = this + .endpoint + .poll_timeout() + .map(|d| d.min(now + this.idle_wake)) + .unwrap_or(now + this.idle_wake); + let target = [ + Some(endpoint_deadline), + this.min_pending_join_deadline(), + this.min_pending_leave_deadline(), + ] + .into_iter() + .flatten() + .min() + .unwrap_or(endpoint_deadline); + if target <= now { + this.endpoint.handle_timeout(now); progress = true; more = true; + } else { + this.arm_timer(target, now); + if let Some(timer) = this.timer.as_mut() + && timer.as_mut().poll(cx).is_ready() + { + this.endpoint.handle_timeout(Instant::now()); + this.timer = None; + this.timer_deadline = None; + progress = true; + more = true; + } } - } - // Reap deadline-expired join / leave waiters (a fired `handle_timeout` may have - // completed exchanges; the deadline path resolves the rest). - this.reap_pending_joins(now); - this.reap_pending_leave(now); + // Reap deadline-expired join / leave waiters (a fired `handle_timeout` may + // have completed exchanges; the deadline path resolves the rest). + this.reap_pending_joins(now); + this.reap_pending_leave(now); + } // Republish the snapshot whenever the pump made progress (the serf endpoint // exposes no cheap version stamp, so — as in serf-compio — a productive poll @@ -1746,3 +1760,6 @@ where keyring, ) } + +#[cfg(all(test, feature = "tokio"))] +mod tests; diff --git a/serf-reactor/src/driver/stream/tests.rs b/serf-reactor/src/driver/stream/tests.rs new file mode 100644 index 00000000..4d077fa2 --- /dev/null +++ b/serf-reactor/src/driver/stream/tests.rs @@ -0,0 +1,295 @@ +//! Stream-driver pump-ordering regression tests. +//! +//! The single `handle_timeout` site and the join / leave deadline reaps must fire +//! only once the pump is quiescent, so a due deadline never times out a probe, +//! await-result join, or graceful leave whose resolving Ack / `ExchangeCompleted` +//! / `LeftCluster` is still queued behind the per-poll `iter_drain_cap`. + +use core::num::NonZeroU8; +use std::task::Waker; + +use agnostic::{RuntimeLite, tokio::TokioRuntime}; +use memberlist_proto::{ + Endpoint, EndpointOptions, Node, PushPullKind, RawRecords, SmallRng, + streams::{LabelOptions, StreamEndpoint as Coordinator}, +}; +use serf_proto::{ + members::{Member, MemberStatus}, + options::Options as SerfOptions, + typed::Tags, +}; +use smol_str::SmolStr; + +use super::*; + +/// Shared loopback cluster label so both coordinators' record-layer handshakes +/// settle. +const CLUSTER: &[u8] = b"serf-reactor-loopback"; + +/// The driver endpoint's advertise identity (distinct from the seed peer). +const DRIVER_ADDR: &str = "127.0.0.1:7946"; + +type TestDriver = StreamDriver; + +fn sa(s: &str) -> SocketAddr { + s.parse().expect("loopback addr") +} + +/// Drive the driver through exactly one `Future::poll`. The manual poll loop +/// re-polls unconditionally, so the no-op waker is a valid, harmless sink. +fn poll_once(driver: &mut TestDriver) -> Poll<()> { + let mut cx = Context::from_waker(Waker::noop()); + Pin::new(driver).poll(&mut cx) +} + +/// Build a serf `StreamEndpoint` rooted at `id` / +/// `advertise`, mirroring the production construction (memberlist inner endpoint → +/// reliable coordinator → serf super-machine). Seeded deterministically; the +/// initial local `NodeJoined` self-event is drained so a caller starts clean. +fn build_endpoint( + id: &str, + advertise: SocketAddr, +) -> StreamEndpoint { + let inner_opts = EndpointOptions::new(SmolStr::new(id), advertise) + .with_user_broadcast_tiers(NonZeroU8::new(3).expect("3 is nonzero")); + let inner = Endpoint::new(inner_opts, SmallRng::seed_from_u64(0)); + let coord = Coordinator::<_, _, RawRecords>::new( + inner, + LabelOptions::new_in(Some(CLUSTER.to_vec()), ()), + Box::new(|_addr: &SocketAddr| None), + Box::new(|addr: &SocketAddr| *addr), + ); + let mut e = StreamEndpoint::new(coord, SerfOptions::new()); + while e.poll_event().is_some() {} + e +} + +/// The initial published snapshot: the local node, `Alive`, zeroed clocks +/// (a test-local copy of `crate::serf::initial_snapshot`). +fn initial_snapshot(id: &str, advertise: SocketAddr) -> SerfSnapshot { + let member = Member::new( + Node::new(SmolStr::new(id), advertise), + Tags::new(), + MemberStatus::Alive, + ); + SerfSnapshot::new( + vec![Arc::new(member)], + &SmolStr::new(id), + SerfState::Alive, + LamportTime::from(0u64), + LamportTime::from(0u64), + LamportTime::from(0u64), + ) +} + +/// Build a real `StreamDriver` over a bound gossip socket + a live accept task, +/// with the scheduling deliberately OFF so no stray coordinator deadline can +/// supply a timer the test means to attribute to the parked join. Returns the +/// driver, the obs receiver (held so the obs channel stays connected), and the +/// shared state. +async fn build_driver( + iter_drain_cap: usize, +) -> ( + TestDriver, + Receiver>, + Arc>, +) { + let socket = <::Net as Net>::UdpSocket::bind("127.0.0.1:0") + .await + .expect("bind gossip socket"); + let endpoint = build_endpoint("drv", sa(DRIVER_ADDR)); + let shared = Arc::new(Shared::new(initial_snapshot("drv", sa(DRIVER_ADDR)))); + let obs_payload_bytes = Arc::new(AtomicU64::new(0)); + let (obs_tx, obs_rx) = flume::unbounded(); + let (accepted_tx, accepted_rx) = flume::bounded(ACCEPT_CAP); + let (accept_shutdown_tx, accept_shutdown_rx) = flume::bounded(1); + let listener = <::Net as Net>::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind accept listener"); + let accept_join = TokioRuntime::spawn(accept_task::( + listener, + accepted_tx, + accept_shutdown_rx, + shared.clone(), + )); + let driver = StreamDriver::::new( + endpoint, + socket, + shared.clone(), + obs_tx, + obs_payload_bytes, + None, + accepted_rx, + accept_shutdown_tx, + accept_join, + RuntimeOptions::new().with_iter_drain_cap(iter_drain_cap), + StreamTransportOptions::new(), + None, + #[cfg(encryption)] + Arc::new(crate::VoidKeyringDelegate), + ); + (driver, obs_rx, shared) +} + +/// Drive one outbound Join push/pull on `driver.endpoint` toward `seed_addr` to a +/// real `Succeeded`, returning the dialer exchange id and the peer's pull response +/// frames (which the caller queues on `inbound_rx` rather than feeding here). A +/// second `StreamEndpoint` stands in for the seed; the dialer sends push + FIN up +/// front, so the exchange completes purely from feeding that response + a peer-FIN +/// EOF back — no live socket needed. +fn drive_push_to_queued_response( + driver: &mut TestDriver, + seed_addr: SocketAddr, + now: Instant, +) -> (ExchangeId, Vec>) { + let mut peer = build_endpoint("seed", seed_addr); + + // Dialer: start the exchange and capture its Connect id + push frames. + driver + .endpoint + .start_push_pull(seed_addr, PushPullKind::Join, now); + let mut eid = None; + let mut push: Vec> = Vec::new(); + for _ in 0..256 { + let mut progressed = false; + while let Some(action) = driver.endpoint.poll_action() { + progressed = true; + if let StreamAction::Connect(info) = action { + eid = Some(info.id()); + } + } + while let Some((id, _peer, bytes)) = driver.endpoint.poll_transport_transmit() { + progressed = true; + if Some(id) == eid { + push.push(bytes.to_vec()); + } + } + if !progressed { + break; + } + } + let eid = eid.expect("the dialer's start_push_pull produced a Connect exchange id"); + assert!( + !push.is_empty(), + "the dialer emitted its push frames up front" + ); + + // Peer: admit the inbound connection, replay push + FIN, collect the pull + // response (ticking the peer so its serf push-pull snapshot resyncs). + let server_eid = peer + .accept_connection(sa(DRIVER_ADDR), now) + .expect("the peer admits the inbound exchange"); + for chunk in &push { + peer.handle_transport_data(server_eid, chunk, false, now); + } + peer.handle_transport_data(server_eid, &[], true, now); // the dialer's FIN + let mut response: Vec> = Vec::new(); + for _ in 0..256 { + let mut progressed = false; + peer.handle_timeout(now); + while peer.poll_action().is_some() { + progressed = true; + } + while let Some((id, _peer, bytes)) = peer.poll_transport_transmit() { + progressed = true; + if id == server_eid { + response.push(bytes.to_vec()); + } + } + while peer.poll_event().is_some() { + progressed = true; + } + if !progressed { + break; + } + } + assert!( + !response.is_empty(), + "the peer produced a pull response to the dialer's push" + ); + (eid, response) +} + +/// Regression: at a DUE await-result-join deadline, the pull `ExchangeCompleted` +/// that resolves the join `Ok` is queued one bridge-inbound item behind the +/// per-poll `iter_drain_cap`. The single `handle_timeout` site and the join +/// deadline reap must wait for that pre-deadline completion to drain — a premature +/// reap would surface a spurious `JoinAllFailed` against a seed that was in fact +/// reached. +/// +/// Pre-fix, the first poll (bridge-inbound cap hit → `more`) still runs the +/// past-due reap and replies `JoinAllFailed`. Post-fix, the timer + reap are gated +/// on quiescence, so the completion drains first and the join resolves `Ok`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn due_deadline_waits_for_join_completion_behind_iter_drain_cap() { + let now = Instant::now(); + // iter_drain_cap = 1: the bridge-inbound loop processes at most one item per + // poll, so the completion staged behind the response chunk(s) is at least one + // poll behind at the due deadline. + let (mut driver, _obs_rx, _shared) = build_driver(1).await; + + let seed_addr = sa("127.0.0.1:7000"); + + // Seed reached: drive its push/pull to a Succeeded whose pull response + peer-FIN + // we queue on inbound_rx (NOT consumed), so the resolving ExchangeCompleted sits + // behind the per-poll cap. + let (eid, response) = drive_push_to_queued_response(&mut driver, seed_addr, now); + let inbound_tx = driver.inbound_tx.as_ref().expect("template alive").clone(); + for bytes in response { + inbound_tx + .try_send(BridgeInbound::Data(BridgeData { + eid, + bytes, + received_at: now, + })) + .expect("queue inbound response"); + } + inbound_tx + .try_send(BridgeInbound::Eof(BridgeEof { + eid, + received_at: now, + })) + .expect("queue inbound EOF"); + + // Park an await-result join awaiting `eid` with a deadline ALREADY in the past, + // so the deadline reap is due on the very first poll — while the completion that + // resolves it Ok is still queued behind the cap. + let (tx, mut rx) = oneshot::channel::(); + let mut pending = HashSet::new(); + pending.insert(eid); + driver.pending_joins.push(PendingJoin { + pending, + contacted: SmallVec::new(), + ignore_streams: SmallVec::new(), + requested: 1, + deadline: now - Duration::from_secs(1), + reply: Some(tx), + }); + + // Drive the pump by hand. The `more` self-wake re-polls, so a bounded loop drains + // the staged completion. Pre-fix, the first poll reaps the past-due deadline + // (JoinAllFailed); post-fix, the reap waits until the completion resolves Ok. + let mut resolved = None; + for _ in 0..256 { + let _ = poll_once(&mut driver); + match rx.try_recv() { + Ok(Some(reply)) => { + resolved = Some(reply); + break; + } + Ok(None) => {} + Err(_) => panic!("join reply sender dropped without resolving"), + } + } + + let reached = resolved + .expect("the join resolved within the bounded poll budget") + .expect( + "the ready ExchangeCompleted resolved the join before the past-due deadline reap fired; \ + a premature reap would surface a spurious JoinAllFailed", + ); + assert!( + reached.contains(&seed_addr), + "the resolved join reached the seed whose completion was queued behind the cap: {reached:?}" + ); +} From d06c71597182a32d008cb050ae624da105cc9b59 Mon Sep 17 00:00:00 2001 From: al8n Date: Thu, 2 Jul 2026 00:15:45 +1200 Subject: [PATCH 029/128] fix(serf-reactor): fixed-point surface drain + bounded-deferral timer for liveness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit drain_surfaces is now a fixed-point (repeat the ordered pass while any surface made progress), so a later surface feeding an earlier one no longer reports false quiescence — the withheld Close and the KeyRequest respond_key gossip are drained the same poll. The timer + deadline reaps fire at quiescence OR after a bounded number of capped-poll deferrals (TIMER_DEFERRAL_LIVENESS_BOUND), so a sustained ingress flood can no longer starve failure-detection / join-leave-query deadlines, while staying non-premature (FIFO drains the pre-deadline backlog first). Regressions: continuous-flood liveness, the fixed-point withheld-Close late surface, and the kept non-premature completion-behind-cap case. --- serf-reactor/src/driver/stream/mod.rs | 148 ++++++++++++++---- serf-reactor/src/driver/stream/tests.rs | 194 ++++++++++++++++++++++++ 2 files changed, 309 insertions(+), 33 deletions(-) diff --git a/serf-reactor/src/driver/stream/mod.rs b/serf-reactor/src/driver/stream/mod.rs index deb8fe7f..bfebc122 100644 --- a/serf-reactor/src/driver/stream/mod.rs +++ b/serf-reactor/src/driver/stream/mod.rs @@ -94,6 +94,18 @@ pub(crate) const ACCEPT_CAP: usize = 256; /// channel (the payload byte budget bounds their bytes; this bounds their count). const OBS_OVERFLOW_MAX: usize = 1024; +/// Upper bound on how many consecutive capped (`more`) polls may DEFER a due +/// timer / deadline reap before it fires regardless. While `more`, ready +/// pre-deadline ingress can sit behind a per-poll cap, so firing immediately could +/// time out a probe / join / leave whose resolving Ack / `ExchangeCompleted` / +/// `LeftCluster` is buffered one poll behind (premature). Deferring re-polls to +/// drain that first — and the pre-deadline backlog is FIFO, so it drains within a +/// few `iter_drain_cap`-sized polls — but an UNBOUNDED defer would let a sustained +/// ingress flood suppress failure detection and join / leave / query deadlines +/// forever (a liveness-denial DoS). This bound caps the deferral, trading a few +/// polls of deadline staleness under flood for guaranteed liveness. +const TIMER_DEFERRAL_LIVENESS_BOUND: u32 = 8; + /// A message from the pump to a bridge's TCP write side. Teardown is signalled out /// of band by dropping the [`BridgeHandle`], not by a variant here, so it can /// preempt even a write stalled on an unresponsive peer. @@ -329,6 +341,11 @@ where iter_drain_cap: usize, timer: Option>>, timer_deadline: Option, + /// Consecutive capped-poll deferrals of a due timer / deadline reap, bounded by + /// [`TIMER_DEFERRAL_LIVENESS_BOUND`]. Accumulates only while a deadline is + /// overdue AND a per-poll cap is being hit; reset the moment the poll reaches + /// quiescence, the deadline fires, or nothing is due. + timer_deferrals: u32, idle_wake: Duration, leave_timeout: Duration, close_timeout: Duration, @@ -396,6 +413,7 @@ where iter_drain_cap: driver_opts.iter_drain_cap().max(1), timer: None, timer_deadline: None, + timer_deferrals: 0, idle_wake: driver_opts.idle_wake_interval(), leave_timeout: driver_opts.leave_timeout(), close_timeout: stream_opts.close_timeout(), @@ -845,14 +863,55 @@ where } } - /// Drains each machine surface up to `iter_drain_cap` items in one pass. - /// Returns `(worked, more)`: whether any surface produced work, and whether any - /// surface hit its cap with work left (self-wake). + /// Repeatedly runs the ordered surface pass to a FIXED POINT. A later surface can + /// create work for an earlier one within the same poll: draining an exchange's + /// final transport transmit releases its withheld [`StreamAction::Close`] for the + /// action surface, and answering an `Event::KeyRequest` (`respond_key`) queues a + /// directed gossip transmit after the transmit surface was already drained. A + /// single ordered pass would leave that late-generated work buffered and report + /// false quiescence (`more == false` with ready machine work undrained), so the + /// timer could fire and the pump return `Pending` with a TCP bridge still open or + /// a key response delayed past its query deadline. + /// + /// Repeating the ordered pass while any surface made progress converges: the + /// machine emits finite output per (finite, already-buffered) input and each + /// surface is cap-bounded, so successive passes strictly drain the buffered work + /// down. On return `more == false` therefore genuinely means no ready machine work + /// remains from this poll's input. A pass that hits a per-surface cap ends the + /// fixed point immediately with `more == true`: the caller self-wakes and re-polls + /// rather than uncapped-draining a surface a fast peer can refill, keeping each + /// poll bounded. fn drain_surfaces(&mut self, cx: &mut Context<'_>) -> (bool, bool) where I: Send + Sync + 'static, { let now = Instant::now(); + let mut worked = false; + loop { + let (pass_worked, pass_more) = self.drain_surfaces_pass(cx, now); + worked |= pass_worked; + if pass_more { + // A per-surface cap was hit: end the fixed point and self-wake (`more`) + // rather than repeat the pass, so the poll stays bounded. + return (worked, true); + } + if !pass_worked { + // No surface made progress: the fixed point is reached and no ready machine + // work remains from this poll's input. + return (worked, false); + } + } + } + + /// One ordered surface pass — inbound-ingress → action → transport → gossip → + /// event — each surface draining up to `iter_drain_cap` items. Returns + /// `(worked, more)`: whether any surface produced work in this pass, and whether + /// any surface hit its cap. [`Self::drain_surfaces`] iterates this to a fixed + /// point so a later surface feeding an earlier one is drained the same poll. + fn drain_surfaces_pass(&mut self, cx: &mut Context<'_>, now: Instant) -> (bool, bool) + where + I: Send + Sync + 'static, + { let budget = self.iter_drain_cap; let mut worked = false; let mut more = false; @@ -1444,38 +1503,41 @@ where more = true; } - // Timer + deadline reaps, gated on quiescence (`!more`). While `more` — the UDP - // recv loop or the bridge-inbound loop hit its per-poll `iter_drain_cap`, or a - // machine surface still had queued work — a ready pre-deadline datagram or - // completion may sit BEHIND that cap, undrained. Firing `handle_timeout` (or a - // deadline reap) now could time out a probe / await-result join / graceful - // leave whose resolving Ack / ExchangeCompleted / LeftCluster is already - // waiting one poll behind, yielding false suspicion, a spurious `JoinAllFailed`, - // or a `LeaveTimeout`. The `more` self-wake below re-polls and drains that work - // first; the single `handle_timeout` site and the join / leave deadline reaps - // run only once the socket is drained to `Poll::Pending` and `drain_surfaces` - // is quiescent. The kernel buffer is finite and each poll makes `iter_drain_cap` - // progress before re-polling, so this defers the timer without starving it. + // Timer + deadline reaps under a BOUNDED-DEFERRAL liveness policy. Fold the + // coordinator's next deadline together with the earliest parked join / leave + // deadline into one `target`; `due` means at least one is overdue. Firing while + // ready pre-deadline ingress is still buffered would be premature (false + // suspicion / a spurious `JoinAllFailed` / a `LeaveTimeout`), but never firing + // while `more` would let a flood starve the deadline — so the two cases split. + let endpoint_deadline = this + .endpoint + .poll_timeout() + .map(|d| d.min(now + this.idle_wake)) + .unwrap_or(now + this.idle_wake); + let target = [ + Some(endpoint_deadline), + this.min_pending_join_deadline(), + this.min_pending_leave_deadline(), + ] + .into_iter() + .flatten() + .min() + .unwrap_or(endpoint_deadline); + let due = target <= now; + // The deferral counter accumulates only while a deadline is actually overdue. + if !due { + this.timer_deferrals = 0; + } + if !more { - // Fire an overdue deadline inline (the single `handle_timeout` site), else arm - // + poll the sleep. Fold in the earliest pending-join / -leave deadline so a - // parked waiter's timeout fires even when the coordinator has no nearer one. - let endpoint_deadline = this - .endpoint - .poll_timeout() - .map(|d| d.min(now + this.idle_wake)) - .unwrap_or(now + this.idle_wake); - let target = [ - Some(endpoint_deadline), - this.min_pending_join_deadline(), - this.min_pending_leave_deadline(), - ] - .into_iter() - .flatten() - .min() - .unwrap_or(endpoint_deadline); - if target <= now { + // Quiescent: the recv / bridge-inbound loops drained to `Poll::Pending` and + // `drain_surfaces` reached its fixed point, so the whole poll-entry backlog is + // processed. Fire an overdue deadline inline (the single `handle_timeout` + // site) — non-prematurely, since nothing that could resolve it is still + // buffered — else arm + poll the sleep so the next deadline wakes the pump. + if due { this.endpoint.handle_timeout(now); + this.timer_deferrals = 0; progress = true; more = true; } else { @@ -1495,6 +1557,26 @@ where // have completed exchanges; the deadline path resolves the rest). this.reap_pending_joins(now); this.reap_pending_leave(now); + } else if due { + // `more`: a per-poll cap was hit, so ready pre-deadline ingress may sit BEHIND + // it — a resolving Ack / `ExchangeCompleted` / `LeftCluster` could be one poll + // behind. DEFER the timer + reaps and re-poll (the `more` self-wake below) so + // that FIFO backlog drains first. Deferral is BOUNDED: after + // `TIMER_DEFERRAL_LIVENESS_BOUND` consecutive deferrals the pre-deadline + // backlog is drained (FIFO, `iter_drain_cap` per poll), so fire regardless — + // an unbounded defer would let a sustained flood starve failure detection and + // join / leave / query deadlines. No timer is armed while deferring; the + // self-wake alone re-polls, so there is no lost wakeup and the counter + // guarantees the deferral terminates. + if this.timer_deferrals >= TIMER_DEFERRAL_LIVENESS_BOUND { + this.endpoint.handle_timeout(now); + this.reap_pending_joins(now); + this.reap_pending_leave(now); + this.timer_deferrals = 0; + progress = true; + } else { + this.timer_deferrals += 1; + } } // Republish the snapshot whenever the pump made progress (the serf endpoint diff --git a/serf-reactor/src/driver/stream/tests.rs b/serf-reactor/src/driver/stream/tests.rs index 4d077fa2..b9ae1f49 100644 --- a/serf-reactor/src/driver/stream/tests.rs +++ b/serf-reactor/src/driver/stream/tests.rs @@ -293,3 +293,197 @@ async fn due_deadline_waits_for_join_completion_behind_iter_drain_cap() { "the resolved join reached the seed whose completion was queued behind the cap: {reached:?}" ); } + +/// Regression (liveness / non-starving): under a SUSTAINED ingress flood that hits +/// `iter_drain_cap` every poll (so `more` never clears), a past-due await-result +/// join deadline must STILL be reaped within a bounded number of polls. A blunt +/// `if !more` gate never fires the reap while `more`, so a flood would suppress the +/// deadline forever (a liveness-denial DoS); the bounded-deferral policy fires it +/// once `TIMER_DEFERRAL_LIVENESS_BOUND` deferrals elapse. +/// +/// Pre-fix, the reap is gated out every poll and the join never resolves within the +/// poll budget. Post-fix, it resolves to `JoinAllFailed` (no seed was contacted) +/// within the deferral bound. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn ingress_flood_does_not_starve_due_join_deadline_reap() { + let now = Instant::now(); + // A small cap makes the per-poll flood cheap. The flood keeps `more` set via the + // inbound-gossip surface: each fed datagram is undecodable garbage the drain + // drops, but it still counts toward the surface's `iter_drain_cap`. + let cap = 4usize; + let (mut driver, _obs_rx, _shared) = build_driver(cap).await; + + // A real, live outbound exchange id whose exchange never completes (no peer ever + // feeds it): the parked join stays pending, so ONLY the past-due deadline reap can + // resolve it — to `JoinAllFailed`. + driver + .endpoint + .start_push_pull(sa("127.0.0.1:7300"), PushPullKind::Join, now); + let mut eid = None; + while let Some(action) = driver.endpoint.poll_action() { + if let StreamAction::Connect(info) = action { + eid = Some(info.id()); + } + } + let eid = eid.expect("start_push_pull emitted a Connect exchange id"); + + let (tx, mut rx) = oneshot::channel::(); + let mut pending = HashSet::new(); + pending.insert(eid); + driver.pending_joins.push(PendingJoin { + pending, + contacted: SmallVec::new(), + ignore_streams: SmallVec::new(), + requested: 1, + deadline: now - Duration::from_secs(1), + reply: Some(tx), + }); + + let flood_src = sa("127.0.0.1:7301"); + let mut resolved = None; + let mut polls = 0usize; + for _ in 0..(TIMER_DEFERRAL_LIVENESS_BOUND as usize + 4) { + // Refill the ingress flood BEFORE each poll so the inbound-gossip surface hits + // its cap and `more` is set for this poll. + for _ in 0..cap { + driver + .endpoint + .handle_gossip(flood_src, &[0xff, 0x00, 0xff], now); + } + let _ = poll_once(&mut driver); + polls += 1; + match rx.try_recv() { + Ok(Some(reply)) => { + resolved = Some(reply); + break; + } + Ok(None) => {} + Err(_) => panic!("join reply sender dropped without resolving"), + } + } + + let reply = resolved.expect( + "the past-due join deadline reap fired despite the sustained ingress flood; \ + a blunt !more gate would starve it indefinitely", + ); + assert!( + polls <= TIMER_DEFERRAL_LIVENESS_BOUND as usize + 1, + "the reap fired within the deferral bound, not later: {polls} polls", + ); + match reply { + Err((ref reached, SerfError::JoinAllFailed(_))) => { + assert!( + reached.is_empty(), + "no seed was contacted, so the all-failed set is empty: {reached:?}" + ); + } + other => panic!("expected JoinAllFailed from the deadline reap, got {other:?}"), + } +} + +/// Drive an INBOUND (server-side) Join push/pull on `driver.endpoint` until its pull +/// response is queued in the coordinator's transmit surface and the terminal +/// `StreamAction::Close` is WITHHELD behind those bytes (the coordinator self-orders +/// a teardown after the exchange's last transmit). A separate dialer endpoint +/// supplies the push frames. The response transmit is deliberately NOT drained here, +/// so the `Close` stays withheld until the pump's own drain runs. Returns the server +/// exchange id. +fn drive_server_to_withheld_close( + driver: &mut TestDriver, + dialer_addr: SocketAddr, + now: Instant, +) -> ExchangeId { + let mut dialer = build_endpoint("dialer", dialer_addr); + dialer.start_push_pull(sa(DRIVER_ADDR), PushPullKind::Join, now); + let mut dialer_eid = None; + let mut push: Vec> = Vec::new(); + for _ in 0..256 { + let mut progressed = false; + while let Some(action) = dialer.poll_action() { + progressed = true; + if let StreamAction::Connect(info) = action { + dialer_eid = Some(info.id()); + } + } + while let Some((id, _peer, bytes)) = dialer.poll_transport_transmit() { + progressed = true; + if Some(id) == dialer_eid { + push.push(bytes.to_vec()); + } + } + if !progressed { + break; + } + } + assert!( + !push.is_empty(), + "the dialer emitted its push frames up front" + ); + + let server_eid = driver + .endpoint + .accept_connection(dialer_addr, now) + .expect("the driver admits the inbound exchange"); + for chunk in &push { + driver + .endpoint + .handle_transport_data(server_eid, chunk, false, now); + } + driver + .endpoint + .handle_transport_data(server_eid, &[], true, now); // the dialer's FIN + + // Tick the server so it generates the pull response and reaps the exchange + // cleanly. Do NOT drain actions or transport transmits: the response stays queued + // and the terminal `Close` stays withheld behind it. + for _ in 0..32 { + driver.endpoint.handle_timeout(now); + } + server_eid +} + +/// Regression (fixed-point drain): a terminal `StreamAction::Close` the coordinator +/// withholds behind an exchange's final transport transmit must be released AND +/// processed within the SAME poll — the transport surface pops the response +/// (releasing the `Close`), and the fixed-point re-pass drains the `Close`, closing +/// the bridge. A single ordered pass would leave the `Close` buffered, so the pump +/// would report false quiescence and return `Pending` with the TCP bridge still open +/// until an unrelated wake. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn fixed_point_drain_releases_withheld_close_same_poll() { + let now = Instant::now(); + // The default (large) cap: no surface hits its cap, so a false quiescence could + // come ONLY from the single-pass ordering — isolating the fixed-point fix. + let cap = RuntimeOptions::new().iter_drain_cap(); + let (mut driver, _obs_rx, _shared) = build_driver(cap).await; + + let dialer_addr = sa("127.0.0.1:7400"); + let server_eid = drive_server_to_withheld_close(&mut driver, dialer_addr, now); + + // Register the pump's bridge handle for the server exchange as the real accept + // path would: the withheld `Close` targets this handle, and the response transmit + // routes to `out_rx`. + let (out_tx, out_rx) = flume::unbounded::(); + let (cancel_tx, _cancel_rx) = oneshot::channel::<()>(); + driver + .bridges + .insert(server_eid, BridgeHandle { out_tx, cancel_tx }); + assert!( + driver.bridges.contains_key(&server_eid), + "precondition: the server bridge is registered" + ); + + // ONE poll. The fixed-point drain must pop the response transmit (releasing the + // withheld `Close`) and then drain that `Close`, removing the bridge — all here. + let _ = poll_once(&mut driver); + + assert!( + !driver.bridges.contains_key(&server_eid), + "the fixed-point surface drain released and processed the withheld Close in the \ + same poll; a single-pass drain would leave the TCP bridge open past this poll" + ); + assert!( + matches!(out_rx.try_recv(), Ok(BridgeOut::Data(_))), + "the pull response transmit routed to the bridge before its Close" + ); +} From 55c76ad43b493e8a30106c27ea19c5fb4e208de3 Mon Sep 17 00:00:00 2001 From: al8n Date: Thu, 2 Jul 2026 00:31:52 +1200 Subject: [PATCH 030/128] test(serf-reactor): encrypted-gossip convergence + mismatched-key enforcement MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ports serf-compio's two encrypted regressions onto the reactor tcp real-node suite: a shared-keyring 2-node cluster joins and converges over the AEAD-sealed gossip plane, and a disjoint-keyring pair must NOT exchange membership — the negative case proves the cfg-gated encrypt_gossip/decrypt_gossip hops are real AEAD enforcement, not an identity transform. Gated #[cfg(encryption)]. --- serf-reactor/tests/tcp.rs | 144 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) diff --git a/serf-reactor/tests/tcp.rs b/serf-reactor/tests/tcp.rs index 5337cb40..8963127f 100644 --- a/serf-reactor/tests/tcp.rs +++ b/serf-reactor/tests/tcp.rs @@ -17,7 +17,11 @@ use std::net::SocketAddr; use agnostic::tokio::TokioRuntime; use bytes::Bytes; use futures_util::{StreamExt, future}; +#[cfg(encryption)] +use serf_proto::event::MemberEventKind; use serf_proto::{event::Event, members::SerfState, options::Options as SerfOptions}; +#[cfg(encryption)] +use serf_reactor::{EncryptionOptions, Keyring, SecretKey, VoidKeyringDelegate}; use serf_reactor::{ FirstAddrResolver, MaybeResolved, RuntimeOptions, Serf, SocketAddrResolver, TcpTransportOptions, VoidDelegate, @@ -354,3 +358,143 @@ async fn remove_failed_node_alias_succeeds() { a.shutdown().await.expect("rfn-a shuts down"); b.shutdown().await.expect("rfn-b shuts down"); } + +/// A deterministic test secret key, selecting whichever AEAD cipher this build +/// compiled so the encrypted tests work under either backend. +#[cfg(encryption)] +fn test_secret_key(fill: u8) -> SecretKey { + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes256([fill; 32]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = SecretKey::ChaCha20Poly1305([fill; 32]); + key +} + +/// Build and spawn a reactor TCP node on an ephemeral loopback port with +/// `encryption` installed as its gossip-and-reliable keyring policy. +#[cfg(encryption)] +async fn spawn_encrypted_node(id: &str, encryption: EncryptionOptions) -> Node { + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(bind)) + .with_encryption(encryption); + Serf::::tcp( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + std::sync::Arc::new(VoidKeyringDelegate), + ) + .await + .expect("spawn encrypted serf tcp node") +} + +/// Two nodes sharing one keyring join and converge over an AEAD-sealed gossip +/// plane: A joins B (await-result over the encrypted reliable push/pull), both +/// reach the two-member cluster, and A surfaces B's membership through its event +/// stream. Proves the keyring reaches the coordinator and that +/// `encrypt_gossip`/`decrypt_gossip` round-trip end-to-end rather than running as +/// identity transforms. +#[cfg(encryption)] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn two_node_join_converges_encrypted() { + let key = EncryptionOptions::new().with_keyring(Keyring::new(test_secret_key(0x42))); + let b = spawn_encrypted_node("enc-b", key.clone()).await; + let a = spawn_encrypted_node("enc-a", key).await; + let b_addr = b.advertise_address(); + let b_id = SmolStr::new("enc-b"); + + // Subscribe before joining so the membership event cannot race the subscription. + let mut a_events = a.events(); + let reached = a + .join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B over the encrypted reliable plane"); + assert_eq!(reached, b_addr, "join returns the reached seed address"); + + converge(&a, &b).await; + + let observed = tokio::time::timeout(Duration::from_secs(20), async { + loop { + match a_events.next().await { + Some(Event::Member(me)) if me.kind() == MemberEventKind::Join => { + if me.members().iter().any(|m| m.node().id_ref() == &b_id) { + break true; + } + } + Some(_) => {} + None => break false, + } + } + }) + .await; + assert!( + matches!(observed, Ok(true)), + "node A should observe node B joining the encrypted cluster within the timeout" + ); + + a.shutdown().await.expect("enc-a shuts down"); + b.shutdown().await.expect("enc-b shuts down"); +} + +/// A node holding one keyring and a node holding a DIFFERENT keyring must NOT +/// exchange membership: the reliable push/pull units and the gossip datagrams are +/// both AEAD-sealed under disjoint keys, so neither side can authenticate the +/// other and the join never merges. Proves the encryption is real enforcement, +/// not an identity pass-through — without this negative case a passing encrypted +/// convergence test could not distinguish real AEAD from an identity transform. +#[cfg(encryption)] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn mismatched_keyring_nodes_do_not_exchange_membership() { + let b = spawn_encrypted_node( + "mis-b", + EncryptionOptions::new().with_keyring(Keyring::new(test_secret_key(0x42))), + ) + .await; + let a = spawn_encrypted_node( + "mis-a", + EncryptionOptions::new().with_keyring(Keyring::new(test_secret_key(0x43))), + ) + .await; + let b_addr = b.advertise_address(); + let b_id = SmolStr::new("mis-b"); + + let mut a_events = a.events(); + + // Fire-and-forget dispatch: an await-result `join` would instead fail here (the + // mismatched-key push/pull never authenticates); the absence probe below is what + // proves membership never merges. + let dispatched = a + .dispatch_join(&SocketAddrResolver, &[MaybeResolved::Resolved(b_addr)]) + .await + .expect("join dispatched"); + assert_eq!(dispatched, 1, "exactly one seed was dispatched"); + + // Absence probe: A must never surface a Join carrying node-b. A short window + // covers several gossip / probe / push-pull rounds on loopback — the positive + // test forms its cluster within ~1-2s, so a clean 3s window is decisive. + let observed = tokio::time::timeout(Duration::from_secs(3), async { + loop { + match a_events.next().await { + Some(Event::Member(me)) if me.kind() == MemberEventKind::Join => { + if me.members().iter().any(|m| m.node().id_ref() == &b_id) { + break true; + } + } + Some(_) => {} + None => break false, + } + } + }) + .await; + assert!( + !matches!(observed, Ok(true)), + "node A must NOT observe node B across a mismatched keyring" + ); + + a.shutdown().await.expect("mis-a shuts down"); + b.shutdown().await.expect("mis-b shuts down"); +} From 55b222003823a2df08b82479ddfd37bab4c68fbb Mon Sep 17 00:00:00 2001 From: al8n Date: Thu, 2 Jul 2026 13:13:36 +1200 Subject: [PATCH 031/128] =?UTF-8?q?fix(serf-reactor):=20residence-scoped?= =?UTF-8?q?=20pump=20gates=20=E2=80=94=20exact=20inbound=20watermark=20+?= =?UTF-8?q?=20duration=20SWIM=20grace?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the fixed-count deferral (premature at low iter_drain_cap / large exchanges) with two residence-scoped gates: reliable reaps + reliable-exchange timeouts gate on the exact inbound_rx backlog depth snapshotted at deadline-crossing (non-premature by FIFO, flood-immune, no force-fire since join completions never ride the UDP flood); the SWIM suspicion tick fires bounded-early under a UDP flood after a duration grace. Uncaps the event surface (folding a completion/LeftCluster before the reaps); accounts parked bridge hand-offs via an exact in-flight counter; keeps the driver deadline a watermark-gated fallback (no serf-proto caller-deadline plumbing yet); the kernel-unread-at-deadline case is the machine's Go-faithful timeout. --- serf-reactor/src/bridge/mod.rs | 26 +- serf-reactor/src/driver/stream/mod.rs | 310 ++++++++++++------ serf-reactor/src/driver/stream/tests.rs | 403 ++++++++++++++++++++---- 3 files changed, 576 insertions(+), 163 deletions(-) diff --git a/serf-reactor/src/bridge/mod.rs b/serf-reactor/src/bridge/mod.rs index 26b2cf33..794494a5 100644 --- a/serf-reactor/src/bridge/mod.rs +++ b/serf-reactor/src/bridge/mod.rs @@ -43,7 +43,15 @@ //! makes NO progress for the full `close_timeout`; the bridge is then torn down, //! dropping the write half so the OS RSTs the stuck stream. -use std::{future::Future, net::Shutdown, sync::Arc, time::Duration}; +use std::{ + future::Future, + net::Shutdown, + sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + }, + time::Duration, +}; use agnostic::{Runtime, net::TcpStream}; use flume::{Receiver, Sender}; @@ -66,6 +74,12 @@ use crate::{ /// Moves bytes between one exchange's TCP stream and the pump, waking it after /// each inbound enqueue. Reads forward to `inbound_tx`; `out_rx` drives writes /// and the write half-close. +/// +/// `inflight` counts frames this bridge has read but the pump has not yet +/// received: bumped BEFORE each (possibly parking) `inbound_tx` send and cleared +/// by the pump on receive, so the pump's reap watermark accounts for a completion +/// still parked on a saturated hand-off — a frame outside `inbound_rx.len()` that +/// a raw depth watermark would miss. #[allow(clippy::too_many_arguments)] pub(crate) async fn bridge_task( stream: S, @@ -73,6 +87,7 @@ pub(crate) async fn bridge_task( out_rx: Receiver, cancel_rx: oneshot::Receiver<()>, inbound_tx: Sender, + inflight: Arc, shared: Arc>, recv_buf_len: usize, close_timeout: Duration, @@ -183,8 +198,12 @@ pub(crate) async fn bridge_task( } else { BridgeInbound::Eof(payload) }; - // Bounded channel: await space (backpressure), then wake the pump. + // Bounded channel: await space (backpressure), then wake the pump. Count + // the frame in-flight BEFORE the (possibly parking) send so the pump's + // watermark covers it while parked; roll back if the pump is gone. + inflight.fetch_add(1, Ordering::Release); if inbound_tx.send_async(msg).await.is_err() { + inflight.fetch_sub(1, Ordering::Release); break; } shared.wake_driver(); @@ -196,7 +215,10 @@ pub(crate) async fn bridge_task( bytes: buf[..n].to_vec(), received_at: Instant::now(), }); + // Count in-flight BEFORE the (possibly parking) send; roll back if gone. + inflight.fetch_add(1, Ordering::Release); if inbound_tx.send_async(msg).await.is_err() { + inflight.fetch_sub(1, Ordering::Release); break; } shared.wake_driver(); diff --git a/serf-reactor/src/driver/stream/mod.rs b/serf-reactor/src/driver/stream/mod.rs index bfebc122..5ea2d7dd 100644 --- a/serf-reactor/src/driver/stream/mod.rs +++ b/serf-reactor/src/driver/stream/mod.rs @@ -94,17 +94,16 @@ pub(crate) const ACCEPT_CAP: usize = 256; /// channel (the payload byte budget bounds their bytes; this bounds their count). const OBS_OVERFLOW_MAX: usize = 1024; -/// Upper bound on how many consecutive capped (`more`) polls may DEFER a due -/// timer / deadline reap before it fires regardless. While `more`, ready -/// pre-deadline ingress can sit behind a per-poll cap, so firing immediately could -/// time out a probe / join / leave whose resolving Ack / `ExchangeCompleted` / -/// `LeftCluster` is buffered one poll behind (premature). Deferring re-polls to -/// drain that first — and the pre-deadline backlog is FIFO, so it drains within a -/// few `iter_drain_cap`-sized polls — but an UNBOUNDED defer would let a sustained -/// ingress flood suppress failure detection and join / leave / query deadlines -/// forever (a liveness-denial DoS). This bound caps the deferral, trading a few -/// polls of deadline staleness under flood for guaranteed liveness. -const TIMER_DEFERRAL_LIVENESS_BOUND: u32 = 8; +/// How long the SWIM suspicion tick may be held back under a sustained UDP flood +/// before it fires bounded-early. The refuting Ack that would clear a false +/// `Suspect` rides ONLY the UDP recv path, which a flood pins every poll, so — +/// unlike the reliable plane — the SWIM tick cannot gate on a drain watermark. It +/// need not: SWIM refutes within its multi-second suspicion window, so a few ms of +/// staleness is the correct freshness-under-load trade. This is a `Duration`, not +/// a poll count: the failure mode of a fixed count is that at a low `iter_drain_cap` +/// (or a large exchange) a count elapses while real pre-deadline work is still +/// buffered, whereas a wall-clock grace bounds the staleness directly. +const SWIM_STALENESS_GRACE: Duration = Duration::from_millis(5); /// A message from the pump to a bridge's TCP write side. Teardown is signalled out /// of band by dropping the [`BridgeHandle`], not by a variant here, so it can @@ -213,6 +212,16 @@ struct PendingJoin { requested: usize, /// Wall-clock instant past which the driver replies with whatever `contacted` /// set it has accumulated even if `pending` is non-empty. + /// + /// This is a driver-local FALLBACK, subordinate to the machine's arrival-time + /// `ExchangeCompleted`: [`Self::resolve_reply`] runs only from the `fire` path, + /// which requires the reliable backlog to have drained (`reap_watermark`), so + /// this deadline can never reap a join whose completing frame — stamped + /// `received_at < deadline` — is still in flight. serf-proto's `StreamEndpoint` + /// exposes no per-exchange CALLER deadline (`start_join_push_pull` takes only + /// `peer, ignore_old, now`; the exchange deadline is the frozen inner FSM's own), + /// so this second clock cannot yet be removed outright; plumbing the caller + /// deadline INTO the exchange is a serf-proto follow-up. deadline: Instant, /// One-shot reply channel back to the caller, taken when the reply resolves. /// `None` once resolved; the waiter then lingers — only to drive ignore-stream @@ -341,11 +350,37 @@ where iter_drain_cap: usize, timer: Option>>, timer_deadline: Option, - /// Consecutive capped-poll deferrals of a due timer / deadline reap, bounded by - /// [`TIMER_DEFERRAL_LIVENESS_BOUND`]. Accumulates only while a deadline is - /// overdue AND a per-poll cap is being hit; reset the moment the poll reaches - /// quiescence, the deadline fires, or nothing is due. - timer_deferrals: u32, + /// Monotone count of bridge-inbound items the pump has removed from + /// `inbound_rx` (the step-6 drain AND the shutdown drain). The reliable-plane + /// watermark is expressed against this counter. + inbound_drained_total: u64, + /// The reliable plane's pre-deadline backlog target: `inbound_drained_total` + /// once every item that was in flight toward `inbound_rx` when a reliable + /// deadline first went due has been drained. Snapshotted ONCE at that crossing; + /// `None` when no reliable deadline is due. The join/leave reaps and the + /// reliable-exchange deadlines inside `handle_timeout` fire only once + /// `inbound_drained_total` reaches it — non-premature by FIFO, and (because a + /// join completion never rides the UDP gossip flood) immune to that flood with + /// no force-fire. + /// + /// Covers only frames a bridge has already READ (`received_at < deadline`, + /// queued or parked). A frame still kernel-resident-but-unread at the deadline + /// is stamped `received_at >= deadline` at its later read, so the FSM's + /// arrival-time gate rejects it as a `Timeout` — a correct, Go-faithful + /// read-deadline outcome (a read completing after the deadline is late), NOT a + /// dropped success, so the watermark does not (and must not) wait for it. + reap_watermark: Option, + /// Wall-clock anchor for the SWIM suspicion plane: the instant the SWIM tick + /// first deferred on a pinned UDP path (reliable backlog already clear). `None` + /// when the SWIM tick is not deferring. The tick fires once + /// [`SWIM_STALENESS_GRACE`] has elapsed since this anchor. + swim_stall_since: Option, + /// Frames a bridge has read but the pump has not yet received — queued in + /// `inbound_rx` OR parked on a bridge's saturated `send_async`. Bridges bump it + /// before each send; the pump clears it on receive. Lets the watermark account + /// for a completion parked OUTSIDE `inbound_rx.len()`, and is exact (drains to + /// `0`) so the watermark is always reachable. + bridge_inbound_inflight: Arc, idle_wake: Duration, leave_timeout: Duration, close_timeout: Duration, @@ -413,7 +448,10 @@ where iter_drain_cap: driver_opts.iter_drain_cap().max(1), timer: None, timer_deadline: None, - timer_deferrals: 0, + inbound_drained_total: 0, + reap_watermark: None, + swim_stall_since: None, + bridge_inbound_inflight: Arc::new(AtomicU64::new(0)), idle_wake: driver_opts.idle_wake_interval(), leave_timeout: driver_opts.leave_timeout(), close_timeout: stream_opts.close_timeout(), @@ -465,6 +503,7 @@ where .as_ref() .expect("a bridge is only spawned while running, before the shutdown freeze drops the template inbound sender") .clone(), + self.bridge_inbound_inflight.clone(), self.shared.clone(), self.bridge_recv_buf_len, self.close_timeout, @@ -881,34 +920,43 @@ where /// fixed point immediately with `more == true`: the caller self-wakes and re-polls /// rather than uncapped-draining a surface a fast peer can refill, keeping each /// poll bounded. - fn drain_surfaces(&mut self, cx: &mut Context<'_>) -> (bool, bool) + fn drain_surfaces(&mut self, cx: &mut Context<'_>) -> (bool, bool, bool) where I: Send + Sync + 'static, { let now = Instant::now(); let mut worked = false; + let mut ingress_capped = false; loop { - let (pass_worked, pass_more) = self.drain_surfaces_pass(cx, now); + let (pass_worked, pass_more, pass_ingress_capped) = self.drain_surfaces_pass(cx, now); worked |= pass_worked; + // `ingress_capped` gates the SWIM staleness plane (a pinned UDP path), so + // surface it across passes even though the machine's inbound ingress is fed + // only by the pre-drain recv loop and so caps at most in the first pass. + ingress_capped |= pass_ingress_capped; if pass_more { // A per-surface cap was hit: end the fixed point and self-wake (`more`) // rather than repeat the pass, so the poll stays bounded. - return (worked, true); + return (worked, true, ingress_capped); } if !pass_worked { // No surface made progress: the fixed point is reached and no ready machine // work remains from this poll's input. - return (worked, false); + return (worked, false, ingress_capped); } } } /// One ordered surface pass — inbound-ingress → action → transport → gossip → - /// event — each surface draining up to `iter_drain_cap` items. Returns - /// `(worked, more)`: whether any surface produced work in this pass, and whether - /// any surface hit its cap. [`Self::drain_surfaces`] iterates this to a fixed - /// point so a later surface feeding an earlier one is drained the same poll. - fn drain_surfaces_pass(&mut self, cx: &mut Context<'_>, now: Instant) -> (bool, bool) + /// event — each capped surface draining up to `iter_drain_cap` items. Returns + /// `(worked, more, ingress_capped)`: whether any surface produced work in this + /// pass, whether any capped surface hit its cap, and whether the inbound-ingress + /// surface specifically hit its cap (the UDP recv path's second stage, which + /// gates the SWIM staleness plane). The event surface is UNCAPPED (drained to + /// empty) so a surfaced terminal is never stranded behind a cap. + /// [`Self::drain_surfaces`] iterates this to a fixed point so a later surface + /// feeding an earlier one is drained the same poll. + fn drain_surfaces_pass(&mut self, cx: &mut Context<'_>, now: Instant) -> (bool, bool, bool) where I: Send + Sync + 'static, { @@ -950,7 +998,8 @@ where } } worked |= ingress > 0; - more |= ingress == budget; + let ingress_capped = ingress == budget; + more |= ingress_capped; // Stream actions: open dials, half-close, or tear down reliable exchanges. let mut actions = 0; @@ -1024,20 +1073,23 @@ where worked |= sent > 0; more |= sent == budget; - // Observation events: retry the overflow first, then drain up to the budget. + // Observation events: retry the overflow first, then drain to EMPTY. UNLIKE + // the other surfaces this one is NOT capped: a surfaced terminal + // (`ExchangeCompleted` / `LeftCluster`) folded by `send_observation` must never + // be stranded behind a per-poll cap under a UDP flood — that residence is + // exactly what let a flood defeat the reap gate. Sound: `pending_events` is + // pump-fed (bounded per poll by the already-capped feeds + an O(members) + // `handle_timeout` burst), `send_observation` is non-blocking (overflow+drop), + // and there is no event→event feedback, so the drain terminates. self.flush_obs_overflow(); - let mut events = 0; - while events < budget { - let Some(ev) = self.endpoint.poll_event() else { - break; - }; - events += 1; + let mut events = false; + while let Some(ev) = self.endpoint.poll_event() { + events = true; self.send_observation(ev); } - worked |= events > 0; - more |= events == budget; + worked |= events; - (worked, more) + (worked, more, ingress_capped) } /// Retries retained overflow events into the obs channel, stopping at the first @@ -1202,6 +1254,21 @@ where self.pending_leave.as_ref().map(|pl| pl.deadline) } + /// Frames that a bridge has read but the pump has not yet drained, expressed as + /// the pair `(queued, parked)`: `queued` is the observable `inbound_rx` depth, + /// `parked` is the residue a bridge has read (`received_at < deadline`) but is + /// still parked delivering on a SATURATED `send_async` — a completion OUTSIDE + /// `inbound_rx.len()` that a raw depth watermark would miss. `bridge_inbound_inflight` + /// counts queued+parked (bridges bump before each send, the pump clears on + /// receive), so `parked = inflight - queued`. Both terms are added to the reap + /// watermark; every counted frame is received exactly once, so the watermark is + /// always reachable (no over-count can strand it above the drain). + fn inbound_backlog_watermark_terms(&self) -> (u64, u64) { + let queued = self.inbound_rx.as_ref().map_or(0, |rx| rx.len() as u64); + let inflight = self.bridge_inbound_inflight.load(Ordering::Acquire); + (queued, inflight.saturating_sub(queued)) + } + /// Publish a fresh [`SerfSnapshot`] of the endpoint's observable membership. /// Skips the publish when the local node is not yet present in the membership /// store (the local `NodeJoined` sieve has not fired), so `SerfSnapshot::new` @@ -1348,18 +1415,24 @@ where bytes, received_at, })) => { + this.inbound_drained_total += 1; + this.bridge_inbound_inflight.fetch_sub(1, Ordering::Release); this .endpoint .handle_transport_data(eid, &bytes, false, received_at); channel_work = true; } Ok(BridgeInbound::Eof(BridgeEof { eid, received_at })) => { + this.inbound_drained_total += 1; + this.bridge_inbound_inflight.fetch_sub(1, Ordering::Release); this .endpoint .handle_transport_data(eid, &[], true, received_at); channel_work = true; } Ok(BridgeInbound::Error(BridgeEof { eid, received_at })) => { + this.inbound_drained_total += 1; + this.bridge_inbound_inflight.fetch_sub(1, Ordering::Release); this.endpoint.handle_transport_error(eid, received_at); channel_work = true; } @@ -1373,7 +1446,7 @@ where } // Pull the resulting machine surfaces to QUIESCENCE; account_event folds // every terminal completion into the matching pending join. - let (_, surf_more) = this.drain_surfaces(cx); + let (_, surf_more, _) = this.drain_surfaces(cx); if channel_work || surf_more { continue; } @@ -1455,9 +1528,11 @@ where if recv_n > 0 { progress = true; } - if recv_n == this.iter_drain_cap { - more = true; - } + // Stage 1 of the UDP path: a full recv batch means the kernel may hold more. + // Folded (with the ingress-decode stage) into `udp_backlog`, which gates ONLY + // the SWIM staleness plane — never the reliable reap. + let recv_capped = recv_n == this.iter_drain_cap; + more |= recv_capped; // Accept inbound connections. Aux tasks wake the driver after enqueueing, so // `try_recv` (no waker registration) is sufficient. @@ -1484,6 +1559,10 @@ where break; }; inbound_n += 1; + // Advance the reliable-plane drain counter and clear the frame's in-flight + // reservation BEFORE folding it, so the watermark reflects post-drain state. + this.inbound_drained_total += 1; + this.bridge_inbound_inflight.fetch_sub(1, Ordering::Release); this.dispatch_bridge_inbound(msg); } if inbound_n > 0 { @@ -1493,8 +1572,8 @@ where more = true; } - // Drain machine surfaces (bounded per surface). - let (drained, drain_more) = this.drain_surfaces(cx); + // Drain machine surfaces (bounded per surface; the event surface uncapped). + let (drained, drain_more, ingress_capped) = this.drain_surfaces(cx); progress |= drained; more |= drain_more; // A conflict `Event::Shutdown` observed during the drain flips the shutdown @@ -1503,79 +1582,108 @@ where more = true; } - // Timer + deadline reaps under a BOUNDED-DEFERRAL liveness policy. Fold the - // coordinator's next deadline together with the earliest parked join / leave - // deadline into one `target`; `due` means at least one is overdue. Firing while - // ready pre-deadline ingress is still buffered would be premature (false - // suspicion / a spurious `JoinAllFailed` / a `LeaveTimeout`), but never firing - // while `more` would let a flood starve the deadline — so the two cases split. + // Timer + deadline reaps under two RESIDENCE-SCOPED gates (replacing the old + // fixed-count deferral, which fired prematurely at a low `iter_drain_cap` or a + // large exchange, and could starve under a flood). Every join/leave-resolving + // input rides the reliable `inbound_rx` FIFO; a UDP gossip flood deposits ZERO + // there. So the reliable reaps + the reliable-exchange deadlines in + // `handle_timeout` gate on the exact `inbound_rx` backlog DEPTH — snapshotted + // once at deadline-crossing (`reap_watermark`) — while the refutable SWIM + // suspicion tick, whose Ack rides only the pinned UDP path, fires bounded-early + // after a wall-clock staleness grace. let endpoint_deadline = this .endpoint .poll_timeout() .map(|d| d.min(now + this.idle_wake)) .unwrap_or(now + this.idle_wake); - let target = [ - Some(endpoint_deadline), + let reap_deadline = [ this.min_pending_join_deadline(), this.min_pending_leave_deadline(), ] .into_iter() .flatten() - .min() - .unwrap_or(endpoint_deadline); - let due = target <= now; - // The deferral counter accumulates only while a deadline is actually overdue. - if !due { - this.timer_deferrals = 0; + .min(); + // `endpoint_deadline` folds `idle_wake`, so `ep_due` is exactly "the coordinator + // has an elapsed SWIM / reliable-exchange deadline"; a bare idle wake is not due + // and takes the idle arm below. + let ep_due = endpoint_deadline <= now; + let reap_due = reap_deadline.is_some_and(|d| d <= now); + + // Snapshot the reliable backlog ONCE, when a reliable deadline first goes due: + // the drain target past which every completion in flight at the deadline (queued + // in `inbound_rx` OR parked on a saturated bridge hand-off) is guaranteed folded. + if (ep_due || reap_due) && this.reap_watermark.is_none() { + let (queued, parked) = this.inbound_backlog_watermark_terms(); + this.reap_watermark = Some(this.inbound_drained_total + queued + parked); } - - if !more { - // Quiescent: the recv / bridge-inbound loops drained to `Poll::Pending` and - // `drain_surfaces` reached its fixed point, so the whole poll-entry backlog is - // processed. Fire an overdue deadline inline (the single `handle_timeout` - // site) — non-prematurely, since nothing that could resolve it is still - // buffered — else arm + poll the sleep so the next deadline wakes the pump. - if due { + if !ep_due && !reap_due { + this.reap_watermark = None; + this.swim_stall_since = None; + } + // GATE 1 (reliable plane): the pre-deadline backlog has drained. FIFO ⇒ a + // completion buffered at the deadline is folded (resolving its join/leave) + // before the counter reaches the watermark; fixed at snapshot ⇒ a concurrent + // flood (UDP, or bridge appends behind the mark) never pushes it away, so the + // pump drains to it in bounded polls with no force-fire. + let tcp_clear = this + .reap_watermark + .is_none_or(|w| this.inbound_drained_total >= w); + // Both UDP stages: a full recv batch (kernel may hold more) or a capped ingress + // decode. Gates ONLY the SWIM plane. + let udp_backlog = recv_capped || ingress_capped; + // Anchor the SWIM staleness grace the first poll the tick is held back purely by + // a pinned UDP path (reliable backlog already clear, nothing else due). + if ep_due && !reap_due && tcp_clear && udp_backlog { + this.swim_stall_since.get_or_insert(now); + } + // GATE 2 (SWIM plane): fire once the UDP path drained this poll, OR the staleness + // grace elapsed. A reap forcing the shared `handle_timeout` also satisfies it + // (`reap_due` short-circuits below). + let swim_ok = !udp_backlog + || this + .swim_stall_since + .is_some_and(|t| now.saturating_duration_since(t) >= SWIM_STALENESS_GRACE); + let fire = tcp_clear && (ep_due || reap_due) && (reap_due || swim_ok); + + if fire { + // Reliable backlog folded: fire the coordinator's elapsed deadlines, then fold + // the UNCAPPED terminal events they emit (`LeftCluster` / `ExchangeCompleted`) + // BEFORE the reaps — a same-poll leave whose `LeftCluster` is unfolded would + // otherwise reap a false `LeaveTimeout` — then reap the deadline residue (a + // no-op for a resolve already folded here). + if ep_due { this.endpoint.handle_timeout(now); - this.timer_deferrals = 0; - progress = true; - more = true; - } else { - this.arm_timer(target, now); - if let Some(timer) = this.timer.as_mut() - && timer.as_mut().poll(cx).is_ready() - { - this.endpoint.handle_timeout(Instant::now()); - this.timer = None; - this.timer_deadline = None; - progress = true; - more = true; - } } - - // Reap deadline-expired join / leave waiters (a fired `handle_timeout` may - // have completed exchanges; the deadline path resolves the rest). + while let Some(ev) = this.endpoint.poll_event() { + this.send_observation(ev); + } this.reap_pending_joins(now); this.reap_pending_leave(now); - } else if due { - // `more`: a per-poll cap was hit, so ready pre-deadline ingress may sit BEHIND - // it — a resolving Ack / `ExchangeCompleted` / `LeftCluster` could be one poll - // behind. DEFER the timer + reaps and re-poll (the `more` self-wake below) so - // that FIFO backlog drains first. Deferral is BOUNDED: after - // `TIMER_DEFERRAL_LIVENESS_BOUND` consecutive deferrals the pre-deadline - // backlog is drained (FIFO, `iter_drain_cap` per poll), so fire regardless — - // an unbounded defer would let a sustained flood starve failure detection and - // join / leave / query deadlines. No timer is armed while deferring; the - // self-wake alone re-polls, so there is no lost wakeup and the counter - // guarantees the deferral terminates. - if this.timer_deferrals >= TIMER_DEFERRAL_LIVENESS_BOUND { - this.endpoint.handle_timeout(now); - this.reap_pending_joins(now); - this.reap_pending_leave(now); - this.timer_deferrals = 0; - progress = true; - } else { - this.timer_deferrals += 1; + this.reap_watermark = None; + this.swim_stall_since = None; + progress = true; + more = true; + } else if ep_due || reap_due { + // A deadline is due but its gate is not yet satisfied (reliable backlog still + // draining, or the SWIM plane inside its grace). DEFER: self-wake and re-poll + // so step-6 drains `inbound_rx` toward the watermark / the grace elapses. No + // timer is armed — the deadline already elapsed, so the `more` self-wake alone + // re-polls (no lost wakeup), and the exact watermark guarantees termination. + more = true; + } else { + // Idle: nothing due. Arm + poll the sleep for the next deadline; NO self-wake + // (return `Pending` — the armed sleep, a bridge `wake_driver`, or a socket + // readiness re-polls us). + let target = reap_deadline.map_or(endpoint_deadline, |d| d.min(endpoint_deadline)); + this.arm_timer(target, now); + if let Some(timer) = this.timer.as_mut() + && timer.as_mut().poll(cx).is_ready() + { + // The sleep elapsed at/just after arming: clear it and self-wake so the next + // poll re-evaluates with `now` advanced and fires through the gates above. + this.timer = None; + this.timer_deadline = None; + more = true; } } diff --git a/serf-reactor/src/driver/stream/tests.rs b/serf-reactor/src/driver/stream/tests.rs index b9ae1f49..d3cdcebc 100644 --- a/serf-reactor/src/driver/stream/tests.rs +++ b/serf-reactor/src/driver/stream/tests.rs @@ -89,6 +89,7 @@ fn initial_snapshot(id: &str, advertise: SocketAddr) -> SerfSnapshot ( TestDriver, Receiver>, @@ -123,7 +124,7 @@ async fn build_driver( accept_shutdown_tx, accept_join, RuntimeOptions::new().with_iter_drain_cap(iter_drain_cap), - StreamTransportOptions::new(), + StreamTransportOptions::new().with_bridge_inbound_cap(bridge_inbound_cap), None, #[cfg(encryption)] Arc::new(crate::VoidKeyringDelegate), @@ -131,6 +132,17 @@ async fn build_driver( (driver, obs_rx, shared) } +/// Queue one bridge-inbound item toward the pump AS A REAL BRIDGE WOULD: bump the +/// in-flight reservation BEFORE the hand-off, then enqueue. Test-injected frames +/// bypass `bridge_task`, so without this the reap watermark would not account for +/// them and could reap a join whose completion is still buffered. +fn queue_inbound(driver: &TestDriver, tx: &Sender, item: BridgeInbound) { + driver + .bridge_inbound_inflight + .fetch_add(1, Ordering::Release); + tx.try_send(item).expect("queue inbound item"); +} + /// Drive one outbound Join push/pull on `driver.endpoint` toward `seed_addr` to a /// real `Succeeded`, returning the dialer exchange id and the peer's pull response /// frames (which the caller queues on `inbound_rx` rather than feeding here). A @@ -210,23 +222,23 @@ fn drive_push_to_queued_response( (eid, response) } -/// Regression: at a DUE await-result-join deadline, the pull `ExchangeCompleted` -/// that resolves the join `Ok` is queued one bridge-inbound item behind the -/// per-poll `iter_drain_cap`. The single `handle_timeout` site and the join -/// deadline reap must wait for that pre-deadline completion to drain — a premature -/// reap would surface a spurious `JoinAllFailed` against a seed that was in fact -/// reached. +/// Regression (Gate 1, non-premature AT DEPTH): at a DUE await-result-join +/// deadline, the pull `ExchangeCompleted` that resolves the join `Ok` is buffered +/// on `inbound_rx` behind a DEEP backlog (far more than the old fixed `8`-poll +/// deferral bound), drained one item per poll at `iter_drain_cap == 1`. The exact +/// inbound-depth watermark must defer the reap the WHOLE way — until the completion +/// drains — so the join resolves `Ok`, not a spurious `JoinAllFailed`. /// -/// Pre-fix, the first poll (bridge-inbound cap hit → `more`) still runs the -/// past-due reap and replies `JoinAllFailed`. Post-fix, the timer + reap are gated -/// on quiescence, so the completion drains first and the join resolves `Ok`. +/// This is the case the old fixed-count deferral got wrong: at depth `> 8` (or a +/// low `iter_drain_cap` / a large exchange) the count elapsed while the resolving +/// completion was still buffered, force-firing a premature reap. The watermark is a +/// DEPTH, not a count, so it is immune. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn due_deadline_waits_for_join_completion_behind_iter_drain_cap() { +async fn due_deadline_waits_for_join_completion_behind_deep_backlog() { let now = Instant::now(); - // iter_drain_cap = 1: the bridge-inbound loop processes at most one item per - // poll, so the completion staged behind the response chunk(s) is at least one - // poll behind at the due deadline. - let (mut driver, _obs_rx, _shared) = build_driver(1).await; + // iter_drain_cap = 1 (one inbound item per poll); a bridge cap large enough to + // hold the whole deep backlog at once. + let (mut driver, _obs_rx, _shared) = build_driver(1, 4096).await; let seed_addr = sa("127.0.0.1:7000"); @@ -235,25 +247,61 @@ async fn due_deadline_waits_for_join_completion_behind_iter_drain_cap() { // behind the per-poll cap. let (eid, response) = drive_push_to_queued_response(&mut driver, seed_addr, now); let inbound_tx = driver.inbound_tx.as_ref().expect("template alive").clone(); + + // A dummy 'connecting' exchange — its Connect is captured but never dialed, so no + // bridge is minted and the machine has no conn for it: transport data keyed by its + // id is ignored. That gives a benign filler id without touching the real join. + driver + .endpoint + .start_push_pull(sa("127.0.0.1:7001"), PushPullKind::Join, now); + let mut dummy = None; + while let Some(action) = driver.endpoint.poll_action() { + if let StreamAction::Connect(info) = action { + dummy.get_or_insert(info.id()); + } + } + let dummy = dummy.expect("the dummy start_push_pull emitted a Connect exchange id"); + + // A DEEP pre-completion backlog — benign `Data` for that dummy exchange, far past + // the old fixed-8 bound — drained one-per-poll AHEAD of the real completion, each + // counted in-flight exactly as a bridge would. + const DEPTH: usize = 40; + for _ in 0..DEPTH { + queue_inbound( + &driver, + &inbound_tx, + BridgeInbound::Data(BridgeData { + eid: dummy, + bytes: vec![0u8; 1], + received_at: now, + }), + ); + } + // The real pull response + peer-FIN EOF (the resolving `ExchangeCompleted` rides + // the EOF), buffered BEHIND the deep backlog. for bytes in response { - inbound_tx - .try_send(BridgeInbound::Data(BridgeData { + queue_inbound( + &driver, + &inbound_tx, + BridgeInbound::Data(BridgeData { eid, bytes, received_at: now, - })) - .expect("queue inbound response"); + }), + ); } - inbound_tx - .try_send(BridgeInbound::Eof(BridgeEof { + queue_inbound( + &driver, + &inbound_tx, + BridgeInbound::Eof(BridgeEof { eid, received_at: now, - })) - .expect("queue inbound EOF"); + }), + ); // Park an await-result join awaiting `eid` with a deadline ALREADY in the past, // so the deadline reap is due on the very first poll — while the completion that - // resolves it Ok is still queued behind the cap. + // resolves it Ok is far behind the deep backlog. let (tx, mut rx) = oneshot::channel::(); let mut pending = HashSet::new(); pending.insert(eid); @@ -266,11 +314,11 @@ async fn due_deadline_waits_for_join_completion_behind_iter_drain_cap() { reply: Some(tx), }); - // Drive the pump by hand. The `more` self-wake re-polls, so a bounded loop drains - // the staged completion. Pre-fix, the first poll reaps the past-due deadline - // (JoinAllFailed); post-fix, the reap waits until the completion resolves Ok. + // The `more` self-wake re-polls; a bounded loop generous enough to drain the whole + // depth (DEPTH + response + EOF at one item per poll). The old fixed-8 deferral + // would have force-reaped `JoinAllFailed` long before this depth drained. let mut resolved = None; - for _ in 0..256 { + for _ in 0..(DEPTH + 64) { let _ = poll_once(&mut driver); match rx.try_recv() { Ok(Some(reply)) => { @@ -285,37 +333,34 @@ async fn due_deadline_waits_for_join_completion_behind_iter_drain_cap() { let reached = resolved .expect("the join resolved within the bounded poll budget") .expect( - "the ready ExchangeCompleted resolved the join before the past-due deadline reap fired; \ - a premature reap would surface a spurious JoinAllFailed", + "the deep-buffered ExchangeCompleted resolved the join Ok before the past-due reap; \ + the old fixed-8 deferral would have force-reaped a spurious JoinAllFailed at depth > 8", ); assert!( reached.contains(&seed_addr), - "the resolved join reached the seed whose completion was queued behind the cap: {reached:?}" + "the resolved join reached the seed whose completion was behind the deep backlog: {reached:?}" ); } -/// Regression (liveness / non-starving): under a SUSTAINED ingress flood that hits -/// `iter_drain_cap` every poll (so `more` never clears), a past-due await-result -/// join deadline must STILL be reaped within a bounded number of polls. A blunt -/// `if !more` gate never fires the reap while `more`, so a flood would suppress the -/// deadline forever (a liveness-denial DoS); the bounded-deferral policy fires it -/// once `TIMER_DEFERRAL_LIVENESS_BOUND` deferrals elapse. +/// Regression (Gate 1 flood-liveness, sub-case i): under a SUSTAINED UDP gossip +/// flood that pins `udp_backlog` every poll, a past-due await-result join whose +/// exchange never completes (nothing ever rides `inbound_rx` for it) must STILL be +/// reaped `JoinAllFailed` PROMPTLY. The reap gates on the reliable `inbound_rx` +/// watermark — clear here, since the flood is UDP and deposits nothing on the +/// reliable plane — NOT on the flood, so `reap_due` fires the very first poll. /// -/// Pre-fix, the reap is gated out every poll and the join never resolves within the -/// poll budget. Post-fix, it resolves to `JoinAllFailed` (no seed was contacted) -/// within the deferral bound. +/// The old fixed-count deferral instead deferred `8` polls under the flood's `more` +/// before firing; the watermark fires immediately because the reliable backlog is +/// already clear (`inflight == 0`). #[tokio::test(flavor = "multi_thread", worker_threads = 2)] -async fn ingress_flood_does_not_starve_due_join_deadline_reap() { +async fn udp_flood_does_not_suppress_stuck_join_reap() { let now = Instant::now(); - // A small cap makes the per-poll flood cheap. The flood keeps `more` set via the - // inbound-gossip surface: each fed datagram is undecodable garbage the drain - // drops, but it still counts toward the surface's `iter_drain_cap`. let cap = 4usize; - let (mut driver, _obs_rx, _shared) = build_driver(cap).await; + let (mut driver, _obs_rx, _shared) = build_driver(cap, 256).await; - // A real, live outbound exchange id whose exchange never completes (no peer ever - // feeds it): the parked join stays pending, so ONLY the past-due deadline reap can - // resolve it — to `JoinAllFailed`. + // A live outbound exchange with NO completion coming (no peer feeds it) and no + // bridge on `inbound_rx`: the reliable in-flight count stays 0, so the watermark + // is clear and only the past-due deadline reap can resolve it — to JoinAllFailed. driver .endpoint .start_push_pull(sa("127.0.0.1:7300"), PushPullKind::Join, now); @@ -342,9 +387,9 @@ async fn ingress_flood_does_not_starve_due_join_deadline_reap() { let flood_src = sa("127.0.0.1:7301"); let mut resolved = None; let mut polls = 0usize; - for _ in 0..(TIMER_DEFERRAL_LIVENESS_BOUND as usize + 4) { - // Refill the ingress flood BEFORE each poll so the inbound-gossip surface hits - // its cap and `more` is set for this poll. + // A tight budget: the watermark-gated reap fires the FIRST poll (reliable backlog + // clear), far inside a budget the old fixed-8 deferral could not meet. + for _ in 0..3 { for _ in 0..cap { driver .endpoint @@ -363,24 +408,97 @@ async fn ingress_flood_does_not_starve_due_join_deadline_reap() { } let reply = resolved.expect( - "the past-due join deadline reap fired despite the sustained ingress flood; \ - a blunt !more gate would starve it indefinitely", + "the past-due join reap fired despite the UDP flood; it gates on the reliable \ + watermark, not the UDP flood, so the flood cannot suppress it", ); assert!( - polls <= TIMER_DEFERRAL_LIVENESS_BOUND as usize + 1, - "the reap fired within the deferral bound, not later: {polls} polls", + polls <= 2, + "the reap fired at once (reliable backlog clear), not after a deferral: {polls} polls", ); match reply { Err((ref reached, SerfError::JoinAllFailed(_))) => { - assert!( - reached.is_empty(), - "no seed was contacted, so the all-failed set is empty: {reached:?}" - ); + assert!(reached.is_empty(), "no seed contacted: {reached:?}"); } other => panic!("expected JoinAllFailed from the deadline reap, got {other:?}"), } } +/// Regression (Gate 1 flood-liveness, sub-case ii): under the SAME sustained UDP +/// flood, an await-result join whose resolving `ExchangeCompleted` is buffered on +/// `inbound_rx` (a FUTURE deadline, so ONLY the completion — never a deadline reap — +/// can resolve it) must resolve `Ok`. The flood pins the UDP path but the reliable +/// inbound drain (step 6) runs every poll regardless, so it is never starved. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn udp_flood_does_not_starve_inbound_join_completion() { + let now = Instant::now(); + let cap = 4usize; + let (mut driver, _obs_rx, _shared) = build_driver(cap, 256).await; + + let seed_addr = sa("127.0.0.1:7002"); + let (eid, response) = drive_push_to_queued_response(&mut driver, seed_addr, now); + let inbound_tx = driver.inbound_tx.as_ref().expect("template alive").clone(); + for bytes in response { + queue_inbound( + &driver, + &inbound_tx, + BridgeInbound::Data(BridgeData { + eid, + bytes, + received_at: now, + }), + ); + } + queue_inbound( + &driver, + &inbound_tx, + BridgeInbound::Eof(BridgeEof { + eid, + received_at: now, + }), + ); + + // FUTURE deadline: no deadline reap can fire, so a starved TCP drain would hang + // the join forever under the flood. Only the completion can resolve it. + let (tx, mut rx) = oneshot::channel::(); + let mut pending = HashSet::new(); + pending.insert(eid); + driver.pending_joins.push(PendingJoin { + pending, + contacted: SmallVec::new(), + ignore_streams: SmallVec::new(), + requested: 1, + deadline: now + Duration::from_secs(30), + reply: Some(tx), + }); + + let flood_src = sa("127.0.0.1:7003"); + let mut resolved = None; + for _ in 0..256 { + for _ in 0..cap { + driver + .endpoint + .handle_gossip(flood_src, &[0xff, 0x00, 0xff], now); + } + let _ = poll_once(&mut driver); + match rx.try_recv() { + Ok(Some(reply)) => { + resolved = Some(reply); + break; + } + Ok(None) => {} + Err(_) => panic!("join reply sender dropped without resolving"), + } + } + + let reached = resolved + .expect("the buffered completion resolved despite the UDP flood") + .expect("the completion resolved the join Ok; the flood did not starve the TCP drain"); + assert!( + reached.contains(&seed_addr), + "the resolved join reached the seed whose completion drained under the flood: {reached:?}" + ); +} + /// Drive an INBOUND (server-side) Join push/pull on `driver.endpoint` until its pull /// response is queued in the coordinator's transmit surface and the terminal /// `StreamAction::Close` is WITHHELD behind those bytes (the coordinator self-orders @@ -455,7 +573,7 @@ async fn fixed_point_drain_releases_withheld_close_same_poll() { // The default (large) cap: no surface hits its cap, so a false quiescence could // come ONLY from the single-pass ordering — isolating the fixed-point fix. let cap = RuntimeOptions::new().iter_drain_cap(); - let (mut driver, _obs_rx, _shared) = build_driver(cap).await; + let (mut driver, _obs_rx, _shared) = build_driver(cap, 256).await; let dialer_addr = sa("127.0.0.1:7400"); let server_eid = drive_server_to_withheld_close(&mut driver, dialer_addr, now); @@ -487,3 +605,168 @@ async fn fixed_point_drain_releases_withheld_close_same_poll() { "the pull response transmit routed to the bridge before its Close" ); } + +/// Regression (leave-before-reap ordering): a same-poll graceful leave whose +/// `LeftCluster` is emitted BY `handle_timeout` inside the fire path must resolve +/// `Ok`, not `LeaveTimeout`. The fire path folds the uncapped `poll_event` surface +/// BETWEEN `handle_timeout` and `reap_pending_leave`, so the fresh `LeftCluster` +/// resolves the parked leave before the reap sees a still-parked leave at a past +/// deadline. Without that intervening fold the reap would fire a false +/// `LeaveTimeout`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn leave_left_cluster_from_handle_timeout_folds_before_reap() { + let real_now = Instant::now(); + // A base well in the past so the (default 1s) leave-propagate deadline, armed + // relative to `base`, is already elapsed at the real-time driver poll — while the + // pump instants below stay UNDER it, so the Leaving→Left transition is withheld + // until the driver poll's `handle_timeout`. + let base = real_now - Duration::from_secs(5); + let (mut driver, _obs_rx, _shared) = build_driver(64, 256).await; + + driver.endpoint.leave(base).expect("leave from Alive"); + // On a lone node the inner memberlist emits `LeftCluster` immediately; the first + // `handle_timeout` sieves it and ARMS the serf leave-complete deadline. Pump at + // instants held below that deadline so serf stays `Leaving` (deadline armed but + // not yet fired), draining the pre-`LeftCluster` serf events as we go. + let mut armed = false; + for i in 0..64u32 { + let t = base + Duration::from_millis(i as u64); + driver.endpoint.handle_timeout(t); + while driver.endpoint.poll_action().is_some() {} + while driver.endpoint.poll_transport_transmit().is_some() {} + while driver.endpoint.poll_memberlist_transmit().is_some() {} + while driver.endpoint.poll_event().is_some() {} + if driver.endpoint.leave_complete_deadline().is_some() { + armed = true; + break; + } + } + assert!( + armed, + "the inner leave armed the serf leave-complete deadline while still Leaving" + ); + + // Park a leave waiter with an ALREADY-PAST deadline: at the driver poll both the + // endpoint deadline (the armed, now-elapsed leave-complete deadline) and this + // leave deadline are due, so the fire path runs handle_timeout (emitting + // LeftCluster) THEN would reap the leave — the ordering under test. + let (tx, mut rx) = oneshot::channel(); + driver.pending_leave = Some(PendingLeave { + repliers: vec![tx], + deadline: base, + }); + + let mut resolved = None; + for _ in 0..64 { + let _ = poll_once(&mut driver); + match rx.try_recv() { + Ok(Some(reply)) => { + resolved = Some(reply); + break; + } + Ok(None) => {} + Err(_) => panic!("leave reply sender dropped without resolving"), + } + } + + resolved + .expect("the leave resolved within the bounded poll budget") + .expect( + "the LeftCluster emitted by handle_timeout was folded before reap_pending_leave, \ + resolving the leave Ok; without the intervening fold the reap fires LeaveTimeout", + ); +} + +/// Regression (parked saturated hand-off): a resolving completion the bridge has +/// READ (`received_at < deadline`) but that is still PARKED on a SATURATED +/// `inbound_rx` hand-off lives OUTSIDE `inbound_rx.len()`. It is accounted in the +/// watermark via the in-flight reservation, so a past-due join deadline does NOT +/// prematurely reap `JoinAllFailed` — the parked completion drains and resolves the +/// join `Ok`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn parked_saturated_handoff_completion_is_accounted_not_reaped() { + let now = Instant::now(); + // bridge_inbound_cap = 1 forces every hand-off after the first to PARK; one inbound + // item per poll makes the parked completion strictly lag the past-due deadline. + let (mut driver, _obs_rx, _shared) = build_driver(1, 1).await; + + let seed_addr = sa("127.0.0.1:7005"); + let (eid, response) = drive_push_to_queued_response(&mut driver, seed_addr, now); + let inbound_tx = driver.inbound_tx.as_ref().expect("template alive").clone(); + + // The frames the parked bridge would deliver: the pull response + the peer-FIN EOF + // that completes the exchange. They stay OUT of inbound_rx (the parked residence), + // delivered one-at-a-time as the pump frees the cap-1 slot below. + let mut parked_frames: std::collections::VecDeque = response + .into_iter() + .map(|bytes| { + BridgeInbound::Data(BridgeData { + eid, + bytes, + received_at: now, + }) + }) + .collect(); + parked_frames.push_back(BridgeInbound::Eof(BridgeEof { + eid, + received_at: now, + })); + + // RESERVE the in-flight count for EVERY held frame up front — exactly the + // reservation a real `bridge_task` accrues via its pre-send bump — while the + // frames themselves are NOT yet in inbound_rx. That is the residence a raw + // `inbound_rx.len()` watermark misses: a completion read (`received_at < deadline`) + // but still parked on a saturated `send_async`, so `inflight > len`. + for _ in 0..parked_frames.len() { + driver + .bridge_inbound_inflight + .fetch_add(1, Ordering::Release); + } + + // Past deadline: only the watermark accounting for the parked completion stands + // between the pump and a premature JoinAllFailed. + let (tx, mut rx) = oneshot::channel::(); + let mut pending = HashSet::new(); + pending.insert(eid); + driver.pending_joins.push(PendingJoin { + pending, + contacted: SmallVec::new(), + ignore_streams: SmallVec::new(), + requested: 1, + deadline: now - Duration::from_secs(1), + reply: Some(tx), + }); + + let mut resolved = None; + for _ in 0..512 { + // Land the next parked frame into the freed cap-1 slot — a deterministic, + // race-free stand-in for a bridge whose `send_async` unparks as the pump drains. + // The watermark sees the identical state a real parked hand-off produces + // (`inflight` reserved for the un-landed frames, `len` only the one in flight). + if !inbound_tx.is_full() + && let Some(frame) = parked_frames.pop_front() + { + inbound_tx.try_send(frame).expect("land parked frame"); + } + let _ = poll_once(&mut driver); + match rx.try_recv() { + Ok(Some(reply)) => { + resolved = Some(reply); + break; + } + Ok(None) => {} + Err(_) => panic!("join reply sender dropped without resolving"), + } + } + + let reached = resolved + .expect("the join resolved within the bounded poll budget") + .expect( + "the parked completion was accounted in the watermark and folded before the past-due reap; \ + a raw inbound_rx.len() watermark would miss it and reap a spurious JoinAllFailed", + ); + assert!( + reached.contains(&seed_addr), + "the resolved join reached the seed whose completion was parked on the saturated hand-off: {reached:?}" + ); +} From be31707c90e11d01db3eab520c39264f7a78d9f0 Mon Sep 17 00:00:00 2001 From: al8n Date: Thu, 2 Jul 2026 13:51:03 +1200 Subject: [PATCH 032/128] fix(serf-reactor): reap watermark covers the latest due deadline, not the first The single sticky watermark snapshotted at the first due-crossing could expire a LATER overlapping deadline against the earlier deadline's backlog target: a t1 completion queued behind W0 was reaped prematurely under a low iter_drain_cap. The watermark now re-snapshots to the current inbound depth whenever the max due deadline advances, so firing at tcp_clear is non-premature for every due deadline while staying live and terminating. Regression: two joins t0= deadline` at its later read, so the FSM's @@ -370,6 +378,12 @@ where /// read-deadline outcome (a read completing after the deadline is late), NOT a /// dropped success, so the watermark does not (and must not) wait for it. reap_watermark: Option, + /// The latest due deadline [`Self::reap_watermark`] currently covers. The + /// watermark re-snapshots to the live inbound depth whenever the greatest due + /// deadline advances past this, so it always covers the backlog as of the + /// latest due deadline — not merely the first that went due. `None` exactly + /// when `reap_watermark` is `None`. + reap_watermark_covers: Option, /// Wall-clock anchor for the SWIM suspicion plane: the instant the SWIM tick /// first deferred on a pinned UDP path (reliable backlog already clear). `None` /// when the SWIM tick is not deferring. The tick fires once @@ -450,6 +464,7 @@ where timer_deadline: None, inbound_drained_total: 0, reap_watermark: None, + reap_watermark_covers: None, swim_stall_since: None, bridge_inbound_inflight: Arc::new(AtomicU64::new(0)), idle_wake: driver_opts.idle_wake_interval(), @@ -1254,6 +1269,22 @@ where self.pending_leave.as_ref().map(|pl| pl.deadline) } + /// The GREATEST join/leave deadline already due (`<= now`), or `None` when none + /// is. Mirrors the min-deadline folds ([`Self::min_pending_join_deadline`] + /// filters to still-unreplied joins) but takes the max, so the reap watermark + /// re-snapshots to cover the backlog as of the LATEST due deadline; the FIFO + /// backlog then covers every earlier due deadline too. + fn max_due_join_leave_deadline(&self, now: Instant) -> Option { + self + .pending_joins + .iter() + .filter(|pj| pj.reply.is_some()) + .map(|pj| pj.deadline) + .chain(self.pending_leave.as_ref().map(|pl| pl.deadline)) + .filter(|&d| d <= now) + .max() + } + /// Frames that a bridge has read but the pump has not yet drained, expressed as /// the pair `(queued, parked)`: `queued` is the observable `inbound_rx` depth, /// `parked` is the residue a bridge has read (`received_at < deadline`) but is @@ -1588,7 +1619,7 @@ where // input rides the reliable `inbound_rx` FIFO; a UDP gossip flood deposits ZERO // there. So the reliable reaps + the reliable-exchange deadlines in // `handle_timeout` gate on the exact `inbound_rx` backlog DEPTH — snapshotted - // once at deadline-crossing (`reap_watermark`) — while the refutable SWIM + // to cover the LATEST due deadline (`reap_watermark`) — while the refutable SWIM // suspicion tick, whose Ack rides only the pinned UDP path, fires bounded-early // after a wall-clock staleness grace. let endpoint_deadline = this @@ -1609,22 +1640,43 @@ where let ep_due = endpoint_deadline <= now; let reap_due = reap_deadline.is_some_and(|d| d <= now); - // Snapshot the reliable backlog ONCE, when a reliable deadline first goes due: - // the drain target past which every completion in flight at the deadline (queued - // in `inbound_rx` OR parked on a saturated bridge hand-off) is guaranteed folded. - if (ep_due || reap_due) && this.reap_watermark.is_none() { - let (queued, parked) = this.inbound_backlog_watermark_terms(); - this.reap_watermark = Some(this.inbound_drained_total + queued + parked); - } - if !ep_due && !reap_due { - this.reap_watermark = None; - this.swim_stall_since = None; + // Snapshot the reliable backlog target to cover the LATEST currently-due + // deadline, not merely the first that went due. `inbound_rx` is FIFO and grows + // only with time, so the tail depth at the greatest due deadline dominates + // every earlier due deadline's pre-deadline completions. Re-snapshot to the + // live inbound depth (queued in `inbound_rx` OR parked on a saturated bridge + // hand-off) whenever that max due deadline advances past what the watermark + // already covers — the endpoint contributes its earliest deadline (`poll_timeout` + // exposes only the min), each still-unreplied join and the leave its own — so a + // later overlapping join/leave deadline re-arms a wider target rather than + // reaping its still-buffered completion against the first deadline's mark. + let max_due = { + let mut m = ep_due.then_some(endpoint_deadline); + if let Some(d) = this.max_due_join_leave_deadline(now) { + m = Some(m.map_or(d, |cur| cur.max(d))); + } + m + }; + match max_due { + Some(md) => { + if this.reap_watermark.is_none() || this.reap_watermark_covers.is_none_or(|c| md > c) { + let (queued, parked) = this.inbound_backlog_watermark_terms(); + this.reap_watermark = Some(this.inbound_drained_total + queued + parked); + this.reap_watermark_covers = Some(md); + } + } + None => { + this.reap_watermark = None; + this.reap_watermark_covers = None; + this.swim_stall_since = None; + } } // GATE 1 (reliable plane): the pre-deadline backlog has drained. FIFO ⇒ a // completion buffered at the deadline is folded (resolving its join/leave) - // before the counter reaches the watermark; fixed at snapshot ⇒ a concurrent - // flood (UDP, or bridge appends behind the mark) never pushes it away, so the - // pump drains to it in bounded polls with no force-fire. + // before the counter reaches the watermark; the mark advances only on a new + // deadline crossing, never on a live append ⇒ a concurrent flood (UDP, or + // bridge appends behind the mark) never pushes it away, so the pump drains to + // it in bounded polls with no force-fire. let tcp_clear = this .reap_watermark .is_none_or(|w| this.inbound_drained_total >= w); @@ -1660,6 +1712,7 @@ where this.reap_pending_joins(now); this.reap_pending_leave(now); this.reap_watermark = None; + this.reap_watermark_covers = None; this.swim_stall_since = None; progress = true; more = true; diff --git a/serf-reactor/src/driver/stream/tests.rs b/serf-reactor/src/driver/stream/tests.rs index d3cdcebc..3f6f8737 100644 --- a/serf-reactor/src/driver/stream/tests.rs +++ b/serf-reactor/src/driver/stream/tests.rs @@ -770,3 +770,287 @@ async fn parked_saturated_handoff_completion_is_accounted_not_reaped() { "the resolved join reached the seed whose completion was parked on the saturated hand-off: {reached:?}" ); } + +/// Regression (overlapping deadlines, Gate 1): a SECOND await-result join whose +/// resolving completion is buffered on `inbound_rx` AFTER the FIRST join's +/// (earlier) deadline crossing snapshotted the reap watermark, but BEFORE the +/// second join's own (later) deadline elapses. A single sticky watermark taken at +/// the first crossing (`W0`) covers only the first join's backlog, so once the +/// later deadline also elapses the shared fire path reaps BOTH joins against `W0` +/// — spuriously failing the later join whose `Ok` completion is still queued +/// behind `W0`. Re-snapshotting the watermark when the latest due deadline +/// advances widens the target to cover the later join's backlog, so both resolve +/// `Ok`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn overlapping_deadlines_later_join_waits_for_its_own_backlog() { + let now = Instant::now(); + // iter_drain_cap = 1: one inbound item per poll, so W0 drains slowly enough that + // the later deadline elapses mid-drain (the fire poll then sees both due). + let (mut driver, _obs_rx, _shared) = build_driver(1, 4096).await; + + // Two reached seeds; each drives an outbound push/pull to a Succeeded whose pull + // response + peer-FIN we queue on inbound_rx. + let seed_a = sa("127.0.0.1:7010"); + let seed_b = sa("127.0.0.1:7011"); + let (eid_a, resp_a) = drive_push_to_queued_response(&mut driver, seed_a, now); + let (eid_b, resp_b) = drive_push_to_queued_response(&mut driver, seed_b, now); + let inbound_tx = driver.inbound_tx.as_ref().expect("template alive").clone(); + + // A dummy 'connecting' exchange for benign filler (its transport data is dropped + // by the machine — no conn), deepening the first join's backlog. + driver + .endpoint + .start_push_pull(sa("127.0.0.1:7012"), PushPullKind::Join, now); + let mut dummy = None; + while let Some(action) = driver.endpoint.poll_action() { + if let StreamAction::Connect(info) = action { + dummy.get_or_insert(info.id()); + } + } + let dummy = dummy.expect("the dummy start_push_pull emitted a Connect exchange id"); + + // Backlog + join A's completion queued FIRST: this is the depth W0 will cover. + // Join B's completion is deliberately NOT queued yet. + const DEPTH: usize = 24; + for _ in 0..DEPTH { + queue_inbound( + &driver, + &inbound_tx, + BridgeInbound::Data(BridgeData { + eid: dummy, + bytes: vec![0u8; 1], + received_at: now, + }), + ); + } + for bytes in resp_a { + queue_inbound( + &driver, + &inbound_tx, + BridgeInbound::Data(BridgeData { + eid: eid_a, + bytes, + received_at: now, + }), + ); + } + queue_inbound( + &driver, + &inbound_tx, + BridgeInbound::Eof(BridgeEof { + eid: eid_a, + received_at: now, + }), + ); + + // Join A: deadline ALREADY past → due on the first poll, snapshotting W0 to the + // depth queued above. Join B: deadline in the near FUTURE (captured fresh so a + // slow setup cannot backdate it), so on that first poll B does NOT yet widen the + // watermark. + let t_ref = Instant::now(); + let future = Duration::from_millis(100); + let (tx_a, mut rx_a) = oneshot::channel::(); + let mut pending_a = HashSet::new(); + pending_a.insert(eid_a); + driver.pending_joins.push(PendingJoin { + pending: pending_a, + contacted: SmallVec::new(), + ignore_streams: SmallVec::new(), + requested: 1, + deadline: t_ref - Duration::from_secs(1), + reply: Some(tx_a), + }); + let (tx_b, mut rx_b) = oneshot::channel::(); + let mut pending_b = HashSet::new(); + pending_b.insert(eid_b); + driver.pending_joins.push(PendingJoin { + pending: pending_b, + contacted: SmallVec::new(), + ignore_streams: SmallVec::new(), + requested: 1, + deadline: t_ref + future, + reply: Some(tx_b), + }); + + // One poll snapshots W0 at join A's crossing (join B still future), covering only + // the depth queued so far — NOT join B's completion. + let _ = poll_once(&mut driver); + + // NOW queue join B's completion, strictly BEHIND W0 and BEFORE its own deadline + // `t_ref + future` (which has not yet elapsed). + for bytes in resp_b { + queue_inbound( + &driver, + &inbound_tx, + BridgeInbound::Data(BridgeData { + eid: eid_b, + bytes, + received_at: now, + }), + ); + } + queue_inbound( + &driver, + &inbound_tx, + BridgeInbound::Eof(BridgeEof { + eid: eid_b, + received_at: now, + }), + ); + + // Let join B's deadline elapse while W0 is still draining: the next fire poll now + // sees BOTH deadlines due. A single sticky W0 reaps join B before its buffered + // completion drains; the re-snapshotted watermark defers until it does. + tokio::time::sleep(future + Duration::from_millis(50)).await; + + let mut reached_a = None; + let mut reached_b = None; + for _ in 0..(DEPTH + 128) { + let _ = poll_once(&mut driver); + if reached_a.is_none() + && let Ok(Some(reply)) = rx_a.try_recv() + { + reached_a = Some(reply); + } + if reached_b.is_none() + && let Ok(Some(reply)) = rx_b.try_recv() + { + reached_b = Some(reply); + } + if reached_a.is_some() && reached_b.is_some() { + break; + } + } + + let reached_a = reached_a + .expect("join A resolved within the poll budget") + .expect("join A (earlier, past deadline) resolved Ok from its buffered completion"); + assert!( + reached_a.contains(&seed_a), + "join A reached seed A: {reached_a:?}" + ); + let reached_b = reached_b + .expect("join B resolved within the poll budget") + .expect( + "join B (later deadline) resolved Ok — its completion, queued behind W0 but before its own \ + deadline, was covered by the re-snapshotted watermark; a single sticky W0 reaps a spurious \ + JoinAllFailed here", + ); + assert!( + reached_b.contains(&seed_b), + "join B reached seed B: {reached_b:?}" + ); +} + +/// Regression (overlapping deadlines, leave analog): the EARLIER due deadline is a +/// graceful-leave deadline and the LATER one an await-result join whose completion +/// is buffered behind the watermark the leave's crossing snapshotted. The max due +/// deadline the re-snapshot tracks must span both planes (join AND leave), so the +/// leave's earlier crossing does not pin the watermark and starve the later join +/// into a spurious `JoinAllFailed`. The leave itself times out on its own past +/// deadline (not under test); the join must resolve `Ok`. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn overlapping_deadlines_leave_then_join_waits_for_join_backlog() { + let now = Instant::now(); + let (mut driver, _obs_rx, _shared) = build_driver(1, 4096).await; + + let seed = sa("127.0.0.1:7020"); + let (eid, resp) = drive_push_to_queued_response(&mut driver, seed, now); + let inbound_tx = driver.inbound_tx.as_ref().expect("template alive").clone(); + + driver + .endpoint + .start_push_pull(sa("127.0.0.1:7021"), PushPullKind::Join, now); + let mut dummy = None; + while let Some(action) = driver.endpoint.poll_action() { + if let StreamAction::Connect(info) = action { + dummy.get_or_insert(info.id()); + } + } + let dummy = dummy.expect("the dummy start_push_pull emitted a Connect exchange id"); + + // Backlog queued FIRST (the depth the leave's crossing snapshots). The join's + // completion is NOT queued yet. + const DEPTH: usize = 24; + for _ in 0..DEPTH { + queue_inbound( + &driver, + &inbound_tx, + BridgeInbound::Data(BridgeData { + eid: dummy, + bytes: vec![0u8; 1], + received_at: now, + }), + ); + } + + // A leave with a PAST deadline (the earlier due deadline that snapshots the + // watermark) and a join with a FUTURE deadline (so its completion, queued after + // the snapshot, must be covered when its own deadline later elapses). + let t_ref = Instant::now(); + let future = Duration::from_millis(100); + let (ltx, _lrx) = oneshot::channel::>(); + driver.pending_leave = Some(PendingLeave { + repliers: vec![ltx], + deadline: t_ref - Duration::from_secs(1), + }); + let (jtx, mut jrx) = oneshot::channel::(); + let mut pending = HashSet::new(); + pending.insert(eid); + driver.pending_joins.push(PendingJoin { + pending, + contacted: SmallVec::new(), + ignore_streams: SmallVec::new(), + requested: 1, + deadline: t_ref + future, + reply: Some(jtx), + }); + + // One poll snapshots the watermark at the leave's past-deadline crossing (the + // join is still future), covering only the filler backlog — not the join. + let _ = poll_once(&mut driver); + + // Queue the join's completion BEHIND the watermark and BEFORE its own deadline. + for bytes in resp { + queue_inbound( + &driver, + &inbound_tx, + BridgeInbound::Data(BridgeData { + eid, + bytes, + received_at: now, + }), + ); + } + queue_inbound( + &driver, + &inbound_tx, + BridgeInbound::Eof(BridgeEof { + eid, + received_at: now, + }), + ); + + // Let the join deadline elapse while the watermark is still draining. + tokio::time::sleep(future + Duration::from_millis(50)).await; + + let mut resolved = None; + for _ in 0..(DEPTH + 128) { + let _ = poll_once(&mut driver); + if let Ok(Some(reply)) = jrx.try_recv() { + resolved = Some(reply); + break; + } + } + + let reached = resolved + .expect("the join resolved within the poll budget") + .expect( + "the join (later deadline) resolved Ok — the leave's earlier crossing did not pin the \ + watermark; the max-due re-snapshot covered the join's backlog before the reap", + ); + assert!( + reached.contains(&seed), + "the join reached its seed despite the overlapping earlier leave deadline: {reached:?}" + ); +} From 033c1bdabf3ca4e12b63138de1eb595554b5f9a8 Mon Sep 17 00:00:00 2001 From: al8n Date: Thu, 2 Jul 2026 14:35:36 +1200 Subject: [PATCH 033/128] fix(serf-reactor): reconcile the caller join/leave deadline with the exchange stream timeout A configured join_deadline above the reliable stream_timeout kept the caller deadline out of the pump's max-due watermark while the join exchange's own internal deadline could fire, reaping a caller-visible premature JoinAllFailed. The join's driver deadline is now clamped to not exceed that exchange stream_timeout, so the watermark already covers a completion that beat it; leave has no push/pull exchange and is unchanged. serf-compio shares this latent gap (its select-loop reap-then-recheck masks it), so it is a shared follow-up rather than a mirror. Regression: a join with join_deadline > stream_timeout and an earlier endpoint deadline snapping the watermark resolves Ok, not a premature JoinAllFailed. --- serf-reactor/src/driver/stream/mod.rs | 43 +++++- serf-reactor/src/driver/stream/tests.rs | 190 +++++++++++++++++++++++- serf-reactor/src/tcp/mod.rs | 5 + 3 files changed, 233 insertions(+), 5 deletions(-) diff --git a/serf-reactor/src/driver/stream/mod.rs b/serf-reactor/src/driver/stream/mod.rs index adac958b..faff939c 100644 --- a/serf-reactor/src/driver/stream/mod.rs +++ b/serf-reactor/src/driver/stream/mod.rs @@ -220,8 +220,10 @@ struct PendingJoin { /// `received_at < deadline` — is still in flight. serf-proto's `StreamEndpoint` /// exposes no per-exchange CALLER deadline (`start_join_push_pull` takes only /// `peer, ignore_old, now`; the exchange deadline is the frozen inner FSM's own), - /// so this second clock cannot yet be removed outright; plumbing the caller - /// deadline INTO the exchange is a serf-proto follow-up. + /// so this second clock cannot yet be removed outright. It is instead reconciled + /// at construction via [`clamp_join_deadline`], which caps it at the exchange + /// deadline so the watermark always covers a completion that beat that exchange; + /// plumbing the caller deadline INTO the exchange is a serf-proto follow-up. deadline: Instant, /// One-shot reply channel back to the caller, taken when the reply resolves. /// `None` once resolved; the waiter then lingers — only to drive ignore-stream @@ -257,6 +259,32 @@ impl PendingJoin { } } +/// Reconcile a caller's await-result join deadline with the machine's push/pull +/// exchange deadline (`now + stream_timeout`), returning the effective +/// [`PendingJoin::deadline`]. +/// +/// A join we initiate carries two clocks: this driver-local fallback deadline and +/// the coordinator's own per-exchange deadline (`now + stream_timeout`). The pump's +/// reap watermark re-snapshots to cover a join's still-buffered completion only when +/// that join's deadline goes due ([`StreamDriver::max_due_join_leave_deadline`]); +/// the exchange deadline, hidden behind any earlier endpoint deadline in +/// `poll_timeout`'s min, never widens the watermark on its own. So a caller deadline +/// LATER than the exchange deadline lets an elapsed exchange emit a terminal +/// `ExchangeCompleted(Failed)` while the watermark still covers only an earlier +/// crossing's backlog — reaping a premature `JoinAllFailed` for a `join_deadline` +/// that has not elapsed. Clamping the driver deadline so it never exceeds the +/// exchange deadline keeps the join's deadline in `max_due` whenever the exchange +/// could fail, so the watermark has already widened to cover a completion that +/// arrived before it. Plumbing the caller deadline INTO the exchange (collapsing the +/// two clocks) is a serf-proto follow-up. +fn clamp_join_deadline( + caller_deadline: Instant, + now: Instant, + stream_timeout: Duration, +) -> Instant { + caller_deadline.min(now + stream_timeout) +} + /// Driver-side state for the single in-flight graceful-leave operation. /// /// A [`Command::Leave`] that finds the endpoint `Alive` initiates the machine's @@ -397,6 +425,11 @@ where bridge_inbound_inflight: Arc, idle_wake: Duration, leave_timeout: Duration, + /// The reliable push/pull exchange timeout the coordinator stamps on each + /// dispatched exchange (`now + stream_timeout`), snapshotted from the same + /// `EndpointOptions` the coordinator is built from. An await-result join's + /// caller deadline is reconciled against it in [`clamp_join_deadline`]. + stream_timeout: Duration, close_timeout: Duration, dial_timeout: Duration, bridge_recv_buf_len: usize, @@ -430,6 +463,7 @@ where driver_opts: RuntimeOptions, stream_opts: StreamTransportOptions, label: Option, + stream_timeout: Duration, #[cfg(encryption)] keyring: Arc, ) -> Self { let buf_len = endpoint @@ -469,6 +503,7 @@ where bridge_inbound_inflight: Arc::new(AtomicU64::new(0)), idle_wake: driver_opts.idle_wake_interval(), leave_timeout: driver_opts.leave_timeout(), + stream_timeout, close_timeout: stream_opts.close_timeout(), dial_timeout: stream_opts.dial_timeout(), bridge_recv_buf_len: stream_opts.bridge_recv_buf_len(), @@ -612,7 +647,7 @@ where contacted: SmallVec::new(), ignore_streams, requested, - deadline, + deadline: clamp_join_deadline(deadline, now, self.stream_timeout), reply: Some(reply), }); } @@ -1939,6 +1974,7 @@ pub(crate) fn spawn_stream_driver( driver_opts: RuntimeOptions, stream_opts: StreamTransportOptions, label: Option, + stream_timeout: Duration, #[cfg(encryption)] keyring: Arc, ) -> StreamDriver where @@ -1999,6 +2035,7 @@ where driver_opts, stream_opts, label, + stream_timeout, #[cfg(encryption)] keyring, ) diff --git a/serf-reactor/src/driver/stream/tests.rs b/serf-reactor/src/driver/stream/tests.rs index 3f6f8737..039af298 100644 --- a/serf-reactor/src/driver/stream/tests.rs +++ b/serf-reactor/src/driver/stream/tests.rs @@ -42,6 +42,11 @@ fn poll_once(driver: &mut TestDriver) -> Poll<()> { Pin::new(driver).poll(&mut cx) } +/// The reliable exchange timeout every non-clamp test uses: large enough that no +/// exchange deadline fires during a fast unit test, so the memberlist default +/// (`10s`) behavior is preserved for the existing pump regressions. +const DEFAULT_TEST_STREAM_TIMEOUT: Duration = Duration::from_secs(10); + /// Build a serf `StreamEndpoint` rooted at `id` / /// `advertise`, mirroring the production construction (memberlist inner endpoint → /// reliable coordinator → serf super-machine). Seeded deterministically; the @@ -49,9 +54,20 @@ fn poll_once(driver: &mut TestDriver) -> Poll<()> { fn build_endpoint( id: &str, advertise: SocketAddr, +) -> StreamEndpoint { + build_endpoint_with_stream_timeout(id, advertise, DEFAULT_TEST_STREAM_TIMEOUT) +} + +/// As [`build_endpoint`], but with an explicit reliable `stream_timeout` so a test +/// can make the coordinator stamp a short exchange deadline on its push/pulls. +fn build_endpoint_with_stream_timeout( + id: &str, + advertise: SocketAddr, + stream_timeout: Duration, ) -> StreamEndpoint { let inner_opts = EndpointOptions::new(SmolStr::new(id), advertise) - .with_user_broadcast_tiers(NonZeroU8::new(3).expect("3 is nonzero")); + .with_user_broadcast_tiers(NonZeroU8::new(3).expect("3 is nonzero")) + .with_stream_timeout(stream_timeout); let inner = Endpoint::new(inner_opts, SmallRng::seed_from_u64(0)); let coord = Coordinator::<_, _, RawRecords>::new( inner, @@ -94,11 +110,32 @@ async fn build_driver( TestDriver, Receiver>, Arc>, +) { + build_driver_with_stream_timeout( + iter_drain_cap, + bridge_inbound_cap, + DEFAULT_TEST_STREAM_TIMEOUT, + ) + .await +} + +/// As [`build_driver`], but with an explicit reliable `stream_timeout` applied to +/// BOTH the driver's endpoint (so its push/pull exchanges carry a short deadline) +/// AND the driver's clamp field (so [`clamp_join_deadline`] reconciles against the +/// same value the coordinator will stamp). +async fn build_driver_with_stream_timeout( + iter_drain_cap: usize, + bridge_inbound_cap: usize, + stream_timeout: Duration, +) -> ( + TestDriver, + Receiver>, + Arc>, ) { let socket = <::Net as Net>::UdpSocket::bind("127.0.0.1:0") .await .expect("bind gossip socket"); - let endpoint = build_endpoint("drv", sa(DRIVER_ADDR)); + let endpoint = build_endpoint_with_stream_timeout("drv", sa(DRIVER_ADDR), stream_timeout); let shared = Arc::new(Shared::new(initial_snapshot("drv", sa(DRIVER_ADDR)))); let obs_payload_bytes = Arc::new(AtomicU64::new(0)); let (obs_tx, obs_rx) = flume::unbounded(); @@ -126,6 +163,7 @@ async fn build_driver( RuntimeOptions::new().with_iter_drain_cap(iter_drain_cap), StreamTransportOptions::new().with_bridge_inbound_cap(bridge_inbound_cap), None, + stream_timeout, #[cfg(encryption)] Arc::new(crate::VoidKeyringDelegate), ); @@ -1054,3 +1092,151 @@ async fn overlapping_deadlines_leave_then_join_waits_for_join_backlog() { "the join reached its seed despite the overlapping earlier leave deadline: {reached:?}" ); } + +/// Regression (two-clock reconciliation): an await-result join configured with a +/// `join_deadline` GREATER than the reliable `stream_timeout` must not resolve a +/// premature `JoinAllFailed` when the join's own exchange deadline elapses ahead of +/// the (public) caller deadline. +/// +/// A join carries two clocks: the driver [`PendingJoin::deadline`] (from the caller) +/// and the coordinator's per-exchange deadline (`now + stream_timeout`). The exchange +/// deadline is HIDDEN behind an earlier endpoint deadline in `poll_timeout`'s min +/// (here a second push/pull started earlier), so it never widens the reap watermark +/// on its own — only a join whose OWN deadline goes due re-snapshots it. With the raw +/// (far-future) caller deadline, that earlier endpoint deadline snapshots `W0`, the +/// shared `handle_timeout` then fires the elapsed exchange deadline as +/// `ExchangeCompleted(Failed)` once `W0` drains, and the join reaps a spurious +/// `JoinAllFailed` while its `Succeeded` completion — queued behind `W0`, arrived +/// before the exchange deadline — is still buffered. [`clamp_join_deadline`] caps the +/// driver deadline at the exchange deadline, so the join's deadline goes due WITH the +/// exchange and re-snapshots the watermark to cover that completion; the join then +/// resolves `Ok`. Reverting the clamp to the raw caller deadline reproduces the +/// premature failure. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn join_deadline_above_stream_timeout_does_not_reap_premature_failure() { + let now = Instant::now(); + // A SHORT stream_timeout so the coordinator stamps a near exchange deadline the + // test can let elapse; applied to BOTH the endpoint and the driver's clamp. + let stream_timeout = Duration::from_millis(200); + // The offset between the earlier (dummy) exchange deadline and the join's, wide + // enough that the `W0`-snapshot poll lands between them under CI jitter. + let gap = Duration::from_millis(300); + // iter_drain_cap = 1: `W0` drains one item per poll, so the join's exchange + // deadline elapses mid-drain (the fire poll then sees it due behind `W0`). + let (mut driver, _obs_rx, _shared) = + build_driver_with_stream_timeout(1, 4096, stream_timeout).await; + + // An EARLIER endpoint deadline: a dummy 'connecting' push/pull started at `now` + // (its Connect captured, never dialed, so its transport data is dropped). Its + // exchange deadline `T_early = now + stream_timeout` is the min `poll_timeout` + // exposes, hiding the later join exchange deadline behind it. + driver + .endpoint + .start_push_pull(sa("127.0.0.1:7031"), PushPullKind::Join, now); + let mut dummy = None; + while let Some(action) = driver.endpoint.poll_action() { + if let StreamAction::Connect(info) = action { + dummy.get_or_insert(info.id()); + } + } + let dummy = dummy.expect("the dummy start_push_pull emitted a Connect exchange id"); + + // The real join push/pull, started `gap` LATER so its exchange deadline + // `T_exch = now + gap + stream_timeout` sits strictly AFTER the dummy's — hidden + // behind it in `poll_timeout`'s min. + let seed = sa("127.0.0.1:7030"); + let (eid, response) = drive_push_to_queued_response(&mut driver, seed, now + gap); + let response_len = response.len(); + let inbound_tx = driver.inbound_tx.as_ref().expect("template alive").clone(); + + // Backlog for the dummy exchange, queued FIRST: the depth `W0` covers. + const DEPTH: usize = 24; + for _ in 0..DEPTH { + queue_inbound( + &driver, + &inbound_tx, + BridgeInbound::Data(BridgeData { + eid: dummy, + bytes: vec![0u8; 1], + received_at: now, + }), + ); + } + + // Park the await-result join with a caller deadline FAR in the future + // (`join_deadline` = 20s > stream_timeout), reconciled by `clamp_join_deadline` to + // the join's exchange deadline (`now + gap + stream_timeout`). The dispatch path + // applies this same clamp; reverting its body to the raw caller deadline + // reproduces the premature `JoinAllFailed`. + let (tx, mut rx) = oneshot::channel::(); + let mut pending = HashSet::new(); + pending.insert(eid); + driver.pending_joins.push(PendingJoin { + pending, + contacted: SmallVec::new(), + ignore_streams: SmallVec::new(), + requested: 1, + deadline: clamp_join_deadline(now + Duration::from_secs(20), now + gap, stream_timeout), + reply: Some(tx), + }); + + // Advance past the dummy's exchange deadline (`T_early`) but BEFORE the join's + // (`T_exch`): one poll snapshots `W0` to the dummy backlog only. cap = 1 keeps + // `handle_timeout` from firing (`W0` not yet drained), so `T_early` stays armed. + tokio::time::sleep(stream_timeout + Duration::from_millis(50)).await; + let _ = poll_once(&mut driver); + + // Queue the join's `Succeeded` completion NOW — strictly BEHIND `W0`, and by + // `received_at` before the join's own exchange deadline. + for bytes in response { + queue_inbound( + &driver, + &inbound_tx, + BridgeInbound::Data(BridgeData { + eid, + bytes, + received_at: now, + }), + ); + } + queue_inbound( + &driver, + &inbound_tx, + BridgeInbound::Eof(BridgeEof { + eid, + received_at: now, + }), + ); + + // Let the join's exchange deadline (`T_exch`) elapse while `W0` is still draining: + // `handle_timeout` would now fire it as `ExchangeCompleted(Failed)`. The clamped + // driver deadline is due WITH it, re-snapshotting the watermark to cover the + // completion queued above; the raw caller deadline would not. + tokio::time::sleep(gap + Duration::from_millis(50)).await; + + let mut resolved = None; + for _ in 0..(DEPTH + response_len + 128) { + let _ = poll_once(&mut driver); + match rx.try_recv() { + Ok(Some(reply)) => { + resolved = Some(reply); + break; + } + Ok(None) => {} + Err(_) => panic!("join reply sender dropped without resolving"), + } + } + + let reached = resolved + .expect("the join resolved within the bounded poll budget") + .expect( + "a join_deadline above the reliable stream_timeout must not reap a premature JoinAllFailed: \ + the clamped driver deadline goes due WITH the exchange deadline, re-snapshotting the \ + watermark to cover the Succeeded completion queued behind W0; the raw caller deadline \ + reaps a spurious failure", + ); + assert!( + reached.contains(&seed), + "the join reached its seed once the clamped deadline widened the watermark: {reached:?}" + ); +} diff --git a/serf-reactor/src/tcp/mod.rs b/serf-reactor/src/tcp/mod.rs index 63e17823..c3bd88f1 100644 --- a/serf-reactor/src/tcp/mod.rs +++ b/serf-reactor/src/tcp/mod.rs @@ -309,6 +309,10 @@ where // inner memberlist endpoint needs at least three broadcast tiers. let inner_opts = EndpointOptions::new(self.local_id, self.advertise_socket) .with_user_broadcast_tiers(NonZeroU8::new(3).expect("3 is nonzero")); + // Snapshot the reliable push/pull exchange timeout from the SAME options the + // coordinator is built from, so the driver reconciles an await-result join's + // caller deadline against the exact deadline the coordinator will stamp. + let stream_timeout = inner_opts.stream_timeout(); let inner = Endpoint::new(inner_opts, gossip_rng); // Plain TCP has no SNI (`|_| None`) and a membership address that IS the // transport socket (`|addr| *addr`). No cluster label at this stage. @@ -342,6 +346,7 @@ where runtime.driver_options, self.stream_options, None, + stream_timeout, #[cfg(encryption)] runtime.keyring, ); From d974d3dc0d076b9f9cfa97df1b7af434b0a9113c Mon Sep 17 00:00:00 2001 From: al8n Date: Thu, 2 Jul 2026 15:23:01 +1200 Subject: [PATCH 034/128] feat(serf-reactor): QUIC driver Agnostic QUIC reactor driver (single UDP socket, QuicEndpoint::new_with_rng, recv-buf sized for gossip+enc vs quic-max-udp-payload) with a readiness pump (recv -> drain_surfaces incl. QUIC ingress decode -> inline handle_timeout) meeting the stream driver's non-premature/live timer-reap invariants adapted to QUIC's shared-UDP path via a recv-quiescence-or-bounded-grace gate; ports serf-compio's QUIC serf-logic; real-node QUIC + encrypted-QUIC tests. --- serf-reactor/Cargo.toml | 8 + serf-reactor/src/driver/mod.rs | 3 + serf-reactor/src/driver/quic/mod.rs | 1354 +++++++++++++++++++++++++ serf-reactor/src/driver/quic/tests.rs | 49 + serf-reactor/src/driver/shared.rs | 5 +- serf-reactor/src/lib.rs | 6 + serf-reactor/src/quic/mod.rs | 357 +++++++ serf-reactor/src/quic/tests.rs | 43 + serf-reactor/src/serf/mod.rs | 101 +- serf-reactor/tests/quic.rs | 531 ++++++++++ 10 files changed, 2454 insertions(+), 3 deletions(-) create mode 100644 serf-reactor/src/driver/quic/mod.rs create mode 100644 serf-reactor/src/driver/quic/tests.rs create mode 100644 serf-reactor/src/quic/mod.rs create mode 100644 serf-reactor/src/quic/tests.rs create mode 100644 serf-reactor/tests/quic.rs diff --git a/serf-reactor/Cargo.toml b/serf-reactor/Cargo.toml index 37cdb841..8895f92a 100644 --- a/serf-reactor/Cargo.toml +++ b/serf-reactor/Cargo.toml @@ -108,6 +108,14 @@ tokio = { version = "1", features = ["rt-multi-thread", "macros", "time"] } name = "tcp" required-features = ["tcp", "tokio"] +# The real-node QUIC test suite (two loopback nodes over a quinn config bundle: +# join/converge, user event, query round-trip, leave/LeftCluster, shutdown, and — +# under an encryption backend — encrypted-gossip convergence + mismatched-key +# enforcement). +[[test]] +name = "quic" +required-features = ["quic-rustls-ring", "tokio"] + [package.metadata.docs.rs] # A single coherent crypto-provider set (ring, not aws-lc-rs; tokio, not smol) so # the doc build documents every gated item without pulling conflicting providers. diff --git a/serf-reactor/src/driver/mod.rs b/serf-reactor/src/driver/mod.rs index 34f7f83b..b9215a78 100644 --- a/serf-reactor/src/driver/mod.rs +++ b/serf-reactor/src/driver/mod.rs @@ -10,5 +10,8 @@ pub(crate) mod options; #[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) mod shared; +#[cfg(feature = "quic")] +pub(crate) mod quic; + #[cfg(feature = "tcp")] pub(crate) mod stream; diff --git a/serf-reactor/src/driver/quic/mod.rs b/serf-reactor/src/driver/quic/mod.rs new file mode 100644 index 00000000..6f8d5923 --- /dev/null +++ b/serf-reactor/src/driver/quic/mod.rs @@ -0,0 +1,1354 @@ +//! The QUIC-plane driver pump: a quinn-style `Future::poll` that solely owns the +//! serf [`QuicEndpoint`], its single UDP socket, and the periodic schedulers. +//! +//! Unlike the stream plane, QUIC carries no per-exchange bridge table: the +//! coordinator (quinn-proto inside `QuicEndpoint`) multiplexes the reliable +//! push/pull streams over the ONE UDP socket, and serf's datagram gossip rides the +//! same socket, both fed through [`QuicEndpoint::handle_udp`]. This is the +//! `Send`/`Arc`/`agnostic` sibling of serf-compio's `!Send` `quic_driver_loop`, +//! restructured onto memberlist-reactor's readiness pump: there is NO top-level +//! `select!` and NO completion-backend drain. Each poll drains queued +//! [`Command`]s, recv-loops the socket to kernel-empty (`poll_recv_from` → +//! `Poll::Pending`), runs [`drain_surfaces`](QuicDriver::drain_surfaces) (decode +//! the buffered gossip ingress — a datagram-carried Ack — route the QUIC/gossip +//! egress, fold events), and then fires `handle_timeout` INLINE at exactly one +//! site. serf-compio's `fire_quic_timeout` / `drain_past_due_udp` chokepoints are +//! deleted — those exist only because io_uring is completion-based; the reactor is +//! readiness-based and `Poll::Pending` from the socket IS the kernel-empty signal. +//! +//! ## The shared-UDP-path timer/reap gate +//! +//! On QUIC the reliable plane and the gossip plane share the one UDP socket, so — +//! unlike the stream driver, whose reliable completions ride a disjoint TCP FIFO a +//! watermark can gate exactly — a join's resolving push/pull completion and a +//! gossip flood arrive on the SAME path. There is thus no per-completion watermark +//! to observe; the non-premature gate is instead UDP-recv QUIESCENCE. When the +//! recv loop stops on `Poll::Pending` (`!recv_capped`), every kernel-ready packet +//! — including any completing QUIC stream packet — was fed through `handle_udp` +//! this poll and its `ExchangeCompleted` folded by `drain_surfaces` BEFORE the +//! reaps run, so a due reap / reliable-exchange `handle_timeout` fires only over a +//! genuinely-absent completion. Under a sustained flood that keeps the batch +//! saturated (`recv_capped`) every poll, the fire is held back only until a +//! bounded wall-clock staleness grace elapses, then fires for liveness (quinn's +//! connection timers must advance and a parked join/leave must resolve). The +//! grace is bounded-early exactly as the stream driver's SWIM tick is; on QUIC it +//! covers the reliable plane too, because that plane shares the flooded path. + +#![cfg(feature = "quic")] + +use std::{ + collections::{HashSet, VecDeque}, + future::Future, + net::SocketAddr, + pin::Pin, + sync::{ + Arc, + atomic::{AtomicU64, Ordering}, + }, + task::{Context, Poll}, + time::Duration, +}; + +use agnostic::{ + Runtime, + net::{Net, UdpSocket}, +}; +use bytes::Bytes; +use flume::{Receiver, Sender, TrySendError}; +use futures_channel::oneshot; +use memberlist_proto::{ + Instant, SeedableRng, StreamId, Transmit, + codec::{ + DecodeOptions, EncodeOptions, decode_incoming, encode_outgoing, encode_outgoing_compound, + parse_messages, + }, +}; +use serf_driver::SerfSnapshot; +use serf_proto::{ + ExchangeKind, ExchangeStatus, LamportTime, QuicEndpoint, event::Event, members::SerfState, +}; +use smallvec::SmallVec; + +#[cfg(encryption)] +use crate::command::{KeyCmd, ListKeysCmd}; +#[cfg(encryption)] +use crate::delegate::KeyringDelegate; +use crate::{ + Channel, + command::{ + Command, ForceLeaveCmd, JoinCmd, JoinKind, JoinReply, LeaveCmd, QueryCmd, RespondCmd, + SetTagsCmd, ShutdownCmd, WaitForCompletionArgs, + }, + delegate::Delegate, + driver::{ + options::RuntimeOptions, + shared::{ExchangeId, dispatch_event_delegate, observation_payload_bytes}, + }, + error::{JoinFailed, Result, SerfError}, + shared::Shared, +}; +#[cfg(encryption)] +use serf_proto::{KeyRequestOperation, KeyResponseArgs, event::KeyRequest}; + +/// Hard ceiling on the gossip-plane contribution to the per-recv UDP buffer — +/// UDP's IPv4 wire payload is capped at 65507 bytes once the IP/UDP headers are +/// deducted, so inflating the gossip path past it just wastes an allocation. The +/// raw-QUIC plane is bounded independently by quinn's `max_udp_payload_size` +/// (≤ 65527) and is intentionally not subject to this gossip cap. +const GOSSIP_RECV_BUF_MAX: usize = 65507; + +/// The largest the encrypted wrapper can inflate a gossip datagram, or `0` when no +/// encryption backend is built in — so an encrypted datagram is not silently +/// truncated by the kernel. +#[cfg(encryption)] +const ENCRYPTED_WRAPPER_OVERHEAD: usize = memberlist_proto::ENCRYPTED_WRAPPER_OVERHEAD; +#[cfg(not(encryption))] +const ENCRYPTED_WRAPPER_OVERHEAD: usize = 0; + +/// Cap on the count of application-data events retained after a full observation +/// channel (the payload byte budget bounds their bytes; this bounds their count). +const OBS_OVERFLOW_MAX: usize = 1024; + +/// How long a due `handle_timeout` / deadline reap may be held back under a +/// sustained UDP flood before it fires bounded-early. +/// +/// On QUIC the reliable push/pull completions and the gossip/probe datagrams share +/// the one UDP recv path (there is no disjoint reliable FIFO to watermark, unlike +/// the stream driver), so a saturated recv batch could hide EITHER a probe Ack OR +/// a reliable-exchange completion behind the flood. Rather than gate each plane on +/// its own drain, the pump gates the single `handle_timeout` site (and the +/// join/leave reaps) on recv quiescence, and — when a flood pins the batch every +/// poll — fires after this bounded staleness. A `Duration`, not a poll count: at a +/// low `iter_drain_cap` (or a large exchange) a fixed count could elapse while real +/// pre-deadline work is still buffered, whereas a wall-clock grace bounds the +/// staleness directly. It is short because a false SWIM `Suspect` self-refutes +/// within the multi-second suspicion window and a prematurely-failed push/pull is +/// retriable, whereas freezing quinn's connection timers under load is not — so +/// liveness wins once the grace elapses. +const SHARED_PATH_STALENESS_GRACE: Duration = Duration::from_millis(5); + +/// Size the per-recv UDP buffer to the larger of the two planes that share this +/// one socket. +/// +/// The gossip plane needs `gossip_mtu` inflated by the encrypted-wrapper overhead +/// (algorithm tag + nonce + AEAD auth tag), clamped at [`GOSSIP_RECV_BUF_MAX`]. +/// The raw-QUIC plane needs whatever max UDP payload the quinn `EndpointConfig` +/// accepts — which a valid caller can set above the gossip MTU (quinn's default +/// 1472 already exceeds the 1400 default `gossip_mtu`, and callers can raise it +/// further). Sizing below either lets the kernel truncate that plane's largest +/// datagram before the coordinator's first-byte demux sees it, corrupting QUIC +/// handshakes/streams while leaving construction silently successful. +/// +/// `quic_max_udp_payload` is quinn-bounded to `[1200, 65527]`, so it always fits +/// `usize`; the conversion fallback is purely defensive. The QUIC plane is NOT +/// clamped at [`GOSSIP_RECV_BUF_MAX`] — quinn already bounds it, and clamping would +/// shrink the buffer below a configured 65508..=65527 ceiling. +fn recv_buf_len_for(gossip_mtu: usize, quic_max_udp_payload: u64) -> usize { + let gossip_path = gossip_mtu + .saturating_add(ENCRYPTED_WRAPPER_OVERHEAD) + .min(GOSSIP_RECV_BUF_MAX); + let quic_path = usize::try_from(quic_max_udp_payload).unwrap_or(GOSSIP_RECV_BUF_MAX); + gossip_path.max(quic_path) +} + +/// Driver-side state for one outstanding await-result join call. +/// +/// Mirrors the stream driver's `PendingJoin`. A [`Command::Join`] carrying +/// [`JoinKind::WaitForCompletion`] dispatches one push/pull per resolved seed and +/// parks the per-call state here. The QUIC coordinator services the dial in-band, +/// so each `start_join_push_pull`'s returned machine `StreamId` coerces directly +/// into the [`ExchangeId`] domain (via `From`) — the same value the +/// bridge-reap path stamps onto its [`Event::ExchangeCompleted`]. Contact +/// accounting is per-OUTBOUND-EXCHANGE, filtered to [`ExchangeKind::PushPull`]. +/// +/// Reply resolution and ignore-stream cleanup are SEPARATE terminal states: the +/// reply resolves on all-exchanges-done OR `deadline` (whichever first); the +/// ignore streams are cleared only once every dispatched exchange has completed +/// (`pending` empty), so a `StreamId` recorded for a still-live exchange stays in +/// the machine's ignore set and a late merge still suppresses the peer's pre-join +/// user events. +struct PendingJoin { + /// Outbound exchange ids this waiter dispatched and is still awaiting a terminal + /// `ExchangeCompleted` for. + pending: HashSet, + /// Peer addresses of the dispatched exchanges that terminated `Succeeded`. + /// Duplicate seeds contribute one entry per successful exchange. + contacted: SmallVec<[SocketAddr; 1]>, + /// The `StreamId`s this join recorded in the machine's per-exchange ignore set + /// (non-empty only for an `ignore_old` join). Cleared via + /// `clear_ignore_join_stream` once every dispatched exchange has completed. + ignore_streams: SmallVec<[StreamId; 1]>, + /// Total outbound-exchange count this call dispatched — the `JoinAllFailed` + /// denominator on a zero-contact resolution. + requested: usize, + /// Wall-clock instant past which the driver replies with whatever `contacted` + /// set it has accumulated even if `pending` is non-empty. + /// + /// This is a driver-local FALLBACK: the join normally resolves the moment every + /// dispatched exchange has surfaced its terminal `ExchangeCompleted` (success or + /// the coordinator's own `stream_timeout` failure), so this deadline only bounds + /// a hung exchange. Its reap runs behind the same shared-UDP-path quiescence gate + /// as the reliable-exchange `handle_timeout` (see the module docs), so it never + /// fires while a pre-deadline completion is still kernel-resident behind a + /// non-saturating recv batch. + deadline: Instant, + /// One-shot reply channel back to the caller, taken when the reply resolves. + /// `None` once resolved; the waiter then lingers — only to drive ignore-stream + /// cleanup — until `pending` empties. + reply: Option>, +} + +impl PendingJoin { + /// Resolve the caller's reply once, from the current `contacted` set. Idempotent: + /// after the first call `reply` is `None` and this is a no-op, so the deadline + /// path and the all-exchanges-done path never double-send. + fn resolve_reply(&mut self) { + if let Some(reply) = self.reply.take() { + let result = if self.contacted.is_empty() { + Err(( + SmallVec::new(), + SerfError::JoinAllFailed(JoinFailed::new(self.requested, 0)), + )) + } else { + Ok(self.contacted.clone()) + }; + // Ignoring Err: caller dropped the reply receiver (the join future was + // cancelled). + let _ = reply.send(result); + } + } + + /// This waiter has reached both terminal states — its reply resolved AND every + /// dispatched exchange completed — so it can be removed and its ignore-stream + /// cleanup run. + fn is_done(&self) -> bool { + self.reply.is_none() && self.pending.is_empty() + } +} + +/// Driver-side state for the single in-flight graceful-leave operation. +/// +/// A [`Command::Leave`] that finds the endpoint `Alive` initiates the machine's +/// `leave()`, which withholds [`Event::LeftCluster`] until the leave notices have +/// drained. The pump parks this and replies only once that `LeftCluster` arrives +/// (success) or `deadline` elapses ([`SerfError::LeaveTimeout`]). Leave is SHARED: +/// a second `Command::Leave` racing an in-flight one joins it by pushing its reply +/// onto `repliers`. +struct PendingLeave { + /// Reply channels of every `leave()` caller that joined this in-flight leave. + repliers: Vec>>, + /// Wall-clock instant past which the pump replies [`SerfError::LeaveTimeout`] to + /// every replier even if `LeftCluster` has not yet fired. + deadline: Instant, +} + +impl PendingLeave { + /// Reply to every joined `leave()` caller with a fresh `Result<()>` from + /// `make_result`. A constructor closure (rather than a cloned value) sidesteps + /// `SerfError` not being `Clone` — every terminal outcome here (`Ok(())`, + /// `LeaveTimeout`, `Shutdown`) is trivially reconstructible. + fn resolve_all(self, mut make_result: impl FnMut() -> Result<()>) { + for replier in self.repliers { + // Ignoring Err: a `leave()` caller dropped its reply receiver. + let _ = replier.send(make_result()); + } + } +} + +/// The single-owner QUIC driver future. Runs until shutdown (a `Shutdown` command, +/// a lost id-conflict `Event::Shutdown`, or the last handle dropped). +pub(crate) struct QuicDriver +where + // Structurally required: `endpoint` names `QuicEndpoint`, whose struct + // declares `I: Eq + Hash`. + I: core::hash::Hash + Eq, + R: Runtime, +{ + endpoint: QuicEndpoint, + /// The shared UDP socket carrying QUIC packets AND plain-UDP gossip. `Option` so + /// the shutdown branch can drop it (releasing the bound port) BEFORE acking; + /// `Some` for the running lifetime, taken only during teardown. + socket: Option<::UdpSocket>, + shared: Arc>, + /// Hand-off to the observation task (delegate dispatch + event-stream fan-out). + obs_tx: Sender>, + /// Bytes of payload-bearing events queued in `obs_tx` — the byte backstop's + /// counter (added on enqueue, subtracted by the obs task on dequeue). + obs_payload_bytes: Arc, + /// Queued-payload byte budget on a bounded obs channel, `None` if unbounded. + obs_payload_budget: Option, + /// Application-data events retained after a full obs channel, retried later. + obs_overflow: VecDeque>, + /// Cluster label threaded into the gossip codec (outbound stamp + inbound + /// verify). + label: Option, + /// Outstanding await-result join waiters. + pending_joins: Vec, + /// The in-flight graceful leave, resolved on `LeftCluster`. + pending_leave: Option, + /// Parked `Shutdown` replies — acked only after the UDP socket drops, so a caller + /// resuming from `shutdown().await` can rebind the same address. A `Vec` because + /// several callers can race `shutdown()`. + shutdown_reply: Vec>>, + recv_buf: Vec, + /// Per-poll cap on each drained surface / recv batch. + iter_drain_cap: usize, + timer: Option>>, + timer_deadline: Option, + /// Wall-clock anchor for the shared-UDP-path staleness grace: the instant a due + /// `handle_timeout` / reap first deferred purely because the recv batch was + /// saturated (the kernel may still hold a pre-deadline completion). `None` when + /// not deferring; the fire proceeds once [`SHARED_PATH_STALENESS_GRACE`] has + /// elapsed since this anchor. + timeout_stall_since: Option, + idle_wake: Duration, + leave_timeout: Duration, + /// The driver's keyring delegate: applies inbound key-management ops and produces + /// the `respond_key` answer. Present only under an encryption backend. + #[cfg(encryption)] + keyring: Arc, +} + +impl QuicDriver +where + I: memberlist_proto::Id + Clone, + R: Runtime, + G: rand::Rng, + SR: rand::Rng + SeedableRng, +{ + /// Build the driver from the endpoint, its bound UDP socket, the shared state, + /// the observation hand-off, and the recv-buffer inputs. + #[allow(clippy::too_many_arguments)] + pub(crate) fn new( + endpoint: QuicEndpoint, + socket: ::UdpSocket, + quic_max_udp_payload: u64, + shared: Arc>, + obs_tx: Sender>, + obs_payload_bytes: Arc, + obs_payload_budget: Option, + driver_opts: RuntimeOptions, + label: Option, + #[cfg(encryption)] keyring: Arc, + ) -> Self { + let buf_len = recv_buf_len_for(endpoint.gossip_mtu(), quic_max_udp_payload); + Self { + endpoint, + socket: Some(socket), + shared, + obs_tx, + obs_payload_bytes, + obs_payload_budget, + obs_overflow: VecDeque::new(), + label, + pending_joins: Vec::new(), + pending_leave: None, + shutdown_reply: Vec::new(), + recv_buf: vec![0u8; buf_len.max(1)], + iter_drain_cap: driver_opts.iter_drain_cap().max(1), + timer: None, + timer_deadline: None, + timeout_stall_since: None, + idle_wake: driver_opts.idle_wake_interval(), + leave_timeout: driver_opts.leave_timeout(), + #[cfg(encryption)] + keyring, + } + } + + /// Applies one handle command to the machine. + fn dispatch(&mut self, cmd: Command, now: Instant) { + let running = self.endpoint.state() == SerfState::Alive; + match cmd { + Command::Join(JoinCmd { + seeds, + kind, + ignore_old, + reply, + }) => { + // Gate on a running node: `leave()` stops the periodic schedulers, so a + // join after leave would leave the node non-participating. + if !running { + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(Err((SmallVec::new(), SerfError::NotRunning))); + return; + } + // Announce the serf-level join intent so peers learn the local join ltime + // without waiting for the next anti-entropy round. + if let Err(e) = self.endpoint.join() { + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(Err((SmallVec::new(), SerfError::from(e)))); + return; + } + // The QUIC coordinator services the dial + flushes the outbound queue + // in-band, so each `start_join_push_pull`'s returned machine `StreamId` + // coerces directly into the `ExchangeId` the bridge-reap path stamps onto + // its `ExchangeCompleted` — no inline action drain / capture is needed + // (unlike the stream driver). + match kind { + JoinKind::Dispatch => { + let mut dispatched: SmallVec<[SocketAddr; 1]> = SmallVec::new(); + for seed in seeds { + // Ignoring StreamId: the Dispatch arm tracks no per-exchange waiter + // state — completion / failure surfaces through `poll_event`. + let _sid = self.endpoint.start_join_push_pull(seed, ignore_old, now); + dispatched.push(seed); + } + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(Ok(dispatched)); + } + JoinKind::WaitForCompletion(WaitForCompletionArgs { deadline }) => { + let requested = seeds.len(); + let mut exchange_ids: HashSet = HashSet::with_capacity(requested); + // An `ignore_old` join records every seed's `StreamId` in the machine; + // the driver owns clearing any that fail to merge. A plain join records + // nothing, so this stays empty. + let mut ignore_streams: SmallVec<[StreamId; 1]> = SmallVec::new(); + for seed in seeds { + let sid = self.endpoint.start_join_push_pull(seed, ignore_old, now); + if ignore_old { + ignore_streams.push(sid); + } + exchange_ids.insert(ExchangeId::from(sid)); + } + if exchange_ids.is_empty() { + // Every seed retired before an exchange (reachable only with a + // zero-length `seeds`, which the handle never sends for an await + // join): resolve now — parking would hang with no terminal incoming. + for s in &ignore_streams { + self.endpoint.clear_ignore_join_stream(*s); + } + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(Err(( + SmallVec::new(), + SerfError::JoinAllFailed(JoinFailed::new(requested, 0)), + ))); + } else { + self.pending_joins.push(PendingJoin { + pending: exchange_ids, + contacted: SmallVec::new(), + ignore_streams, + requested, + deadline, + reply: Some(reply), + }); + } + } + } + } + Command::Leave(LeaveCmd { reply }) => { + // Leave is a SHARED in-flight operation. If one is in flight, JOIN it (do + // not re-invoke `leave()`, a terminal no-op once `Leaving`/`Left` that + // emits no second `LeftCluster`). Otherwise INITIATE: snapshot `Alive` + // before the call, then park (was Alive) or reply immediately (no-op / + // error). + if let Some(pl) = self.pending_leave.as_mut() { + pl.repliers.push(reply); + } else { + let was_alive = running; + let leave_timeout = self.leave_timeout; + let res: Result<()> = self.endpoint.leave(now).map_err(SerfError::from); + match res { + Ok(()) if was_alive => { + self.pending_leave = Some(PendingLeave { + repliers: vec![reply], + deadline: now + leave_timeout, + }); + } + other => { + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(other); + } + } + } + } + Command::ForceLeave(ForceLeaveCmd { + id, + prune, + now: at, + reply, + }) => { + let res = if running { + self + .endpoint + .force_leave(id, prune, at) + .map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(res); + } + Command::UserEvent(cmd) => { + let res = if running { + let name = cmd.name().clone(); + let payload = cmd.payload().clone(); + self + .endpoint + .user_event(name, payload, cmd.coalesce) + .map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = cmd.reply.send(res); + } + Command::Query(cmd) => { + let res = if running { + let name = cmd.name().clone(); + let payload = cmd.payload().clone(); + let QueryCmd { + params, now: at, .. + } = &cmd; + self + .endpoint + .query(name, payload, params.clone(), *at) + .map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = cmd.reply.send(res); + } + Command::Respond(cmd) => { + let res = if running { + let payload = cmd.payload().clone(); + self + .endpoint + .respond(&cmd.token, payload, cmd.now) + .map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = cmd.reply.send(res); + } + Command::SetTags(SetTagsCmd { tags, reply }) => { + let res = if running { + self.endpoint.set_tags(tags).map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(res); + } + #[cfg(encryption)] + Command::InstallKey(KeyCmd { + key, + now: at, + reply, + }) => { + let res = if running { + self.endpoint.install_key(key, at).map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(res); + } + #[cfg(encryption)] + Command::UseKey(KeyCmd { + key, + now: at, + reply, + }) => { + let res = if running { + self.endpoint.use_key(key, at).map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(res); + } + #[cfg(encryption)] + Command::RemoveKey(KeyCmd { + key, + now: at, + reply, + }) => { + let res = if running { + self.endpoint.remove_key(key, at).map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(res); + } + #[cfg(encryption)] + Command::ListKeys(ListKeysCmd { now: at, reply }) => { + let res = if running { + self.endpoint.list_keys(at).map_err(SerfError::from) + } else { + Err(SerfError::NotRunning) + }; + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(res); + } + Command::Shutdown(ShutdownCmd { reply }) => { + // Do NOT ack inline: the UDP socket is still bound. Flag shutdown and park + // the reply; the teardown branch acks every parked caller only AFTER it + // drops the socket, so an immediate rebind on the same address after + // `shutdown().await` succeeds. + self.shared.begin_shutdown(); + self.shutdown_reply.push(reply); + } + } + } + + /// Repeatedly runs the ordered surface pass to a FIXED POINT. A later surface can + /// create work for an earlier one within the same poll: answering an inbound + /// `Event::KeyRequest` (`respond_key`) queues a directed gossip transmit after + /// the transmit surfaces were already drained, and a fed inbound message can + /// enqueue a fresh transmit + event. Repeating the ordered pass while any surface + /// made progress converges (the machine emits finite output per already-buffered + /// input and each egress surface is cap-bounded), so on return `more == false` + /// genuinely means no ready machine work remains from this poll's input. A pass + /// that hits a per-surface cap ends the fixed point with `more == true` (after + /// completing the pass, so no terminal event is stranded): the caller self-wakes + /// rather than uncapped-draining an egress a fast peer can refill. + fn drain_surfaces(&mut self, cx: &mut Context<'_>) -> (bool, bool) { + let mut worked = false; + loop { + let (pass_worked, pass_more) = self.drain_surfaces_pass(cx); + worked |= pass_worked; + if pass_more { + return (worked, true); + } + if !pass_worked { + return (worked, false); + } + } + } + + /// One ordered surface pass — inbound-gossip-ingress → outbound-gossip → raw-QUIC + /// egress → events. Returns `(worked, more)`: whether any surface produced work, + /// and whether a capped egress surface hit its cap (with work left). The ingress + /// decode and the event drain are UNCAPPED (drained to empty): the ingress + /// decode-to-empty guarantees a datagram-carried Ack is applied before the inline + /// timer, and the uncapped event drain guarantees a surfaced terminal + /// (`ExchangeCompleted` / `LeftCluster`) is folded before the reaps — neither may + /// sit behind a cap under a flood. Both are bounded per poll: the ingress by the + /// coordinator's `mem_ingress` cap, the events by the (already cap-bounded) feeds + /// plus an O(members) `handle_timeout` burst. + fn drain_surfaces_pass(&mut self, cx: &mut Context<'_>) -> (bool, bool) { + let budget = self.iter_drain_cap; + let now = Instant::now(); + let mut worked = false; + let mut more = false; + + // Inbound gossip: decrypt + strip-label + parse, inline on the pump. Drained to + // EMPTY (bounded by the coordinator's `mem_ingress` cap) so a datagram-carried + // probe Ack is decoded and applied through `handle_message` BEFORE the inline + // `handle_timeout` can mark the peer suspect. A QUIC stream packet is NOT here + // — `handle_udp` processes reliable stream data in-band; this surface carries + // only the buffered gossip frames. + let decode_opts = DecodeOptions::new(self.label.clone()); + let mut ingress = false; + while let Some((from, raw)) = self.endpoint.poll_memberlist_ingress() { + ingress = true; + // Reverse the wire transform stack: with an encryption backend built in, + // `decrypt_gossip` strips (and authenticates) the encryption wrapper; with + // none the serf gossip plane carries no transforms so the raw bytes are the + // plain label frame. A dropped datagram self-heals on the next gossip round. + #[cfg(encryption)] + let plain = match self.endpoint.decrypt_gossip(&raw) { + Ok(p) => Bytes::from(p), + Err(_) => continue, + }; + #[cfg(not(encryption))] + let plain = raw; + let inner = match decode_incoming(plain, &decode_opts) { + Ok(b) => b, + Err(_) => continue, + }; + let msgs = match parse_messages::(inner) { + Ok(m) => m, + Err(_) => continue, + }; + for msg in msgs { + self.endpoint.handle_message(from, msg, now); + } + } + worked |= ingress; + + // Outbound gossip: encode (plain or compound) + encrypt, then send on the UDP + // socket. Popping the last transmit is the endpoint's leave-completion fence + // (it emits `LeftCluster`), so the leave/shutdown datagrams reach the socket + // before that fence fires. + let encode_opts = EncodeOptions::new(self.label.clone()); + let mut sent = 0; + while sent < budget { + let Some(transmit) = self.endpoint.poll_memberlist_transmit() else { + break; + }; + sent += 1; + let (peer, plain): (SocketAddr, Bytes) = match transmit { + Transmit::Packet(pkt) => { + let (to, msg) = pkt.into_parts(); + match encode_outgoing(&msg, &encode_opts) { + Ok(b) => (to, b), + Err(_) => continue, + } + } + Transmit::Compound(cmp) => { + let (to, msgs) = cmp.into_parts(); + match encode_outgoing_compound(&msgs, &encode_opts) { + Ok(b) => (to, b), + Err(_) => continue, + } + } + }; + #[allow(unused_mut)] + let mut on_wire: Vec = plain.to_vec(); + #[cfg(encryption)] + { + on_wire = match self.endpoint.encrypt_gossip(&on_wire) { + Ok(bytes) => bytes, + Err(_) => continue, + }; + } + if let Some(socket) = self.socket.as_ref() { + // Ignoring Poll: gossip is best-effort — a full or errored UDP send drops + // the datagram and SWIM recovers on the next round. + let _ = socket.poll_send_to(cx, &on_wire, peer); + } + } + worked |= sent > 0; + more |= sent == budget; + + // Raw QUIC datagrams: already wire-framed by quinn-proto (handshake, acks, + // reliable stream data, datagram-mode gossip), no codec wrap. + let mut raw_sent = 0; + while raw_sent < budget { + let Some((dest, bytes)) = self.endpoint.poll_transmit() else { + break; + }; + raw_sent += 1; + if let Some(socket) = self.socket.as_ref() { + // Ignoring Poll: a dropped QUIC datagram is retransmitted by quinn-proto. + let _ = socket.poll_send_to(cx, &bytes, dest); + } + } + worked |= raw_sent > 0; + more |= raw_sent == budget; + + // Observation events: retry the overflow first, then drain to EMPTY (uncapped) + // so a surfaced terminal folded by `send_observation` is never stranded behind + // a per-poll cap under a flood. + self.flush_obs_overflow(); + let mut events = false; + while let Some(ev) = self.endpoint.poll_event() { + events = true; + self.send_observation(ev); + } + worked |= events; + + (worked, more) + } + + /// Retries retained overflow events into the obs channel, stopping at the first + /// `Full`. + fn flush_obs_overflow(&mut self) { + while let Some(ev) = self.obs_overflow.pop_front() { + match self.obs_tx.try_send(ev) { + Ok(()) => {} + Err(TrySendError::Full(ev)) => { + self.obs_overflow.push_front(ev); + break; + } + Err(TrySendError::Disconnected(ev)) => { + // The obs task is gone: reclaim this event's reserved payload bytes. + if let Some(bytes) = observation_payload_bytes(&ev) { + self.obs_payload_bytes.fetch_sub(bytes, Ordering::Relaxed); + } + } + } + } + } + + /// Hands one event to the obs task. Applies the synchronous protocol accounting + /// first (join/leave/conflict/key). A full channel retains application data for + /// retry (bounded by the payload byte budget and `OBS_OVERFLOW_MAX`) and drops + /// recoverable membership/control events, counting them. + fn send_observation(&mut self, ev: Event) { + self.account_event(&ev); + let payload = observation_payload_bytes(&ev); + // Byte backstop: refuse a payload event if enqueuing it would push the queued + // payload bytes over budget. + if let (Some(budget), Some(bytes)) = (self.obs_payload_budget, payload) + && self + .obs_payload_bytes + .load(Ordering::Relaxed) + .saturating_add(bytes) + > budget + { + self.shared.add_observation_dropped(1); + return; + } + // Reserve the payload bytes before the event becomes visible to the obs task, + // so its release (subtract on receive) can never run ahead of the reservation. + if let Some(bytes) = payload { + self.obs_payload_bytes.fetch_add(bytes, Ordering::Relaxed); + } + match self.obs_tx.try_send(ev) { + Ok(()) => {} + Err(TrySendError::Full(ev)) => match payload { + // Application data the event stream cannot reconstruct: retain (still + // reserved) for a retry. + Some(_) if self.obs_overflow.len() < OBS_OVERFLOW_MAX => { + self.obs_overflow.push_back(ev); + } + // Recoverable membership/control, or the overflow is full: drop, count, and + // roll back any reservation. + _ => { + if let Some(bytes) = payload { + self.obs_payload_bytes.fetch_sub(bytes, Ordering::Relaxed); + } + self.shared.add_observation_dropped(1); + } + }, + // The obs task is gone: roll back the reservation. + Err(TrySendError::Disconnected(_)) => { + if let Some(bytes) = payload { + self.obs_payload_bytes.fetch_sub(bytes, Ordering::Relaxed); + } + } + } + } + + /// Synchronous protocol accounting for a surfaced event: reduce the matching + /// await-result join on a push/pull `ExchangeCompleted`, resolve a parked leave + /// on `LeftCluster`, begin teardown on a lost id-conflict `Shutdown`, and answer + /// an inbound key-management request. + fn account_event(&mut self, ev: &Event) { + if let Event::ExchangeCompleted(c) = ev + && c.kind() == ExchangeKind::PushPull + { + let Self { + endpoint, + pending_joins, + .. + } = self; + complete_join_exchange( + endpoint, + pending_joins, + c.eid(), + *c.peer(), + matches!(c.outcome(), ExchangeStatus::Succeeded), + ); + } + if matches!(ev, Event::LeftCluster) + && let Some(pl) = self.pending_leave.take() + { + pl.resolve_all(|| Ok(())); + } + // A lost id-conflict vote means the local node MUST stop, exactly as for a + // `Command::Shutdown`. Flag shutdown; the pump self-wakes into the teardown + // branch. The event still reaches subscribers through the obs hand-off. + if matches!(ev, Event::Shutdown) { + self.shared.begin_shutdown(); + } + #[cfg(encryption)] + if let Event::KeyRequest(req) = ev { + let resp = apply_key_request(&*self.keyring, req); + // Ignoring Err: `respond_key` fails only when the response cannot be routed; + // the key op has already applied locally. + let _ = self.endpoint.respond_key(req, resp, Instant::now()); + } + } + + /// Reap await-result join waiters on the deadline timer (the reply terminal), + /// then remove any waiter that has reached BOTH terminals (reply resolved AND + /// `pending` empty), clearing its still-recorded ignore-join streams. + fn reap_pending_joins(&mut self, now: Instant) { + let Self { + endpoint, + pending_joins, + .. + } = self; + let mut i = 0; + while i < pending_joins.len() { + if now >= pending_joins[i].deadline { + pending_joins[i].resolve_reply(); + } + if pending_joins[i].is_done() { + let pj = pending_joins.swap_remove(i); + for s in &pj.ignore_streams { + endpoint.clear_ignore_join_stream(*s); + } + } else { + i += 1; + } + } + } + + /// Reap a deadline-expired graceful-leave waiter. + fn reap_pending_leave(&mut self, now: Instant) { + if let Some(pl) = self.pending_leave.as_ref() + && now >= pl.deadline + { + let pl = self.pending_leave.take().expect("checked Some above"); + pl.resolve_all(|| Err(SerfError::LeaveTimeout)); + } + } + + /// Earliest pending-join deadline of a still-unreplied waiter, folded into the + /// per-poll timer target so it fires by the first expiring join. + fn min_pending_join_deadline(&self) -> Option { + self + .pending_joins + .iter() + .filter(|pj| pj.reply.is_some()) + .map(|pj| pj.deadline) + .min() + } + + /// Earliest pending-leave deadline, folded into the per-poll timer target. + fn min_pending_leave_deadline(&self) -> Option { + self.pending_leave.as_ref().map(|pl| pl.deadline) + } + + /// Publish a fresh [`SerfSnapshot`] of the endpoint's observable membership. + /// Skips the publish when the local node is not yet present in the membership + /// store (the local `NodeJoined` sieve has not fired), so `SerfSnapshot::new` + /// (which requires the local node) is never called with it absent. + fn refresh_snapshot(&self) { + let members = self.endpoint.members_snapshot(); + let local_id = self.endpoint.local_id(); + if !members.iter().any(|m| m.node().id_ref() == local_id) { + return; + } + let snap = SerfSnapshot::new( + members, + local_id, + self.endpoint.state(), + LamportTime::from(self.endpoint.member_time()), + LamportTime::from(self.endpoint.event_time()), + LamportTime::from(self.endpoint.query_time()), + ); + self.shared.publish(snap); + } + + /// (Re)arms the wakeup timer for `target` if it is not already armed for it. + fn arm_timer(&mut self, target: Instant, now: Instant) { + if self.timer_deadline != Some(target) { + self.timer = Some(Box::pin(R::sleep(target.saturating_duration_since(now)))); + self.timer_deadline = Some(target); + } + } + + /// Reply `Err(Shutdown)` to a command drained during teardown. + fn reply_shutdown(cmd: Command) { + match cmd { + Command::Join(JoinCmd { reply, .. }) => { + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(Err((SmallVec::new(), SerfError::Shutdown))); + } + Command::Leave(LeaveCmd { reply }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + Command::ForceLeave(ForceLeaveCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + Command::UserEvent(cmd) => { + let _ = cmd.reply.send(Err(SerfError::Shutdown)); + } + Command::Query(QueryCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + Command::Respond(RespondCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + Command::SetTags(SetTagsCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + #[cfg(encryption)] + Command::InstallKey(KeyCmd { reply, .. }) + | Command::UseKey(KeyCmd { reply, .. }) + | Command::RemoveKey(KeyCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + #[cfg(encryption)] + Command::ListKeys(ListKeysCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + Command::Shutdown(ShutdownCmd { reply }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } + } + } +} + +impl Future for QuicDriver +where + I: memberlist_proto::Id + Clone + Send + Sync + Unpin + 'static, + R: Runtime, + G: rand::Rng + Unpin, + SR: rand::Rng + SeedableRng + Unpin, + QuicEndpoint: Unpin, +{ + type Output = (); + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<()> { + let this = self.get_mut(); + let now = Instant::now(); + let mut progress = false; + let mut more = false; + + // Drain queued commands (parks the waker for the next push). + for cmd in this.shared.drain_commands(cx.waker()) { + this.dispatch(cmd, now); + progress = true; + } + + // Shutdown: best-effort leave, flush to quiescence (the leave's `Dead`-self + // notices must reach the wire before the socket drops), fail every parked + // waiter and queued command, release the bound port, then ack. The completion + // latch promises the bind address is free, not that every QUIC connection has + // closed. + if this.shared.is_shutdown() { + // Ignoring Err: best-effort leave during shutdown. + let _ = this.endpoint.leave(Instant::now()); + // Drain endpoint surfaces to quiescence before reaping: a single + // `drain_surfaces` pass is egress-capped, so a large batch of already-queued + // `ExchangeCompleted` events would be partially skipped, leaving contacted + // addresses unaccounted in the `Err` tuple. `account_event` folds every + // terminal completion into the matching pending join as it drains. + loop { + let (_, drain_more) = this.drain_surfaces(cx); + if !drain_more { + break; + } + } + // Close the command queue and fail any still-queued commands. + for cmd in this.shared.close_and_drain() { + if let Command::Shutdown(ShutdownCmd { reply }) = cmd { + // A straggler `Shutdown`: park it too, so it is acked after the socket + // drops like every other caller. + this.shutdown_reply.push(reply); + } else { + Self::reply_shutdown(cmd); + } + } + for mut pj in this.pending_joins.drain(..) { + for s in &pj.ignore_streams { + this.endpoint.clear_ignore_join_stream(*s); + } + if let Some(reply) = pj.reply.take() { + // Ignoring Err: the join caller dropped its reply receiver. Carry the + // addresses reached before shutdown raced this waiter. + let _ = reply.send(Err(( + std::mem::take(&mut pj.contacted), + SerfError::Shutdown, + ))); + } + } + if let Some(pl) = this.pending_leave.take() { + pl.resolve_all(|| Err(SerfError::Shutdown)); + } + // Release the bound port BEFORE acking: dropping the agnostic UDP socket + // closes its FD synchronously, so a caller resuming from `shutdown().await` + // can immediately rebind the same address. + drop(this.socket.take()); + for reply in this.shutdown_reply.drain(..) { + // Ignoring Err: the caller dropped its reply receiver. + let _ = reply.send(Ok(())); + } + this.shared.mark_shutdown_complete(); + return Poll::Ready(()); + } + + // Receive QUIC/gossip (bounded; a full batch means the kernel may hold more). + // The socket is always `Some` here — the shutdown branch above (which takes it) + // returned before reaching this point. `Poll::Pending` from the socket IS the + // kernel-empty signal, and — because `handle_udp` processes a QUIC stream + // packet in-band — a `!recv_capped` stop proves every completing reliable + // packet ready this poll was fed to the machine. + let mut recv_n = 0; + while recv_n < this.iter_drain_cap { + let Some(socket) = this.socket.as_ref() else { + break; + }; + match socket.poll_recv_from(cx, &mut this.recv_buf) { + Poll::Ready(Ok((n, src))) => { + this.endpoint.handle_udp(src, &this.recv_buf[..n], now); + recv_n += 1; + } + // Ignoring Err: a transient recv error is non-fatal; re-armed next poll. + Poll::Ready(Err(_)) => break, + Poll::Pending => break, + } + } + if recv_n > 0 { + progress = true; + } + // A saturated batch means the kernel may still hold datagrams — possibly a + // reliable-exchange completion or a probe Ack. This is the ONLY backlog signal + // (the ingress decode drains to empty), and it gates the shared `handle_timeout` + // site below. + let recv_capped = recv_n == this.iter_drain_cap; + more |= recv_capped; + + // Drain machine surfaces (gossip/QUIC egress capped, ingress + events uncapped). + let (drained, drain_more) = this.drain_surfaces(cx); + progress |= drained; + more |= drain_more; + // A conflict `Event::Shutdown` observed during the drain flips the shutdown + // latch; self-wake so the next poll enters the teardown branch. + if this.shared.is_shutdown() { + more = true; + } + + // Timer + deadline reaps under the shared-UDP-path quiescence gate. On QUIC + // every resolving input (a reliable push/pull completion, a probe Ack) rides + // the one UDP recv, so — with no disjoint FIFO to watermark — the non-premature + // gate is recv QUIESCENCE: a due `handle_timeout` / join / leave reap fires only + // when the recv batch drained (`!recv_capped`, so every kernel-ready completing + // packet was fed through `handle_udp` and its terminal folded by + // `drain_surfaces` above), OR — under a flood that saturates the batch every + // poll — after a bounded staleness grace, for liveness. + let endpoint_deadline = this + .endpoint + .poll_timeout() + .map(|d| d.min(now + this.idle_wake)) + .unwrap_or(now + this.idle_wake); + let reap_deadline = [ + this.min_pending_join_deadline(), + this.min_pending_leave_deadline(), + ] + .into_iter() + .flatten() + .min(); + // `endpoint_deadline` folds `idle_wake`, so `ep_due` is exactly "the coordinator + // has an elapsed deadline"; a bare idle wake is not due and takes the idle arm. + let ep_due = endpoint_deadline <= now; + let reap_due = reap_deadline.is_some_and(|d| d <= now); + + if ep_due || reap_due { + // Anchor the staleness grace the first poll the fire is held back purely by a + // saturated recv batch (the kernel may still hold a pre-deadline completion). + if recv_capped { + this.timeout_stall_since.get_or_insert(now); + } + let quiescent = !recv_capped; + let grace_ok = quiescent + || this + .timeout_stall_since + .is_some_and(|t| now.saturating_duration_since(t) >= SHARED_PATH_STALENESS_GRACE); + if grace_ok { + // The recv is quiescent (every kernel-ready completing packet was folded by + // `drain_surfaces` above) or the grace elapsed. Fire the coordinator's + // elapsed deadlines, fold the UNCAPPED terminal events they emit + // (`LeftCluster` / `ExchangeCompleted`) BEFORE the reaps — a same-poll leave + // whose `LeftCluster` went unfolded would otherwise reap a false + // `LeaveTimeout` — then reap the deadline residue. + if ep_due { + this.endpoint.handle_timeout(now); + } + while let Some(ev) = this.endpoint.poll_event() { + this.send_observation(ev); + } + this.reap_pending_joins(now); + this.reap_pending_leave(now); + this.timeout_stall_since = None; + progress = true; + more = true; + } else { + // A deadline is due but the recv batch was saturated and the grace has not + // elapsed: DEFER (self-wake) so the next poll drains the recv toward + // quiescence. No timer is armed — the deadline already elapsed, so the + // `more` self-wake alone re-polls (no lost wakeup), and the wall-clock grace + // bounds the deferral. + more = true; + } + } else { + // Idle: nothing due. Clear the stall anchor, then arm + poll the sleep for the + // next deadline; NO self-wake (an armed sleep or socket readiness re-polls). + this.timeout_stall_since = None; + let target = reap_deadline.map_or(endpoint_deadline, |d| d.min(endpoint_deadline)); + this.arm_timer(target, now); + if let Some(timer) = this.timer.as_mut() + && timer.as_mut().poll(cx).is_ready() + { + this.timer = None; + this.timer_deadline = None; + more = true; + } + } + + // Republish the snapshot whenever the pump made progress (the serf endpoint + // exposes no cheap version stamp, so — as in serf-compio — a productive poll + // rebuilds and republishes the observable membership). + if progress { + this.refresh_snapshot(); + } + + // Yield to other tasks, but re-poll promptly while work remains. + if more { + cx.waker().wake_by_ref(); + } + Poll::Pending + } +} + +/// Apply one terminal `ExchangeCompleted` to its await-result join waiter (if any): +/// remove `eid` from `pending`, push the peer into `contacted` on success, resolve +/// the caller's reply the instant `pending` empties (ahead of the obs hand-off, so +/// a slow delegate cannot delay it), and once fully done clear the still-recorded +/// ignore-join streams and reap the waiter. +fn complete_join_exchange( + endpoint: &mut QuicEndpoint, + pending_joins: &mut Vec, + eid: ExchangeId, + peer: SocketAddr, + succeeded: bool, +) where + I: memberlist_proto::Id + Clone, + G: rand::Rng, + SR: rand::Rng + SeedableRng, +{ + let Some(idx) = pending_joins + .iter() + .position(|pj| pj.pending.contains(&eid)) + else { + return; + }; + let pj = &mut pending_joins[idx]; + pj.pending.remove(&eid); + if succeeded { + pj.contacted.push(peer); + } + // Resolve the reply the moment every dispatched exchange has terminated. If the + // deadline already replied, `reply` is `None` and this is a no-op. + if pj.pending.is_empty() { + pj.resolve_reply(); + } + if pending_joins[idx].is_done() { + let pj = pending_joins.swap_remove(idx); + for s in &pj.ignore_streams { + endpoint.clear_ignore_join_stream(*s); + } + } +} + +/// Apply one inbound [`KeyRequest`] to the driver's keyring delegate, producing the +/// [`KeyResponseArgs`] the pump forwards to `respond_key`. +#[cfg(encryption)] +fn apply_key_request( + keyring: &dyn KeyringDelegate, + req: &KeyRequest, +) -> KeyResponseArgs { + match (req.op(), req.key()) { + (KeyRequestOperation::Install, Some(key)) => keyring.install(*key), + (KeyRequestOperation::Use, Some(key)) => keyring.use_key(*key), + (KeyRequestOperation::Remove, Some(key)) => keyring.remove(*key), + (KeyRequestOperation::List, _) => keyring.list(), + (_, None) => KeyResponseArgs { + result: false, + message: "key-management request missing its required key".into(), + keys: Vec::new(), + primary_key: None, + }, + } +} + +/// The observation task: drains machine events off the pump, invokes the +/// [`Delegate`] hooks, and forwards every serf event to the +/// [`EventStream`](crate::EventStream). The forward is best-effort (a full queue +/// drops + counts, never blocks). +async fn observation_task( + obs_rx: Receiver>, + delegate: D, + events_tx: Sender>, + shared: Arc>, + obs_payload_bytes: Arc, +) where + I: Clone + Send + Sync + 'static, + D: Delegate, +{ + use futures_util::FutureExt; + use std::panic::AssertUnwindSafe; + while let Ok(ev) = obs_rx.recv_async().await { + // Reclaim the byte-backstop budget this event occupied, before the (possibly + // slow) delegate hook, so the pump's enqueue side sees it promptly. + let payload = observation_payload_bytes(&ev); + if let Some(bytes) = payload { + obs_payload_bytes.fetch_sub(bytes, Ordering::Relaxed); + } + // Contain a panicking delegate hook so the task survives and keeps releasing the + // reservations of still-queued events. Ignoring the unwind result: the panic is + // contained and the event is still forwarded to subscribers below. + let _ = AssertUnwindSafe(dispatch_event_delegate(&delegate, &ev)) + .catch_unwind() + .await; + if events_tx + .try_send(ev) + .is_err_and(|e| matches!(e, TrySendError::Full(_))) + { + shared.add_events_dropped(1); + } + } +} + +/// Set up the QUIC driver: arm the periodic schedulers, spawn the observation task, +/// then build the [`QuicDriver`] future. The caller (`Transport::run`) awaits it. +#[allow(clippy::too_many_arguments)] +pub(crate) fn spawn_quic_driver( + mut endpoint: QuicEndpoint, + socket: ::UdpSocket, + quic_max_udp_payload: u64, + shared: Arc>, + events_tx: Sender>, + delegate: D, + driver_opts: RuntimeOptions, + label: Option, + #[cfg(encryption)] keyring: Arc, +) -> QuicDriver +where + I: memberlist_proto::Id + Clone + Send + Sync + Unpin + 'static, + R: Runtime, + G: rand::Rng, + SR: rand::Rng + SeedableRng, + D: Delegate, +{ + // Arm the periodic probe / gossip / push-pull schedulers. Without this the + // coordinator's schedulers stay unset, so failure detection, dissemination, and + // anti-entropy never run. + endpoint.start_scheduling(Instant::now()); + + // Observation byte backstop: bound the queued payload bytes a slow delegate can + // pin (the obs-channel count cap alone does not). + let obs_payload_bytes = Arc::new(AtomicU64::new(0)); + let obs_payload_budget: Option = match driver_opts.observation_channel() { + Channel::Bounded(_) => Some((endpoint.max_stream_frame_size() as u64).saturating_mul(4)), + Channel::Unbounded => None, + }; + let (obs_tx, obs_rx) = match driver_opts.observation_channel() { + Channel::Bounded(n) => flume::bounded(n), + Channel::Unbounded => flume::unbounded(), + }; + R::spawn_detach(observation_task::( + obs_rx, + delegate, + events_tx, + shared.clone(), + obs_payload_bytes.clone(), + )); + + QuicDriver::::new( + endpoint, + socket, + quic_max_udp_payload, + shared, + obs_tx, + obs_payload_bytes, + obs_payload_budget, + driver_opts, + label, + #[cfg(encryption)] + keyring, + ) +} + +#[cfg(test)] +mod tests; diff --git a/serf-reactor/src/driver/quic/tests.rs b/serf-reactor/src/driver/quic/tests.rs new file mode 100644 index 00000000..2a6c750a --- /dev/null +++ b/serf-reactor/src/driver/quic/tests.rs @@ -0,0 +1,49 @@ +//! Unit tests for the QUIC driver's recv-buffer sizing — the pure decision that +//! keeps neither the gossip plane nor the raw-QUIC plane truncated on the shared +//! UDP socket. The end-to-end pump behaviour (join / converge / user-event / query +//! / leave, encrypted convergence, mismatched-key enforcement) is covered by the +//! real-node suite in `tests/quic.rs`. + +use super::{ENCRYPTED_WRAPPER_OVERHEAD, GOSSIP_RECV_BUF_MAX, recv_buf_len_for}; + +/// The recv buffer follows the LARGER of the two planes sharing the one socket. A +/// caller can set a quinn `max_udp_payload_size` above the gossip MTU — quinn's own +/// default 1472 already exceeds the 1400 default `gossip_mtu` — so the buffer must +/// be sized from the QUIC plane, not silently left at the gossip size that would +/// truncate a full-size QUIC packet. +#[test] +fn recv_buf_sizes_to_the_larger_plane() { + // QUIC plane far above the gossip plane (jumbo `max_udp_payload_size`): the buffer + // follows the QUIC plane regardless of the small AEAD overhead. + assert_eq!(recv_buf_len_for(1400, 9000), 9000); + + // quinn's default max UDP payload (1472) exceeds the default gossip MTU (1400), so + // the buffer is sized to 1472, not 1400 — a full-size QUIC packet is not + // truncated. Holds with or without an AEAD backend since + // 1472 > 1400 + ENCRYPTED_WRAPPER_OVERHEAD. + assert_eq!(recv_buf_len_for(1400, 1472), 1472); +} + +/// When the gossip plane is the larger of the two, the QUIC ceiling never shrinks it +/// below the AEAD-inflated gossip requirement. +#[test] +fn recv_buf_keeps_the_gossip_requirement() { + // Large configured gossip MTU, default-ish QUIC payload: the gossip plane wins and + // keeps its encrypted-wrapper headroom. + assert_eq!( + recv_buf_len_for(16_000, 1472), + 16_000 + ENCRYPTED_WRAPPER_OVERHEAD + ); + + // The gossip plane is capped at the IPv4 UDP maximum; a small QUIC payload cannot + // shrink it below that cap. + assert_eq!(recv_buf_len_for(70_000, 1472), GOSSIP_RECV_BUF_MAX); +} + +/// quinn permits a `max_udp_payload_size` up to 65527, just above the gossip plane's +/// 65507 cap; the QUIC plane is NOT clamped to that cap, so such a packet is +/// buffered in full. +#[test] +fn recv_buf_does_not_clamp_quic_below_quinn_max() { + assert_eq!(recv_buf_len_for(70_000, 65_527), 65_527); +} diff --git a/serf-reactor/src/driver/shared.rs b/serf-reactor/src/driver/shared.rs index dd42e6dc..a78e9e18 100644 --- a/serf-reactor/src/driver/shared.rs +++ b/serf-reactor/src/driver/shared.rs @@ -11,8 +11,9 @@ /// Coordinator-allocated handle for one in-flight reliable exchange. /// /// Shared by the stream driver and the per-bridge task so they agree on the same -/// opaque id without the rest of the crate naming the machine's streams module. -#[cfg(feature = "tcp")] +/// opaque id without the rest of the crate naming the machine's streams module; the +/// QUIC driver correlates its await-result joins on the same id domain. +#[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) type ExchangeId = memberlist_proto::event::ExchangeId; /// Byte-backstop weight of a serf event. Delegates to diff --git a/serf-reactor/src/lib.rs b/serf-reactor/src/lib.rs index 42201c76..d12b6974 100644 --- a/serf-reactor/src/lib.rs +++ b/serf-reactor/src/lib.rs @@ -54,6 +54,8 @@ mod delegate; mod driver; mod error; mod events; +#[cfg(feature = "quic")] +mod quic; mod resolver; #[cfg(any(feature = "tcp", feature = "quic"))] mod serf; @@ -127,6 +129,10 @@ pub use transport::{Transport, TransportRuntime}; #[cfg_attr(docsrs, doc(cfg(feature = "tcp")))] pub use tcp::{TcpTransport, TcpTransportOptions}; +#[cfg(feature = "quic")] +#[cfg_attr(docsrs, doc(cfg(feature = "quic")))] +pub use quic::{QuicOptions, QuicTransport, QuicTransportOptions}; + #[cfg(any(feature = "tcp", feature = "quic"))] #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] pub use delegate::{ diff --git a/serf-reactor/src/quic/mod.rs b/serf-reactor/src/quic/mod.rs new file mode 100644 index 00000000..7b38125f --- /dev/null +++ b/serf-reactor/src/quic/mod.rs @@ -0,0 +1,357 @@ +//! QUIC-backed serf driver over the agnostic runtime. +//! +//! [`QuicTransport`] owns a single bound UDP socket — QUIC carries no separate TCP +//! listener: the coordinator (quinn-proto inside `serf_proto::QuicEndpoint`) +//! multiplexes the reliable push/pull streams over that one socket, and serf's +//! datagram gossip rides the same socket. The machine-layer +//! `serf_proto::QuicEndpoint` is built inside [`QuicTransport::run`] from +//! the stored [`QuicOptions`] and the serf `Options` carried by the +//! [`TransportRuntime`](crate::TransportRuntime). This is the `Send`/`agnostic` +//! sibling of serf-compio's `!Send`, compio-bound `QuicTransport`. +//! +//! ## TLS server name +//! +//! QUIC's TLS 1.3 handshake requires a server name to verify the peer's +//! certificate against. [`QuicOptions::new`] installs a cluster-uniform string used +//! for every peer; deployments whose certs name each peer's hostname/IP supply a +//! per-peer SNI closure via `QuicOptions::new_with_sni_provider`. + +#![cfg(feature = "quic")] + +use core::num::NonZeroU8; +use std::{io::ErrorKind, net::SocketAddr}; + +use agnostic::{ + Runtime, + net::{Net, UdpSocket}, +}; +use hostaddr::HostAddr; +use memberlist_proto::{ + CheapClone, Data, EndpointOptions, Id, MaybeResolved, QuicEndpoint as Coordinator, +}; +use rand::rngs::StdRng; +use smol_str::SmolStr; + +#[cfg(encryption)] +use memberlist_proto::EncryptionOptions; + +/// QUIC config bundle handed to [`QuicTransport`]. Re-exported from +/// `memberlist-proto` so callers don't need a direct dep. +pub use memberlist_proto::QuicOptions; + +use crate::{ + SerfError, + delegate::Delegate, + resolver::{AdvertiseAddrResolver, Resolver}, + transport::{Transport, TransportRuntime}, +}; + +/// Per-backend QUIC-specific transport options. +/// +/// Embedded into the transport constructor. Bundles the local node identifier, the +/// (possibly-unresolved) advertise address, and the caller-built [`QuicOptions`] +/// (quinn-proto `EndpointConfig` / `ServerConfig` / `ClientConfig` / +/// `TransportConfig` bundle plus SNI provider). The cluster label and +/// inbound-label-check policy are supplied via the serf `Options` block (not here), +/// feeding both planes from a single validated source. +pub struct QuicTransportOptions> { + local_id: Option, + advertise_addr: Option>, + quic_config: Option, + /// Gossip-encryption policy. The default (no keyring) leaves the gossip datagrams + /// plaintext; attaching a keyring via [`with_encryption`](Self::with_encryption) + /// makes the coordinator's `encrypt_gossip`/`decrypt_gossip` AEAD-protect them. + /// The reliable plane rides quinn's own TLS, so the keyring covers only the + /// gossip datagrams. + #[cfg(encryption)] + encryption: EncryptionOptions, +} + +impl QuicTransportOptions { + /// Construct with defaults. Caller MUST chain + /// [`with_local_id`](Self::with_local_id), + /// [`with_advertise_addr`](Self::with_advertise_addr), and + /// [`with_quic_config`](Self::with_quic_config) before passing to + /// `QuicTransport::new`. + #[inline] + pub fn new() -> Self { + Self { + local_id: None, + advertise_addr: None, + quic_config: None, + #[cfg(encryption)] + encryption: EncryptionOptions::new(), + } + } + + /// Builder: local node identifier. + #[must_use] + #[inline] + pub fn with_local_id(mut self, id: I) -> Self { + self.local_id = Some(id); + self + } + + /// Builder: advertise address (resolved or unresolved). + #[must_use] + #[inline] + pub fn with_advertise_addr(mut self, addr: MaybeResolved) -> Self { + self.advertise_addr = Some(addr); + self + } + + /// Builder: QUIC config bundle (caller-built quinn-proto configs + SNI). + #[must_use] + #[inline] + pub fn with_quic_config(mut self, cfg: QuicOptions) -> Self { + self.quic_config = Some(cfg); + self + } + + /// Builder: gossip-encryption policy. + /// + /// The default (no keyring) keeps the gossip datagrams plaintext, so an + /// unencrypted node still builds and interoperates. Attach a keyring + /// (`EncryptionOptions::new().with_keyring(Keyring::new(primary_key))`) to + /// AEAD-protect the gossip datagrams; every node sharing the cluster MUST carry + /// the same keyring to interop. The reliable plane is quinn TLS and is unaffected + /// by this keyring. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + #[must_use] + #[inline] + pub fn with_encryption(mut self, encryption: EncryptionOptions) -> Self { + self.encryption = encryption; + self + } + + /// Local node identifier, if set. + #[inline] + pub const fn local_id(&self) -> Option<&I> { + self.local_id.as_ref() + } + + /// Advertise address, if set. + #[inline] + pub const fn advertise_addr(&self) -> Option<&MaybeResolved> { + self.advertise_addr.as_ref() + } + + /// QUIC config bundle, if set. + #[inline] + pub const fn quic_config(&self) -> Option<&QuicOptions> { + self.quic_config.as_ref() + } + + /// Gossip-encryption policy. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + #[inline] + pub const fn encryption(&self) -> &EncryptionOptions { + &self.encryption + } +} + +impl Default for QuicTransportOptions { + #[inline] + fn default() -> Self { + Self::new() + } +} + +/// QUIC-backed serf transport. +/// +/// Owns the bound `UdpSocket` only — the coordinator multiplexes the reliable +/// push/pull streams over that one socket (no separate listener), and serf's +/// datagram gossip shares it. The machine-layer `serf_proto::QuicEndpoint` +/// is built inside [`Transport::run`] from the stored `quic_config` and the serf +/// options sourced from [`TransportRuntime`](crate::TransportRuntime). +pub struct QuicTransport +where + R: Runtime, +{ + local_id: I, + local_address: MaybeResolved, + advertise_socket: SocketAddr, + gossip_socket: ::UdpSocket, + quic_config: QuicOptions, + /// Independent OS-seeded seed for the serf core's RNG, drawn once per node in + /// [`Transport::new`] and consumed when [`Transport::run`] builds the endpoint via + /// `new_with_rng`. Distinct from the coordinator's gossip RNG so serf's query IDs + /// and relay choices are not correlated across nodes. + serf_rng: StdRng, + /// Gossip-encryption policy applied to the coordinator built in + /// [`Transport::run`]. Absent keyring ⇒ plaintext gossip (the default). + #[cfg(encryption)] + encryption: EncryptionOptions, +} + +impl Transport for QuicTransport +where + R: Runtime, + I: + Id + CheapClone + Clone + core::fmt::Debug + core::fmt::Display + Send + Sync + Unpin + 'static, + A: Data + Clone + Send + Sync + 'static, +{ + type Error = SerfError; + type Id = I; + type Address = A; + type Options = QuicTransportOptions; + + async fn new( + options: Self::Options, + resolver: &RES, + advertise_resolver: &AR, + ) -> Result + where + RES: Resolver
, + AR: AdvertiseAddrResolver, + { + let local_id = options.local_id.ok_or_else(|| { + SerfError::Io(std::io::Error::new( + ErrorKind::InvalidInput, + "local_id required", + )) + })?; + let advertise_input = options.advertise_addr.ok_or_else(|| { + SerfError::Io(std::io::Error::new( + ErrorKind::InvalidInput, + "advertise_addr required", + )) + })?; + let quic_config = options.quic_config.ok_or_else(|| { + SerfError::Io(std::io::Error::new( + ErrorKind::InvalidInput, + "quic_config required", + )) + })?; + + let advertise_socket = match &advertise_input { + MaybeResolved::Resolved(s) => *s, + MaybeResolved::Unresolved(a) => { + let candidates = resolver + .resolve(a) + .await + .map_err(|e| SerfError::Resolve(std::io::Error::other(e.to_string())))?; + advertise_resolver.pick(candidates).map_err(|e| { + SerfError::Resolve(std::io::Error::new( + ErrorKind::AddrNotAvailable, + e.to_string(), + )) + })? + } + }; + + // QUIC multiplexes streams + gossip over a single UDP socket; there is no + // separate listener to claim a port, so a plain bind suffices (no TCP/UDP + // port-space race, hence no ephemeral-retry pair). + let gossip_socket = ::UdpSocket::bind(advertise_socket) + .await + .map_err(SerfError::Io)?; + + // The socket is now bound. Read the bound address back (an ephemeral `:0` + // resolves to a concrete OS-assigned port here, which the node gossips to its + // peers, while a wildcard `0.0.0.0:0` bind yields `0.0.0.0:` — rejected by + // `post_bind_setup` as an unspecified IP peers could not route serf traffic + // back to), then draw the OS-seeded serf-core RNG. On ANY error the bound socket + // is dropped (which closes its FD synchronously for an agnostic socket) before + // returning, so a failed construction never leaks the bound UDP port to race an + // immediate same-address rebind into `AddrInUse`. + let bound = gossip_socket.local_addr().map_err(SerfError::Io)?; + let serf_rng = match crate::transport::post_bind_setup(&bound) { + Ok(rng) => rng, + Err(e) => { + drop(gossip_socket); + return Err(e); + } + }; + + Ok(Self { + local_id, + local_address: advertise_input, + advertise_socket: bound, + gossip_socket, + quic_config, + serf_rng, + #[cfg(encryption)] + encryption: options.encryption, + }) + } + + #[inline] + fn local_id(&self) -> &Self::Id { + &self.local_id + } + + #[inline] + fn local_address(&self) -> &MaybeResolved { + &self.local_address + } + + #[inline] + fn advertise_address(&self) -> &SocketAddr { + &self.advertise_socket + } + + async fn run(self, runtime: TransportRuntime, gossip_rng: G) + where + D: Delegate, + G: rand::Rng + Send + Unpin + 'static, + { + // `Serf::new` is generic over `T` and cannot build the QUIC endpoint (it needs + // the quinn-proto config bundle); build it here from `self`'s stored config. + // Serf ranks its user broadcasts on three tiers (intent / event / query → + // ranks 0 / 1 / 2), so the inner memberlist endpoint needs at least three + // broadcast tiers. + let inner_opts = EndpointOptions::new(self.local_id, self.advertise_socket) + .with_user_broadcast_tiers(NonZeroU8::new(3).expect("3 is nonzero")); + // The shared UDP socket also carries raw QUIC packets, whose size is governed by + // the quinn `EndpointConfig`'s accepted max UDP payload — which a caller can set + // above the serf gossip MTU (quinn's default 1472 already exceeds the 1400 + // default `gossip_mtu`). Read it off the config here, before it moves into the + // coordinator, so the driver sizes its recv buffer for the larger of the two + // planes and does not truncate a full-size QUIC packet. + let quic_max_udp_payload = self.quic_config.endpoint_ref().get_max_udp_payload_size(); + let inner = memberlist_proto::Endpoint::new(inner_opts, gossip_rng); + // The QUIC coordinator owns the quinn endpoint and the per-peer connection pool. + #[allow(unused_mut)] + let mut coord = Coordinator::new(inner, self.quic_config); + // Install the gossip-encryption keyring so the coordinator's + // `encrypt_gossip`/`decrypt_gossip` AEAD-protect the gossip datagrams. A + // no-keyring policy is the identity transform; the reliable plane is quinn TLS + // and is unaffected either way. + #[cfg(encryption)] + coord.set_encryption_options(self.encryption); + // Serf's core RNG is seeded from its own OS-drawn entropy (`self.serf_rng`), + // independent of the coordinator's gossip RNG, so two nodes never share the + // query-ID / relay-selection stream. + let endpoint = serf_proto::QuicEndpoint::::new_with_rng( + coord, + runtime.serf_options, + self.serf_rng, + ); + + let driver = crate::driver::quic::spawn_quic_driver::( + endpoint, + self.gossip_socket, + quic_max_udp_payload, + runtime.shared, + runtime.events_tx, + runtime.delegate, + runtime.driver_options, + None, + #[cfg(encryption)] + runtime.keyring, + ); + driver.await; + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-reactor/src/quic/tests.rs b/serf-reactor/src/quic/tests.rs new file mode 100644 index 00000000..84c50011 --- /dev/null +++ b/serf-reactor/src/quic/tests.rs @@ -0,0 +1,43 @@ +//! Unit tests for the QUIC transport options builder — the pure accessor / builder +//! wiring that feeds `QuicTransport::new`. Real-node construction (advertise +//! validation, single-socket bind, freed-port rebind) is exercised end-to-end by +//! the tokio suite in `tests/quic.rs`. + +use super::QuicTransportOptions; +use memberlist_proto::MaybeResolved; +use smol_str::SmolStr; +use std::net::SocketAddr; + +/// A fresh `QuicTransportOptions` carries no id / advertise / config until the +/// builder sets them. +#[test] +fn new_starts_empty() { + let opts = QuicTransportOptions::::new(); + assert!(opts.local_id().is_none()); + assert!(opts.advertise_addr().is_none()); + assert!(opts.quic_config().is_none()); +} + +/// The builder setters round-trip through the accessors (the id and advertise the +/// transport constructor requires). +#[test] +fn builders_round_trip() { + let addr: SocketAddr = "127.0.0.1:8300".parse().expect("addr"); + let opts = QuicTransportOptions::::new() + .with_local_id(SmolStr::new("node-a")) + .with_advertise_addr(MaybeResolved::Resolved(addr)); + + assert_eq!(opts.local_id(), Some(&SmolStr::new("node-a"))); + match opts.advertise_addr() { + Some(MaybeResolved::Resolved(s)) => assert_eq!(*s, addr), + other => panic!("expected a resolved advertise addr, got {other:?}"), + } +} + +/// `Default` delegates to `new` — the empty starting point. +#[test] +fn default_matches_new() { + let opts = QuicTransportOptions::::default(); + assert!(opts.local_id().is_none()); + assert!(opts.quic_config().is_none()); +} diff --git a/serf-reactor/src/serf/mod.rs b/serf-reactor/src/serf/mod.rs index b5ee293d..29cf1967 100644 --- a/serf-reactor/src/serf/mod.rs +++ b/serf-reactor/src/serf/mod.rs @@ -33,6 +33,8 @@ use smol_str::SmolStr; use crate::command::{KeyCmd, ListKeysCmd}; #[cfg(encryption)] use crate::delegate::KeyringDelegate; +#[cfg(feature = "quic")] +use crate::quic::{QuicTransport, QuicTransportOptions}; #[cfg(feature = "tcp")] use crate::tcp::{TcpTransport, TcpTransportOptions}; use crate::{ @@ -52,7 +54,7 @@ use crate::{ }; #[cfg(encryption)] use memberlist_proto::SecretKey; -#[cfg(feature = "tcp")] +#[cfg(any(feature = "tcp", feature = "quic"))] use memberlist_proto::{CheapClone, Data}; /// The initial published snapshot: the local node, `Alive`, with empty tags and @@ -299,6 +301,103 @@ where } } +// Ergonomic QUIC constructor: instantiate the QUIC transport for the caller so a +// node can be built without naming the generic `Serf::new::` machinery. +#[cfg(feature = "quic")] +#[cfg_attr(docsrs, doc(cfg(feature = "quic")))] +impl Serf +where + I: memberlist_proto::Id + + CheapClone + + Clone + + core::fmt::Debug + + core::fmt::Display + + Send + + Sync + + Unpin + + 'static, + A: Data + Clone + Send + Sync + 'static, + R: Runtime, +{ + /// Build a QUIC-backed serf node and spawn its driver on the runtime `R`. + /// + /// The ergonomic wrapper over [`Serf::new`] that instantiates the + /// [`QuicTransport`](crate::QuicTransport) for the caller: it binds a single UDP + /// socket on the advertise address (resolved once via `resolver` / + /// `advertise_resolver`) over which the coordinator multiplexes the reliable + /// push/pull streams and serf's datagram gossip, then spawns the QUIC driver. The + /// caller supplies the quinn-proto config bundle through + /// [`QuicTransportOptions::with_quic_config`](crate::QuicTransportOptions::with_quic_config). + /// The gossip RNG is drawn from OS entropy via [`gossip_rng`](crate::gossip_rng); + /// use [`quic_with_rng`](Self::quic_with_rng) to supply your own. + /// + /// Under an encryption backend, pass an + /// [`Arc`](crate::KeyringDelegate) + /// (`Arc::new(VoidKeyringDelegate)` for a node that manages no keys); the keyring + /// AEAD-protects the gossip datagrams (the reliable plane rides quinn's own TLS). + #[allow(clippy::too_many_arguments)] + pub async fn quic( + options: QuicTransportOptions, + resolver: &RES, + advertise_resolver: &AR, + delegate: D, + runtime_options: RuntimeOptions, + serf_options: SerfOptions, + #[cfg(encryption)] keyring: Arc, + ) -> Result + where + RES: Resolver
, + AR: AdvertiseAddrResolver, + D: Delegate, + { + Self::quic_with_rng( + options, + resolver, + advertise_resolver, + delegate, + runtime_options, + serf_options, + crate::gossip_rng()?, + #[cfg(encryption)] + keyring, + ) + .await + } + + /// Like [`quic`](Self::quic) but with a caller-supplied gossip RNG `G` — draw it + /// via [`gossip_rng`](crate::gossip_rng) for fork-safe OS entropy. + #[allow(clippy::too_many_arguments)] + pub async fn quic_with_rng( + options: QuicTransportOptions, + resolver: &RES, + advertise_resolver: &AR, + delegate: D, + runtime_options: RuntimeOptions, + serf_options: SerfOptions, + gossip_rng: G, + #[cfg(encryption)] keyring: Arc, + ) -> Result + where + RES: Resolver
, + AR: AdvertiseAddrResolver, + D: Delegate, + G: rand::Rng + Send + Unpin + 'static, + { + Self::new::, RES, AR, D, G>( + options, + resolver, + advertise_resolver, + delegate, + runtime_options, + serf_options, + gossip_rng, + #[cfg(encryption)] + keyring, + ) + .await + } +} + // Handle operations that read a cached snapshot or push a command over the queue — // none touch node identity directly, so they impose no bound and stay callable on // a `Serf` of any id type. diff --git a/serf-reactor/tests/quic.rs b/serf-reactor/tests/quic.rs new file mode 100644 index 00000000..d7d0db74 --- /dev/null +++ b/serf-reactor/tests/quic.rs @@ -0,0 +1,531 @@ +//! Real-node QUIC serf tests on tokio: two loopback nodes exercising the reactor +//! QUIC driver end-to-end over a quinn-proto config bundle. Each test spins up +//! ephemeral `127.0.0.1:0` nodes via the ergonomic [`Serf::quic`] constructor and +//! drives the full pump — QUIC push/pull join over a real quinn handshake, +//! coordinator merge, datagram gossip, user events, queries, and graceful +//! leave/shutdown — end-to-end proof the reactor QUIC driver works over a concrete +//! runtime, meeting the same behaviour bar as the TCP suite. +//! +//! Mirrors serf-compio's QUIC smoke tests and the reactor's `tests/tcp.rs`, +//! adapted to the reactor's `Send`/`agnostic` model and QUIC's single-socket, +//! stream-multiplexed transport. + +#![cfg(all(feature = "quic", feature = "tokio"))] + +use core::time::Duration; +use std::{net::SocketAddr, sync::Arc}; + +use agnostic::tokio::TokioRuntime; +use bytes::Bytes; +use futures_util::{StreamExt, future}; +use memberlist_proto::UnreliableTransport; +use rustls::{ + client::danger::{HandshakeSignatureValid, ServerCertVerified}, + version::TLS13, +}; +use rustls_pki_types::{CertificateDer, PrivateKeyDer}; +use serf_proto::{event::Event, members::SerfState, options::Options as SerfOptions}; +#[cfg(encryption)] +use serf_reactor::{EncryptionOptions, Keyring, SecretKey, VoidKeyringDelegate}; +use serf_reactor::{ + FirstAddrResolver, MaybeResolved, QuicOptions, QuicTransportOptions, RuntimeOptions, Serf, + SocketAddrResolver, VoidDelegate, +}; +use smol_str::SmolStr; + +/// A tokio-backed reactor QUIC node handle. +type Node = Serf; + +/// A self-signed cert + key for `localhost`, for the test TLS bundle. +fn self_signed() -> (Vec>, PrivateKeyDer<'static>) { + let ck = rcgen::generate_simple_self_signed(vec!["localhost".into()]).expect("self-signed cert"); + let cert = CertificateDer::from(ck.cert.der().to_vec()); + let key = PrivateKeyDer::Pkcs8(ck.signing_key.serialize_der().into()); + (vec![cert], key) +} + +fn test_endpoint_config(reset_key: &[u8]) -> quinn_proto::EndpointConfig { + let hmac = ring::hmac::Key::new(ring::hmac::HMAC_SHA256, reset_key); + quinn_proto::EndpointConfig::new(Arc::new(hmac)) +} + +fn test_server() -> quinn_proto::ServerConfig { + let (chain, key) = self_signed(); + let provider = Arc::new(rustls::crypto::ring::default_provider()); + let rustls_server = rustls::ServerConfig::builder_with_provider(provider) + .with_protocol_versions(&[&TLS13]) + .expect("tls13 server") + .with_no_client_auth() + .with_single_cert(chain, key) + .expect("single cert"); + let qsc = quinn_proto::crypto::rustls::QuicServerConfig::try_from(Arc::new(rustls_server)) + .expect("quic server config"); + quinn_proto::ServerConfig::with_crypto(Arc::new(qsc)) +} + +/// Accept-any server-cert verifier — test only. +#[derive(Debug)] +struct AnyServer; + +impl rustls::client::danger::ServerCertVerifier for AnyServer { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer, + _intermediates: &[CertificateDer], + _server_name: &rustls_pki_types::ServerName, + _ocsp_response: &[u8], + _now: rustls_pki_types::UnixTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + _message: &[u8], + _cert: &CertificateDer, + _dss: &rustls::DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + + fn verify_tls13_signature( + &self, + _message: &[u8], + _cert: &CertificateDer, + _dss: &rustls::DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + + fn supported_verify_schemes(&self) -> Vec { + rustls::crypto::ring::default_provider() + .signature_verification_algorithms + .supported_schemes() + } +} + +fn test_client() -> quinn_proto::ClientConfig { + let provider = Arc::new(rustls::crypto::ring::default_provider()); + let cfg = rustls::ClientConfig::builder_with_provider(provider) + .with_protocol_versions(&[&TLS13]) + .expect("tls13 client") + .dangerous() + .with_custom_certificate_verifier(Arc::new(AnyServer)) + .with_no_client_auth(); + let qcc = + quinn_proto::crypto::rustls::QuicClientConfig::try_from(Arc::new(cfg)).expect("quic client"); + quinn_proto::ClientConfig::new(Arc::new(qcc)) +} + +/// A fresh QUIC config bundle with a 20s idle timeout (well past a localhost +/// handshake) and datagram-mode unreliable transport. A fresh bundle is built per +/// node so each owns its own cert and quinn endpoint config. +fn test_quic_options() -> QuicOptions { + let mut transport = quinn_proto::TransportConfig::default(); + transport.max_idle_timeout(Some( + quinn_proto::IdleTimeout::try_from(Duration::from_secs(20)).expect("idle timeout"), + )); + QuicOptions::new( + test_endpoint_config(&[0x5au8; 32]), + test_server(), + test_client(), + transport, + "localhost", + UnreliableTransport::Datagram, + ) +} + +/// Build and spawn a reactor QUIC node on an ephemeral loopback port through the +/// ergonomic `Serf::quic` constructor. +async fn spawn_node(id: &str) -> Node { + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let opts = QuicTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(bind)) + .with_quic_config(test_quic_options()); + Serf::::quic( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + #[cfg(encryption)] + std::sync::Arc::new(VoidKeyringDelegate), + ) + .await + .expect("spawn serf quic node") +} + +/// Poll both nodes until each reports the full two-member cluster, or fail on a +/// generous timeout so a convergence regression surfaces as a timeout, not a hang. +async fn converge(a: &Node, b: &Node) { + tokio::time::timeout(Duration::from_secs(30), async { + loop { + if a.num_members() == 2 && b.num_members() == 2 { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("both nodes converge to a 2-member cluster"); +} + +/// Two nodes on loopback: A joins B (await-result over a real QUIC push/pull), then +/// BOTH converge to a two-member cluster and shut down cleanly. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn two_node_quic_join_converges() { + let b = spawn_node("conv-b").await; + let a = spawn_node("conv-a").await; + let b_addr = b.advertise_address(); + + let reached = a + .join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B over QUIC"); + assert_eq!(reached, b_addr, "join returns the reached seed address"); + + converge(&a, &b).await; + assert_eq!(a.num_members(), 2, "A sees the 2-member cluster"); + assert_eq!(b.num_members(), 2, "B sees the 2-member cluster"); + + a.shutdown().await.expect("conv-a shuts down"); + b.shutdown().await.expect("conv-b shuts down"); +} + +/// After a two-node QUIC join, a user event broadcast by B is delivered to A's event +/// stream carrying the original name and payload (datagram gossip over the shared +/// socket). +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn user_event_delivered() { + let b = spawn_node("ue-b").await; + let a = spawn_node("ue-a").await; + let b_addr = b.advertise_address(); + + // Subscribe before joining so the user event cannot race the subscription. + let mut a_events = a.events(); + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + converge(&a, &b).await; + + b.user_event("greet", Bytes::from_static(b"hello"), false) + .await + .expect("user event dispatched"); + + let got = tokio::time::timeout(Duration::from_secs(30), async { + loop { + match a_events.next().await { + Some(Event::User(u)) if u.name.as_str() == "greet" => break Some(u.payload.clone()), + Some(_) => {} + None => break None, + } + } + }) + .await + .expect("A observes B's user event within the timeout"); + assert_eq!( + got, + Some(Bytes::from_static(b"hello")), + "A receives B's user-event payload" + ); + + a.shutdown().await.expect("ue-a shuts down"); + b.shutdown().await.expect("ue-b shuts down"); +} + +/// After a two-node QUIC join, a query issued by A round-trips: B receives the +/// `Event::Query`, responds, and A surfaces the matching `Event::QueryResponse`. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn query_round_trip() { + let b = spawn_node("q-b").await; + let a = spawn_node("q-a").await; + let b_addr = b.advertise_address(); + + // Subscribe both before the join so neither the query nor its response races ahead + // of a subscription. + let mut b_events = b.events(); + let mut a_events = a.events(); + + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + converge(&a, &b).await; + + let want = Bytes::from_static(b"pong"); + a.query( + "ping", + Bytes::from_static(b"ping-payload"), + a.default_query_param(), + ) + .await + .expect("query issued"); + + // B answers the first "ping" query it sees; A collects the matching response. + let responder = async { + loop { + match b_events.next().await { + Some(Event::Query(qe)) if qe.name() == "ping" => { + b.respond(qe, want.clone()) + .await + .expect("B responds to the query"); + break; + } + Some(_) => {} + None => panic!("B's event stream closed before the query arrived"), + } + } + }; + let collector = async { + loop { + match a_events.next().await { + Some(Event::QueryResponse(qr)) if qr.payload() == &want => break true, + Some(_) => {} + None => break false, + } + } + }; + + let got = tokio::time::timeout(Duration::from_secs(30), async { + let (_, got) = future::join(responder, collector).await; + got + }) + .await + .expect("query round-trip completes within the timeout"); + assert!(got, "A must receive B's query response"); + + a.shutdown().await.expect("q-a shuts down"); + b.shutdown().await.expect("q-b shuts down"); +} + +/// A graceful leave completes the machine's leave chain: `leave()` resolves only +/// once `LeftCluster` fires (the reactor gates the reply on it), that event surfaces +/// on the leaver's own stream, and the local endpoint settles at `Left`. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn leave_emits_left_cluster() { + let b = spawn_node("lv-b").await; + let a = spawn_node("lv-a").await; + let b_addr = b.advertise_address(); + + let mut a_events = a.events(); + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + converge(&a, &b).await; + + // The reactor resolves `leave()` only once the machine's `LeftCluster` fires, so a + // successful return already proves the graceful-leave chain completed. + a.leave().await.expect("A leaves the cluster"); + + // `LeftCluster` is also forwarded to A's own subscribers. + let saw = tokio::time::timeout(Duration::from_secs(30), async { + loop { + match a_events.next().await { + Some(Event::LeftCluster) => break true, + Some(_) => {} + None => break false, + } + } + }) + .await + .expect("A observes LeftCluster within the timeout"); + assert!(saw, "A must surface Event::LeftCluster after leave()"); + + // The local endpoint state settles at `Left` (poll to absorb the snapshot-refresh + // race after the leave chain completes). + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if a.state() == SerfState::Left { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("A's endpoint state becomes Left"); + + a.shutdown().await.expect("lv-a shuts down"); + b.shutdown().await.expect("lv-b shuts down"); +} + +/// The QUIC driver binds a single UDP socket and drops it (releasing its FD) before +/// acking shutdown, so `shutdown().await` releases the bound port before it +/// resolves: a second QUIC node binding the SAME advertise address the instant the +/// first shuts down must construct successfully, not fail with `AddrInUse`. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn quic_shutdown_releases_bound_address_for_rebind() { + let first = spawn_node("rebind-first").await; + let addr = first.advertise_address(); + first.shutdown().await.expect("first node shuts down"); + + let opts = QuicTransportOptions::::new() + .with_local_id(SmolStr::new("rebind-second")) + .with_advertise_addr(MaybeResolved::Resolved(addr)) + .with_quic_config(test_quic_options()); + let second = Serf::::quic( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + #[cfg(encryption)] + std::sync::Arc::new(VoidKeyringDelegate), + ) + .await + .expect("rebinding the freed UDP address must succeed, not AddrInUse"); + assert_eq!( + second.advertise_address(), + addr, + "the second node rebinds the exact freed address" + ); + second.shutdown().await.expect("second node shuts down"); +} + +/// A deterministic test secret key, selecting whichever AEAD cipher this build +/// compiled so the encrypted tests work under either backend. +#[cfg(encryption)] +fn test_secret_key(fill: u8) -> SecretKey { + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes256([fill; 32]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = SecretKey::ChaCha20Poly1305([fill; 32]); + key +} + +/// Build and spawn a reactor QUIC node on an ephemeral loopback port with +/// `encryption` installed as its gossip keyring policy. +#[cfg(encryption)] +async fn spawn_encrypted_node(id: &str, encryption: EncryptionOptions) -> Node { + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let opts = QuicTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(bind)) + .with_quic_config(test_quic_options()) + .with_encryption(encryption); + Serf::::quic( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + std::sync::Arc::new(VoidKeyringDelegate), + ) + .await + .expect("spawn encrypted serf quic node") +} + +/// Two QUIC nodes sharing one gossip keyring converge AND exchange gossip: A joins B +/// (the reliable push/pull rides quinn's own TLS, so it merges membership +/// regardless of the keyring), both reach the two-member cluster, and then a user +/// event B broadcasts — which rides the AEAD-sealed GOSSIP plane, not the reliable +/// push/pull — reaches A. The user-event delivery is the discriminating check: on +/// QUIC the gossip keyring seals only the datagram plane (the reliable plane is +/// quinn TLS), so it is a gossip-carried event, not the membership merge, that +/// proves `encrypt_gossip`/`decrypt_gossip` round-trip end-to-end rather than +/// running as identity transforms. +#[cfg(encryption)] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn two_node_quic_gossip_convergence_encrypted() { + let key = EncryptionOptions::new().with_keyring(Keyring::new(test_secret_key(0x42))); + let b = spawn_encrypted_node("enc-b", key.clone()).await; + let a = spawn_encrypted_node("enc-a", key).await; + let b_addr = b.advertise_address(); + + // Subscribe before joining so the user event cannot race the subscription. + let mut a_events = a.events(); + let reached = a + .join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B over the encrypted QUIC cluster"); + assert_eq!(reached, b_addr, "join returns the reached seed address"); + + converge(&a, &b).await; + + // The gossip-plane discriminator: B broadcasts a user event, disseminated over the + // AEAD-sealed gossip datagrams. With a shared keyring A decrypts and surfaces it. + b.user_event("greet", Bytes::from_static(b"hello"), false) + .await + .expect("user event dispatched"); + + let got = tokio::time::timeout(Duration::from_secs(30), async { + loop { + match a_events.next().await { + Some(Event::User(u)) if u.name.as_str() == "greet" => break Some(u.payload.clone()), + Some(_) => {} + None => break None, + } + } + }) + .await + .expect("A observes B's user event over the shared-key gossip plane within the timeout"); + assert_eq!( + got, + Some(Bytes::from_static(b"hello")), + "A receives B's user-event payload across the encrypted gossip plane" + ); + + a.shutdown().await.expect("enc-a shuts down"); + b.shutdown().await.expect("enc-b shuts down"); +} + +/// A node holding one gossip keyring and a node holding a DIFFERENT keyring share no +/// GOSSIP: on QUIC the reliable push/pull rides quinn's own TLS, so the join still +/// merges membership (the gossip keyring does not gate that plane) — but a user +/// event, which is disseminated only over the AEAD-sealed gossip datagrams, cannot +/// cross a disjoint key. Within a window shorter than the 30s anti-entropy +/// push/pull interval a user event rides gossip ALONE, so its ABSENCE at A proves +/// the gossip encryption is real enforcement, not an identity pass-through — the +/// discriminating negative the positive test above pairs with (both turn on a +/// gossip-carried event, since the membership merge crosses regardless of the key). +#[cfg(encryption)] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn mismatched_keyring_gossip_does_not_cross() { + let b = spawn_encrypted_node( + "mis-b", + EncryptionOptions::new().with_keyring(Keyring::new(test_secret_key(0x42))), + ) + .await; + let a = spawn_encrypted_node( + "mis-a", + EncryptionOptions::new().with_keyring(Keyring::new(test_secret_key(0x43))), + ) + .await; + let b_addr = b.advertise_address(); + + // The reliable push/pull (quinn TLS) merges membership regardless of the gossip + // keyring, so the await-result join still completes — B is now a known member A + // gossips to. + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("the reliable push/pull merges membership over quinn TLS"); + + // Subscribe, then have B broadcast a user event over the gossip plane. Under the + // mismatched keyring A cannot decrypt B's gossip datagrams, so it never surfaces + // the event. The window is well under the 30s anti-entropy push/pull interval, so + // the event rides gossip ALONE — a later push/pull cannot carry it into A here. + let mut a_events = a.events(); + b.user_event("secret", Bytes::from_static(b"hidden"), false) + .await + .expect("user event dispatched"); + + let observed = tokio::time::timeout(Duration::from_secs(5), async { + loop { + match a_events.next().await { + Some(Event::User(u)) if u.name.as_str() == "secret" => break true, + Some(_) => {} + None => break false, + } + } + }) + .await; + assert!( + !matches!(observed, Ok(true)), + "node A must NOT surface B's gossip-carried user event across a mismatched keyring" + ); + + a.shutdown().await.expect("mis-a shuts down"); + b.shutdown().await.expect("mis-b shuts down"); +} From b91ac3dfffbeb3b7bcf0cc32060824114ffdeb2d Mon Sep 17 00:00:00 2001 From: al8n Date: Thu, 2 Jul 2026 16:14:03 +1200 Subject: [PATCH 035/128] fix(serf-reactor): route QUIC gossip through the configured datagram mode; recv-error is not quiescence Outbound QUIC gossip is now routed through queue_unreliable_datagram/flush_outbound_transmits in Datagram mode (TLS-protected quinn datagrams), falling back to plain UDP only on NotReady/TooLarge, mirroring memberlist reactor/compio; the prior direct poll_send_to bypassed the configured mode and made the datagram tests vacuous. The pump's non-premature timer/reap gate now treats a recv-error stop as backlog-uncertain (defer + self-wake, bounded by the staleness grace) rather than quiescence, so a due join/leave reap cannot fire with a completing QUIC packet unread behind the error. Adds serf-proto QUIC forwarders (unreliable_transport / queue_unreliable_datagram / flush_outbound_transmits) plus a datagrams_sent counter surfaced on the Serf handle. Regressions: a non-vacuous Datagram-mode gossip test (the datagrams_sent discriminator) and a recv-error pump-gate test, both verified fail-on-revert. --- serf-proto/src/quic_endpoint/mod.rs | 47 ++++- serf-reactor/src/driver/quic/mod.rs | 188 +++++++++++++----- serf-reactor/src/driver/quic/tests.rs | 268 +++++++++++++++++++++++++- serf-reactor/src/serf/mod.rs | 9 + serf-reactor/src/shared/mod.rs | 16 ++ serf-reactor/tests/quic.rs | 66 +++++++ 6 files changed, 538 insertions(+), 56 deletions(-) diff --git a/serf-proto/src/quic_endpoint/mod.rs b/serf-proto/src/quic_endpoint/mod.rs index 996b3ed7..a176500b 100644 --- a/serf-proto/src/quic_endpoint/mod.rs +++ b/serf-proto/src/quic_endpoint/mod.rs @@ -34,8 +34,9 @@ use core::net::SocketAddr; use std::sync::Arc; use memberlist_proto::{ - Data, Id, Instant, PushPullKind, QuicEndpoint as Coordinator, Rng, SeedableRng, SmallRng, - Transmit, event::StreamId, parse_message, typed::Message, + Data, DatagramSendStatus, Id, Instant, PushPullKind, QuicEndpoint as Coordinator, Rng, + SeedableRng, SmallRng, Transmit, UnreliableTransport, event::StreamId, parse_message, + typed::Message, }; use smol_str::SmolStr; @@ -319,6 +320,48 @@ where self.transport.gossip_mtu() } + /// Which wire the coordinator's unreliable (gossip + probe) path rides — + /// [`Datagram`](UnreliableTransport::Datagram) (a QUIC datagram over the peer's + /// pooled, TLS-protected connection) or [`Udp`](UnreliableTransport::Udp) (the + /// shared UDP socket). The driver reads this to route each outbound gossip + /// transmit: queue a QUIC datagram in `Datagram` mode, or send on the plain-UDP + /// path. Forwards to + /// [`memberlist_proto::QuicEndpoint::unreliable_transport`]. + pub fn unreliable_transport(&self) -> UnreliableTransport { + self.transport.unreliable_transport() + } + + /// Offer one already-encoded (label-framed and, under an encryption backend, + /// AEAD-sealed) gossip datagram to `peer` over its pooled QUIC connection. + /// + /// Forwards to [`memberlist_proto::QuicEndpoint::queue_unreliable_datagram`]. + /// The returned [`DatagramSendStatus`] tells the driver whether the payload was + /// accepted onto an established connection + /// ([`Queued`](DatagramSendStatus::Queued)) or it must fall back to the + /// plain-UDP path ([`NotReady`](DatagramSendStatus::NotReady) / + /// [`TooLarge`](DatagramSendStatus::TooLarge)). Connection liveness is never a + /// membership signal — a dropped or refused datagram becomes a probe timeout, + /// not a `Suspect`. + pub fn queue_unreliable_datagram( + &mut self, + peer: SocketAddr, + bytes: Bytes, + now: Instant, + ) -> DatagramSendStatus { + self.transport.queue_unreliable_datagram(peer, bytes, now) + } + + /// Flush quinn's queued outbound — including datagrams just handed to + /// [`queue_unreliable_datagram`](Self::queue_unreliable_datagram) — into the + /// [`poll_transmit`](Self::poll_transmit) queue at `now` WITHOUT advancing any + /// membership timer, so a datagram leaves on the same tick it was queued (a + /// datagram-borne probe whose timeout is armed this tick must not wait for the + /// next driver wake). Forwards to + /// [`memberlist_proto::QuicEndpoint::flush_outbound_transmits`]. + pub fn flush_outbound_transmits(&mut self, now: Instant) { + self.transport.flush_outbound_transmits(now); + } + /// Encrypt one outbound gossip datagram for the wire, applying the /// coordinator's configured encryption keyring. /// diff --git a/serf-reactor/src/driver/quic/mod.rs b/serf-reactor/src/driver/quic/mod.rs index 6f8d5923..1e037baf 100644 --- a/serf-reactor/src/driver/quic/mod.rs +++ b/serf-reactor/src/driver/quic/mod.rs @@ -23,16 +23,17 @@ //! watermark can gate exactly — a join's resolving push/pull completion and a //! gossip flood arrive on the SAME path. There is thus no per-completion watermark //! to observe; the non-premature gate is instead UDP-recv QUIESCENCE. When the -//! recv loop stops on `Poll::Pending` (`!recv_capped`), every kernel-ready packet +//! recv loop stops on `Poll::Pending` (`recv_quiescent`), every kernel-ready packet //! — including any completing QUIC stream packet — was fed through `handle_udp` //! this poll and its `ExchangeCompleted` folded by `drain_surfaces` BEFORE the //! reaps run, so a due reap / reliable-exchange `handle_timeout` fires only over a -//! genuinely-absent completion. Under a sustained flood that keeps the batch -//! saturated (`recv_capped`) every poll, the fire is held back only until a -//! bounded wall-clock staleness grace elapses, then fires for liveness (quinn's -//! connection timers must advance and a parked join/leave must resolve). The -//! grace is bounded-early exactly as the stream driver's SWIM tick is; on QUIC it -//! covers the reliable plane too, because that plane shares the flooded path. +//! genuinely-absent completion. A stop that is NOT `Poll::Pending` — a saturated +//! batch (`recv_capped`) OR a recv error (which is not a kernel-empty signal: a +//! completing packet may sit behind it) — is backlog-uncertain and holds the fire +//! back until a bounded wall-clock staleness grace elapses, then fires for liveness +//! (quinn's connection timers must advance and a parked join/leave must resolve). +//! The grace is bounded-early exactly as the stream driver's SWIM tick is; on QUIC +//! it covers the reliable plane too, because that plane shares the same path. #![cfg(feature = "quic")] @@ -57,7 +58,7 @@ use bytes::Bytes; use flume::{Receiver, Sender, TrySendError}; use futures_channel::oneshot; use memberlist_proto::{ - Instant, SeedableRng, StreamId, Transmit, + DatagramSendStatus, Instant, SeedableRng, StreamId, Transmit, UnreliableTransport, codec::{ DecodeOptions, EncodeOptions, decode_incoming, encode_outgoing, encode_outgoing_compound, parse_messages, @@ -303,6 +304,13 @@ where timeout_stall_since: Option, idle_wake: Duration, leave_timeout: Duration, + /// Test-only: the number of upcoming recv-loop socket polls that must report a + /// recv ERROR stop instead of reading the real socket. A bound UDP socket cannot + /// be made to error on demand, so a pump test decrements this to drive the + /// recv-error gate deterministically (an `Err` stop, then a real `Poll::Pending` + /// quiescent stop). + #[cfg(test)] + recv_errors_remaining: usize, /// The driver's keyring delegate: applies inbound key-management ops and produces /// the `respond_key` answer. Present only under an encryption backend. #[cfg(encryption)] @@ -351,11 +359,31 @@ where timeout_stall_since: None, idle_wake: driver_opts.idle_wake_interval(), leave_timeout: driver_opts.leave_timeout(), + #[cfg(test)] + recv_errors_remaining: 0, #[cfg(encryption)] keyring, } } + /// Poll the shared UDP socket once for the recv loop, returning one datagram, a + /// recv error, or `Poll::Pending` (the kernel-empty signal). A `#[cfg(test)]` + /// hook can script an `Err` stop here so a pump test can drive the recv-error + /// gate deterministically — a bound socket cannot be made to error on demand. + fn poll_recv_once(&mut self, cx: &mut Context<'_>) -> Poll> { + #[cfg(test)] + if self.recv_errors_remaining > 0 { + self.recv_errors_remaining -= 1; + return Poll::Ready(Err(std::io::Error::from( + std::io::ErrorKind::ConnectionRefused, + ))); + } + let Some(socket) = self.socket.as_ref() else { + return Poll::Pending; + }; + socket.poll_recv_from(cx, &mut self.recv_buf) + } + /// Applies one handle command to the machine. fn dispatch(&mut self, cmd: Command, now: Instant) { let running = self.endpoint.state() == SerfState::Alive; @@ -671,11 +699,15 @@ where } worked |= ingress; - // Outbound gossip: encode (plain or compound) + encrypt, then send on the UDP - // socket. Popping the last transmit is the endpoint's leave-completion fence - // (it emits `LeftCluster`), so the leave/shutdown datagrams reach the socket - // before that fence fires. + // Outbound gossip: encode (plain or compound) + encrypt, then route onto the + // unreliable wire the endpoint is configured for — a QUIC datagram over the + // peer's pooled (quinn-TLS-protected) connection in `Datagram` mode, or the + // shared UDP socket in `Udp` mode. Popping the last transmit is the endpoint's + // leave-completion fence (it emits `LeftCluster`), so the leave/shutdown + // datagrams reach the wire before that fence fires. let encode_opts = EncodeOptions::new(self.label.clone()); + let unreliable = self.endpoint.unreliable_transport(); + let mut needs_flush = false; let mut sent = 0; while sent < budget { let Some(transmit) = self.endpoint.poll_memberlist_transmit() else { @@ -707,15 +739,64 @@ where Err(_) => continue, }; } - if let Some(socket) = self.socket.as_ref() { - // Ignoring Poll: gossip is best-effort — a full or errored UDP send drops - // the datagram and SWIM recovers on the next round. - let _ = socket.poll_send_to(cx, &on_wire, peer); + // `Bytes` so the datagram-queue path and the UDP fallback can share the + // encoded (and, under an encryption backend, already-sealed) payload without + // a second copy — `clone` is an O(1) refcount bump. + let on_wire = Bytes::from(on_wire); + match unreliable { + UnreliableTransport::Udp => { + if let Some(socket) = self.socket.as_ref() { + // Ignoring Poll: gossip is best-effort — a full or errored UDP send + // drops the datagram and SWIM recovers on the next round. + let _ = socket.poll_send_to(cx, &on_wire, peer); + } + } + UnreliableTransport::Datagram => { + match self + .endpoint + .queue_unreliable_datagram(peer, on_wire.clone(), now) + { + // Accepted onto an established QUIC connection: flush it into + // `poll_transmit` this pass (below) so a datagram-borne probe leaves on + // the tick its timeout is armed. + DatagramSendStatus::Queued => { + needs_flush = true; + self.shared.add_datagrams_sent(1); + } + // NotReady may mean the queue just initiated a cold dial: flush this + // pass so the connection's Initial is emitted now (else it does not warm + // until the next driver wake). The gossip itself still goes out + // immediately over the plain-UDP fallback. + DatagramSendStatus::NotReady => { + needs_flush = true; + if let Some(socket) = self.socket.as_ref() { + // Ignoring Poll: gossip is best-effort — SWIM recovers next round. + let _ = socket.poll_send_to(cx, &on_wire, peer); + } + } + // TooLarge: the connection is already Established (max_size was Some), so + // there is no pending Initial to flush; fall back to plain UDP. + DatagramSendStatus::TooLarge => { + if let Some(socket) = self.socket.as_ref() { + // Ignoring Poll: gossip is best-effort — SWIM recovers next round. + let _ = socket.poll_send_to(cx, &on_wire, peer); + } + } + } + } } } worked |= sent > 0; more |= sent == budget; + // Flush any datagrams queued above into `poll_transmit` THIS pass so the + // raw-QUIC loop below sends them now — a datagram-borne probe whose timeout is + // armed this same tick must not wait for the next driver wake (that wake can be + // the timeout). + if needs_flush { + self.endpoint.flush_outbound_transmits(now); + } + // Raw QUIC datagrams: already wire-framed by quinn-proto (handshake, acks, // reliable stream data, datagram-mode gossip), no codec wrap. let mut raw_sent = 0; @@ -1060,34 +1141,42 @@ where // Receive QUIC/gossip (bounded; a full batch means the kernel may hold more). // The socket is always `Some` here — the shutdown branch above (which takes it) - // returned before reaching this point. `Poll::Pending` from the socket IS the - // kernel-empty signal, and — because `handle_udp` processes a QUIC stream - // packet in-band — a `!recv_capped` stop proves every completing reliable - // packet ready this poll was fed to the machine. + // returned before reaching this point. Only `Poll::Pending` from the socket IS + // the kernel-empty signal, and — because `handle_udp` processes a QUIC stream + // packet in-band — a `Poll::Pending` stop proves every completing reliable + // packet ready this poll was fed to the machine. A recv-ERROR stop is NOT + // kernel-empty (the kernel may still hold a completing packet behind the + // error), so it is tracked separately and gates the reaps like a saturated + // batch. let mut recv_n = 0; + let mut recv_errored = false; while recv_n < this.iter_drain_cap { - let Some(socket) = this.socket.as_ref() else { - break; - }; - match socket.poll_recv_from(cx, &mut this.recv_buf) { + match this.poll_recv_once(cx) { Poll::Ready(Ok((n, src))) => { this.endpoint.handle_udp(src, &this.recv_buf[..n], now); recv_n += 1; } - // Ignoring Err: a transient recv error is non-fatal; re-armed next poll. - Poll::Ready(Err(_)) => break, + // Ignoring Err: a transient recv error is non-fatal (the datagram is + // dropped, re-armed next poll) — but it is NOT quiescence, so record it for + // the timer/reap gate below. + Poll::Ready(Err(_)) => { + recv_errored = true; + break; + } Poll::Pending => break, } } if recv_n > 0 { progress = true; } - // A saturated batch means the kernel may still hold datagrams — possibly a - // reliable-exchange completion or a probe Ack. This is the ONLY backlog signal - // (the ingress decode drains to empty), and it gates the shared `handle_timeout` - // site below. + // Either non-quiescent stop means the kernel may still hold datagrams — possibly + // a reliable-exchange completion or a probe Ack: a saturated batch, or a recv + // error behind which a completing packet may sit. This is the ONLY backlog + // signal (the ingress decode drains to empty), and it gates the shared + // `handle_timeout` site below. let recv_capped = recv_n == this.iter_drain_cap; - more |= recv_capped; + let recv_quiescent = !recv_capped && !recv_errored; + more |= recv_capped || recv_errored; // Drain machine surfaces (gossip/QUIC egress capped, ingress + events uncapped). let (drained, drain_more) = this.drain_surfaces(cx); @@ -1103,10 +1192,12 @@ where // every resolving input (a reliable push/pull completion, a probe Ack) rides // the one UDP recv, so — with no disjoint FIFO to watermark — the non-premature // gate is recv QUIESCENCE: a due `handle_timeout` / join / leave reap fires only - // when the recv batch drained (`!recv_capped`, so every kernel-ready completing - // packet was fed through `handle_udp` and its terminal folded by - // `drain_surfaces` above), OR — under a flood that saturates the batch every - // poll — after a bounded staleness grace, for liveness. + // when the recv loop stopped on `Poll::Pending` (`recv_quiescent`, so every + // kernel-ready completing packet was fed through `handle_udp` and its terminal + // folded by `drain_surfaces` above), OR — under a flood that saturates the batch + // (or a persistent recv error) every poll — after a bounded staleness grace, for + // liveness. A saturated batch AND a recv-error stop are both backlog-uncertain: + // neither proves the kernel is empty, so both defer the reaps. let endpoint_deadline = this .endpoint .poll_timeout() @@ -1126,21 +1217,21 @@ where if ep_due || reap_due { // Anchor the staleness grace the first poll the fire is held back purely by a - // saturated recv batch (the kernel may still hold a pre-deadline completion). - if recv_capped { + // non-quiescent recv stop — a saturated batch OR a recv error, either of which + // may hide a pre-deadline completion still in the kernel. + if !recv_quiescent { this.timeout_stall_since.get_or_insert(now); } - let quiescent = !recv_capped; - let grace_ok = quiescent + let grace_ok = recv_quiescent || this .timeout_stall_since .is_some_and(|t| now.saturating_duration_since(t) >= SHARED_PATH_STALENESS_GRACE); if grace_ok { - // The recv is quiescent (every kernel-ready completing packet was folded by - // `drain_surfaces` above) or the grace elapsed. Fire the coordinator's - // elapsed deadlines, fold the UNCAPPED terminal events they emit - // (`LeftCluster` / `ExchangeCompleted`) BEFORE the reaps — a same-poll leave - // whose `LeftCluster` went unfolded would otherwise reap a false + // The recv stopped on `Poll::Pending` (every kernel-ready completing packet + // was folded by `drain_surfaces` above) or the grace elapsed. Fire the + // coordinator's elapsed deadlines, fold the UNCAPPED terminal events they + // emit (`LeftCluster` / `ExchangeCompleted`) BEFORE the reaps — a same-poll + // leave whose `LeftCluster` went unfolded would otherwise reap a false // `LeaveTimeout` — then reap the deadline residue. if ep_due { this.endpoint.handle_timeout(now); @@ -1154,11 +1245,12 @@ where progress = true; more = true; } else { - // A deadline is due but the recv batch was saturated and the grace has not - // elapsed: DEFER (self-wake) so the next poll drains the recv toward - // quiescence. No timer is armed — the deadline already elapsed, so the + // A deadline is due but the recv stop was non-quiescent (a saturated batch or + // a recv error, either backlog-uncertain) and the grace has not elapsed: + // DEFER (self-wake) so the next poll drains the recv toward a `Poll::Pending` + // quiescent stop. No timer is armed — the deadline already elapsed, so the // `more` self-wake alone re-polls (no lost wakeup), and the wall-clock grace - // bounds the deferral. + // bounds the deferral so a persistent error cannot starve the timer forever. more = true; } } else { diff --git a/serf-reactor/src/driver/quic/tests.rs b/serf-reactor/src/driver/quic/tests.rs index 2a6c750a..6d275212 100644 --- a/serf-reactor/src/driver/quic/tests.rs +++ b/serf-reactor/src/driver/quic/tests.rs @@ -1,10 +1,12 @@ -//! Unit tests for the QUIC driver's recv-buffer sizing — the pure decision that -//! keeps neither the gossip plane nor the raw-QUIC plane truncated on the shared -//! UDP socket. The end-to-end pump behaviour (join / converge / user-event / query -//! / leave, encrypted convergence, mismatched-key enforcement) is covered by the -//! real-node suite in `tests/quic.rs`. +//! Unit tests for the QUIC driver's recv-buffer sizing and the shared-UDP-path +//! timer/reap gate. The recv-buffer sizing is a pure decision that keeps neither the +//! gossip plane nor the raw-QUIC plane truncated on the shared UDP socket; the gate +//! tests drive the real pump and prove a recv-ERROR stop is not read as quiescence. +//! The full end-to-end pump behaviour (join / converge / user-event / query / leave, +//! encrypted convergence, mismatched-key enforcement, datagram egress) is covered by +//! the real-node suite in `tests/quic.rs`. -use super::{ENCRYPTED_WRAPPER_OVERHEAD, GOSSIP_RECV_BUF_MAX, recv_buf_len_for}; +use super::*; /// The recv buffer follows the LARGER of the two planes sharing the one socket. A /// caller can set a quinn `max_udp_payload_size` above the gossip MTU — quinn's own @@ -47,3 +49,257 @@ fn recv_buf_keeps_the_gossip_requirement() { fn recv_buf_does_not_clamp_quic_below_quinn_max() { assert_eq!(recv_buf_len_for(70_000, 65_527), 65_527); } + +/// Pump-level regressions for the shared-UDP-path timer/reap gate, driven over a +/// real `QuicDriver::poll` on a tokio runtime with a real bound socket. +#[cfg(all(feature = "quic-rustls-ring", feature = "tokio"))] +mod gate { + use super::*; + + use core::num::NonZeroU8; + use std::{sync::Arc, task::Waker}; + + use agnostic::tokio::TokioRuntime; + use memberlist_proto::{ + Endpoint, EndpointOptions, Node, QuicEndpoint as Coordinator, QuicOptions, + }; + use rand::rngs::StdRng; + use rustls::{ + client::danger::{HandshakeSignatureValid, ServerCertVerified}, + version::TLS13, + }; + use rustls_pki_types::{CertificateDer, PrivateKeyDer}; + use serf_proto::{ + members::{Member, MemberStatus}, + options::Options as SerfOptions, + typed::Tags, + }; + use smol_str::SmolStr; + + /// The tokio-backed reactor QUIC driver under test. + type TestDriver = QuicDriver; + + fn sa(s: &str) -> SocketAddr { + s.parse().expect("loopback addr") + } + + /// Drive the driver through exactly one `Future::poll`. The pump re-polls + /// unconditionally on `more`, so the no-op waker is a valid, harmless sink. + fn poll_once(driver: &mut TestDriver) -> Poll<()> { + let mut cx = Context::from_waker(Waker::noop()); + Pin::new(driver).poll(&mut cx) + } + + /// A self-signed `localhost` cert + key for the test quinn `ServerConfig`. + fn self_signed() -> (Vec>, PrivateKeyDer<'static>) { + let ck = + rcgen::generate_simple_self_signed(vec!["localhost".into()]).expect("self-signed cert"); + let cert = CertificateDer::from(ck.cert.der().to_vec()); + let key = PrivateKeyDer::Pkcs8(ck.signing_key.serialize_der().into()); + (vec![cert], key) + } + + fn test_endpoint_config(reset_key: &[u8]) -> quinn_proto::EndpointConfig { + let hmac = ring::hmac::Key::new(ring::hmac::HMAC_SHA256, reset_key); + quinn_proto::EndpointConfig::new(Arc::new(hmac)) + } + + fn test_server() -> quinn_proto::ServerConfig { + let (chain, key) = self_signed(); + let provider = Arc::new(rustls::crypto::ring::default_provider()); + let rustls_server = rustls::ServerConfig::builder_with_provider(provider) + .with_protocol_versions(&[&TLS13]) + .expect("tls13 server") + .with_no_client_auth() + .with_single_cert(chain, key) + .expect("single cert"); + let qsc = quinn_proto::crypto::rustls::QuicServerConfig::try_from(Arc::new(rustls_server)) + .expect("quic server config"); + quinn_proto::ServerConfig::with_crypto(Arc::new(qsc)) + } + + /// Accept-any server-cert verifier — test only. + #[derive(Debug)] + struct AnyServer; + + impl rustls::client::danger::ServerCertVerifier for AnyServer { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer, + _intermediates: &[CertificateDer], + _server_name: &rustls_pki_types::ServerName, + _ocsp_response: &[u8], + _now: rustls_pki_types::UnixTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + _message: &[u8], + _cert: &CertificateDer, + _dss: &rustls::DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + + fn verify_tls13_signature( + &self, + _message: &[u8], + _cert: &CertificateDer, + _dss: &rustls::DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + + fn supported_verify_schemes(&self) -> Vec { + rustls::crypto::ring::default_provider() + .signature_verification_algorithms + .supported_schemes() + } + } + + fn test_client() -> quinn_proto::ClientConfig { + let provider = Arc::new(rustls::crypto::ring::default_provider()); + let cfg = rustls::ClientConfig::builder_with_provider(provider) + .with_protocol_versions(&[&TLS13]) + .expect("tls13 client") + .dangerous() + .with_custom_certificate_verifier(Arc::new(AnyServer)) + .with_no_client_auth(); + let qcc = + quinn_proto::crypto::rustls::QuicClientConfig::try_from(Arc::new(cfg)).expect("quic client"); + quinn_proto::ClientConfig::new(Arc::new(qcc)) + } + + /// A datagram-mode QUIC config bundle (the mode is irrelevant to the recv gate, + /// but matches the production default the endpoint is built with). + fn test_quic_options() -> QuicOptions { + QuicOptions::new( + test_endpoint_config(&[0x5au8; 32]), + test_server(), + test_client(), + quinn_proto::TransportConfig::default(), + "localhost", + UnreliableTransport::Datagram, + ) + } + + /// A minimal initial published snapshot (the local node, `Alive`, zeroed clocks); + /// the gate tests never assert on it. + fn initial_snapshot(id: &str, advertise: SocketAddr) -> SerfSnapshot { + let member = Member::new( + Node::new(SmolStr::new(id), advertise), + Tags::new(), + MemberStatus::Alive, + ); + SerfSnapshot::new( + vec![Arc::new(member)], + &SmolStr::new(id), + SerfState::Alive, + LamportTime::from(0u64), + LamportTime::from(0u64), + LamportTime::from(0u64), + ) + } + + /// Build a real `QuicDriver` over a bound UDP socket, scheduling deliberately OFF + /// (built via `QuicDriver::new`, not `spawn_quic_driver`) so no stray coordinator + /// deadline supplies a timer the test means to attribute to the parked join. + async fn build_driver( + iter_drain_cap: usize, + ) -> ( + TestDriver, + Receiver>, + Arc>, + ) { + let advertise = sa("127.0.0.1:7946"); + let socket = <::Net as Net>::UdpSocket::bind("127.0.0.1:0") + .await + .expect("bind udp socket"); + let inner_opts = EndpointOptions::new(SmolStr::new("drv"), advertise) + .with_user_broadcast_tiers(NonZeroU8::new(3).expect("3 is nonzero")); + let quic_config = test_quic_options(); + let quic_max_udp_payload = quic_config.endpoint_ref().get_max_udp_payload_size(); + let inner = Endpoint::new(inner_opts, StdRng::seed_from_u64(0)); + let coord = Coordinator::new(inner, quic_config); + let endpoint = QuicEndpoint::::new_with_rng( + coord, + SerfOptions::new(), + StdRng::seed_from_u64(1), + ); + let shared = Arc::new(Shared::new(initial_snapshot("drv", advertise))); + let obs_payload_bytes = Arc::new(AtomicU64::new(0)); + let (obs_tx, obs_rx) = flume::unbounded(); + let driver = QuicDriver::::new( + endpoint, + socket, + quic_max_udp_payload, + shared.clone(), + obs_tx, + obs_payload_bytes, + None, + RuntimeOptions::new().with_iter_drain_cap(iter_drain_cap), + None, + #[cfg(encryption)] + Arc::new(crate::VoidKeyringDelegate), + ); + (driver, obs_rx, shared) + } + + /// A recv-ERROR stop is NOT the kernel-empty (`Poll::Pending`) signal, so the + /// shared-UDP-path gate must NOT read it as quiescence and fire the past-due + /// join/leave reap: a completing QUIC push/pull packet may still sit in the kernel + /// behind the error. The reap must defer until a real `Poll::Pending` stop. + /// + /// A live outbound push/pull with no peer feeds `pending` with an exchange id that + /// never completes, so ONLY the past-due deadline reap can resolve the join — the + /// clean signal for whether the gate fired. The first poll's recv is scripted to + /// report an error stop (a bound UDP socket cannot be made to error on demand); the + /// second poll's recv is a genuine `Poll::Pending`. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn recv_error_stop_is_not_quiescence() { + let now = Instant::now(); + let (mut driver, _obs_rx, _shared) = build_driver(8).await; + + let sid = driver + .endpoint + .start_join_push_pull(sa("127.0.0.1:7300"), false, now); + let eid = ExchangeId::from(sid); + let (tx, mut rx) = oneshot::channel::(); + let mut pending = HashSet::new(); + pending.insert(eid); + driver.pending_joins.push(PendingJoin { + pending, + contacted: SmallVec::new(), + ignore_streams: SmallVec::new(), + requested: 1, + // Past-due, so the deadline reap is due on the very first poll. + deadline: now - Duration::from_secs(1), + reply: Some(tx), + }); + + // Poll A: the recv loop stops on an ERROR. The past-due reap must NOT fire — + // the error is backlog-uncertain, so the join stays parked. + driver.recv_errors_remaining = 1; + let _ = poll_once(&mut driver); + assert!( + rx.try_recv().expect("reply channel live").is_none(), + "a recv-ERROR stop was read as quiescence: the past-due reap fired and resolved \ + the join, but a completing packet may sit behind the error", + ); + + // Poll B: the recv loop now stops on a genuine `Poll::Pending` (nothing was ever + // sent to the bound socket). The gate is quiescent, so the past-due reap fires and + // resolves the stuck join `JoinAllFailed`. + let _ = poll_once(&mut driver); + match rx.try_recv().expect("reply channel live") { + Some(Err((reached, SerfError::JoinAllFailed(_)))) => { + assert!(reached.is_empty(), "no seed was contacted: {reached:?}"); + } + other => { + panic!("a quiescent (Poll::Pending) recv stop must fire the past-due reap: {other:?}") + } + } + } +} diff --git a/serf-reactor/src/serf/mod.rs b/serf-reactor/src/serf/mod.rs index 29cf1967..ad672d34 100644 --- a/serf-reactor/src/serf/mod.rs +++ b/serf-reactor/src/serf/mod.rs @@ -518,6 +518,15 @@ impl Serf { self.shared.observation_dropped() } + /// The cumulative count of gossip payloads sent over the QUIC datagram plane + /// (a datagram queued onto the peer's pooled, TLS-protected connection) rather + /// than the plain-UDP fallback. Always `0` on the stream transports and on a + /// QUIC endpoint configured for `UnreliableTransport::Udp`. + #[must_use] + pub fn datagrams_sent(&self) -> u64 { + self.shared.datagrams_sent() + } + /// Send `cmd` to the driver, failing fast if the node has shut down. fn send(&self, cmd: Command) -> Result<()> { if self.shared.is_shutdown() { diff --git a/serf-reactor/src/shared/mod.rs b/serf-reactor/src/shared/mod.rs index 8064aab9..7d81ee07 100644 --- a/serf-reactor/src/shared/mod.rs +++ b/serf-reactor/src/shared/mod.rs @@ -48,6 +48,11 @@ pub(crate) struct Shared { events_dropped: AtomicU64, /// Observation-channel drops (a slow delegate; may lose application data). observation_dropped: AtomicU64, + /// Cumulative gossip payloads that rode the QUIC datagram plane (a + /// `DatagramSendStatus::Queued`), as opposed to the plain-UDP fallback. Zero on + /// the stream transports and on a QUIC endpoint in `UnreliableTransport::Udp` + /// mode. + datagrams_sent: AtomicU64, /// Set once the driver is shutting down; handle command methods observe it. shutdown: AtomicBool, /// Count of live `Serf` handles. The last to drop flips `shutdown` and wakes @@ -78,6 +83,7 @@ impl Shared { snapshot: ArcSwap::from_pointee(initial), events_dropped: AtomicU64::new(0), observation_dropped: AtomicU64::new(0), + datagrams_sent: AtomicU64::new(0), shutdown: AtomicBool::new(false), handles: AtomicUsize::new(1), shutdown_complete_tx: Mutex::new(Some(shutdown_complete_tx)), @@ -139,6 +145,11 @@ impl Shared { self.observation_dropped.fetch_add(n, Ordering::Relaxed); } + /// Records `n` gossip payloads sent over the QUIC datagram plane. + pub(crate) fn add_datagrams_sent(&self, n: u64) { + self.datagrams_sent.fetch_add(n, Ordering::Relaxed); + } + /// The cumulative recoverable EventStream-forward drop count. pub(crate) fn events_dropped(&self) -> u64 { self.events_dropped.load(Ordering::Relaxed) @@ -149,6 +160,11 @@ impl Shared { self.observation_dropped.load(Ordering::Relaxed) } + /// The cumulative count of gossip payloads sent over the QUIC datagram plane. + pub(crate) fn datagrams_sent(&self) -> u64 { + self.datagrams_sent.load(Ordering::Relaxed) + } + /// Whether the driver is shutting down. pub(crate) fn is_shutdown(&self) -> bool { self.shutdown.load(Ordering::Acquire) diff --git a/serf-reactor/tests/quic.rs b/serf-reactor/tests/quic.rs index d7d0db74..c89294ac 100644 --- a/serf-reactor/tests/quic.rs +++ b/serf-reactor/tests/quic.rs @@ -235,6 +235,72 @@ async fn user_event_delivered() { b.shutdown().await.expect("ue-b shuts down"); } +/// Datagram-mode non-vacuity: with `UnreliableTransport::Datagram` the outbound +/// gossip is routed through the QUIC datagram plane (`queue_unreliable_datagram` + +/// `flush_outbound_transmits`), NOT the plain-UDP fallback. Two nodes join and +/// converge, B broadcasts a user event that A receives over gossip, and the sender's +/// `datagrams_sent` counter proves the gossip actually rode QUIC datagrams over the +/// pooled, TLS-protected connection. +/// +/// This is the discriminator the plain-UDP fallback would otherwise mask: a driver +/// that bypassed the configured mode (the pre-fix always-`poll_send_to` path) still +/// delivers the event over UDP and converges, but leaves `datagrams_sent` at `0`. +/// `datagrams_sent` advances only on a `DatagramSendStatus::Queued`, so asserting it +/// is non-zero fails on that revert while the convergence assertions alone would not. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn datagram_mode_gossip_rides_quic_datagrams() { + let b = spawn_node("dg-b").await; + let a = spawn_node("dg-a").await; + let b_addr = b.advertise_address(); + + let mut a_events = a.events(); + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B over QUIC"); + converge(&a, &b).await; + + b.user_event("greet", Bytes::from_static(b"hello"), false) + .await + .expect("user event dispatched"); + + let got = tokio::time::timeout(Duration::from_secs(30), async { + loop { + match a_events.next().await { + Some(Event::User(u)) if u.name.as_str() == "greet" => break Some(u.payload.clone()), + Some(_) => {} + None => break None, + } + } + }) + .await + .expect("A observes B's user event within the timeout"); + assert_eq!( + got, + Some(Bytes::from_static(b"hello")), + "A receives B's user-event payload over datagram-mode gossip" + ); + + // The discriminator: the gossip that crossed rode QUIC datagrams, not the plain-UDP + // fallback. Both nodes hold a warm pooled connection after the join, so their + // periodic gossip is queued as datagrams; `datagrams_sent` advances only on a + // `DatagramSendStatus::Queued`. + tokio::time::timeout(Duration::from_secs(30), async { + loop { + if b.datagrams_sent() > 0 { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect( + "B's gossip must ride the QUIC datagram plane (datagrams_sent > 0), not the plain-UDP fallback", + ); + + a.shutdown().await.expect("dg-a shuts down"); + b.shutdown().await.expect("dg-b shuts down"); +} + /// After a two-node QUIC join, a query issued by A round-trips: B receives the /// `Event::Query`, responds, and A surfaces the matching `Event::QueryResponse`. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] From eaccc148ba667805fda8d044c61be1b1e292591e Mon Sep 17 00:00:00 2001 From: al8n Date: Thu, 2 Jul 2026 16:42:52 +1200 Subject: [PATCH 036/128] fix(serf-reactor): QUIC recv error parks on a bounded backoff instead of busy-spinning The recv-error stop stayed non-quiescent for the timer/reap gate (correct) but also drove an unconditional self-wake, so a persistent recv error re-polled every tick and burned a core between deadlines. A recv error with no due deadline now arms a bounded backoff timer and parks (recovering transient errors, bounding the retry rate) while the due-deadline gate still defers to the grace. Regression asserts the pump parks (no immediate self-wake) under repeated recv errors. --- serf-reactor/src/driver/quic/mod.rs | 88 ++++++++++++++++++++++----- serf-reactor/src/driver/quic/tests.rs | 78 +++++++++++++++++++++++- 2 files changed, 149 insertions(+), 17 deletions(-) diff --git a/serf-reactor/src/driver/quic/mod.rs b/serf-reactor/src/driver/quic/mod.rs index 1e037baf..56bf9d05 100644 --- a/serf-reactor/src/driver/quic/mod.rs +++ b/serf-reactor/src/driver/quic/mod.rs @@ -128,6 +128,23 @@ const OBS_OVERFLOW_MAX: usize = 1024; /// liveness wins once the grace elapses. const SHARED_PATH_STALENESS_GRACE: Duration = Duration::from_millis(5); +/// How long the pump parks after a UDP recv ERROR stop before retrying the recv, +/// when there is nothing else bounded to make progress on (no saturated batch, no +/// due deadline). +/// +/// A recv error is not the kernel-empty `Poll::Pending` signal, so it must not be +/// read as quiescence — it gates the timer/reap like a saturated batch (see the +/// module docs). But it is ALSO not a reason to self-wake: an immediate +/// `wake_by_ref` on every errored poll would busy-spin a core between deadlines, and +/// — because a `Poll::Ready(Err(_))` from the socket registers NO readiness waker — +/// no socket wake will re-poll the pump either. So a recv error with nothing else +/// pending arms a real sleep for this bounded backoff and parks; the timer alone +/// re-polls and retries the errored recv. Small, because a UDP recv error is +/// typically transient (a stale ICMP port-unreachable surfacing as `ECONNREFUSED`, +/// or a momentary `ENOBUFS`) and clears on the next poll, and the shared socket's +/// gossip/QUIC recv is stalled only for this window. +const RECV_ERROR_BACKOFF: Duration = Duration::from_millis(5); + /// Size the per-recv UDP buffer to the larger of the two planes that share this /// one socket. /// @@ -1018,6 +1035,23 @@ where } } + /// Arms the wakeup timer for `target` and polls it once so its waker is registered + /// and the pump re-polls when it fires. Returns whether it fired already (an + /// already-elapsed target), in which case the caller self-wakes via `more`. Parking + /// on the returned `false` re-polls ONLY on the armed timer — no self-wake — which + /// is how a recv-error backoff avoids a busy-spin. + fn arm_and_poll_timer(&mut self, target: Instant, now: Instant, cx: &mut Context<'_>) -> bool { + self.arm_timer(target, now); + if let Some(timer) = self.timer.as_mut() + && timer.as_mut().poll(cx).is_ready() + { + self.timer = None; + self.timer_deadline = None; + return true; + } + false + } + /// Reply `Err(Shutdown)` to a command drained during teardown. fn reply_shutdown(cmd: Command) { match cmd { @@ -1176,7 +1210,14 @@ where // `handle_timeout` site below. let recv_capped = recv_n == this.iter_drain_cap; let recv_quiescent = !recv_capped && !recv_errored; - more |= recv_capped || recv_errored; + // WAKE and GATE are separate concerns. Both a saturated batch and a recv error + // are non-quiescent for the reap gate above (`recv_quiescent`), but their WAKE + // policy differs: a saturated batch is bounded backlog to drain next poll, so it + // self-wakes; a recv ERROR is not bounded progress — a persistent one would + // self-wake every poll and burn a core, and the socket registered no readiness + // waker on its `Ready(Err)` — so it does NOT self-wake here. The deadline gate + // below instead arms a bounded `RECV_ERROR_BACKOFF` timer for it and parks. + more |= recv_capped; // Drain machine surfaces (gossip/QUIC egress capped, ingress + events uncapped). let (drained, drain_more) = this.drain_surfaces(cx); @@ -1244,26 +1285,41 @@ where this.timeout_stall_since = None; progress = true; more = true; - } else { - // A deadline is due but the recv stop was non-quiescent (a saturated batch or - // a recv error, either backlog-uncertain) and the grace has not elapsed: - // DEFER (self-wake) so the next poll drains the recv toward a `Poll::Pending` - // quiescent stop. No timer is armed — the deadline already elapsed, so the - // `more` self-wake alone re-polls (no lost wakeup), and the wall-clock grace - // bounds the deferral so a persistent error cannot starve the timer forever. + } else if recv_capped { + // A deadline is due but the recv stop was a SATURATED BATCH (real backlog) and + // the grace has not elapsed: DEFER by self-waking so the next poll drains the + // recv toward a `Poll::Pending` quiescent stop. No timer is armed — the + // deadline already elapsed, so the `more` self-wake alone re-polls (no lost + // wakeup), and the wall-clock grace bounds the deferral. more = true; + } else { + // A deadline is due but the recv stop was a recv ERROR (the only remaining + // non-quiescent case) and the grace has not elapsed: DEFER. There is nothing + // bounded to drain, so an immediate self-wake would busy-spin the whole grace; + // instead park on a timer at the grace expiry, when the force-fire (grace + // elapsed) re-polls the recv. The grace still bounds the hold-back, so a + // persistent error cannot starve the timer. `timeout_stall_since` is `Some` + // here (anchored just above for this non-quiescent stop). + let grace_deadline = this + .timeout_stall_since + .map_or(now, |t| t + SHARED_PATH_STALENESS_GRACE); + if this.arm_and_poll_timer(grace_deadline, now, cx) { + more = true; + } } } else { // Idle: nothing due. Clear the stall anchor, then arm + poll the sleep for the - // next deadline; NO self-wake (an armed sleep or socket readiness re-polls). + // next deadline; NO self-wake (an armed sleep or socket readiness re-polls). A + // recv ERROR folds a bounded `RECV_ERROR_BACKOFF` into the target so the errored + // recv is retried within the backoff even when the next deadline is far off (the + // idle wake is 60s) — a `Ready(Err)` socket registered no readiness waker, so + // only this timer re-polls. this.timeout_stall_since = None; - let target = reap_deadline.map_or(endpoint_deadline, |d| d.min(endpoint_deadline)); - this.arm_timer(target, now); - if let Some(timer) = this.timer.as_mut() - && timer.as_mut().poll(cx).is_ready() - { - this.timer = None; - this.timer_deadline = None; + let mut target = reap_deadline.map_or(endpoint_deadline, |d| d.min(endpoint_deadline)); + if recv_errored { + target = target.min(now + RECV_ERROR_BACKOFF); + } + if this.arm_and_poll_timer(target, now, cx) { more = true; } } diff --git a/serf-reactor/src/driver/quic/tests.rs b/serf-reactor/src/driver/quic/tests.rs index 6d275212..b2bbda8b 100644 --- a/serf-reactor/src/driver/quic/tests.rs +++ b/serf-reactor/src/driver/quic/tests.rs @@ -57,7 +57,10 @@ mod gate { use super::*; use core::num::NonZeroU8; - use std::{sync::Arc, task::Waker}; + use std::{ + sync::{Arc, atomic::AtomicBool}, + task::{Wake, Waker}, + }; use agnostic::tokio::TokioRuntime; use memberlist_proto::{ @@ -90,6 +93,25 @@ mod gate { Pin::new(driver).poll(&mut cx) } + /// A `Waker` that records whether it was woken. A SYNCHRONOUS wake during a poll is + /// the busy-spin signal: it means the pump requested an immediate re-poll + /// (`wake_by_ref`) rather than parking on a timer. Timer / channel registrations do + /// NOT wake synchronously, so the flag stays clear when the pump correctly parks. + #[derive(Default)] + struct SpinFlag { + woken: AtomicBool, + } + + impl Wake for SpinFlag { + fn wake(self: Arc) { + self.woken.store(true, Ordering::SeqCst); + } + + fn wake_by_ref(self: &Arc) { + self.woken.store(true, Ordering::SeqCst); + } + } + /// A self-signed `localhost` cert + key for the test quinn `ServerConfig`. fn self_signed() -> (Vec>, PrivateKeyDer<'static>) { let ck = @@ -302,4 +324,58 @@ mod gate { } } } + + /// A recv-ERROR stop is (correctly) non-quiescent for the timer/reap GATE, but that + /// must NOT drive an immediate self-wake: with nothing bounded to make progress on — + /// no saturated batch, and only a FUTURE deadline pending — a `wake_by_ref` every + /// poll would busy-spin a core between deadlines. The pump must instead PARK on the + /// bounded `RECV_ERROR_BACKOFF` timer: it returns `Poll::Pending` WITHOUT waking its + /// waker synchronously, and the timer alone re-polls (retrying the errored recv) + /// after the backoff. + /// + /// Fail-on-revert: with the recv-error stop feeding an unconditional `more`, the + /// pump self-wakes on the very first poll and this asserts false. + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn persistent_recv_error_does_not_spin() { + let now = Instant::now(); + let (mut driver, _obs_rx, _shared) = build_driver(8).await; + + // A FUTURE (not-yet-due) deadline pending: the join sits parked, so nothing is due + // this poll and the only thing that could wake the pump is the (buggy) recv-error + // self-wake. `pending` empty + `reply` live keeps the waiter in the timer horizon + // without a real exchange whose egress could set `more` and mask the spin. + let (tx, _rx) = oneshot::channel::(); + driver.pending_joins.push(PendingJoin { + pending: HashSet::new(), + contacted: SmallVec::new(), + ignore_streams: SmallVec::new(), + requested: 1, + deadline: now + Duration::from_secs(30), + reply: Some(tx), + }); + + // Script every recv-loop poll to report an ERROR stop for the whole test. + driver.recv_errors_remaining = usize::MAX; + + let flag = Arc::new(SpinFlag::default()); + let waker = Waker::from(flag.clone()); + + // Repeatedly poll: each must park (Pending) WITHOUT a synchronous self-wake. The + // 5ms backoff timer cannot fire in the microseconds before the flag is read, and + // each poll re-arms it (cancelling the prior), so only a busy-spin bug trips this. + for _ in 0..5 { + flag.woken.store(false, Ordering::SeqCst); + let mut cx = Context::from_waker(&waker); + let poll = Pin::new(&mut driver).poll(&mut cx); + assert!( + poll.is_pending(), + "the pump must stay pending under a persistent recv error", + ); + assert!( + !flag.woken.load(Ordering::SeqCst), + "a persistent recv error self-woke the pump (busy-spin) instead of parking on \ + the bounded RECV_ERROR_BACKOFF timer", + ); + } + } } From faf6fa443594e76527e795612d0044934577e3a5 Mon Sep 17 00:00:00 2001 From: al8n Date: Thu, 2 Jul 2026 17:22:53 +1200 Subject: [PATCH 037/128] feat(serf-reactor): TLS record-layer transport + DNS/getifs resolvers The record-layer Labeled TLS transport drives rustls over the plain agnostic TCP stream (no runtime TLS stream; the rustls provider forwards to memberlist-proto's tls-rustls-ring/aws-lc-rs per flagged decision #1) with the ergonomic Serf::tls/tls_with_rng constructor. Adds the DNS (TCP-first DNS over R-agnostic I/O) and getifs advertise resolvers ported from serf-compio, Send-bounded. Real-node TLS suite + DNS/getifs unit tests; mirrors memberlist-reactor's TLS wiring and serf-compio's resolvers. --- serf-reactor/Cargo.toml | 6 + serf-reactor/src/lib.rs | 14 + serf-reactor/src/resolver/dns/mod.rs | 278 +++++++++++++ serf-reactor/src/resolver/dns/tests.rs | 413 +++++++++++++++++++ serf-reactor/src/resolver/getifs/mod.rs | 184 +++++++++ serf-reactor/src/resolver/getifs/tests.rs | 129 ++++++ serf-reactor/src/resolver/mod.rs | 21 +- serf-reactor/src/serf/mod.rs | 100 +++++ serf-reactor/src/shared/mod.rs | 4 +- serf-reactor/src/tls/mod.rs | 462 ++++++++++++++++++++++ serf-reactor/src/tls/tests.rs | 94 +++++ serf-reactor/tests/tls.rs | 323 +++++++++++++++ 12 files changed, 2024 insertions(+), 4 deletions(-) create mode 100644 serf-reactor/src/resolver/dns/mod.rs create mode 100644 serf-reactor/src/resolver/dns/tests.rs create mode 100644 serf-reactor/src/resolver/getifs/mod.rs create mode 100644 serf-reactor/src/resolver/getifs/tests.rs create mode 100644 serf-reactor/src/tls/mod.rs create mode 100644 serf-reactor/src/tls/tests.rs create mode 100644 serf-reactor/tests/tls.rs diff --git a/serf-reactor/Cargo.toml b/serf-reactor/Cargo.toml index 8895f92a..4b486b68 100644 --- a/serf-reactor/Cargo.toml +++ b/serf-reactor/Cargo.toml @@ -108,6 +108,12 @@ tokio = { version = "1", features = ["rt-multi-thread", "macros", "time"] } name = "tcp" required-features = ["tcp", "tokio"] +# The real-node TLS test suite (two loopback nodes over a self-signed cert + +# accept-any verifier: join/converge, user event, query round-trip, leave/shutdown). +[[test]] +name = "tls" +required-features = ["tls-rustls-ring", "tokio"] + # The real-node QUIC test suite (two loopback nodes over a quinn config bundle: # join/converge, user event, query round-trip, leave/LeftCluster, shutdown, and — # under an encryption backend — encrypted-gossip convergence + mismatched-key diff --git a/serf-reactor/src/lib.rs b/serf-reactor/src/lib.rs index d12b6974..23ca0d88 100644 --- a/serf-reactor/src/lib.rs +++ b/serf-reactor/src/lib.rs @@ -64,6 +64,8 @@ mod shared; mod snapshot; #[cfg(feature = "tcp")] mod tcp; +#[cfg(feature = "tls")] +mod tls; #[cfg(any(feature = "tcp", feature = "quic"))] mod transport; @@ -112,6 +114,14 @@ pub use resolver::{ Ipv6PreferringResolver, OsResolver, Resolver, SocketAddrResolver, }; +#[cfg(feature = "dns")] +#[cfg_attr(docsrs, doc(cfg(feature = "dns")))] +pub use resolver::{DEFAULT_DNS_TIMEOUT, DnsError, DnsResolver}; + +#[cfg(feature = "getifs")] +#[cfg_attr(docsrs, doc(cfg(feature = "getifs")))] +pub use resolver::{LocalAddrResolver, LocalAddrScope, local_advertise}; + #[cfg(any(feature = "tcp", feature = "quic"))] #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] pub use serf::Serf; @@ -133,6 +143,10 @@ pub use tcp::{TcpTransport, TcpTransportOptions}; #[cfg_attr(docsrs, doc(cfg(feature = "quic")))] pub use quic::{QuicOptions, QuicTransport, QuicTransportOptions}; +#[cfg(feature = "tls")] +#[cfg_attr(docsrs, doc(cfg(feature = "tls")))] +pub use tls::{SniProvider, TlsOptions, TlsTransport, TlsTransportOptions}; + #[cfg(any(feature = "tcp", feature = "quic"))] #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] pub use delegate::{ diff --git a/serf-reactor/src/resolver/dns/mod.rs b/serf-reactor/src/resolver/dns/mod.rs new file mode 100644 index 00000000..42198c14 --- /dev/null +++ b/serf-reactor/src/resolver/dns/mod.rs @@ -0,0 +1,278 @@ +//! TCP-first DNS resolver — hickory-proto codec over the agnostic runtime's +//! `R`-generic TCP stream. Mirrors Go memberlist's `tcpLookupIP` algorithm +//! (`hashicorp/memberlist/memberlist.go:308-417`). +//! +//! Why TCP-first: UDP DNS responses are capped at 512 bytes (without EDNS), which +//! can truncate the answer list for cluster-discovery hostnames resolving to many +//! A/AAAA records. TCP-DNS has no such cap, so it gives the largest possible join +//! set on a single query. +//! +//! This is the `Send`/`agnostic` sibling of serf-compio's `!Send`, compio-bound +//! `DnsResolver`: the transport is `::TcpStream` and the per-query +//! deadline is armed with `R::sleep`, so resolution runs on any `agnostic::Runtime`. + +#![cfg(feature = "dns")] + +use core::future::Future; +use std::{ + io::{self, Read}, + net::{IpAddr, SocketAddr}, + path::Path, + time::Duration, +}; + +use agnostic::{ + Runtime, + net::{Net, TcpStream}, +}; +use futures_util::{AsyncReadExt, AsyncWriteExt, FutureExt, pin_mut, select_biased}; +use hickory_proto::{ + ProtoError, + op::{Message, Query}, + rr::{Name, RData, RecordType}, + serialize::binary::{BinEncodable, BinEncoder, DecodeError}, +}; +use hostaddr::{Host, HostAddr}; +use smol_str::SmolStr; + +use crate::resolver::{OsResolver, Resolver}; + +/// Default wall-clock upper bound on a single TCP-DNS query (connect + write + +/// read length-prefix + read response). Matches the default DNS query timeout used +/// by most stub resolvers (Go's `net.Resolver` uses 5s, glibc's resolver uses 5s +/// per attempt). Configured per-resolver via [`DnsResolver::with_timeout`]; without +/// a bound the query inherits the kernel's TCP timeouts (~3 minutes connect, +/// infinite read), which would let a slow or hostile nameserver hang the caller's +/// `join` future indefinitely. +pub const DEFAULT_DNS_TIMEOUT: Duration = Duration::from_secs(5); + +/// TCP-first DNS resolver — queries configured nameservers over TCP and falls back +/// to the OS resolver (which is UDP-first with TCP retry on truncation) if TCP +/// returns nothing. +/// +/// Constructed from a resolv.conf-format file. For hostnames that lack a `.` (short +/// names, likely resolved via the host's search-domain list) the TCP path is +/// skipped entirely and the OS resolver is used directly, matching the upstream +/// behavior. +/// +/// Generic over the runtime `R` (the same `agnostic::Runtime` the node's driver is +/// spawned on): the TCP transport and the per-query timer are both `R`'s. +pub struct DnsResolver { + servers: Vec, + fallback: OsResolver, + timeout: Duration, +} + +/// Errors returned by [`DnsResolver`]. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum DnsError { + /// I/O error from the TCP transport or the OS-resolver fallback. + #[error("I/O error: {0}")] + Io(#[from] io::Error), + + /// hickory-proto encoding error (malformed query construction). + #[error("DNS encode error: {0}")] + Encode(#[from] ProtoError), + + /// hickory-proto decoding error (malformed response from the server). + #[error("DNS decode error: {0}")] + Decode(#[from] DecodeError), + + /// The hostname could not be parsed into a wire-format DNS name. + /// Carries the hickory `ProtoError` via `#[source]` rather than `#[from]`, since + /// [`Self::Encode`] already owns the `From` conversion. + #[error("hostname parse error: {0}")] + Hostname(#[source] ProtoError), +} + +impl From for io::Error { + fn from(e: DnsError) -> Self { + Self::other(e) + } +} + +impl DnsResolver { + /// Construct from a resolv.conf-format file path. Reads the file, parses the + /// nameserver list (each pinned to port 53), and stores the OS resolver as the + /// fallback. + pub fn from_resolv_conf(path: impl AsRef) -> Result { + let mut file = std::fs::File::open(path)?; + let mut buf = String::new(); + file.read_to_string(&mut buf)?; + let cfg = resolv_conf::Config::parse(&buf) + .map_err(|e| io::Error::other(format!("resolv.conf parse: {e}")))?; + + let servers: Vec = cfg + .nameservers + .iter() + .map(|ns| SocketAddr::new(IpAddr::from(ns), 53)) + .collect(); + + Ok(Self { + servers, + fallback: OsResolver::new(), + timeout: DEFAULT_DNS_TIMEOUT, + }) + } + + /// Construct from an explicit nameserver list. The OS resolver is used as the + /// fallback path. + pub fn from_servers(servers: Vec) -> Self { + Self { + servers, + fallback: OsResolver::new(), + timeout: DEFAULT_DNS_TIMEOUT, + } + } + + /// Builder: override the per-query timeout. Defaults to [`DEFAULT_DNS_TIMEOUT`]. + #[must_use] + #[inline] + pub const fn with_timeout(mut self, d: Duration) -> Self { + self.timeout = d; + self + } + + /// The configured per-query timeout. + #[inline] + pub const fn timeout(&self) -> Duration { + self.timeout + } +} + +impl DnsResolver +where + R: Runtime, +{ + /// Send a single TCP-DNS query for TYPE ANY against the first configured + /// nameserver and return the collected A + AAAA records. Returns an empty vec + /// when no servers are configured. + /// + /// Bounded by `self.timeout` (default [`DEFAULT_DNS_TIMEOUT`]): a slow or hostile + /// nameserver cannot hang the caller's `join` future beyond this wall-clock + /// budget. On timeout returns [`DnsError::Io`]`(io::ErrorKind::TimedOut)`; the + /// resolver's caller falls back to the OS resolver via the standard error path + /// (see [`Resolver::resolve`] below). + async fn tcp_query(&self, host: &str, port: u16) -> Result, DnsError> { + let query = self.tcp_query_inner(host, port).fuse(); + let timeout = R::sleep(self.timeout).fuse(); + pin_mut!(query, timeout); + select_biased! { + res = query => res, + _ = timeout => Err(DnsError::Io(io::Error::new( + io::ErrorKind::TimedOut, + "TCP-DNS query exceeded the configured timeout", + ))), + } + } + + /// Inner unbounded TCP-DNS query — invoked by [`Self::tcp_query`] inside the + /// deadline select. Kept separate so the deadline wrapper owns the timer arm + /// without complicating the protocol logic. + async fn tcp_query_inner(&self, host: &str, port: u16) -> Result, DnsError> { + let Some(&server) = self.servers.first() else { + return Ok(Vec::new()); + }; + + // Build a TYPE ANY query message. `Message::query()` initializes a fresh ID + // with the standard query flags; we add the question. + let name = Name::from_ascii(host).map_err(DnsError::Hostname)?; + let mut msg = Message::query(); + msg.add_query(Query::query(name, RecordType::ANY)); + + // Encode to bytes via BinEncoder over an owned Vec. + let mut payload: Vec = Vec::with_capacity(512); + { + let mut encoder = BinEncoder::new(&mut payload); + msg.emit(&mut encoder)?; + } + + // TCP-DNS (RFC 1035 §4.2.2) prepends a 2-byte big-endian length. + let payload_len = u16::try_from(payload.len()) + .map_err(|_| DnsError::Io(io::Error::other("DNS query exceeds 65535 bytes")))?; + let mut framed = Vec::with_capacity(2 + payload.len()); + framed.extend_from_slice(&payload_len.to_be_bytes()); + framed.extend_from_slice(&payload); + + // Connect and send the framed query. The agnostic stream reads/writes into + // borrowed buffers via the futures `AsyncRead`/`AsyncWrite` ext traits. + let mut stream = ::TcpStream::connect(server).await?; + stream.write_all(&framed).await?; + + // Read the 2-byte length prefix, then the body of exactly that length. + let mut len_buf = [0u8; 2]; + stream.read_exact(&mut len_buf).await?; + let response_len = u16::from_be_bytes(len_buf) as usize; + + let mut resp_buf = vec![0u8; response_len]; + stream.read_exact(&mut resp_buf).await?; + + // Decode and collect A + AAAA answers. CNAME and other RR types are ignored to + // match the upstream behavior (see Go reference above). + let response = Message::from_vec(&resp_buf)?; + let mut addrs = Vec::new(); + for record in &response.answers { + // `Record` exposes the rdata as a public field `data`; the same-named + // accessor is shadowed when the field is the same name. + match &record.data { + RData::A(ipv4) => addrs.push(SocketAddr::new(IpAddr::V4(ipv4.0), port)), + RData::AAAA(ipv6) => addrs.push(SocketAddr::new(IpAddr::V6(ipv6.0), port)), + _ => {} + } + } + Ok(addrs) + } +} + +impl Resolver for DnsResolver +where + R: Runtime, +{ + type Address = HostAddr; + type Error = DnsError; + + fn resolve( + &self, + addr: &Self::Address, + ) -> impl Future, Self::Error>> + Send + '_ { + // Clone the input up front so the returned future borrows only `self`, not the + // shorter-lived `addr` param (the `Resolver` contract binds the future to + // `&self`'s lifetime; the fallback resolves the owned clone, which lives inside + // the future). + let addr = addr.clone(); + async move { + let port = addr.port().unwrap_or(0); + + // IP literal: short-circuit, no DNS at all. + if let Host::Ip(ip) = addr.host() { + return Ok(vec![SocketAddr::new(*ip, port)]); + } + + let host_str: &str = match addr.host() { + Host::Domain(d) => d.as_ref(), + Host::Ip(_) => unreachable!("handled above"), + }; + + // TCP-first only for names that look fully qualified (contain a `.`) and only + // when we have at least one nameserver configured. Short names will be + // resolved through the OS resolver's search-domain list. + if host_str.contains('.') && !self.servers.is_empty() { + // Ignoring Err: TCP-first is best-effort per the upstream spec ("If this + // fails it's not fatal since this isn't a standard way to query DNS, and we + // have a fallback below.", memberlist.go:404). We unconditionally fall + // through to the OS resolver on any error or empty answer. + if let Ok(addrs) = self.tcp_query(host_str, port).await + && !addrs.is_empty() + { + return Ok(addrs); + } + } + + self.fallback.resolve(&addr).await.map_err(DnsError::Io) + } + } +} + +#[cfg(all(test, feature = "tokio"))] +mod tests; diff --git a/serf-reactor/src/resolver/dns/tests.rs b/serf-reactor/src/resolver/dns/tests.rs new file mode 100644 index 00000000..dec5648e --- /dev/null +++ b/serf-reactor/src/resolver/dns/tests.rs @@ -0,0 +1,413 @@ +use std::{ + fs, + net::{Ipv4Addr, Ipv6Addr}, + path::PathBuf, + sync::atomic::{AtomicU64, Ordering}, +}; + +use agnostic::{ + Runtime, + net::{Net, TcpListener}, + tokio::TokioRuntime, +}; +use hickory_proto::rr::Name; + +use super::*; + +/// The tokio-backed DNS resolver under test. +type Dns = DnsResolver; + +/// The DNS resolver's user-facing address type — the hostname-or-IP type the OS +/// resolver also consumes. +type Address = HostAddr; + +/// A nameserver that is guaranteed never to answer: `240.0.0.1` is in the reserved +/// 240.0.0.0/4 block (RFC 1112 §4) and is unroutable. Pairing it with a fast-path +/// test proves the IP-literal branch never opens a socket — a real connect would +/// hang well past any test budget. +fn dead_server() -> SocketAddr { + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(240, 0, 0, 1)), 53) +} + +/// Build a resolver pointed at the unroutable nameserver with a sub-second +/// timeout, so any test that *did* hit the network would fail loudly rather than +/// silently fall through. +fn resolver_with_dead_server() -> Dns { + Dns::from_servers(vec![dead_server()]).with_timeout(Duration::from_millis(50)) +} + +/// Allocate a unique temp path for a resolv.conf fixture. Avoids a `tempfile` +/// dev-dependency; the counter + PID keep parallel test threads from colliding. +fn temp_resolv_conf_path(tag: &str) -> PathBuf { + static COUNTER: AtomicU64 = AtomicU64::new(0); + let n = COUNTER.fetch_add(1, Ordering::Relaxed); + let mut p = std::env::temp_dir(); + p.push(format!( + "serf-reactor-dns-{tag}-{}-{n}.conf", + std::process::id() + )); + p +} + +#[test] +fn from_servers_uses_default_timeout() { + let r = Dns::from_servers(vec![dead_server()]); + assert_eq!(r.timeout(), DEFAULT_DNS_TIMEOUT); + assert_eq!(DEFAULT_DNS_TIMEOUT, Duration::from_secs(5)); +} + +#[test] +fn from_empty_servers_uses_default_timeout() { + let r = Dns::from_servers(Vec::new()); + assert_eq!(r.timeout(), DEFAULT_DNS_TIMEOUT); +} + +#[test] +fn with_timeout_overrides_default() { + let custom = Duration::from_millis(250); + let r = Dns::from_servers(vec![dead_server()]).with_timeout(custom); + assert_eq!(r.timeout(), custom); + + // The builder is chainable and last-write-wins. + let r = r.with_timeout(Duration::from_secs(30)); + assert_eq!(r.timeout(), Duration::from_secs(30)); +} + +#[test] +fn from_resolv_conf_parses_valid_file() { + let path = temp_resolv_conf_path("valid"); + fs::write( + &path, + "# a comment\nnameserver 8.8.8.8\nnameserver 1.1.1.1\nsearch example.com\n", + ) + .expect("write resolv.conf fixture"); + + let r = Dns::from_resolv_conf(&path).expect("parse valid resolv.conf"); + // No accessor exposes the parsed server list, but a successful parse must still + // leave the default per-query timeout in place. + assert_eq!(r.timeout(), DEFAULT_DNS_TIMEOUT); + + // Ignoring Err: best-effort cleanup of the temp fixture; a leaked file in the OS + // temp dir is harmless and must not fail the test. + let _ = fs::remove_file(&path); +} + +#[test] +fn from_resolv_conf_parses_empty_file() { + let path = temp_resolv_conf_path("empty"); + fs::write(&path, "").expect("write empty resolv.conf fixture"); + + // An empty file is a valid resolv.conf with zero nameservers. + let r = Dns::from_resolv_conf(&path).expect("parse empty resolv.conf"); + assert_eq!(r.timeout(), DEFAULT_DNS_TIMEOUT); + + // Ignoring Err: best-effort temp-fixture cleanup. + let _ = fs::remove_file(&path); +} + +#[test] +fn from_resolv_conf_rejects_invalid_nameserver() { + let path = temp_resolv_conf_path("invalid"); + // A `nameserver` line whose argument is not an IP makes + // `resolv_conf::Config::parse` surface an `InvalidIp` error, which the + // constructor maps to `io::Error::other`. + fs::write(&path, "nameserver not-an-ip-address\n").expect("write bad resolv.conf fixture"); + + // `DnsResolver` is not `Debug`, so map the `Ok` side to `()` before `expect_err` + // can report it. + let err = Dns::from_resolv_conf(&path) + .map(|_| ()) + .expect_err("malformed resolv.conf must fail"); + assert_eq!(err.kind(), io::ErrorKind::Other); + assert!( + err.to_string().contains("resolv.conf parse"), + "error should carry the parse context, got: {err}" + ); + + // Ignoring Err: best-effort temp-fixture cleanup. + let _ = fs::remove_file(&path); +} + +#[test] +fn from_resolv_conf_missing_file_is_io_error() { + let path = temp_resolv_conf_path("does-not-exist"); + // Guard against an accidentally pre-existing path. + // Ignoring Err: removal is precautionary; absence is the desired state. + let _ = fs::remove_file(&path); + + let err = Dns::from_resolv_conf(&path) + .map(|_| ()) + .expect_err("missing file must fail"); + assert_eq!(err.kind(), io::ErrorKind::NotFound); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn resolve_ipv4_literal_skips_dns() { + // An IPv4 literal must short-circuit before any socket is opened, even though the + // resolver is pointed at an unroutable nameserver. + let r = resolver_with_dead_server(); + let addr: Address = HostAddr::from_sock_addr("127.0.0.1:7946".parse().unwrap()); + let resolved = r.resolve(&addr).await.expect("IP literal resolves"); + assert_eq!( + resolved, + vec![SocketAddr::new( + IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)), + 7946 + )] + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn resolve_ipv6_literal_skips_dns() { + let r = resolver_with_dead_server(); + let addr: Address = HostAddr::from_ip_addr(IpAddr::V6(Ipv6Addr::LOCALHOST)).with_port(443); + let resolved = r.resolve(&addr).await.expect("IPv6 literal resolves"); + assert_eq!( + resolved, + vec![SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 443)] + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn resolve_ip_literal_without_port_uses_zero() { + // `HostAddr::port()` is `None` for a bare IP; the resolver substitutes 0. + let r = resolver_with_dead_server(); + let addr: Address = HostAddr::from_ip_addr(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 5))); + assert_eq!(addr.port(), None); + let resolved = r + .resolve(&addr) + .await + .expect("portless IP literal resolves"); + assert_eq!( + resolved, + vec![SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 5)), 0)] + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn resolve_ip_literal_works_with_empty_server_list() { + // The fast path does not consult `self.servers`, so an empty list is irrelevant + // for IP literals. + let r = Dns::from_servers(Vec::new()); + let addr: Address = HostAddr::from_sock_addr("192.0.2.1:8080".parse().unwrap()); + let resolved = r.resolve(&addr).await.expect("IP literal resolves"); + assert_eq!( + resolved, + vec![SocketAddr::new( + IpAddr::V4(Ipv4Addr::new(192, 0, 2, 1)), + 8080 + )] + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn resolve_short_name_bypasses_tcp_and_uses_os_fallback() { + // `localhost` has no `.`, so the TCP-first branch is skipped entirely and + // resolution goes straight to the OS resolver. `localhost` maps to loopback via + // the hosts file, so this stays off the external network even though the + // configured nameserver is unroutable. + let r = resolver_with_dead_server(); + let addr: Address = "localhost:9000".parse().expect("parse localhost:9000"); + // Confirm the test input is actually a short domain, not an IP literal. + assert!(matches!(addr.host(), Host::Domain(_))); + + let resolved = r.resolve(&addr).await.expect("localhost resolves via OS"); + assert!( + !resolved.is_empty(), + "localhost must resolve to at least one address" + ); + for sa in &resolved { + assert_eq!(sa.port(), 9000); + assert!( + sa.ip().is_loopback(), + "localhost must only map to loopback, got {sa}" + ); + } +} + +#[test] +fn dns_error_display_and_debug() { + let io_err = DnsError::Io(io::Error::new(io::ErrorKind::TimedOut, "boom")); + assert!(io_err.to_string().starts_with("I/O error:")); + assert!(io_err.to_string().contains("boom")); + assert!(!format!("{io_err:?}").is_empty()); + + // A 64-octet single label exceeds the 63-octet DNS limit, yielding a real + // hickory `ProtoError`. + let proto_err = Name::from_ascii("x".repeat(64)).unwrap_err(); + let host_err = DnsError::Hostname(proto_err); + assert!(host_err.to_string().starts_with("hostname parse error: ")); + assert!(format!("{host_err:?}").contains("Hostname")); +} + +#[test] +fn dns_error_from_io_error() { + // `#[from]` wires `io::Error` into the `Io` variant. + let converted: DnsError = io::Error::new(io::ErrorKind::ConnectionRefused, "nope").into(); + assert!(matches!(converted, DnsError::Io(_))); + assert_eq!(converted.to_string(), "I/O error: nope"); +} + +#[test] +fn dns_error_into_io_error_roundtrips() { + // `From for io::Error` wraps via `io::Error::other`, so the kind is + // `Other` and the source string is preserved. + let original = DnsError::Hostname(Name::from_ascii("x".repeat(64)).unwrap_err()); + let io_err: io::Error = original.into(); + assert_eq!(io_err.kind(), io::ErrorKind::Other); + assert!(io_err.to_string().starts_with("hostname parse error: ")); +} + +#[test] +fn dns_error_decode_variant_displays() { + // Exercise the `Decode` variant's `Display` prefix by forcing a decode failure + // from a too-short DNS message buffer. + let decode_err = Message::from_vec(&[0x00]).expect_err("truncated DNS message must fail"); + let err = DnsError::from(decode_err); + assert!( + err.to_string().starts_with("DNS decode error:"), + "got: {err}" + ); + assert!(matches!(err, DnsError::Decode(_))); +} + +/// A loopback TCP server that speaks just enough of the TCP-DNS wire protocol (RFC +/// 1035 §4.2.2) to answer ONE query: read the 2-byte length prefix + query body, +/// then write back a framed DNS response carrying the supplied A and AAAA answers. +/// Returns the bound address so the resolver can target it. Stays entirely on +/// loopback — no external DNS is contacted. +/// +/// The response echoes no relationship to the query name; the resolver's +/// `tcp_query_inner` decodes the message and harvests A/AAAA answers without +/// validating the question, so a fixed answer set exercises the full frame → +/// connect → write → read-length → read-body → decode → collect path. +async fn spawn_tcp_dns_server(answers: Vec) -> (SocketAddr, tokio::task::JoinHandle<()>) { + use futures_util::{AsyncReadExt, AsyncWriteExt}; + use hickory_proto::{ + op::{Message, OpCode}, + rr::{Name, Record}, + }; + + let listener = <::Net as Net>::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind loopback DNS server"); + let addr = listener.local_addr().expect("server local_addr"); + + let handle = tokio::spawn(async move { + let Ok((mut stream, _)) = listener.accept().await else { + return; + }; + + // Read the 2-byte big-endian query length, then the query body. + let mut len_buf = [0u8; 2]; + if stream.read_exact(&mut len_buf).await.is_err() { + return; + } + let qlen = u16::from_be_bytes(len_buf) as usize; + let mut qbuf = vec![0u8; qlen]; + if stream.read_exact(&mut qbuf).await.is_err() { + return; + } + + // Build a response message carrying the fixed answers. The id is irrelevant to + // the resolver (it does not match the query id). + let mut resp = Message::response(0, OpCode::Query); + let name = Name::from_ascii("seed.cluster.test.").expect("answer name"); + for rdata in answers { + resp.add_answer(Record::from_rdata(name.clone(), 60, rdata)); + } + let body = resp.to_vec().expect("encode response"); + let mut framed = Vec::with_capacity(2 + body.len()); + framed.extend_from_slice(&(body.len() as u16).to_be_bytes()); + framed.extend_from_slice(&body); + + // Ignoring Err: best-effort single write into a test fixture; if the client + // hung up the resolver test will surface the failure itself. + let _ = stream.write_all(&framed).await; + }); + + (addr, handle) +} + +// The TCP-first success path: a FQDN (contains a `.`) with a configured nameserver +// drives `tcp_query` → `tcp_query_inner` end to end against a loopback DNS server, +// and the harvested A + AAAA answers (carrying the query port) are returned without +// consulting the OS fallback. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn tcp_query_collects_a_and_aaaa_from_loopback_server() { + use hickory_proto::rr::{ + RData, + rdata::{A, AAAA}, + }; + + let answers = vec![ + RData::A(A(Ipv4Addr::new(203, 0, 113, 7))), + RData::AAAA(AAAA(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 0x1))), + // A CNAME-like non-address record is ignored by the collector; model it with a + // TXT to prove the `_ => {}` arm is taken. + RData::TXT(hickory_proto::rr::rdata::TXT::new(vec![ + "ignored".to_string(), + ])), + ]; + let (server_addr, handle) = spawn_tcp_dns_server(answers).await; + + let r = Dns::from_servers(vec![server_addr]).with_timeout(Duration::from_secs(5)); + // A fully-qualified name (has a `.`) so the TCP-first branch is taken. + let addr: Address = "seed.cluster.test:8300".parse().expect("parse FQDN:port"); + assert!(matches!(addr.host(), Host::Domain(_))); + + let resolved = r.resolve(&addr).await.expect("TCP-DNS resolve"); + // Both address answers surface, each carrying the requested port; the TXT is + // dropped. + assert_eq!( + resolved.len(), + 2, + "only A + AAAA are collected: {resolved:?}" + ); + assert!(resolved.contains(&SocketAddr::new( + IpAddr::V4(Ipv4Addr::new(203, 0, 113, 7)), + 8300 + ))); + assert!(resolved.contains(&SocketAddr::new( + IpAddr::V6(Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 0x1)), + 8300 + ))); + + handle.await.expect("DNS server task"); +} + +// When the TCP-DNS server returns an EMPTY answer set, the resolver treats the TCP +// attempt as unproductive and falls through to the OS fallback. With a short +// loopback name target the fallback resolves loopback, proving the empty-TCP-answer +// → fallback path (the `!addrs.is_empty()` guard going false) rather than surfacing +// the empty TCP result. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn tcp_query_empty_answers_falls_through_to_os_fallback() { + // Server answers with zero address records. + let (server_addr, handle) = spawn_tcp_dns_server(Vec::new()).await; + + let r = Dns::from_servers(vec![server_addr]).with_timeout(Duration::from_secs(5)); + // `localhost.` is fully-qualified (trailing dot ⇒ contains `.`), so the TCP + // branch is attempted; the empty answer makes it fall through to the OS resolver, + // which maps localhost to loopback. + let addr: Address = "localhost.:9100".parse().expect("parse localhost.:9100"); + assert!(matches!(addr.host(), Host::Domain(_))); + + let resolved = r + .resolve(&addr) + .await + .expect("empty TCP answer falls back to OS resolver"); + assert!( + !resolved.is_empty(), + "OS fallback must resolve localhost to loopback" + ); + for sa in &resolved { + assert_eq!(sa.port(), 9100); + assert!( + sa.ip().is_loopback(), + "localhost maps to loopback, got {sa}" + ); + } + + handle.await.expect("DNS server task"); +} diff --git a/serf-reactor/src/resolver/getifs/mod.rs b/serf-reactor/src/resolver/getifs/mod.rs new file mode 100644 index 00000000..5ea65be7 --- /dev/null +++ b/serf-reactor/src/resolver/getifs/mod.rs @@ -0,0 +1,184 @@ +//! getifs-backed advertise auto-detection (feature `getifs`). +//! +//! [`LocalAddrResolver`] enumerates the host's own interface addresses when the +//! configured advertise address is a wildcard (`0.0.0.0` / `[::]`) and substitutes +//! a concrete one. This driver uses a single address for both the socket bind and +//! the advertised contact — and rejects an unspecified advertise at construction — +//! so the substituted address becomes the bind address too. Unlike HashiCorp +//! memberlist (which binds `0.0.0.0` across every interface and advertises a +//! separately-detected IP), the node binds and advertises only the chosen interface +//! address. +//! +//! This is the `Send`/`agnostic` sibling of serf-compio's `LocalAddrResolver`; the +//! resolution itself is a synchronous OS query, so the two are identical apart from +//! the crate they live in. + +use core::future::Future; +use std::{ + io, + net::{IpAddr, SocketAddr}, +}; + +use crate::resolver::Resolver; + +/// Which of the host's own interface addresses to enumerate when resolving a +/// wildcard (`0.0.0.0` / `[::]`) advertise address via [`LocalAddrResolver`]. +#[cfg_attr(docsrs, doc(cfg(feature = "getifs")))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LocalAddrScope { + /// Private (RFC 1918 / RFC 4193) addresses — the usual choice for a LAN cluster. + Private, + /// Globally-routable public addresses. + Public, + /// Every interface address except loopback, unspecified, and link-local. + All, +} + +/// A [`Resolver`] that auto-detects the host's advertise address from its network +/// interfaces, via [`getifs`](https://crates.io/crates/getifs). +/// +/// `resolve` enumerates the host's own interface addresses — filtered by the +/// configured [`LocalAddrScope`], IPv4 first — **only when the input address is a +/// wildcard** (`0.0.0.0` / `[::]`). A concrete address is passed through unchanged, +/// so the same resolver also resolves `join` seeds. +/// +/// The chosen address becomes the node's single bind **and** advertise address; on +/// a multi-homed host the node binds only that one interface, not all of them (this +/// driver has no separate all-interfaces bind). +#[cfg_attr(docsrs, doc(cfg(feature = "getifs")))] +#[derive(Debug, Clone, Copy)] +pub struct LocalAddrResolver { + scope: LocalAddrScope, +} + +impl LocalAddrResolver { + /// A resolver for the given scope. + pub const fn new(scope: LocalAddrScope) -> Self { + Self { scope } + } + + /// Private addresses (the default). + pub const fn private() -> Self { + Self::new(LocalAddrScope::Private) + } + + /// Public addresses. + pub const fn public() -> Self { + Self::new(LocalAddrScope::Public) + } + + /// Every interface address except loopback, unspecified, and link-local. + pub const fn all() -> Self { + Self::new(LocalAddrScope::All) + } +} + +impl Default for LocalAddrResolver { + fn default() -> Self { + Self::private() + } +} + +impl Resolver for LocalAddrResolver { + type Address = SocketAddr; + type Error = io::Error; + + fn resolve( + &self, + address: &SocketAddr, + ) -> impl Future, io::Error>> + Send + '_ { + // Extract owned copies up front so the returned future borrows neither `self` + // nor `address` (the `Resolver` contract binds the future to `&self`'s + // lifetime, so it must not capture the shorter-lived `address`). + let addr = *address; + let scope = self.scope; + async move { + if addr.ip().is_unspecified() { + // Honor the wildcard's family: `0.0.0.0` yields only IPv4 candidates, `[::]` + // only IPv6 — never substitute the other family for the address the caller + // asked to bind + advertise. + local_socket_addrs(scope, addr.port(), Some(addr.is_ipv6())) + } else { + Ok(vec![addr]) + } + } + } +} + +/// Detect a single advertise [`SocketAddr`] directly — the first interface address +/// matching `scope` (IPv4 first), with `port` attached — for callers who would +/// rather compute the address up front and pass +/// [`MaybeResolved::Resolved`](memberlist_proto::MaybeResolved::Resolved). +/// +/// Family-agnostic: prefers IPv4. To pin a family, drive [`LocalAddrResolver`] with +/// a `0.0.0.0` / `[::]` wildcard instead. +#[cfg_attr(docsrs, doc(cfg(feature = "getifs")))] +pub fn local_advertise(scope: LocalAddrScope, port: u16) -> io::Result { + local_socket_addrs(scope, port, None)? + .into_iter() + .next() + .ok_or_else(|| { + io::Error::new( + io::ErrorKind::AddrNotAvailable, + "no local interface address found", + ) + }) +} + +/// Enumerate the host's interface addresses matching `scope`, IPv4 first, each with +/// `port` attached. `only_ipv6` filters to one family (`Some(true)` = IPv6, +/// `Some(false)` = IPv4); `None` keeps both, IPv4 first. +fn local_socket_addrs( + scope: LocalAddrScope, + port: u16, + only_ipv6: Option, +) -> io::Result> { + // Enumerate ALL interface addresses and classify them here. getifs's own + // `private_addrs` / `public_addrs` use the broad RFC 6890 special-purpose registry + // (CGNAT, benchmarking, NAT64, ...), not the RFC 1918 / RFC 4193 "reachable LAN + // contact" set we want to advertise. + let mut ips: Vec = getifs::local_addrs()? + .iter() + .map(|n| n.addr()) + .filter(|ip| scope.accepts(*ip)) + .filter(|ip| only_ipv6.is_none_or(|v6| ip.is_ipv6() == v6)) + .collect(); + // IPv4 first — Go memberlist advertises an IPv4 private address by default. + ips.sort_by_key(|ip| ip.is_ipv6()); + Ok( + ips + .into_iter() + .map(|ip| SocketAddr::new(ip, port)) + .collect(), + ) +} + +// Classification via the `iprfc` RFC registry getifs itself uses (depended on +// directly so the version floor — iprfc 0.2.2, which fixes the RFC6890 IPv6 table — +// is enforced in our own dependency graph). Mirrored in the std memberlist +// resolvers; keep the two in sync. +impl LocalAddrScope { + /// Whether `ip` is an acceptable advertise candidate for this scope. + fn accepts(self, ip: IpAddr) -> bool { + use iprfc::{RFC1918, RFC4193, RFC6890}; + match self { + // RFC 1918 (IPv4) + RFC 4193 unique-local (IPv6). + LocalAddrScope::Private => RFC1918.contains(&ip) || RFC4193.contains(&ip), + // Globally-routable = not in the RFC 6890 special-purpose registry — the exact + // definition getifs uses for its own `public_addrs`. + LocalAddrScope::Public => !RFC6890.contains(&ip), + LocalAddrScope::All => !ip.is_loopback() && !ip.is_unspecified() && !is_link_local(ip), + } + } +} + +/// Link-local (IPv4 169.254/16, IPv6 fe80::/10) — usable only on the local link. +fn is_link_local(ip: IpAddr) -> bool { + match ip { + IpAddr::V4(a) => a.is_link_local(), + IpAddr::V6(a) => (a.segments()[0] & 0xffc0) == 0xfe80, + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-reactor/src/resolver/getifs/tests.rs b/serf-reactor/src/resolver/getifs/tests.rs new file mode 100644 index 00000000..c5e29076 --- /dev/null +++ b/serf-reactor/src/resolver/getifs/tests.rs @@ -0,0 +1,129 @@ +use super::*; + +#[tokio::test] +async fn concrete_address_passes_through() { + let r = LocalAddrResolver::all(); + let addr: SocketAddr = "10.0.0.5:7946".parse().unwrap(); + assert_eq!(r.resolve(&addr).await.unwrap(), vec![addr]); +} + +#[tokio::test] +async fn wildcard_enumerates_interface_addrs() { + let r = LocalAddrResolver::all(); + let wild: SocketAddr = "0.0.0.0:7946".parse().unwrap(); + let out = r.resolve(&wild).await.unwrap(); + // The wildcard was replaced by real interface addresses, all on the asked port; + // none is the wildcard itself. + assert!(!out.contains(&wild)); + assert!( + out + .iter() + .all(|s| s.port() == 7946 && !s.ip().is_unspecified()) + ); +} + +#[tokio::test] +async fn wildcard_respects_address_family() { + let r = LocalAddrResolver::all(); + // `0.0.0.0` must never substitute an IPv6 address, and `[::]` never an IPv4 one — + // the resolved bind+advertise address keeps the requested family. + let v4 = r.resolve(&"0.0.0.0:7946".parse().unwrap()).await.unwrap(); + assert!( + v4.iter().all(|s| s.is_ipv4()), + "0.0.0.0 must yield only IPv4" + ); + let v6 = r.resolve(&"[::]:7946".parse().unwrap()).await.unwrap(); + assert!(v6.iter().all(|s| s.is_ipv6()), "[::] must yield only IPv6"); +} + +#[test] +fn local_advertise_attaches_port() { + // Best-effort: a host may have no address in a given scope, but when one is found + // the helper attaches the requested port. + if let Ok(addr) = local_advertise(LocalAddrScope::All, 7946) { + assert_eq!(addr.port(), 7946); + } +} + +#[test] +fn private_scope_is_rfc1918_and_ula_only() { + use LocalAddrScope::Private; + for ip in [ + "10.0.0.1", + "172.16.5.4", + "192.168.1.1", + "fc00::1", + "fd12::3", + ] { + assert!( + Private.accepts(ip.parse().unwrap()), + "{ip} should be private" + ); + } + // RFC 6890 special-purpose that getifs::private_addrs would wrongly include: + for ip in [ + "100.64.0.1", // CGNAT + "198.18.0.1", // benchmarking + "192.0.2.1", // documentation + "8.8.8.8", // public + "169.254.0.1", // link-local + "2001:db8::1", // documentation v6 + "64:ff9b::1", // NAT64 + "2606:4700::1", // public v6 + "fe80::1", // link-local v6 + ] { + assert!( + !Private.accepts(ip.parse().unwrap()), + "{ip} must NOT be private" + ); + } +} + +#[test] +fn public_scope_excludes_special_purpose() { + use LocalAddrScope::Public; + assert!(Public.accepts("8.8.8.8".parse().unwrap())); + assert!(Public.accepts("2606:4700::1".parse().unwrap())); + // Real global unicast inside 2001::/16 — must be public (not the whole /16 + // special-purpose block; only 2001::/23 + 2001:db8::/32 are). + assert!(Public.accepts("2001:4860:4860::8888".parse().unwrap())); + for ip in [ + "10.0.0.1", + "100.64.0.1", // CGNAT + "198.18.0.1", // benchmarking + "192.0.2.1", // documentation + "192.0.0.1", // IETF protocol assignments + "240.0.0.1", // reserved / future use + "127.0.0.1", + "169.254.0.1", + "fc00::1", + "2001:db8::1", // documentation v6 + "2001::1", // 2001::/23 (Teredo) + "2001:2::1", // benchmarking v6 + "2001:10::1", // ORCHID + "fe80::1", + "::1", + ] { + assert!( + !Public.accepts(ip.parse().unwrap()), + "{ip} must NOT be public" + ); + } +} + +#[test] +fn all_scope_excludes_loopback_unspecified_linklocal() { + use LocalAddrScope::All; + assert!(All.accepts("10.0.0.1".parse().unwrap())); + assert!(All.accepts("8.8.8.8".parse().unwrap())); + for ip in [ + "127.0.0.1", + "0.0.0.0", + "169.254.0.1", + "::1", + "::", + "fe80::1", + ] { + assert!(!All.accepts(ip.parse().unwrap()), "{ip} must NOT be in All"); + } +} diff --git a/serf-reactor/src/resolver/mod.rs b/serf-reactor/src/resolver/mod.rs index 72adbb20..5be2d27b 100644 --- a/serf-reactor/src/resolver/mod.rs +++ b/serf-reactor/src/resolver/mod.rs @@ -10,14 +10,21 @@ //! Built-in resolvers: //! - [`OsResolver`]: `getaddrinfo`-backed via the runtime's blocking pool. //! - [`SocketAddrResolver`]: identity pass-through for already-resolved addrs. -//! -//! The `hickory`-backed `DnsResolver` (`dns` feature) and the getifs -//! `LocalAddrResolver` (`getifs` feature) are added in a later chunk. +//! - [`DnsResolver`] (`dns` feature): TCP-first `hickory`-backed DNS over the +//! agnostic runtime's TCP stream. +//! - [`LocalAddrResolver`] (`getifs` feature): auto-detect the advertise address +//! from the host's own network interfaces. mod advertise; mod os; mod socket_addr; +#[cfg(feature = "dns")] +mod dns; + +#[cfg(feature = "getifs")] +mod getifs; + pub use advertise::{ AdvertiseAddrResolver, AdvertiseResolutionError, FirstAddrResolver, Ipv4PreferringResolver, Ipv6PreferringResolver, @@ -25,6 +32,14 @@ pub use advertise::{ pub use os::OsResolver; pub use socket_addr::SocketAddrResolver; +#[cfg(feature = "dns")] +#[cfg_attr(docsrs, doc(cfg(feature = "dns")))] +pub use dns::{DEFAULT_DNS_TIMEOUT, DnsError, DnsResolver}; + +#[cfg(feature = "getifs")] +#[cfg_attr(docsrs, doc(cfg(feature = "getifs")))] +pub use getifs::{LocalAddrResolver, LocalAddrScope, local_advertise}; + use core::future::Future; use std::net::SocketAddr; diff --git a/serf-reactor/src/serf/mod.rs b/serf-reactor/src/serf/mod.rs index ad672d34..ba24f432 100644 --- a/serf-reactor/src/serf/mod.rs +++ b/serf-reactor/src/serf/mod.rs @@ -37,6 +37,8 @@ use crate::delegate::KeyringDelegate; use crate::quic::{QuicTransport, QuicTransportOptions}; #[cfg(feature = "tcp")] use crate::tcp::{TcpTransport, TcpTransportOptions}; +#[cfg(feature = "tls")] +use crate::tls::{TlsTransport, TlsTransportOptions}; use crate::{ MaybeResolved, command::{ @@ -301,6 +303,104 @@ where } } +// Ergonomic TLS constructor: instantiate the TLS transport for the caller so a node +// can be built without naming the generic `Serf::new::` machinery. TLS rides +// the same stream driver as plain TCP, differing only in the record layer. +#[cfg(feature = "tls")] +#[cfg_attr(docsrs, doc(cfg(feature = "tls")))] +impl Serf +where + I: memberlist_proto::Id + + CheapClone + + Clone + + core::fmt::Debug + + core::fmt::Display + + Send + + Sync + + Unpin + + 'static, + A: Data + Clone + Send + Sync + 'static, + R: Runtime, +{ + /// Build a TLS-backed serf node and spawn its driver on the runtime `R`. + /// + /// The ergonomic wrapper over [`Serf::new`] that instantiates the + /// [`TlsTransport`](crate::TlsTransport) for the caller: it binds a UDP gossip + /// socket and a TCP reliable listener on the advertise address (resolved once via + /// `resolver` / `advertise_resolver`), then spawns the stream driver whose + /// reliable record layer drives rustls over the plain agnostic TCP stream. The + /// caller supplies the rustls server/client bundle and the per-peer SNI provider + /// through [`TlsTransportOptions`](crate::TlsTransportOptions). The gossip RNG is + /// drawn from OS entropy via [`gossip_rng`](crate::gossip_rng); use + /// [`tls_with_rng`](Self::tls_with_rng) to supply your own. + /// + /// Under an encryption backend, pass an + /// [`Arc`](crate::KeyringDelegate) + /// (`Arc::new(VoidKeyringDelegate)` for a node that manages no keys); the keyring + /// AEAD-protects the gossip datagrams (the reliable plane rides the TLS session). + #[allow(clippy::too_many_arguments)] + pub async fn tls( + options: TlsTransportOptions, + resolver: &RES, + advertise_resolver: &AR, + delegate: D, + runtime_options: RuntimeOptions, + serf_options: SerfOptions, + #[cfg(encryption)] keyring: Arc, + ) -> Result + where + RES: Resolver
, + AR: AdvertiseAddrResolver, + D: Delegate, + { + Self::tls_with_rng( + options, + resolver, + advertise_resolver, + delegate, + runtime_options, + serf_options, + crate::gossip_rng()?, + #[cfg(encryption)] + keyring, + ) + .await + } + + /// Like [`tls`](Self::tls) but with a caller-supplied gossip RNG `G` — draw it via + /// [`gossip_rng`](crate::gossip_rng) for fork-safe OS entropy. + #[allow(clippy::too_many_arguments)] + pub async fn tls_with_rng( + options: TlsTransportOptions, + resolver: &RES, + advertise_resolver: &AR, + delegate: D, + runtime_options: RuntimeOptions, + serf_options: SerfOptions, + gossip_rng: G, + #[cfg(encryption)] keyring: Arc, + ) -> Result + where + RES: Resolver
, + AR: AdvertiseAddrResolver, + D: Delegate, + G: rand::Rng + Send + Unpin + 'static, + { + Self::new::, RES, AR, D, G>( + options, + resolver, + advertise_resolver, + delegate, + runtime_options, + serf_options, + gossip_rng, + #[cfg(encryption)] + keyring, + ) + .await + } +} + // Ergonomic QUIC constructor: instantiate the QUIC transport for the caller so a // node can be built without naming the generic `Serf::new::` machinery. #[cfg(feature = "quic")] diff --git a/serf-reactor/src/shared/mod.rs b/serf-reactor/src/shared/mod.rs index 7d81ee07..47155ef6 100644 --- a/serf-reactor/src/shared/mod.rs +++ b/serf-reactor/src/shared/mod.rs @@ -145,7 +145,9 @@ impl Shared { self.observation_dropped.fetch_add(n, Ordering::Relaxed); } - /// Records `n` gossip payloads sent over the QUIC datagram plane. + /// Records `n` gossip payloads sent over the QUIC datagram plane. Only the QUIC + /// driver reports datagram sends; the stream backends leave the counter at zero. + #[cfg(feature = "quic")] pub(crate) fn add_datagrams_sent(&self, n: u64) { self.datagrams_sent.fetch_add(n, Ordering::Relaxed); } diff --git a/serf-reactor/src/tls/mod.rs b/serf-reactor/src/tls/mod.rs new file mode 100644 index 00000000..899e0f44 --- /dev/null +++ b/serf-reactor/src/tls/mod.rs @@ -0,0 +1,462 @@ +//! TLS-backed serf driver over the agnostic runtime — the TLS sibling of the +//! plain-TCP plane. +//! +//! [`TlsTransport`] owns the bound UDP gossip socket and TCP reliable listener, +//! exactly like [`TcpTransport`](crate::TcpTransport); it differs only in the +//! reliable record layer, which drives rustls's record layer manually over the +//! plain, `R`-agnostic TCP stream the shared stream driver already uses — there is +//! NO runtime TLS stream. The machine-layer +//! `serf_proto::StreamEndpoint>` is built inside +//! [`Transport::run`] from the stored [`TlsOptions`](memberlist_proto::TlsOptions) +//! (cert/key/verifier bundle) and the per-peer SNI provider; the gossip datagram +//! plane stays plain UDP. This is the `Send`/`agnostic` sibling of serf-compio's +//! `!Send`, compio-bound `TlsTransport`. +//! +//! TLS secures the *reliable* push-pull plane. The unreliable gossip plane is +//! still AEAD-protected by the optional encryption keyring (carried through to the +//! coordinator the same way the TCP plane carries it), so an encrypted cluster +//! protects both planes. +//! +//! ## Server name +//! +//! TLS verifies the peer's certificate against a server name. The `sni_provider` +//! closure on [`TlsTransportOptions`] is called per dial with the peer's +//! membership address; it must return `Some(name)` matching the peer cert's +//! SAN/CN. Returning `None` aborts the dial before the handshake. The default +//! closure returns `Some("localhost".to_string())` for every peer — matching the +//! bundled smoke test's self-signed localhost-SAN cert. Production operators +//! supply a closure mapping each peer to its actual DNS name or SAN. + +#![cfg(feature = "tls")] + +use core::num::NonZeroU8; +use std::{io::ErrorKind, net::SocketAddr}; + +use agnostic::{ + Runtime, + net::{Net, TcpListener, UdpSocket}, +}; +use hostaddr::HostAddr; +use memberlist_proto::{ + CheapClone, Data, Endpoint, EndpointOptions, Id, MaybeResolved, TlsRecords, + streams::{LabelOptions, Labeled, StreamEndpoint as Coordinator}, +}; +use rand::rngs::StdRng; +use smol_str::SmolStr; + +/// TLS machine-options bundle (server + client `rustls` config) handed to +/// [`TlsTransport`]. Re-exported from `memberlist-proto` so callers don't need a +/// direct dep on it. +pub use memberlist_proto::TlsOptions; + +#[cfg(encryption)] +use memberlist_proto::EncryptionOptions; + +use crate::{ + SerfError, + delegate::Delegate, + driver::options::StreamTransportOptions, + resolver::{AdvertiseAddrResolver, Resolver}, + transport::{Transport, TransportRuntime}, +}; + +/// Boxed SNI provider closure: maps a peer's `SocketAddr` to the expected TLS +/// server name used for certificate verification. Returns `None` to abort the +/// dial before the handshake. +pub type SniProvider = Box Option + Send + Sync>; + +/// Per-backend TLS-specific transport options. +/// +/// Embedded into the transport constructor. Bundles the local node identifier, +/// the (possibly-unresolved) advertise address, the stream-transport tuning +/// knobs, the per-peer SNI provider closure, the machine-layer [`TlsOptions`] +/// bundle (cert/key/verifier), and the optional gossip-encryption policy. The +/// cluster label and inbound-label-check policy are supplied via the serf +/// `Options` block (not here), feeding both planes from a single validated +/// source. +pub struct TlsTransportOptions> { + local_id: Option, + advertise_addr: Option>, + stream: StreamTransportOptions, + sni_provider: SniProvider, + tls_options: Option, + /// Gossip encryption policy. The default (no keyring) leaves the gossip + /// datagrams plaintext; attaching a keyring via + /// [`with_encryption`](Self::with_encryption) makes the coordinator's + /// `encrypt_gossip`/`decrypt_gossip` AEAD-protect them. The reliable plane + /// rides the TLS session, so the keyring covers only the gossip datagrams. + #[cfg(encryption)] + encryption: EncryptionOptions, +} + +impl TlsTransportOptions { + /// Construct with defaults. Caller MUST chain + /// [`with_local_id`](Self::with_local_id), + /// [`with_advertise_addr`](Self::with_advertise_addr), and + /// [`with_tls_options`](Self::with_tls_options) before passing to + /// `TlsTransport::new`. The default `sni_provider` returns + /// `Some("localhost".to_string())` for every peer — matching the bundled smoke + /// test's self-signed localhost-SAN cert. + #[inline] + pub fn new() -> Self { + Self { + local_id: None, + advertise_addr: None, + stream: StreamTransportOptions::new(), + sni_provider: Box::new(|_addr: &SocketAddr| Some("localhost".to_string())), + tls_options: None, + #[cfg(encryption)] + encryption: EncryptionOptions::new(), + } + } + + /// Builder: local node identifier. + #[must_use] + #[inline] + pub fn with_local_id(mut self, id: I) -> Self { + self.local_id = Some(id); + self + } + + /// Builder: advertise address (resolved or unresolved). + #[must_use] + #[inline] + pub fn with_advertise_addr(mut self, addr: MaybeResolved) -> Self { + self.advertise_addr = Some(addr); + self + } + + /// Builder: stream-transport tuning knobs. + #[must_use] + #[inline] + pub fn with_stream(mut self, opts: StreamTransportOptions) -> Self { + self.stream = opts; + self + } + + /// Builder: SNI provider closure. Default returns + /// `Some("localhost".to_string())` for every peer; for deployments with + /// per-peer SAN certs, supply a closure mapping each dialed `SocketAddr` to its + /// expected SNI string. Returning `None` for a peer causes the outbound TLS dial + /// to fail before the handshake. + #[must_use] + #[inline] + pub fn with_sni_provider(mut self, f: SniProvider) -> Self { + self.sni_provider = f; + self + } + + /// Builder: TLS machine options (cert/key/verifier). Must be set before + /// `TlsTransport::new`. + #[must_use] + #[inline] + pub fn with_tls_options(mut self, opts: TlsOptions) -> Self { + self.tls_options = Some(opts); + self + } + + /// Builder: gossip-encryption policy. + /// + /// The default (no keyring) keeps the gossip datagrams plaintext, so an + /// unencrypted node still builds and interoperates. Attach a keyring + /// (`EncryptionOptions::new().with_keyring(Keyring::new(primary_key))`) to + /// AEAD-protect the gossip datagrams — every node sharing the cluster MUST carry + /// the same keyring to interop. The reliable plane is secured by the TLS session + /// independently of this keyring. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + #[must_use] + #[inline] + pub fn with_encryption(mut self, encryption: EncryptionOptions) -> Self { + self.encryption = encryption; + self + } + + /// Local node identifier, if set. + #[inline] + pub const fn local_id(&self) -> Option<&I> { + self.local_id.as_ref() + } + + /// Advertise address, if set. + #[inline] + pub const fn advertise_addr(&self) -> Option<&MaybeResolved> { + self.advertise_addr.as_ref() + } + + /// Stream-transport tuning knobs. + #[inline] + pub const fn stream(&self) -> &StreamTransportOptions { + &self.stream + } + + /// SNI provider closure. + #[inline] + pub fn sni_provider(&self) -> &(dyn Fn(&SocketAddr) -> Option + Send + Sync) { + self.sni_provider.as_ref() + } + + /// TLS machine options bundle, if set. + #[inline] + pub const fn tls_options(&self) -> Option<&TlsOptions> { + self.tls_options.as_ref() + } + + /// Gossip-encryption policy. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + #[inline] + pub const fn encryption(&self) -> &EncryptionOptions { + &self.encryption + } +} + +impl Default for TlsTransportOptions { + #[inline] + fn default() -> Self { + Self::new() + } +} + +/// TLS-backed serf transport. +/// +/// Owns the bound `UdpSocket` (gossip unreliable plane) and `TcpListener` +/// (reliable coordinator; TLS handshake-on-accept). The machine-layer +/// `serf_proto::StreamEndpoint>` is built +/// inside [`Transport::run`] from the stored config (`tls_options` + +/// `sni_provider`) and the cluster options sourced from +/// [`TransportRuntime`](crate::TransportRuntime). +pub struct TlsTransport +where + R: Runtime, +{ + local_id: I, + local_address: MaybeResolved, + advertise_socket: SocketAddr, + gossip_socket: ::UdpSocket, + tcp_listener: ::TcpListener, + stream_options: StreamTransportOptions, + sni_provider: SniProvider, + tls_options: TlsOptions, + /// Independent OS-seeded seed for the serf core's RNG, drawn once per node in + /// [`Transport::new`] and consumed when [`Transport::run`] builds the endpoint + /// via `new_with_rng`. Distinct from the coordinator's gossip RNG so serf's + /// query IDs and relay choices are not correlated across nodes. + serf_rng: StdRng, + /// Gossip-encryption policy applied to the coordinator built in + /// [`Transport::run`]. Absent keyring ⇒ plaintext gossip (the default). + #[cfg(encryption)] + encryption: EncryptionOptions, +} + +impl Transport for TlsTransport +where + R: Runtime, + I: + Id + CheapClone + Clone + core::fmt::Debug + core::fmt::Display + Send + Sync + Unpin + 'static, + A: Data + Clone + Send + Sync + 'static, +{ + type Error = SerfError; + type Id = I; + type Address = A; + type Options = TlsTransportOptions; + + async fn new( + options: Self::Options, + resolver: &RES, + advertise_resolver: &AR, + ) -> Result + where + RES: Resolver
, + AR: AdvertiseAddrResolver, + { + // Validate stream knobs that would deterministically break the backend (e.g. a + // zero `bridge_recv_buf_len` makes every bridge read return a false EOF) BEFORE + // binding any socket. + options.stream.validate()?; + + let local_id = options.local_id.ok_or_else(|| { + SerfError::Io(std::io::Error::new( + ErrorKind::InvalidInput, + "local_id required", + )) + })?; + let advertise_input = options.advertise_addr.ok_or_else(|| { + SerfError::Io(std::io::Error::new( + ErrorKind::InvalidInput, + "advertise_addr required", + )) + })?; + let tls_options = options.tls_options.ok_or_else(|| { + SerfError::Io(std::io::Error::new( + ErrorKind::InvalidInput, + "tls_options required", + )) + })?; + + let advertise_socket = match &advertise_input { + MaybeResolved::Resolved(s) => *s, + MaybeResolved::Unresolved(a) => { + let candidates = resolver + .resolve(a) + .await + .map_err(|e| SerfError::Resolve(std::io::Error::other(e.to_string())))?; + advertise_resolver.pick(candidates).map_err(|e| { + SerfError::Resolve(std::io::Error::new( + ErrorKind::AddrNotAvailable, + e.to_string(), + )) + })? + } + }; + + // Bind the TCP listener first to claim an OS-assigned free port, then bind the + // gossip UDP socket to that same port. TCP and UDP port spaces are independent, + // so for an ephemeral (`:0`) advertise we retry the pair on a fresh port when + // the UDP bind fails transiently: AddrInUse from the port-space race, or + // PermissionDenied when the TCP-claimed port falls in a UDP-excluded range. A + // dropped agnostic socket closes its FD synchronously, so an abandoned attempt + // never leaks a bound port. + const EPHEMERAL_BIND_RETRIES: usize = 16; + let ephemeral = advertise_socket.port() == 0; + let (tcp_listener, bound, gossip_socket) = { + let mut attempt = 0usize; + loop { + let tcp_listener = ::TcpListener::bind(advertise_socket) + .await + .map_err(SerfError::Io)?; + let bound = tcp_listener.local_addr().map_err(SerfError::Io)?; + match ::UdpSocket::bind(bound).await { + Ok(gossip_socket) => break (tcp_listener, bound, gossip_socket), + Err(e) + if ephemeral + && matches!(e.kind(), ErrorKind::AddrInUse | ErrorKind::PermissionDenied) + && attempt < EPHEMERAL_BIND_RETRIES => + { + // Release the claimed TCP port (drop closes the FD) and retry a fresh + // ephemeral pair. + drop(tcp_listener); + attempt += 1; + } + Err(e) => return Err(SerfError::Io(e)), + } + } + }; + + // Both sockets are now bound. The readback resolves an ephemeral `:0` to a + // concrete port but keeps an unspecified IP: `post_bind_setup` rejects an + // advertise address peers could not route serf traffic back to, then draws the + // OS-seeded serf-core RNG. Either failure drops BOTH bound sockets (dropping an + // agnostic socket closes its FD) before returning. + let serf_rng = match crate::transport::post_bind_setup(&bound) { + Ok(rng) => rng, + Err(e) => { + drop(tcp_listener); + drop(gossip_socket); + return Err(e); + } + }; + + Ok(Self { + local_id, + local_address: advertise_input, + advertise_socket: bound, + gossip_socket, + tcp_listener, + stream_options: options.stream, + sni_provider: options.sni_provider, + tls_options, + serf_rng, + #[cfg(encryption)] + encryption: options.encryption, + }) + } + + #[inline] + fn local_id(&self) -> &Self::Id { + &self.local_id + } + + #[inline] + fn local_address(&self) -> &MaybeResolved { + &self.local_address + } + + #[inline] + fn advertise_address(&self) -> &SocketAddr { + &self.advertise_socket + } + + async fn run(self, runtime: TransportRuntime, gossip_rng: G) + where + D: Delegate, + G: rand::Rng + Send + Unpin + 'static, + { + // `Serf::new` is generic over `T` and cannot build the record-layer-specific + // endpoint; build it here from `self`'s stored config. Serf ranks its user + // broadcasts on three tiers (intent / event / query → ranks 0 / 1 / 2), so the + // inner memberlist endpoint needs at least three broadcast tiers. + let inner_opts = EndpointOptions::new(self.local_id, self.advertise_socket) + .with_user_broadcast_tiers(NonZeroU8::new(3).expect("3 is nonzero")); + // Snapshot the reliable push/pull exchange timeout from the SAME options the + // coordinator is built from, so the driver reconciles an await-result join's + // caller deadline against the exact deadline the coordinator will stamp. + let stream_timeout = inner_opts.stream_timeout(); + let inner = Endpoint::new(inner_opts, gossip_rng); + // The TLS coordinator carries the per-peer SNI provider and the cert/key bundle + // (ridden as the inner options on `LabelOptions`); the membership address IS the + // transport socket (`|addr| *addr`). Like the plain-TCP plane this stage carries + // no cluster label (`None`) — TLS isolation comes from the record-layer cert + // verification and SNI. + #[allow(unused_mut)] + let mut coord = Coordinator::<_, _, Labeled, G>::new( + inner, + LabelOptions::new_in(None::>, self.tls_options), + self.sni_provider, + Box::new(|addr: &SocketAddr| *addr), + ); + // Install the gossip-encryption keyring so the coordinator's + // `encrypt_gossip`/`decrypt_gossip` (forwarded from the serf endpoint pump) + // become real on the unreliable plane. A no-keyring policy is the identity + // transform, so an unencrypted node is unaffected. The reliable plane is secured + // by the TLS session regardless. + #[cfg(encryption)] + coord.set_encryption_options(self.encryption); + // Serf's core RNG is seeded from its own OS-drawn entropy (`self.serf_rng`), + // independent of the coordinator's gossip RNG, so two nodes never share the + // query-ID / relay-selection stream. + let endpoint = serf_proto::StreamEndpoint::< + Self::Id, + SocketAddr, + Labeled, + G, + StdRng, + >::new_with_rng(coord, runtime.serf_options, self.serf_rng); + + let driver = + crate::driver::stream::spawn_stream_driver::, D, G, StdRng>( + endpoint, + self.gossip_socket, + self.tcp_listener, + runtime.shared, + runtime.events_tx, + runtime.delegate, + runtime.driver_options, + self.stream_options, + None, + stream_timeout, + #[cfg(encryption)] + runtime.keyring, + ); + driver.await; + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-reactor/src/tls/tests.rs b/serf-reactor/src/tls/tests.rs new file mode 100644 index 00000000..f76ccf59 --- /dev/null +++ b/serf-reactor/src/tls/tests.rs @@ -0,0 +1,94 @@ +//! Unit tests for the TLS transport options builder — the pure accessor / builder +//! wiring that feeds `TlsTransport::new`. Real-node construction (the rustls +//! record-layer handshake, join/converge, freed-port rebind) is exercised +//! end-to-end by the tokio suite in `tests/tls.rs`. + +use std::{net::SocketAddr, sync::Arc}; + +use memberlist_proto::MaybeResolved; +use rustls::{ClientConfig, RootCertStore, ServerConfig, crypto::CryptoProvider, version::TLS13}; +use rustls_pki_types::{CertificateDer, PrivateKeyDer}; +use smol_str::SmolStr; + +use super::{TlsOptions, TlsTransportOptions}; + +/// The process-default crypto provider, falling back to ring for the dev build. +fn crypto_provider() -> Arc { + CryptoProvider::get_default() + .cloned() + .unwrap_or_else(|| Arc::new(rustls::crypto::ring::default_provider())) +} + +/// Build a self-signed localhost-SAN `ServerConfig` + a root-verifying +/// `ClientConfig` bundle. The option-accessor tests only need a valid +/// [`TlsOptions`] to prove the builder round-trips it; the real-node handshake +/// (which uses an accept-any client verifier) lives in `tests/tls.rs`. +fn test_tls_options() -> TlsOptions { + let ck = rcgen::generate_simple_self_signed(vec!["localhost".into()]) + .expect("rcgen generate_simple_self_signed"); + let cert = CertificateDer::from(ck.cert.der().to_vec()); + let key = PrivateKeyDer::Pkcs8(ck.signing_key.serialize_der().into()); + let provider = crypto_provider(); + + let server = ServerConfig::builder_with_provider(provider.clone()) + .with_protocol_versions(&[&TLS13]) + .expect("TLS 1.3 supported") + .with_no_client_auth() + .with_single_cert(vec![cert.clone()], key) + .expect("valid self-signed cert"); + + let mut roots = RootCertStore::empty(); + roots.add(cert).expect("add root cert"); + let client = ClientConfig::builder_with_provider(provider) + .with_protocol_versions(&[&TLS13]) + .expect("TLS 1.3 supported") + .with_root_certificates(roots) + .with_no_client_auth(); + + TlsOptions::new(server, client) +} + +/// The `TlsTransportOptions` getters reflect what the builders set, including the +/// SNI provider closure and the TLS options bundle. The pre-build state is `None` +/// for the required fields the `new()` `ok_or_else` checks arm. +#[test] +fn options_accessors_reflect_builders() { + let addr: SocketAddr = "127.0.0.1:7946".parse().unwrap(); + // Default options: required fields unset, but the SNI provider has a default + // (`localhost`) so `sni_provider()` returns a live closure even before build. + let empty = TlsTransportOptions::::new(); + assert!(empty.local_id().is_none()); + assert!(empty.advertise_addr().is_none()); + assert!(empty.tls_options().is_none()); + assert!(empty.stream().validate().is_ok()); + assert_eq!((empty.sni_provider())(&addr), Some("localhost".to_string())); + + let opts = TlsTransportOptions::::new() + .with_local_id(SmolStr::new("acc-node")) + .with_advertise_addr(MaybeResolved::Resolved(addr)) + .with_tls_options(test_tls_options()) + .with_sni_provider(Box::new(|_| Some("peer.example".to_string()))); + assert_eq!(opts.local_id().map(|s| s.as_str()), Some("acc-node")); + match opts.advertise_addr() { + Some(MaybeResolved::Resolved(s)) => assert_eq!(*s, addr), + other => panic!("expected a resolved advertise addr, got {other:?}"), + } + assert!(opts.tls_options().is_some()); + // The custom SNI provider overrides the default for every peer. + assert_eq!( + (opts.sni_provider())(&addr), + Some("peer.example".to_string()) + ); +} + +/// `Default` is the `new()` state: required fields `None`, default SNI provider +/// installed. +#[test] +fn default_matches_new() { + let addr: SocketAddr = "127.0.0.1:0".parse().unwrap(); + let d = TlsTransportOptions::::default(); + assert!(d.local_id().is_none()); + assert!(d.advertise_addr().is_none()); + assert!(d.tls_options().is_none()); + assert_eq!((d.sni_provider())(&addr), Some("localhost".to_string())); +} diff --git a/serf-reactor/tests/tls.rs b/serf-reactor/tests/tls.rs new file mode 100644 index 00000000..13431b8c --- /dev/null +++ b/serf-reactor/tests/tls.rs @@ -0,0 +1,323 @@ +//! Real-node TLS serf tests on tokio: two loopback nodes exercising the reactor +//! stream driver with the rustls record layer end-to-end. Each test spins up +//! ephemeral `127.0.0.1:0` nodes via the ergonomic [`Serf::tls`] constructor and +//! drives the full pump — TLS handshake-on-dial, join push/pull, coordinator merge, +//! gossip, user events, queries, and graceful leave/shutdown. +//! +//! Mirrors `tests/tcp.rs` (TLS rides the same stream driver as plain TCP, differing +//! only in the record layer) and serf-compio's / memberlist-reactor's TLS harness: +//! each node presents a fresh self-signed localhost-SAN cert and the client side +//! accepts whatever the server presents, so the handshake completes without a real +//! trust anchor. The default SNI provider (`Some("localhost")`) matches the cert +//! SAN. + +#![cfg(all(feature = "tls", feature = "tokio"))] + +use core::time::Duration; +use std::{net::SocketAddr, sync::Arc}; + +use agnostic::tokio::TokioRuntime; +use bytes::Bytes; +use futures_util::{StreamExt, future}; +use rustls::{ + client::danger::{HandshakeSignatureValid, ServerCertVerified}, + crypto::CryptoProvider, + pki_types::CertificateDer, + version::TLS13, +}; +use serf_proto::{event::Event, members::SerfState, options::Options as SerfOptions}; +use serf_reactor::{ + FirstAddrResolver, MaybeResolved, RuntimeOptions, Serf, SocketAddrResolver, TlsOptions, + TlsTransportOptions, VoidDelegate, +}; +use smol_str::SmolStr; + +/// A tokio-backed reactor TLS node handle. +type Node = Serf; + +/// Accept-any server-cert verifier for the loopback tests. +/// +/// Each node presents its own fresh self-signed localhost-SAN cert; the client side +/// accepts whatever the server presents so the handshake completes without a real +/// trust anchor. NEVER use this outside a test. +#[derive(Debug)] +struct AcceptAnyServer(Arc); + +impl rustls::client::danger::ServerCertVerifier for AcceptAnyServer { + fn verify_server_cert( + &self, + _e: &CertificateDer<'_>, + _i: &[CertificateDer<'_>], + _n: &rustls::pki_types::ServerName<'_>, + _o: &[u8], + _t: rustls::pki_types::UnixTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } + fn verify_tls12_signature( + &self, + _m: &[u8], + _c: &CertificateDer<'_>, + _d: &rustls::DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + fn verify_tls13_signature( + &self, + _m: &[u8], + _c: &CertificateDer<'_>, + _d: &rustls::DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + fn supported_verify_schemes(&self) -> Vec { + self.0.signature_verification_algorithms.supported_schemes() + } +} + +fn crypto_provider() -> Arc { + CryptoProvider::get_default() + .cloned() + .unwrap_or_else(|| Arc::new(rustls::crypto::ring::default_provider())) +} + +/// Build a self-signed localhost-SAN `ServerConfig` + accept-any `ClientConfig` +/// bundle. A fresh bundle is built per node so each owns its own cert. +fn test_tls_options() -> TlsOptions { + let ck = rcgen::generate_simple_self_signed(vec!["localhost".into()]) + .expect("rcgen generate_simple_self_signed"); + let chain = vec![CertificateDer::from(ck.cert.der().to_vec())]; + let key = rustls::pki_types::PrivateKeyDer::Pkcs8(ck.signing_key.serialize_der().into()); + + let provider = crypto_provider(); + + let server_cfg = rustls::ServerConfig::builder_with_provider(provider.clone()) + .with_protocol_versions(&[&TLS13]) + .expect("TLS 1.3 supported") + .with_no_client_auth() + .with_single_cert(chain, key) + .expect("valid self-signed cert"); + + let client_cfg = rustls::ClientConfig::builder_with_provider(provider.clone()) + .with_protocol_versions(&[&TLS13]) + .expect("TLS 1.3 supported") + .dangerous() + .with_custom_certificate_verifier(Arc::new(AcceptAnyServer(provider))) + .with_no_client_auth(); + + TlsOptions::new(server_cfg, client_cfg) +} + +/// Build and spawn a reactor TLS node on an ephemeral loopback port through the +/// ergonomic `Serf::tls` constructor. The default SNI provider (`Some("localhost")`) +/// matches the self-signed cert SAN. +async fn spawn_node(id: &str) -> Node { + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let opts = TlsTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(bind)) + .with_tls_options(test_tls_options()); + Serf::::tls( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + #[cfg(encryption)] + std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), + ) + .await + .expect("spawn serf tls node") +} + +/// Poll both nodes until each reports the full two-member cluster, or fail on a +/// generous timeout so a convergence regression surfaces as a timeout, not a hang. +async fn converge(a: &Node, b: &Node) { + tokio::time::timeout(Duration::from_secs(20), async { + loop { + if a.num_members() == 2 && b.num_members() == 2 { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("both nodes converge to a 2-member cluster"); +} + +/// Two nodes on loopback: A joins B over a real TLS push-pull exchange, then BOTH +/// converge to a two-member cluster and shut down cleanly. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn two_node_tls_join_converges() { + let b = spawn_node("conv-b").await; + let a = spawn_node("conv-a").await; + let b_addr = b.advertise_address(); + + let reached = a + .join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B over TLS"); + assert_eq!(reached, b_addr, "join returns the reached seed address"); + + converge(&a, &b).await; + assert_eq!(a.num_members(), 2, "A sees the 2-member cluster"); + assert_eq!(b.num_members(), 2, "B sees the 2-member cluster"); + + a.shutdown().await.expect("conv-a shuts down"); + b.shutdown().await.expect("conv-b shuts down"); +} + +/// After a two-node TLS join, a user event broadcast by B is delivered to A's event +/// stream carrying the original name and payload. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn user_event_delivered() { + let b = spawn_node("ue-b").await; + let a = spawn_node("ue-a").await; + let b_addr = b.advertise_address(); + + // Subscribe before joining so the user event cannot race the subscription. + let mut a_events = a.events(); + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + converge(&a, &b).await; + + b.user_event("greet", Bytes::from_static(b"hello"), false) + .await + .expect("user event dispatched"); + + let got = tokio::time::timeout(Duration::from_secs(20), async { + loop { + match a_events.next().await { + Some(Event::User(u)) if u.name.as_str() == "greet" => break Some(u.payload.clone()), + Some(_) => {} + None => break None, + } + } + }) + .await + .expect("A observes B's user event within the timeout"); + assert_eq!( + got, + Some(Bytes::from_static(b"hello")), + "A receives B's user-event payload" + ); + + a.shutdown().await.expect("ue-a shuts down"); + b.shutdown().await.expect("ue-b shuts down"); +} + +/// After a two-node TLS join, a query issued by A round-trips: B receives the +/// `Event::Query`, responds, and A surfaces the matching `Event::QueryResponse`. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn query_round_trip() { + let b = spawn_node("q-b").await; + let a = spawn_node("q-a").await; + let b_addr = b.advertise_address(); + + // Subscribe both before the join so neither the query nor its response races ahead + // of a subscription. + let mut b_events = b.events(); + let mut a_events = a.events(); + + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + converge(&a, &b).await; + + let want = Bytes::from_static(b"pong"); + a.query( + "ping", + Bytes::from_static(b"ping-payload"), + a.default_query_param(), + ) + .await + .expect("query issued"); + + // B answers the first "ping" query it sees; A collects the matching response. + let responder = async { + loop { + match b_events.next().await { + Some(Event::Query(qe)) if qe.name() == "ping" => { + b.respond(qe, want.clone()) + .await + .expect("B responds to the query"); + break; + } + Some(_) => {} + None => panic!("B's event stream closed before the query arrived"), + } + } + }; + let collector = async { + loop { + match a_events.next().await { + Some(Event::QueryResponse(qr)) if qr.payload() == &want => break true, + Some(_) => {} + None => break false, + } + } + }; + + let got = tokio::time::timeout(Duration::from_secs(20), async { + let (_, got) = future::join(responder, collector).await; + got + }) + .await + .expect("query round-trip completes within the timeout"); + assert!(got, "A must receive B's query response"); + + a.shutdown().await.expect("q-a shuts down"); + b.shutdown().await.expect("q-b shuts down"); +} + +/// A graceful leave completes the machine's leave chain: `leave()` resolves only +/// once `LeftCluster` fires (the reactor gates the reply on it), that event surfaces +/// on the leaver's own stream, and the local endpoint settles at `Left`. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn leave_emits_left_cluster() { + let b = spawn_node("lv-b").await; + let a = spawn_node("lv-a").await; + let b_addr = b.advertise_address(); + + let mut a_events = a.events(); + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + converge(&a, &b).await; + + // The reactor resolves `leave()` only once the machine's `LeftCluster` fires, so a + // successful return already proves the graceful-leave chain completed. + a.leave().await.expect("A leaves the cluster"); + + // `LeftCluster` is also forwarded to A's own subscribers. + let saw = tokio::time::timeout(Duration::from_secs(20), async { + loop { + match a_events.next().await { + Some(Event::LeftCluster) => break true, + Some(_) => {} + None => break false, + } + } + }) + .await + .expect("A observes LeftCluster within the timeout"); + assert!(saw, "A must surface Event::LeftCluster after leave()"); + + // The local endpoint state settles at `Left` (poll to absorb the snapshot-refresh + // race after the leave chain completes). + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if a.state() == SerfState::Left { + break; + } + tokio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("A's endpoint state becomes Left"); + + a.shutdown().await.expect("lv-a shuts down"); + b.shutdown().await.expect("lv-b shuts down"); +} From 9ff719fb84f4ae83e8c5a2a5026a168581253c3e Mon Sep 17 00:00:00 2001 From: al8n Date: Thu, 2 Jul 2026 17:51:54 +1200 Subject: [PATCH 038/128] fix(serf-reactor): DNS resolution timeout is not bypassed by the OS fallback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DNS: a TimedOut TCP-DNS error now returns without the unbounded OS fallback (which runs outside the deadline), so a slow/hostile configured resolver can no longer burn the TCP budget then hang bootstrap in OS DNS; genuine unavailability still falls back. Regression: a stalling loopback DNS server resolves within the timeout (fails on revert). Label: push-back — serf exposes no memberlist cluster label anywhere (TCP/TLS/QUIC transport options and serf Options have no label field; both the reliable and gossip planes pass None; serf-compio is identical). memberlist-reactor surfaces a label at its lower layer but serf does not re-expose it. The reliable-plane cluster boundary is the TLS/QUIC trust anchor (cert verification + SNI) plus the gossip keyring by design; corrected the misleading "supplied via the serf Options block" doc claim on tcp/tls/quic to state this. --- serf-reactor/src/quic/mod.rs | 10 ++- serf-reactor/src/resolver/dns/mod.rs | 35 +++++++---- serf-reactor/src/resolver/dns/tests.rs | 84 ++++++++++++++++++++++++++ serf-reactor/src/tcp/mod.rs | 16 +++-- serf-reactor/src/tls/mod.rs | 21 ++++--- 5 files changed, 141 insertions(+), 25 deletions(-) diff --git a/serf-reactor/src/quic/mod.rs b/serf-reactor/src/quic/mod.rs index 7b38125f..a15994dc 100644 --- a/serf-reactor/src/quic/mod.rs +++ b/serf-reactor/src/quic/mod.rs @@ -51,9 +51,13 @@ use crate::{ /// Embedded into the transport constructor. Bundles the local node identifier, the /// (possibly-unresolved) advertise address, and the caller-built [`QuicOptions`] /// (quinn-proto `EndpointConfig` / `ServerConfig` / `ClientConfig` / -/// `TransportConfig` bundle plus SNI provider). The cluster label and -/// inbound-label-check policy are supplied via the serf `Options` block (not here), -/// feeding both planes from a single validated source. +/// `TransportConfig` bundle plus SNI provider). +/// +/// Serf exposes no memberlist cluster label: neither this block nor the serf +/// `Options` carries one, so both planes run unlabeled. The reliable-plane cluster +/// boundary is the QUIC TLS trust anchor (peer-certificate verification + SNI); +/// gossip is segregated by the encryption keyring. memberlist-reactor's lower-level +/// options DO surface a label; serf, layered on top, does not. pub struct QuicTransportOptions> { local_id: Option, advertise_addr: Option>, diff --git a/serf-reactor/src/resolver/dns/mod.rs b/serf-reactor/src/resolver/dns/mod.rs index 42198c14..8aa905ac 100644 --- a/serf-reactor/src/resolver/dns/mod.rs +++ b/serf-reactor/src/resolver/dns/mod.rs @@ -151,9 +151,11 @@ where /// /// Bounded by `self.timeout` (default [`DEFAULT_DNS_TIMEOUT`]): a slow or hostile /// nameserver cannot hang the caller's `join` future beyond this wall-clock - /// budget. On timeout returns [`DnsError::Io`]`(io::ErrorKind::TimedOut)`; the - /// resolver's caller falls back to the OS resolver via the standard error path - /// (see [`Resolver::resolve`] below). + /// budget. On timeout returns [`DnsError::Io`]`(io::ErrorKind::TimedOut)`, which + /// [`Resolver::resolve`] surfaces WITHOUT the OS fallback — that fallback runs + /// outside this deadline, so escalating a timeout into it would defeat the bound. + /// A genuine unavailability (connect refused, unreachable nameserver, malformed + /// response) or an empty answer does fall through to the OS resolver. async fn tcp_query(&self, host: &str, port: u16) -> Result, DnsError> { let query = self.tcp_query_inner(host, port).fuse(); let timeout = R::sleep(self.timeout).fuse(); @@ -258,14 +260,25 @@ where // when we have at least one nameserver configured. Short names will be // resolved through the OS resolver's search-domain list. if host_str.contains('.') && !self.servers.is_empty() { - // Ignoring Err: TCP-first is best-effort per the upstream spec ("If this - // fails it's not fatal since this isn't a standard way to query DNS, and we - // have a fallback below.", memberlist.go:404). We unconditionally fall - // through to the OS resolver on any error or empty answer. - if let Ok(addrs) = self.tcp_query(host_str, port).await - && !addrs.is_empty() - { - return Ok(addrs); + // TCP-first is best-effort per the upstream spec ("If this fails it's not + // fatal since this isn't a standard way to query DNS, and we have a fallback + // below.", memberlist.go:404), so a genuine unavailability (connect refused, + // unreachable nameserver, malformed response) or an empty answer falls + // through to the OS resolver below. + match self.tcp_query(host_str, port).await { + // A productive TCP answer wins outright; no fallback needed. + Ok(addrs) if !addrs.is_empty() => return Ok(addrs), + // A configured-resolver TIMEOUT must NOT escalate into the OS resolver: the + // OS path runs OUTSIDE `self.timeout` (its DNS is unbounded / + // runtime-dependent), so falling through would let a slow or hostile + // nameserver burn the TCP deadline and THEN hang bootstrap in unbounded OS + // DNS — contradicting the timeout contract. Surface the timeout so the + // configured budget bounds the whole resolution. + Err(DnsError::Io(err)) if err.kind() == io::ErrorKind::TimedOut => { + return Err(DnsError::Io(err)); + } + // Empty answer or a genuine unavailability: fall through to the OS resolver. + Ok(_) | Err(_) => {} } } diff --git a/serf-reactor/src/resolver/dns/tests.rs b/serf-reactor/src/resolver/dns/tests.rs index dec5648e..120bb502 100644 --- a/serf-reactor/src/resolver/dns/tests.rs +++ b/serf-reactor/src/resolver/dns/tests.rs @@ -411,3 +411,87 @@ async fn tcp_query_empty_answers_falls_through_to_os_fallback() { handle.await.expect("DNS server task"); } + +/// A loopback TCP server that ACCEPTS the connection and reads the TCP-DNS query but +/// then STALLS forever without writing a response — modeling a slow or hostile +/// nameserver that keeps the socket open past any deadline. The accepted stream is +/// held alive inside the parked task (a dropped stream would send FIN and let the +/// resolver's `read_exact` return an early EOF instead of blocking on its own +/// timer). Returns the bound address plus the JoinHandle; the caller aborts the +/// handle to tear the server down. +async fn spawn_stalling_tcp_dns_server() -> (SocketAddr, tokio::task::JoinHandle<()>) { + use futures_util::AsyncReadExt; + + let listener = <::Net as Net>::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind loopback DNS server"); + let addr = listener.local_addr().expect("server local_addr"); + + let handle = tokio::spawn(async move { + let Ok((mut stream, _)) = listener.accept().await else { + return; + }; + // Drain the 2-byte length prefix + query body so the resolver's write completes, + // then never respond: hold the stream open and park forever so the resolver's + // `read_exact` on the response blocks until its OWN `R::sleep(timeout)` fires. + let mut len_buf = [0u8; 2]; + if stream.read_exact(&mut len_buf).await.is_err() { + return; + } + let qlen = u16::from_be_bytes(len_buf) as usize; + let mut qbuf = vec![0u8; qlen]; + // Ignoring Err: a truncated read just means the client hung up early; the test's + // assertion still holds because the resolver never received a response either way. + let _ = stream.read_exact(&mut qbuf).await; + // Park forever, keeping `stream` (and thus the open connection) alive. The test + // aborts this task via its JoinHandle once the resolver has timed out. + core::future::pending::<()>().await; + }); + + (addr, handle) +} + +// A configured nameserver that ACCEPTS the TCP-DNS query and then STALLS (never +// writes a response) must not hang the caller: the per-query `timeout` bounds the +// WHOLE resolution, and the timeout is surfaced as an error rather than silently +// escalating into the unbounded OS-resolver fallback (which runs outside the +// deadline). Regression for the bypass where any `tcp_query` error — including +// `TimedOut` — fell through to `self.fallback.resolve`, letting a hostile resolver +// burn the TCP budget and THEN hang bootstrap in OS DNS. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn stalling_server_times_out_without_os_fallback() { + let (server_addr, handle) = spawn_stalling_tcp_dns_server().await; + + let timeout = Duration::from_millis(100); + let r = Dns::from_servers(vec![server_addr]).with_timeout(timeout); + // A fully-qualified name (contains a `.`) so the TCP-first branch is taken; the + // reserved `.test` TLD (RFC 6761) never resolves, so were the buggy OS fallback + // reached it would surface a DIFFERENT lookup error (or hang) rather than our + // synthetic timeout — the assertion below discriminates the fix from the regression. + let addr: Address = "seed.cluster.test:8300".parse().expect("parse FQDN:port"); + assert!(matches!(addr.host(), Host::Domain(_))); + + let start = std::time::Instant::now(); + let err = r + .resolve(&addr) + .await + .map(|_| ()) + .expect_err("a stalling resolver must surface the timeout, not fall through to OS DNS"); + let elapsed = start.elapsed(); + + // The synthetic TCP-DNS timeout is surfaced verbatim, proving the resolution did + // NOT escalate into the OS fallback (which does not produce a `TimedOut`). + assert!( + matches!(&err, DnsError::Io(io_err) if io_err.kind() == io::ErrorKind::TimedOut), + "expected a TimedOut error bounding the whole resolution, got: {err:?}" + ); + // And it returned promptly — bounded by the configured timeout, not the unbounded + // OS resolver. A generous ceiling (20x the 100ms budget) stays robust under CI + // load while still separating a bounded timeout from a fall-through hang. + assert!( + elapsed < Duration::from_secs(2), + "resolve must return within the configured timeout budget, took {elapsed:?}" + ); + + handle.abort(); +} diff --git a/serf-reactor/src/tcp/mod.rs b/serf-reactor/src/tcp/mod.rs index c3bd88f1..732923a5 100644 --- a/serf-reactor/src/tcp/mod.rs +++ b/serf-reactor/src/tcp/mod.rs @@ -38,9 +38,15 @@ use crate::{ /// Per-backend TCP-specific transport options. /// /// Bundles the local node identifier, the (possibly-unresolved) advertise -/// address, and the stream-transport tuning knobs. The cluster label and -/// inbound-label-check policy are supplied via the serf `Options` block (not -/// here), feeding both planes from a single validated source. +/// address, and the stream-transport tuning knobs. +/// +/// Serf exposes no memberlist cluster label: neither this block nor the serf +/// `Options` carries one, so both the reliable and gossip planes run unlabeled (the +/// coordinator built in `Transport::run` passes `None`). On the plain-TCP transport +/// the cluster boundary is the encryption keyring alone — there is no TLS trust +/// anchor, so an unencrypted plain-TCP cluster is segregated only by network +/// reachability. memberlist-reactor's lower-level options DO surface a label; serf, +/// layered on top, does not. pub struct TcpTransportOptions> { local_id: Option, advertise_addr: Option>, @@ -315,7 +321,9 @@ where let stream_timeout = inner_opts.stream_timeout(); let inner = Endpoint::new(inner_opts, gossip_rng); // Plain TCP has no SNI (`|_| None`) and a membership address that IS the - // transport socket (`|addr| *addr`). No cluster label at this stage. + // transport socket (`|addr| *addr`). Serf threads no memberlist cluster label — + // there is none in its options — so the reliable record layer runs unlabeled + // (`None`); on plain TCP the cluster boundary is the encryption keyring alone. #[allow(unused_mut)] let mut coord = Coordinator::<_, _, RawRecords, G>::new( inner, diff --git a/serf-reactor/src/tls/mod.rs b/serf-reactor/src/tls/mod.rs index 899e0f44..341314d6 100644 --- a/serf-reactor/src/tls/mod.rs +++ b/serf-reactor/src/tls/mod.rs @@ -70,10 +70,15 @@ pub type SniProvider = Box Option + Send + Sync>; /// Embedded into the transport constructor. Bundles the local node identifier, /// the (possibly-unresolved) advertise address, the stream-transport tuning /// knobs, the per-peer SNI provider closure, the machine-layer [`TlsOptions`] -/// bundle (cert/key/verifier), and the optional gossip-encryption policy. The -/// cluster label and inbound-label-check policy are supplied via the serf -/// `Options` block (not here), feeding both planes from a single validated -/// source. +/// bundle (cert/key/verifier), and the optional gossip-encryption policy. +/// +/// Serf exposes no memberlist cluster label: neither this block nor the serf +/// `Options` carries one, so both planes run unlabeled (the coordinator built in +/// `Transport::run` passes `None`). The reliable-plane cluster boundary is the TLS +/// trust anchor — peer-certificate verification plus the per-peer SNI — while the +/// gossip plane is segregated by the encryption keyring. memberlist-reactor's +/// lower-level options DO surface a label; serf, layered on top, relies on the TLS +/// trust + keyring instead. pub struct TlsTransportOptions> { local_id: Option, advertise_addr: Option>, @@ -411,9 +416,11 @@ where let inner = Endpoint::new(inner_opts, gossip_rng); // The TLS coordinator carries the per-peer SNI provider and the cert/key bundle // (ridden as the inner options on `LabelOptions`); the membership address IS the - // transport socket (`|addr| *addr`). Like the plain-TCP plane this stage carries - // no cluster label (`None`) — TLS isolation comes from the record-layer cert - // verification and SNI. + // transport socket (`|addr| *addr`). Serf threads no memberlist cluster label — + // there is none in its options — so, like the plain-TCP plane, the reliable + // record layer runs unlabeled (`None`); the cluster boundary here is the TLS + // trust anchor (peer-cert verification + SNI), with gossip segregated by the + // keyring. #[allow(unused_mut)] let mut coord = Coordinator::<_, _, Labeled, G>::new( inner, From 0e2def7ae53cf1d38efa2ac13b5ffd125c676f4f Mon Sep 17 00:00:00 2001 From: al8n Date: Thu, 2 Jul 2026 18:08:20 +1200 Subject: [PATCH 039/128] docs(serf-reactor): state the TLS reliable-plane boundary precisely per client-auth mode The reliable-plane boundary doc claimed TLS peer-cert verification unconditionally, but that holds only under mTLS; with server-auth-only TLS (with_no_client_auth) the acceptor does not authenticate the inbound peer and the gossip keyring does not protect the reliable plane, so inbound cluster membership relies on network policy. The tcp/tls/quic boundary docs now distinguish mTLS from server-auth-only and note serf exposes no cluster label (a deferred serf-wide feature). --- serf-reactor/src/quic/mod.rs | 22 +++++++++++--- serf-reactor/src/tcp/mod.rs | 15 ++++++---- serf-reactor/src/tls/mod.rs | 57 ++++++++++++++++++++++++++---------- 3 files changed, 68 insertions(+), 26 deletions(-) diff --git a/serf-reactor/src/quic/mod.rs b/serf-reactor/src/quic/mod.rs index a15994dc..6fd95631 100644 --- a/serf-reactor/src/quic/mod.rs +++ b/serf-reactor/src/quic/mod.rs @@ -54,10 +54,24 @@ use crate::{ /// `TransportConfig` bundle plus SNI provider). /// /// Serf exposes no memberlist cluster label: neither this block nor the serf -/// `Options` carries one, so both planes run unlabeled. The reliable-plane cluster -/// boundary is the QUIC TLS trust anchor (peer-certificate verification + SNI); -/// gossip is segregated by the encryption keyring. memberlist-reactor's lower-level -/// options DO surface a label; serf, layered on top, does not. +/// `Options` carries one, so both planes run unlabeled. +/// +/// QUIC always runs TLS 1.3, but its reliable-plane inbound cluster boundary still +/// depends on the client-auth mode of the supplied [`QuicOptions`]: +/// +/// - **mTLS** (the quinn `ServerConfig` carries a client-certificate verifier): the +/// boundary IS the QUIC TLS trust anchor — mutual peer-certificate verification +/// plus SNI — so only a peer holding a cluster-trusted client cert can drive a +/// reliable membership merge. +/// - **Server-auth-only** (no client-cert verifier): the acceptor does NOT +/// authenticate the inbound peer, so the reliable plane has no cryptographic +/// inbound cluster-membership check; inbound membership then relies on network +/// policy (firewall / segmentation). The gossip keyring protects only the gossip +/// datagrams, never the QUIC reliable streams. +/// +/// memberlist-reactor's lower-level options DO surface a label; a label-equivalent +/// separation would be a serf-wide product feature (both runtimes), out of scope for +/// this port. pub struct QuicTransportOptions> { local_id: Option, advertise_addr: Option>, diff --git a/serf-reactor/src/tcp/mod.rs b/serf-reactor/src/tcp/mod.rs index 732923a5..aa2a10e5 100644 --- a/serf-reactor/src/tcp/mod.rs +++ b/serf-reactor/src/tcp/mod.rs @@ -42,11 +42,13 @@ use crate::{ /// /// Serf exposes no memberlist cluster label: neither this block nor the serf /// `Options` carries one, so both the reliable and gossip planes run unlabeled (the -/// coordinator built in `Transport::run` passes `None`). On the plain-TCP transport -/// the cluster boundary is the encryption keyring alone — there is no TLS trust -/// anchor, so an unencrypted plain-TCP cluster is segregated only by network -/// reachability. memberlist-reactor's lower-level options DO surface a label; serf, -/// layered on top, does not. +/// coordinator built in `Transport::run` passes `None`). Plain TCP has no +/// transport-layer peer authentication at all — there is no TLS trust anchor — so +/// the reliable-plane cluster boundary is the encryption keyring (a shared symmetric +/// secret that on plain TCP AEAD-protects both planes) plus network policy; an +/// unencrypted plain-TCP cluster is segregated by network reachability alone. +/// memberlist-reactor's lower-level options DO surface a label; serf, layered on +/// top, does not. pub struct TcpTransportOptions> { local_id: Option, advertise_addr: Option>, @@ -323,7 +325,8 @@ where // Plain TCP has no SNI (`|_| None`) and a membership address that IS the // transport socket (`|addr| *addr`). Serf threads no memberlist cluster label — // there is none in its options — so the reliable record layer runs unlabeled - // (`None`); on plain TCP the cluster boundary is the encryption keyring alone. + // (`None`). Plain TCP has no transport-layer peer auth, so the reliable-plane + // boundary is the encryption keyring (or, absent one, network policy alone). #[allow(unused_mut)] let mut coord = Coordinator::<_, _, RawRecords, G>::new( inner, diff --git a/serf-reactor/src/tls/mod.rs b/serf-reactor/src/tls/mod.rs index 341314d6..4cf69c33 100644 --- a/serf-reactor/src/tls/mod.rs +++ b/serf-reactor/src/tls/mod.rs @@ -12,10 +12,13 @@ //! plane stays plain UDP. This is the `Send`/`agnostic` sibling of serf-compio's //! `!Send`, compio-bound `TlsTransport`. //! -//! TLS secures the *reliable* push-pull plane. The unreliable gossip plane is -//! still AEAD-protected by the optional encryption keyring (carried through to the -//! coordinator the same way the TCP plane carries it), so an encrypted cluster -//! protects both planes. +//! TLS gives the *reliable* push-pull plane channel confidentiality and integrity. +//! Whether it also authenticates the inbound peer — the reliable-plane cluster +//! boundary — depends on the client-auth mode of the supplied `rustls` config (mTLS +//! vs. `with_no_client_auth`); see [`TlsTransportOptions`]. The unreliable gossip +//! plane is separately AEAD-protected by the optional encryption keyring (carried +//! through to the coordinator the same way the TCP plane carries it); that keyring +//! never protects the TLS reliable plane. //! //! ## Server name //! @@ -47,6 +50,12 @@ use smol_str::SmolStr; /// TLS machine-options bundle (server + client `rustls` config) handed to /// [`TlsTransport`]. Re-exported from `memberlist-proto` so callers don't need a /// direct dep on it. +/// +/// The client-auth mode of the `rustls` `ServerConfig` in this bundle sets the +/// reliable-plane cluster boundary: a config carrying a client-certificate verifier +/// (mTLS) authenticates every inbound peer, whereas a `with_no_client_auth` +/// (server-auth-only) config does not — see [`TlsTransportOptions`] for the full +/// per-mode contract. pub use memberlist_proto::TlsOptions; #[cfg(encryption)] @@ -74,11 +83,25 @@ pub type SniProvider = Box Option + Send + Sync>; /// /// Serf exposes no memberlist cluster label: neither this block nor the serf /// `Options` carries one, so both planes run unlabeled (the coordinator built in -/// `Transport::run` passes `None`). The reliable-plane cluster boundary is the TLS -/// trust anchor — peer-certificate verification plus the per-peer SNI — while the -/// gossip plane is segregated by the encryption keyring. memberlist-reactor's -/// lower-level options DO surface a label; serf, layered on top, relies on the TLS -/// trust + keyring instead. +/// `Transport::run` passes `None`). +/// +/// The reliable-plane inbound cluster boundary then depends on the client-auth mode +/// of the supplied [`TlsOptions`]: +/// +/// - **mTLS** (the `ServerConfig` carries a client-certificate verifier): the +/// boundary IS the TLS trust anchor — mutual peer-certificate verification plus +/// the per-peer SNI — so only a peer holding a cluster-trusted client cert can +/// drive a reliable membership merge. +/// - **Server-auth-only** (`with_no_client_auth`): the acceptor does NOT +/// authenticate the inbound peer, so the reliable plane has no cryptographic +/// inbound cluster-membership check; inbound membership then relies on network +/// policy (firewall / segmentation), not on the TLS layer. The gossip keyring +/// protects only the gossip (unreliable) plane, never the TLS reliable push/pull. +/// +/// The gossip plane is separately segregated by the encryption keyring under both +/// modes. memberlist-reactor's lower-level options DO surface a label; a +/// label-equivalent separation across shared / server-auth-only TLS deployments +/// would be a serf-wide product feature (both runtimes), out of scope for this port. pub struct TlsTransportOptions> { local_id: Option, advertise_addr: Option>, @@ -166,8 +189,8 @@ impl TlsTransportOptions { /// unencrypted node still builds and interoperates. Attach a keyring /// (`EncryptionOptions::new().with_keyring(Keyring::new(primary_key))`) to /// AEAD-protect the gossip datagrams — every node sharing the cluster MUST carry - /// the same keyring to interop. The reliable plane is secured by the TLS session - /// independently of this keyring. + /// the same keyring to interop. The reliable plane rides the TLS session + /// independently of this keyring, which never covers it. #[cfg(encryption)] #[cfg_attr( docsrs, @@ -418,9 +441,11 @@ where // (ridden as the inner options on `LabelOptions`); the membership address IS the // transport socket (`|addr| *addr`). Serf threads no memberlist cluster label — // there is none in its options — so, like the plain-TCP plane, the reliable - // record layer runs unlabeled (`None`); the cluster boundary here is the TLS - // trust anchor (peer-cert verification + SNI), with gossip segregated by the - // keyring. + // record layer runs unlabeled (`None`). The reliable-plane inbound cluster + // boundary is the TLS trust anchor (mutual peer-cert verification + SNI) ONLY + // under mTLS; a server-auth-only config (`with_no_client_auth`) does not + // authenticate the inbound peer, leaving that check to network policy. Gossip is + // segregated separately by the keyring, which never covers the TLS reliable plane. #[allow(unused_mut)] let mut coord = Coordinator::<_, _, Labeled, G>::new( inner, @@ -431,8 +456,8 @@ where // Install the gossip-encryption keyring so the coordinator's // `encrypt_gossip`/`decrypt_gossip` (forwarded from the serf endpoint pump) // become real on the unreliable plane. A no-keyring policy is the identity - // transform, so an unencrypted node is unaffected. The reliable plane is secured - // by the TLS session regardless. + // transform, so an unencrypted node is unaffected. The reliable plane rides the + // TLS session regardless of the keyring. #[cfg(encryption)] coord.set_encryption_options(self.encryption); // Serf's core RNG is seeded from its own OS-drawn entropy (`self.serf_rng`), From 8c3eae9388d20a98fb9b64b26e0ace3e29e2853e Mon Sep 17 00:00:00 2001 From: Al Liu Date: Thu, 2 Jul 2026 19:30:14 +1200 Subject: [PATCH 040/128] feat(serf-reactor): deterministic non-vacuous negative-encryption test + push_pull_interval knob (#70) --- serf-reactor/src/quic/mod.rs | 42 +++++++++++++++++- serf-reactor/src/tcp/mod.rs | 39 ++++++++++++++++- serf-reactor/src/tls/mod.rs | 39 ++++++++++++++++- serf-reactor/tests/quic.rs | 85 +++++++++++++++++++++++++++++------- 4 files changed, 183 insertions(+), 22 deletions(-) diff --git a/serf-reactor/src/quic/mod.rs b/serf-reactor/src/quic/mod.rs index 6fd95631..65ada43d 100644 --- a/serf-reactor/src/quic/mod.rs +++ b/serf-reactor/src/quic/mod.rs @@ -18,7 +18,7 @@ #![cfg(feature = "quic")] -use core::num::NonZeroU8; +use core::{num::NonZeroU8, time::Duration}; use std::{io::ErrorKind, net::SocketAddr}; use agnostic::{ @@ -76,6 +76,10 @@ pub struct QuicTransportOptions> { local_id: Option, advertise_addr: Option>, quic_config: Option, + /// Override for the memberlist anti-entropy push/pull interval. `None` keeps the + /// coordinator default; `Some(Duration::ZERO)` disables periodic push/pull + /// entirely. See [`with_push_pull_interval`](Self::with_push_pull_interval). + push_pull_interval: Option, /// Gossip-encryption policy. The default (no keyring) leaves the gossip datagrams /// plaintext; attaching a keyring via [`with_encryption`](Self::with_encryption) /// makes the coordinator's `encrypt_gossip`/`decrypt_gossip` AEAD-protect them. @@ -97,6 +101,7 @@ impl QuicTransportOptions { local_id: None, advertise_addr: None, quic_config: None, + push_pull_interval: None, #[cfg(encryption)] encryption: EncryptionOptions::new(), } @@ -126,6 +131,22 @@ impl QuicTransportOptions { self } + /// Builder: override the memberlist anti-entropy push/pull interval. + /// + /// `None` (the default) keeps the coordinator's built-in interval. A positive + /// duration re-tunes the periodic full-state sync; `Duration::ZERO` disables + /// periodic push/pull entirely — join-time and explicit exchanges still run, but + /// no background anti-entropy is scheduled. Disabling it isolates the gossip + /// datagram plane as the sole carrier of ongoing user events and membership + /// deltas, which is exactly what a gossip-encryption conformance test wants to + /// observe. + #[must_use] + #[inline] + pub const fn with_push_pull_interval(mut self, interval: Duration) -> Self { + self.push_pull_interval = Some(interval); + self + } + /// Builder: gossip-encryption policy. /// /// The default (no keyring) keeps the gossip datagrams plaintext, so an @@ -164,6 +185,12 @@ impl QuicTransportOptions { self.quic_config.as_ref() } + /// The push/pull interval override, if set. + #[inline] + pub const fn push_pull_interval(&self) -> Option { + self.push_pull_interval + } + /// Gossip-encryption policy. #[cfg(encryption)] #[cfg_attr( @@ -199,6 +226,10 @@ where advertise_socket: SocketAddr, gossip_socket: ::UdpSocket, quic_config: QuicOptions, + /// Push/pull interval override, applied to the coordinator's `EndpointOptions` in + /// [`Transport::run`]. `None` keeps the default; `Some(Duration::ZERO)` disables + /// periodic anti-entropy. + push_pull_interval: Option, /// Independent OS-seeded seed for the serf core's RNG, drawn once per node in /// [`Transport::new`] and consumed when [`Transport::run`] builds the endpoint via /// `new_with_rng`. Distinct from the coordinator's gossip RNG so serf's query IDs @@ -296,6 +327,7 @@ where advertise_socket: bound, gossip_socket, quic_config, + push_pull_interval: options.push_pull_interval, serf_rng, #[cfg(encryption)] encryption: options.encryption, @@ -327,8 +359,14 @@ where // Serf ranks its user broadcasts on three tiers (intent / event / query → // ranks 0 / 1 / 2), so the inner memberlist endpoint needs at least three // broadcast tiers. - let inner_opts = EndpointOptions::new(self.local_id, self.advertise_socket) + let mut inner_opts = EndpointOptions::new(self.local_id, self.advertise_socket) .with_user_broadcast_tiers(NonZeroU8::new(3).expect("3 is nonzero")); + // A caller-supplied push/pull interval re-tunes (or, at `Duration::ZERO`, + // disables) the periodic anti-entropy full-state sync. Left unset, the + // coordinator keeps its own default. + if let Some(interval) = self.push_pull_interval { + inner_opts = inner_opts.with_push_pull_interval(interval); + } // The shared UDP socket also carries raw QUIC packets, whose size is governed by // the quinn `EndpointConfig`'s accepted max UDP payload — which a caller can set // above the serf gossip MTU (quinn's default 1472 already exceeds the 1400 diff --git a/serf-reactor/src/tcp/mod.rs b/serf-reactor/src/tcp/mod.rs index aa2a10e5..bb45d675 100644 --- a/serf-reactor/src/tcp/mod.rs +++ b/serf-reactor/src/tcp/mod.rs @@ -9,7 +9,7 @@ #![cfg(feature = "tcp")] -use core::num::NonZeroU8; +use core::{num::NonZeroU8, time::Duration}; use std::{io::ErrorKind, net::SocketAddr}; use agnostic::{ @@ -53,6 +53,10 @@ pub struct TcpTransportOptions> { local_id: Option, advertise_addr: Option>, stream: StreamTransportOptions, + /// Override for the memberlist anti-entropy push/pull interval. `None` keeps the + /// coordinator default; `Some(Duration::ZERO)` disables periodic push/pull + /// entirely. See [`with_push_pull_interval`](Self::with_push_pull_interval). + push_pull_interval: Option, /// Gossip-and-reliable encryption policy. The default (no keyring) leaves both /// planes plaintext; attaching a keyring via [`with_encryption`](Self::with_encryption) /// makes the coordinator's `encrypt_gossip`/`decrypt_gossip` (and the plain-TCP @@ -72,6 +76,7 @@ impl TcpTransportOptions { local_id: None, advertise_addr: None, stream: StreamTransportOptions::new(), + push_pull_interval: None, #[cfg(encryption)] encryption: EncryptionOptions::new(), } @@ -101,6 +106,19 @@ impl TcpTransportOptions { self } + /// Builder: override the memberlist anti-entropy push/pull interval. + /// + /// `None` (the default) keeps the coordinator's built-in interval. A positive + /// duration re-tunes the periodic full-state sync; `Duration::ZERO` disables + /// periodic push/pull entirely — join-time and explicit exchanges still run, but + /// no background anti-entropy is scheduled. + #[must_use] + #[inline] + pub const fn with_push_pull_interval(mut self, interval: Duration) -> Self { + self.push_pull_interval = Some(interval); + self + } + /// Builder: gossip-and-reliable encryption policy. #[cfg(encryption)] #[cfg_attr( @@ -132,6 +150,12 @@ impl TcpTransportOptions { &self.stream } + /// The push/pull interval override, if set. + #[inline] + pub const fn push_pull_interval(&self) -> Option { + self.push_pull_interval + } + /// Gossip-and-reliable encryption policy. #[cfg(encryption)] #[cfg_attr( @@ -168,6 +192,10 @@ where gossip_socket: ::UdpSocket, tcp_listener: ::TcpListener, stream_options: StreamTransportOptions, + /// Push/pull interval override, applied to the coordinator's `EndpointOptions` in + /// [`Transport::run`]. `None` keeps the default; `Some(Duration::ZERO)` disables + /// periodic anti-entropy. + push_pull_interval: Option, /// Independent OS-seeded seed for the serf core's RNG, drawn once per node in /// [`Transport::new`] and consumed when [`Transport::run`] builds the endpoint. serf_rng: StdRng, @@ -285,6 +313,7 @@ where gossip_socket, tcp_listener, stream_options: options.stream, + push_pull_interval: options.push_pull_interval, serf_rng, #[cfg(encryption)] encryption: options.encryption, @@ -315,8 +344,14 @@ where // endpoint; build it here from `self`'s stored config. Serf ranks its user // broadcasts on three tiers (intent / event / query → ranks 0 / 1 / 2), so the // inner memberlist endpoint needs at least three broadcast tiers. - let inner_opts = EndpointOptions::new(self.local_id, self.advertise_socket) + let mut inner_opts = EndpointOptions::new(self.local_id, self.advertise_socket) .with_user_broadcast_tiers(NonZeroU8::new(3).expect("3 is nonzero")); + // A caller-supplied push/pull interval re-tunes (or, at `Duration::ZERO`, + // disables) the periodic anti-entropy full-state sync. Left unset, the + // coordinator keeps its own default. + if let Some(interval) = self.push_pull_interval { + inner_opts = inner_opts.with_push_pull_interval(interval); + } // Snapshot the reliable push/pull exchange timeout from the SAME options the // coordinator is built from, so the driver reconciles an await-result join's // caller deadline against the exact deadline the coordinator will stamp. diff --git a/serf-reactor/src/tls/mod.rs b/serf-reactor/src/tls/mod.rs index 4cf69c33..5c63572c 100644 --- a/serf-reactor/src/tls/mod.rs +++ b/serf-reactor/src/tls/mod.rs @@ -32,7 +32,7 @@ #![cfg(feature = "tls")] -use core::num::NonZeroU8; +use core::{num::NonZeroU8, time::Duration}; use std::{io::ErrorKind, net::SocketAddr}; use agnostic::{ @@ -108,6 +108,10 @@ pub struct TlsTransportOptions> { stream: StreamTransportOptions, sni_provider: SniProvider, tls_options: Option, + /// Override for the memberlist anti-entropy push/pull interval. `None` keeps the + /// coordinator default; `Some(Duration::ZERO)` disables periodic push/pull + /// entirely. See [`with_push_pull_interval`](Self::with_push_pull_interval). + push_pull_interval: Option, /// Gossip encryption policy. The default (no keyring) leaves the gossip /// datagrams plaintext; attaching a keyring via /// [`with_encryption`](Self::with_encryption) makes the coordinator's @@ -133,6 +137,7 @@ impl TlsTransportOptions { stream: StreamTransportOptions::new(), sni_provider: Box::new(|_addr: &SocketAddr| Some("localhost".to_string())), tls_options: None, + push_pull_interval: None, #[cfg(encryption)] encryption: EncryptionOptions::new(), } @@ -183,6 +188,19 @@ impl TlsTransportOptions { self } + /// Builder: override the memberlist anti-entropy push/pull interval. + /// + /// `None` (the default) keeps the coordinator's built-in interval. A positive + /// duration re-tunes the periodic full-state sync; `Duration::ZERO` disables + /// periodic push/pull entirely — join-time and explicit exchanges still run, but + /// no background anti-entropy is scheduled. + #[must_use] + #[inline] + pub const fn with_push_pull_interval(mut self, interval: Duration) -> Self { + self.push_pull_interval = Some(interval); + self + } + /// Builder: gossip-encryption policy. /// /// The default (no keyring) keeps the gossip datagrams plaintext, so an @@ -221,6 +239,12 @@ impl TlsTransportOptions { &self.stream } + /// The push/pull interval override, if set. + #[inline] + pub const fn push_pull_interval(&self) -> Option { + self.push_pull_interval + } + /// SNI provider closure. #[inline] pub fn sni_provider(&self) -> &(dyn Fn(&SocketAddr) -> Option + Send + Sync) { @@ -272,6 +296,10 @@ where stream_options: StreamTransportOptions, sni_provider: SniProvider, tls_options: TlsOptions, + /// Push/pull interval override, applied to the coordinator's `EndpointOptions` in + /// [`Transport::run`]. `None` keeps the default; `Some(Duration::ZERO)` disables + /// periodic anti-entropy. + push_pull_interval: Option, /// Independent OS-seeded seed for the serf core's RNG, drawn once per node in /// [`Transport::new`] and consumed when [`Transport::run`] builds the endpoint /// via `new_with_rng`. Distinct from the coordinator's gossip RNG so serf's @@ -400,6 +428,7 @@ where stream_options: options.stream, sni_provider: options.sni_provider, tls_options, + push_pull_interval: options.push_pull_interval, serf_rng, #[cfg(encryption)] encryption: options.encryption, @@ -430,8 +459,14 @@ where // endpoint; build it here from `self`'s stored config. Serf ranks its user // broadcasts on three tiers (intent / event / query → ranks 0 / 1 / 2), so the // inner memberlist endpoint needs at least three broadcast tiers. - let inner_opts = EndpointOptions::new(self.local_id, self.advertise_socket) + let mut inner_opts = EndpointOptions::new(self.local_id, self.advertise_socket) .with_user_broadcast_tiers(NonZeroU8::new(3).expect("3 is nonzero")); + // A caller-supplied push/pull interval re-tunes (or, at `Duration::ZERO`, + // disables) the periodic anti-entropy full-state sync. Left unset, the + // coordinator keeps its own default. + if let Some(interval) = self.push_pull_interval { + inner_opts = inner_opts.with_push_pull_interval(interval); + } // Snapshot the reliable push/pull exchange timeout from the SAME options the // coordinator is built from, so the driver reconciles an await-result join's // caller deadline against the exact deadline the coordinator will stamp. diff --git a/serf-reactor/tests/quic.rs b/serf-reactor/tests/quic.rs index c89294ac..9d5ae976 100644 --- a/serf-reactor/tests/quic.rs +++ b/serf-reactor/tests/quic.rs @@ -462,6 +462,16 @@ fn test_secret_key(fill: u8) -> SecretKey { /// Build and spawn a reactor QUIC node on an ephemeral loopback port with /// `encryption` installed as its gossip keyring policy. +/// +/// Periodic anti-entropy push/pull is disabled (`with_push_pull_interval(ZERO)`). +/// On QUIC the reliable push/pull rides quinn's own TLS — NOT the gossip keyring — +/// and a `PushPullMessage` carries the buffered user events, so a background +/// full-state sync would smuggle a user event across a mismatched gossip keyring, +/// bypassing the AEAD. Disabling it leaves the gossip datagram plane as the sole +/// carrier of ongoing user events, which is exactly the plane these gossip-encryption +/// tests mean to exercise: the positive test then proves the event rode gossip (not +/// an incidental push/pull), and the negative test's absence is decisive rather than +/// a race against the next scheduled sync. Join-time exchanges are unaffected. #[cfg(encryption)] async fn spawn_encrypted_node(id: &str, encryption: EncryptionOptions) -> Node { let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); @@ -469,6 +479,7 @@ async fn spawn_encrypted_node(id: &str, encryption: EncryptionOptions) -> Node { .with_local_id(SmolStr::new(id)) .with_advertise_addr(MaybeResolved::Resolved(bind)) .with_quic_config(test_quic_options()) + .with_push_pull_interval(Duration::ZERO) .with_encryption(encryption); Serf::::quic( opts, @@ -483,6 +494,14 @@ async fn spawn_encrypted_node(id: &str, encryption: EncryptionOptions) -> Node { .expect("spawn encrypted serf quic node") } +/// The window within which a gossip-carried user event is proved to arrive on loopback +/// under a MATCHED keyring (the positive test) — and therefore the window its ABSENCE +/// is decisive over under a MISMATCHED keyring (the negative test). Shared by both so +/// the negative's absence is measured against the exact window the positive proves +/// delivery within, rather than an arbitrarily shorter one a real cross could outlast. +#[cfg(encryption)] +const GOSSIP_DELIVERY_WINDOW: Duration = Duration::from_secs(10); + /// Two QUIC nodes sharing one gossip keyring converge AND exchange gossip: A joins B /// (the reliable push/pull rides quinn's own TLS, so it merges membership /// regardless of the keyring), both reach the two-member cluster, and then a user @@ -516,7 +535,7 @@ async fn two_node_quic_gossip_convergence_encrypted() { .await .expect("user event dispatched"); - let got = tokio::time::timeout(Duration::from_secs(30), async { + let got = tokio::time::timeout(GOSSIP_DELIVERY_WINDOW, async { loop { match a_events.next().await { Some(Event::User(u)) if u.name.as_str() == "greet" => break Some(u.payload.clone()), @@ -526,7 +545,7 @@ async fn two_node_quic_gossip_convergence_encrypted() { } }) .await - .expect("A observes B's user event over the shared-key gossip plane within the timeout"); + .expect("A observes B's user event over the shared-key gossip plane within the shared window"); assert_eq!( got, Some(Bytes::from_static(b"hello")), @@ -541,11 +560,20 @@ async fn two_node_quic_gossip_convergence_encrypted() { /// GOSSIP: on QUIC the reliable push/pull rides quinn's own TLS, so the join still /// merges membership (the gossip keyring does not gate that plane) — but a user /// event, which is disseminated only over the AEAD-sealed gossip datagrams, cannot -/// cross a disjoint key. Within a window shorter than the 30s anti-entropy -/// push/pull interval a user event rides gossip ALONE, so its ABSENCE at A proves -/// the gossip encryption is real enforcement, not an identity pass-through — the -/// discriminating negative the positive test above pairs with (both turn on a -/// gossip-carried event, since the membership merge crosses regardless of the key). +/// cross a disjoint key. Its ABSENCE at A proves the gossip encryption is real +/// enforcement, not an identity pass-through — the discriminating negative the +/// positive test above pairs with (both turn on a gossip-carried event, since the +/// membership merge crosses regardless of the key). +/// +/// Determinism rests on the gossip plane being the event's SOLE carrier. Both nodes +/// run with periodic push/pull disabled (see `spawn_encrypted_node`): a background +/// full-state sync rides quinn TLS and replays a peer's buffered user events, so +/// left enabled it would carry the event over the reliable plane at a random point in +/// its interval — bypassing the gossip AEAD and racing any bounded window. With it +/// off, the join-time exchange (which precedes the broadcast, when B's event buffer +/// is still empty) is the only reliable exchange, and every ongoing user event must +/// ride gossip. A matched key WOULD surface the event within this window (the +/// positive test proves exactly that), so the absence is not vacuous. #[cfg(encryption)] #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn mismatched_keyring_gossip_does_not_cross() { @@ -562,22 +590,28 @@ async fn mismatched_keyring_gossip_does_not_cross() { let b_addr = b.advertise_address(); // The reliable push/pull (quinn TLS) merges membership regardless of the gossip - // keyring, so the await-result join still completes — B is now a known member A - // gossips to. + // keyring, so the await-result join completes and then BOTH nodes hold each other + // as members. Converging to that defined stable state first is what makes the + // later absence "the event was blocked", not "it had not arrived yet". a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) .await .expect("the reliable push/pull merges membership over quinn TLS"); + converge(&a, &b).await; // Subscribe, then have B broadcast a user event over the gossip plane. Under the - // mismatched keyring A cannot decrypt B's gossip datagrams, so it never surfaces - // the event. The window is well under the 30s anti-entropy push/pull interval, so - // the event rides gossip ALONE — a later push/pull cannot carry it into A here. + // mismatched keyring A cannot decrypt B's gossip datagrams, and periodic push/pull + // is disabled, so no plane can carry the event to A. let mut a_events = a.events(); b.user_event("secret", Bytes::from_static(b"hidden"), false) .await .expect("user event dispatched"); - let observed = tokio::time::timeout(Duration::from_secs(5), async { + // Poll over the SAME window the paired positive test proves delivery within, so the + // absence is measured against a proven-sufficient window. A crossed event fails the + // negation; a CLOSED stream (`None`) is NOT success — it would mean A's driver died + // on the bad ciphertext, which cannot prove the event was blocked, so it too is a + // failure. Only a clean timeout (the event never surfaces) is the blocked outcome. + let outcome = tokio::time::timeout(GOSSIP_DELIVERY_WINDOW, async { loop { match a_events.next().await { Some(Event::User(u)) if u.name.as_str() == "secret" => break true, @@ -587,9 +621,28 @@ async fn mismatched_keyring_gossip_does_not_cross() { } }) .await; - assert!( - !matches!(observed, Ok(true)), - "node A must NOT surface B's gossip-carried user event across a mismatched keyring" + match outcome { + Ok(true) => { + panic!("node A surfaced B's gossip-carried user event across a mismatched keyring") + } + Ok(false) => panic!( + "node A's event stream closed before the window elapsed — cannot conclude the \ + mismatched-key event was blocked" + ), + Err(_) => {} + } + // A clean timeout must mean "blocked", not "A died": both nodes must still hold the + // 2-member cluster, so the absence was gossip-AEAD enforcement under a healthy, + // converged pair. + assert_eq!( + a.num_members(), + 2, + "A remains converged after the absence window" + ); + assert_eq!( + b.num_members(), + 2, + "B remains converged after the absence window" ); a.shutdown().await.expect("mis-a shuts down"); From a93a59c59f8f1e2aacdbff5a4b6ce40dee937843 Mon Sep 17 00:00:00 2001 From: Al Liu Date: Thu, 2 Jul 2026 20:42:48 +1200 Subject: [PATCH 041/128] ci: port memberlist Sans-I/O workflows to serf's new crate architecture (#71) --- .github/workflows/ci-compio.yml | 95 +++++++++ .github/workflows/ci-core.yml | 202 +++++++++++++++++++ .github/workflows/ci-smol.yml | 90 +++++++++ .github/workflows/ci-tokio.yml | 82 ++++++++ .github/workflows/ci.yml | 117 ----------- .github/workflows/coverage.yml | 84 ++++---- .github/workflows/doc.yml | 56 ++++++ .github/workflows/embedded.yml.bk | 115 +++++++++++ .github/workflows/fuzz.yml | 12 +- .github/workflows/loc.yml | 49 +++-- .github/workflows/msrv.yml | 53 +++++ .github/workflows/net.yml | 73 ------- .github/workflows/vopr.yml.bk | 89 +++++++++ serf-compio/src/lib.rs | 1 + serf-driver/src/lib.rs | 1 + serf-proto/src/lib.rs | 2 + serf-reactor/Cargo.toml | 16 +- serf-reactor/src/driver/quic/mod.rs | 13 ++ serf-reactor/src/driver/quic/tests.rs | 9 +- serf-reactor/src/lib.rs | 1 + serf-reactor/tests/quic.rs | 247 +++++++++++++++++------ serf-reactor/tests/tcp.rs | 269 ++++++++++++++++++++------ serf-reactor/tests/tls.rs | 145 ++++++++++---- 23 files changed, 1413 insertions(+), 408 deletions(-) create mode 100644 .github/workflows/ci-compio.yml create mode 100644 .github/workflows/ci-core.yml create mode 100644 .github/workflows/ci-smol.yml create mode 100644 .github/workflows/ci-tokio.yml delete mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/doc.yml create mode 100644 .github/workflows/embedded.yml.bk create mode 100644 .github/workflows/msrv.yml delete mode 100644 .github/workflows/net.yml create mode 100644 .github/workflows/vopr.yml.bk diff --git a/.github/workflows/ci-compio.yml b/.github/workflows/ci-compio.yml new file mode 100644 index 00000000..92d9108e --- /dev/null +++ b/.github/workflows/ci-compio.yml @@ -0,0 +1,95 @@ +name: ci-compio + +# The compio async driver: serf-compio's unit tests on compio's single runtime. +# Nodes bind ephemeral loopback ports, so no subnet setup is required. + +on: + push: + branches: + - main + - refactor/sansio + paths-ignore: + - "README.md" + - "COPYRIGHT" + - "LICENSE*" + - "**.md" + - "**.txt" + - "art" + pull_request: + paths-ignore: + - "README.md" + - "COPYRIGHT" + - "LICENSE*" + - "**.md" + - "**.txt" + - "art" + workflow_dispatch: + schedule: [cron: "0 1 */30 * *"] + +env: + # Every transport plus both AEAD backends, so every gated cell runs. Both AEAD + # backends are on so the encryption-validation usability probe sees a present + # backend for every SecretKey variant (it trial-encrypts AES and + # ChaCha20-Poly1305 keys). compio's own `tls` feature drives the record layer + # via compio/rustls; QUIC uses quic-rustls-ring. + FEATURES: "tcp,tls,quic-rustls-ring,aes-gcm,chacha20-poly1305,coordinates,tag-regex,tracing,serde,clap,dns,getifs" + +jobs: + test: + name: ${{ matrix.os }} - compio + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - macos-latest + - windows-latest + steps: + - uses: actions/checkout@v7 + + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: nightly + components: clippy, rustfmt + + # Cache BEFORE the build steps so the restore actually warms this run; a + # cache placed after the test step only saves, never restores. + - name: Cache Cargo registry + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-compio + + - name: Install protoc + uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Format + run: cargo fmt -p serf-compio --check + + - name: Clippy + # --no-deps: lint only our own crate, not its external dependencies. + run: cargo clippy -p serf-compio --all-targets --features "$FEATURES" --no-deps -- -D warnings + + - name: Test + # --no-fail-fast: run every test binary even if one fails, so a single + # OS-specific failure surfaces the rest in the same run. + # + # Windows runs single-threaded: each compio runtime is single-threaded, so + # libtest's default per-core parallelism oversubscribes the CPU and the + # SWIM convergence scenarios can miss their timers under the contention. + run: | + if [ "${{ matrix.os }}" = "windows-latest" ]; then + cargo test -p serf-compio --features "$FEATURES" --no-fail-fast -- --test-threads=1 + else + cargo test -p serf-compio --features "$FEATURES" --no-fail-fast + fi diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml new file mode 100644 index 00000000..396be658 --- /dev/null +++ b/.github/workflows/ci-core.yml @@ -0,0 +1,202 @@ +name: ci-core + +# The runtime-independent core: the serf-proto wire codec + Sans-I/O machine +# and its runtime-agnostic glue crate serf-driver. Both are pure-CPU and bind no +# sockets. This job also builds and tests the preserved `legacy/` serf-core crate +# (excluded from the workspace) so the old implementation can't silently rot. + +on: + push: + branches: + - main + - refactor/sansio + paths-ignore: + - "README.md" + - "COPYRIGHT" + - "LICENSE*" + - "**.md" + - "**.txt" + - "art" + pull_request: + paths-ignore: + - "README.md" + - "COPYRIGHT" + - "LICENSE*" + - "**.md" + - "**.txt" + - "art" + workflow_dispatch: + schedule: [cron: "0 1 */30 * *"] + +jobs: + proto: + name: ${{ matrix.os }} - proto + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash + env: + FEATURES: "tcp,tls,quic-rustls-ring,coordinates,aes-gcm,chacha20-poly1305,tag-regex" + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - macos-latest + - windows-latest + steps: + - uses: actions/checkout@v7 + + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: nightly + components: clippy, rustfmt + + - name: Cache Cargo registry + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-proto + + # serf-proto's build.rs runs protoc (via buffa-build) for the wire codegen, + # and its memberlist-proto dependency's build.rs also runs protoc. + - name: Install protoc + uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Format + run: cargo fmt -p serf-proto -p serf-driver --check + + - name: Clippy + run: cargo clippy -p serf-proto --all-targets --features "$FEATURES" --no-deps -- -D warnings + + - name: Test + run: cargo test -p serf-proto --features "$FEATURES" + + # serf-proto is pure-CPU, so exercise the interesting feature permutations on a + # single OS (default, the coordinate + AEAD combos, all-features, no-default, + # coordinates+aes+tcp, and quic-rustls-ring alone). + proto-features: + name: proto - ${{ matrix.name }} + runs-on: ubuntu-latest + defaults: + run: + shell: bash + strategy: + fail-fast: false + matrix: + include: + - { name: "default", flags: "" } + - { name: "coords-aes-gcm", flags: "--features coordinates,aes-gcm" } + - { name: "coords-chacha", flags: "--features coordinates,chacha20-poly1305" } + - { name: "all-features", flags: "--all-features" } + - { name: "no-default", flags: "--no-default-features" } + - { name: "coords-aes-gcm-tcp", flags: "--features coordinates,aes-gcm,tcp" } + - { name: "quic-rustls-ring", flags: "--no-default-features --features quic-rustls-ring" } + steps: + - uses: actions/checkout@v7 + + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: nightly + components: clippy, rustfmt + + - name: Cache Cargo registry + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-proto-features + + - name: Install protoc + uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Clippy + run: cargo clippy -p serf-proto --all-targets ${{ matrix.flags }} --no-deps -- -D warnings + + - name: Test + run: cargo test -p serf-proto ${{ matrix.flags }} + + driver: + name: driver + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v7 + + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: nightly + components: clippy, rustfmt + + - name: Cache Cargo registry + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-driver + + - name: Install protoc + uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Format + run: cargo fmt -p serf-driver --check + + - name: Clippy + run: cargo clippy -p serf-driver --all-targets --all-features --no-deps -- -D warnings + + - name: Test + run: cargo test -p serf-driver --all-features + + # The preserved pre-Sans-I/O implementation (legacy/serf-core). It is excluded + # from the workspace, so build + test it from its own directory on the feature + # set the old serf CI used, to keep it from silently rotting. + legacy: + name: legacy - serf-core + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v7 + + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: nightly + components: clippy, rustfmt + + - name: Cache Cargo registry + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + legacy/target + key: ${{ runner.os }}-cargo-${{ hashFiles('legacy/**/Cargo.lock') }}-legacy + + - name: Install protoc + uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Test legacy serf-core + run: cargo test --no-default-features --features "test,encryption,serde,quickcheck" + working-directory: legacy/serf-core diff --git a/.github/workflows/ci-smol.yml b/.github/workflows/ci-smol.yml new file mode 100644 index 00000000..76637ef8 --- /dev/null +++ b/.github/workflows/ci-smol.yml @@ -0,0 +1,90 @@ +name: ci-smol + +# The reactor async driver under smol. serf-reactor's real-node integration suites +# (tcp / tls / quic) have runtime-generic scenario bodies with both a `#[tokio::test]` +# cell (run in ci-tokio) and a `_smol` cell driven by `SmolRuntime::block_on`. This +# workflow is the smol gate: it clippy-lints the crate with the `smol` runtime feature +# (proving the `agnostic/smol` code path compiles clean), runs the runtime-independent +# unit tests, and runs the `_smol` integration cells (`--test {tcp,tls,quic} -- smol`, +# where `-- smol` selects only the smol cells across the suite binaries). Nodes bind +# ephemeral loopback ports, so no subnet setup is required. + +on: + push: + branches: + - main + - refactor/sansio + paths-ignore: + - "README.md" + - "COPYRIGHT" + - "LICENSE*" + - "**.md" + - "**.txt" + - "art" + pull_request: + paths-ignore: + - "README.md" + - "COPYRIGHT" + - "LICENSE*" + - "**.md" + - "**.txt" + - "art" + workflow_dispatch: + schedule: [cron: "0 1 */30 * *"] + +env: + FEATURES: "smol,tcp,tls-rustls-ring,quic-rustls-ring,aes-gcm,chacha20-poly1305,coordinates,tag-regex,tracing,serde,clap,dns,getifs" + +jobs: + test: + name: ${{ matrix.os }} - reactor - smol + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - macos-latest + - windows-latest + steps: + - uses: actions/checkout@v7 + + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: nightly + components: clippy, rustfmt + + # Cache BEFORE the build steps so the restore actually warms this run; a + # cache placed after the test step only saves, never restores. + - name: Cache Cargo registry + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-reactor-smol + + - name: Install protoc + uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Clippy (smol runtime path) + # --no-deps: lint only our own crate, not its external dependencies. + run: cargo clippy -p serf-reactor --all-targets --features "$FEATURES" --no-deps -- -D warnings + + - name: Test (runtime-independent unit tests) + run: cargo test -p serf-reactor --lib --features "$FEATURES" + + - name: Test (smol integration cells) + # `-- smol` selects only the `_smol` cells in each suite binary; the tokio + # cells over the same runtime-generic scenarios run in ci-tokio. + run: | + cargo test -p serf-reactor --features "smol,tcp,tls-rustls-ring,quic-rustls-ring,aes-gcm,chacha20-poly1305,coordinates,tag-regex,dns,getifs" --test tcp -- smol + cargo test -p serf-reactor --features "smol,tcp,tls-rustls-ring,quic-rustls-ring,aes-gcm,chacha20-poly1305,coordinates,tag-regex,dns,getifs" --test tls -- smol + cargo test -p serf-reactor --features "smol,tcp,tls-rustls-ring,quic-rustls-ring,aes-gcm,chacha20-poly1305,coordinates,tag-regex,dns,getifs" --test quic -- smol diff --git a/.github/workflows/ci-tokio.yml b/.github/workflows/ci-tokio.yml new file mode 100644 index 00000000..6c7db465 --- /dev/null +++ b/.github/workflows/ci-tokio.yml @@ -0,0 +1,82 @@ +name: ci-tokio + +# The reactor async driver under tokio: the crate's unit tests plus its real-node +# integration suites (tcp / tls / quic), which are written as `#[tokio::test]` +# and gated on the `tokio` feature. Nodes bind ephemeral loopback ports, so no +# subnet setup is required. The smol runtime path is a build/clippy gate in +# ci-smol (the integration suites are tokio-only). + +on: + push: + branches: + - main + - refactor/sansio + paths-ignore: + - "README.md" + - "COPYRIGHT" + - "LICENSE*" + - "**.md" + - "**.txt" + - "art" + pull_request: + paths-ignore: + - "README.md" + - "COPYRIGHT" + - "LICENSE*" + - "**.md" + - "**.txt" + - "art" + workflow_dispatch: + schedule: [cron: "0 1 */30 * *"] + +env: + FEATURES: "tokio,tcp,tls-rustls-ring,quic-rustls-ring,aes-gcm,chacha20-poly1305,coordinates,tag-regex,tracing,serde,clap,dns,getifs" + +jobs: + test: + name: ${{ matrix.os }} - reactor - tokio + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash + strategy: + fail-fast: false + matrix: + os: + - ubuntu-latest + - macos-latest + - windows-latest + steps: + - uses: actions/checkout@v7 + + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: nightly + components: clippy, rustfmt + + # Cache BEFORE the build steps so the restore actually warms this run; a + # cache placed after the test step only saves, never restores. + - name: Cache Cargo registry + uses: actions/cache@v6 + with: + path: | + ~/.cargo/registry + ~/.cargo/git + target + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-reactor-tokio + + - name: Install protoc + uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Format + run: cargo fmt -p serf-reactor --check + + - name: Clippy + # --no-deps: lint only our own crate, not its external dependencies. + run: cargo clippy -p serf-reactor --all-targets --features "$FEATURES" --no-deps -- -D warnings + + - name: Test + run: cargo test -p serf-reactor --features "$FEATURES" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index d54f610b..00000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,117 +0,0 @@ -name: ci - -on: - push: - branches: - - main - paths-ignore: - - "README.md" - - "COPYRIGHT" - - "LICENSE*" - - "**.md" - - "**.txt" - - "art" - pull_request: - paths-ignore: - - "README.md" - - "COPYRIGHT" - - "LICENSE*" - - "**.md" - - "**.txt" - - "art" - workflow_dispatch: - schedule: - - cron: "0 1 */30 * *" - -jobs: - test: - name: ${{ matrix.os }} - ${{ matrix.runtime }} - ${{ matrix.stream_layer }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: - - ubuntu-latest - # - macos-latest, - # - windows-latest - runtime: [tokio, smol] - stream_layer: - # - tls - # - native-tls - - tcp - steps: - - uses: actions/checkout@v6 - - - name: Install Rust - run: | - rustup update stable && rustup default stable - rustup component add clippy - rustup component add rustfmt - - - name: Setup loopback interface (Windows) - if: matrix.os == 'windows-latest' - shell: powershell - run: ci\setup_subnet_windows.ps1 - - name: Setup loopback interface (MacOS) - if: matrix.os == 'macos-latest' - run: ci/setup_subnet_macos.sh - - name: Setup loopback interface (Ubuntu) - if: matrix.os == 'ubuntu-latest' - run: ci/setup_subnet_ubuntu.sh - - - name: Run Unit Tests for core - run: | - cargo test --no-default-features --features "test,encryption,serde,quickcheck" - working-directory: serf-core - - - name: Cache Cargo registry - uses: actions/cache@v5 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-${{ matrix.runtime }}-${{ matrix.stream_layer }} - - serf-proto: - name: serf-proto - ${{ matrix.name }} - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - { name: "default", flags: "" } - - { name: "coords-aes-gcm", flags: "--features coordinates,aes-gcm" } - - { name: "coords-chacha", flags: "--features coordinates,chacha20-poly1305" } - - { name: "all-features", flags: "--all-features" } - - { name: "no-default", flags: "--no-default-features" } - - { name: "coords-aes-gcm-tcp", flags: "--features coordinates,aes-gcm,tcp" } - - { name: "quic-rustls-ring", flags: "--no-default-features --features quic-rustls-ring" } - steps: - - uses: actions/checkout@v6 - - - name: Install Rust - run: | - rustup update stable && rustup default stable - rustup component add clippy - rustup component add rustfmt - - - name: Format check - if: matrix.name == 'default' - run: cargo fmt --check - working-directory: serf-proto - - - name: Clippy - run: cargo clippy -p serf-proto --all-targets ${{ matrix.flags }} -- -D warnings - - - name: Test - run: cargo test -p serf-proto ${{ matrix.flags }} - - - name: Cache Cargo registry - uses: actions/cache@v5 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-serf-proto-${{ hashFiles('**/Cargo.lock') }}-${{ matrix.name }} diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index ee2d1b2d..f1f99098 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - refactor/sansio paths-ignore: - "README.md" - "COPYRIGHT" @@ -25,55 +26,54 @@ env: CARGO_TERM_COLOR: always jobs: - coverage-report: - name: Generate Coverage for ${{ matrix.crate }}-${{ matrix.name }} + coverage: + name: coverage runs-on: ubuntu-latest - strategy: - matrix: - include: - - crate: serf-core - features: "test,metrics,quickcheck,encryption" - name: "serf-core" - - crate: serf - features: "test,tokio,tcp,encryption,metrics" - name: "serf-tcp-encryption" - - crate: serf-proto - features: "encryption,coordinates,tag-regex" - name: "serf-proto" + defaults: + run: + shell: bash steps: - - uses: actions/checkout@v6 - - name: Install latest nightly - run: rustup toolchain install nightly && rustup default nightly - - name: Setup loopback interface - run: ci/setup_subnet_ubuntu.sh - - name: Install cargo-tarpaulin - run: cargo install cargo-tarpaulin - - name: Run Coverage and Generate Report - run: | - cargo tarpaulin --out xml --no-default-features --features ${{ matrix.features }} --no-fail-fast --output-dir "$PWD" -j 1 - working-directory: ${{ matrix.crate }} - continue-on-error: true - - name: Upload Coverage Report as Artifact - uses: actions/upload-artifact@v7 + - uses: actions/checkout@v7 + + - name: Install Rust + uses: dtolnay/rust-toolchain@master with: - name: ${{ matrix.name }} - path: ${{ matrix.crate }}/cobertura.xml + toolchain: nightly + components: llvm-tools-preview - upload-codecov: - needs: coverage-report - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - - name: Download all coverage reports - uses: actions/download-artifact@v8 + - uses: Swatinem/rust-cache@v2 + + - name: Install protoc + uses: arduino/setup-protoc@v3 with: - path: reports/ - - name: List contents of the reports directory - run: ls -a reports + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@v2 + with: + tool: cargo-llvm-cov + + - name: Collect coverage + run: | + # Accumulate per-crate runs into a shared profile, then emit one lcov + # report. Each crate runs a representative feature combo that maximizes + # covered code (every transport plus both AEAD backends and the + # coordinate estimator). serf-reactor needs a concrete runtime to run + # its real-node integration suites, so it runs on tokio; serf-compio is + # self-contained on its own runtime. + cargo llvm-cov --no-report -p serf-proto \ + --features "tcp,tls,quic-rustls-ring,coordinates,aes-gcm,chacha20-poly1305,tag-regex" + cargo llvm-cov --no-report -p serf-driver --all-features + cargo llvm-cov --no-report -p serf-reactor \ + --features "tokio,tcp,tls-rustls-ring,quic-rustls-ring,aes-gcm,chacha20-poly1305,coordinates,tag-regex,tracing,serde,clap,dns,getifs" + cargo llvm-cov --no-report -p serf-compio \ + --features "tcp,tls,quic-rustls-ring,aes-gcm,chacha20-poly1305,coordinates,tag-regex,tracing,serde,clap,dns,getifs" + cargo llvm-cov report --lcov --output-path lcov.info + - name: Upload to codecov.io - uses: codecov/codecov-action@v6 + uses: codecov/codecov-action@v7 with: - directory: reports + files: lcov.info fail_ci_if_error: false slug: ${{ github.repository }} env: diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml new file mode 100644 index 00000000..8532cef8 --- /dev/null +++ b/.github/workflows/doc.yml @@ -0,0 +1,56 @@ +name: doc + +# Build the rustdoc for the crates with the docs.rs cfg, so a doc break (a bad +# intra-doc link target, an unbuildable feature combination, a rustdoc +# regression) is caught BEFORE publish instead of on docs.rs. Not +# warnings-as-errors: serf-proto may carry pre-existing intra-doc-link warnings; +# this gate catches hard failures and can ratchet to -D warnings later. + +on: + push: + branches: + - main + - refactor/sansio + paths-ignore: + - "**.txt" + - "art" + pull_request: + paths-ignore: + - "**.txt" + - "art" + workflow_dispatch: + +env: + RUSTDOCFLAGS: "--cfg docsrs" + +jobs: + doc: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: nightly + - uses: Swatinem/rust-cache@v2 + - name: Install protoc + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler + + - name: Doc serf-proto + run: > + cargo doc -p serf-proto --no-deps + --features tcp,tls,quic-rustls-ring,coordinates,aes-gcm,chacha20-poly1305,tag-regex + + - name: Doc serf-driver + run: cargo doc -p serf-driver --no-deps --all-features + + - name: Doc serf-reactor + run: > + cargo doc -p serf-reactor --no-deps + --features tokio,tcp,tls-rustls-ring,quic-rustls-ring,coordinates,aes-gcm,chacha20-poly1305,tag-regex,tracing,serde,clap,dns,getifs + + - name: Doc serf-compio + run: > + cargo doc -p serf-compio --no-deps + --features tcp,tls,quic-rustls-ring,coordinates,aes-gcm,chacha20-poly1305,tag-regex,tracing,serde,clap,dns,getifs diff --git a/.github/workflows/embedded.yml.bk b/.github/workflows/embedded.yml.bk new file mode 100644 index 00000000..36fe6960 --- /dev/null +++ b/.github/workflows/embedded.yml.bk @@ -0,0 +1,115 @@ +# INACTIVE PLACEHOLDER — GitHub Actions does NOT run `.bk` files. +# +# This is a ready-to-activate template for FUTURE serf no_std / embedded driver +# support. serf has no embedded/smoltcp/embassy crates yet; the crate names below +# (serf-embedded / serf-smoltcp / serf-embassy) are anticipated equivalents of +# memberlist's embedded stack. When those crates exist, rename this file +# `embedded.yml.bk` -> `embedded.yml` and adjust the crate names / feature flags +# to match what the crates actually expose. + +name: embedded + +on: + push: + branches: + - main + - refactor/sansio + paths-ignore: + - "README.md" + - "COPYRIGHT" + - "LICENSE*" + - "**.md" + - "**.txt" + - "art" + pull_request: + paths-ignore: + - "README.md" + - "COPYRIGHT" + - "LICENSE*" + - "**.md" + - "**.txt" + - "art" + workflow_dispatch: + +jobs: + # Bare-metal compile gate: the `no_std` + `alloc` engine and both embedded + # drivers must build for a real Cortex-M target. A `no_std` lib also compiles on + # a `std` host, so the host jobs don't exercise the no_std path; this does. + # `cargo check` doesn't link, so no embedded runtime or cross-linker is needed. + # The gossip RNG's default entropy draw needs a backend on a bare-metal target, + # so the custom-backend cfg is set (the symbol itself is only required when an + # actual binary links — the qemu job below). + embedded-check: + runs-on: ubuntu-latest + env: + RUSTFLAGS: -D warnings --cfg getrandom_backend="custom" + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: nightly + targets: thumbv7em-none-eabihf + - uses: Swatinem/rust-cache@v2 + # serf-proto's build.rs runs protoc for the wire codegen, so protoc is + # needed even for a bare-metal `cargo check` (build scripts run on the host). + - name: Install protoc + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler + - run: > + cargo check + -p serf-embedded + -p serf-smoltcp + -p serf-embassy + --no-default-features --features alloc + --target thumbv7em-none-eabihf + + # Host test gate: the smoltcp and embassy drivers carry host-runnable test + # suites (block_on + bounded virtual time) that the bare-metal check above does + # not exercise. Run them on a std host, plus fmt + clippy for both libs. + host-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: nightly + components: clippy, rustfmt + - uses: Swatinem/rust-cache@v2 + - name: Install protoc + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler + - name: Format + run: cargo fmt -p serf-smoltcp -p serf-embassy --check + - name: Clippy + run: cargo clippy -p serf-smoltcp -p serf-embassy --all-targets --features aes-gcm,coordinates --no-deps -- -D warnings + - name: Test + run: cargo test -p serf-smoltcp -p serf-embassy --features aes-gcm,coordinates + + # Execution proof: build the `serf-embassy-qemu` firmware and run it under QEMU. + # The crate's `.cargo/config.toml` supplies the target, the linker script, the + # custom getrandom cfg, and a `qemu-system-arm` runner with semihosting on, so + # `cargo run` boots two serf nodes on an emulated Cortex-M4 and the firmware's + # `semihosting::process::exit` code becomes the job's pass/fail. + # + # No workflow `RUSTFLAGS` is set here: a `RUSTFLAGS` env var replaces (does not + # append to) the crate's `.cargo/config.toml` rustflags, which would drop the + # linker script and the getrandom cfg and break the link. The crate is excluded + # from the workspace, so it is built only by this `cargo run`. + qemu: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: nightly + targets: thumbv7em-none-eabihf + - uses: Swatinem/rust-cache@v2 + - name: Install QEMU and protoc + run: | + sudo apt-get update + sudo apt-get install -y qemu-system-arm protobuf-compiler + - name: Two nodes join under QEMU + run: cargo run + working-directory: tests/serf-embassy-qemu diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 0179048b..d1059e47 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -1,8 +1,14 @@ name: Fuzz Testing + +# Fuzz the preserved legacy serf-core wire codec. The fuzz crate (serf-types-fuzz, +# target `messages`) lives in the excluded `legacy/` workspace and fuzzes serf's +# message (de)serialization; it is built + briefly run from that workspace root. + on: push: branches: - main + - refactor/sansio paths-ignore: - "README.md" - "COPYRIGHT" @@ -24,8 +30,12 @@ on: jobs: fuzz: runs-on: ubuntu-latest + defaults: + run: + shell: bash + working-directory: legacy steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Install Rust uses: dtolnay/rust-toolchain@master diff --git a/.github/workflows/loc.yml b/.github/workflows/loc.yml index d0730a26..439c4614 100644 --- a/.github/workflows/loc.yml +++ b/.github/workflows/loc.yml @@ -4,6 +4,7 @@ on: push: branches: - main + - refactor/sansio paths-ignore: - "README.md" - "COPYRIGHT" @@ -25,36 +26,50 @@ jobs: loc: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@v7 - name: Install Rust - run: | - rustup update stable && rustup default stable - rustup component add clippy - rustup component add rustfmt + uses: dtolnay/rust-toolchain@master + with: + toolchain: nightly + components: clippy, rustfmt - name: Install tokeit run: | cargo install tokeit --force - - name: Count lines of code + - name: Count lines of code (workspace total + each crate) run: | + set -euo pipefail tokeit + mv tokeit.json loc-serf.json + for c in serf-proto serf-driver serf-compio serf-reactor; do + ( cd "$c" && tokeit ) + mv "$c/tokeit.json" "loc-$c.json" + done - - name: Upload loc to GitHub Gist + - name: Upload all loc to the gist in one request uses: actions/github-script@v9 with: github-token: ${{ secrets.GIST_PAT }} script: | const fs = require('fs'); - const output = fs.readFileSync('tokeit.json', 'utf8'); const gistId = 'd29ceff54c025fe4e8b144a51efb9324'; - await github.rest.gists.update({ - gist_id: gistId, - files: { - "serf": { - content: output - } - } - }); - console.log("Gist updated"); + // Gist file `serf` is the workspace total (the main README's LoC + // badge reads …/raw/serf); the rest are per-crate, ready for + // per-crate READMEs. The gists API accepts every file in one PATCH, + // so one request avoids the per-crate rate-limit storm. + const crates = [ + 'serf-proto', + 'serf-driver', + 'serf-compio', + 'serf-reactor', + ]; + const files = { + serf: { content: fs.readFileSync('loc-serf.json', 'utf8') }, + }; + for (const c of crates) { + files[c] = { content: fs.readFileSync(`loc-${c}.json`, 'utf8') }; + } + await github.rest.gists.update({ gist_id: gistId, files }); + console.log('Gist updated'); diff --git a/.github/workflows/msrv.yml b/.github/workflows/msrv.yml new file mode 100644 index 00000000..99bb6929 --- /dev/null +++ b/.github/workflows/msrv.yml @@ -0,0 +1,53 @@ +name: msrv + +# Validate the crates build on the stable channel and on the declared MSRV. +# MSRV is 1.96.0: serf depends on memberlist-proto, whose dependency tree floors +# at 1.96 (compio-buf adopts the just-stabilized maybe_uninit_slice), so the +# workspace does not build below this regardless of serf's own rust-version. +# Every other workflow pins a specific nightly, so without this nothing proves +# the crates compile on stable or at their stated minimum. + +on: + push: + branches: + - main + - refactor/sansio + paths-ignore: + - "**.md" + - "**.txt" + - "art" + pull_request: + paths-ignore: + - "**.md" + - "**.txt" + - "art" + workflow_dispatch: + +jobs: + check: + name: ${{ matrix.toolchain }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + toolchain: + - stable + - "1.96.0" + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: ${{ matrix.toolchain }} + - uses: Swatinem/rust-cache@v2 + - name: Install protoc + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler + - name: Check crates + run: > + cargo check + -p serf-proto + -p serf-driver + -p serf-reactor + -p serf-compio + --features serf-reactor/tokio,serf-reactor/tcp,serf-reactor/quic-rustls-ring,serf-compio/tcp,serf-compio/quic-rustls-ring diff --git a/.github/workflows/net.yml b/.github/workflows/net.yml deleted file mode 100644 index 0b08611a..00000000 --- a/.github/workflows/net.yml +++ /dev/null @@ -1,73 +0,0 @@ -name: net - -on: - push: - branches: - - main - paths-ignore: - - "README.md" - - "COPYRIGHT" - - "LICENSE*" - - "**.md" - - "**.txt" - - "art" - pull_request: - paths-ignore: - - "README.md" - - "COPYRIGHT" - - "LICENSE*" - - "**.md" - - "**.txt" - - "art" - workflow_dispatch: - schedule: - - cron: "0 1 1 * *" - -jobs: - test: - name: ${{ matrix.os }} - ${{ matrix.runtime }} - ${{ matrix.stream_layer }} - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - os: - - ubuntu-latest - # - macos-latest, - # - windows-latest - runtime: [tokio, smol] - stream_layer: - - tls - - tcp - steps: - - uses: actions/checkout@v6 - - - name: Install Rust - run: | - rustup update stable && rustup default stable - rustup component add clippy - rustup component add rustfmt - - - name: Setup loopback interface (Windows) - if: matrix.os == 'windows-latest' - shell: powershell - run: ci\setup_subnet_windows.ps1 - - name: Setup loopback interface (MacOS) - if: matrix.os == 'macos-latest' - run: ci/setup_subnet_macos.sh - - name: Setup loopback interface (Ubuntu) - if: matrix.os == 'ubuntu-latest' - run: ci/setup_subnet_ubuntu.sh - - - name: Run Unit Tests for serf based on net transport - run: | - cargo test --no-default-features --features "test,encryption,net,metrics,${{ matrix.runtime }},${{ matrix.stream_layer }}" -- --test-threads=1 - working-directory: serf - - - name: Cache Cargo registry - uses: actions/cache@v5 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - target - key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }}-${{ matrix.runtime }}-${{ matrix.stream_layer }} diff --git a/.github/workflows/vopr.yml.bk b/.github/workflows/vopr.yml.bk new file mode 100644 index 00000000..f50c0d58 --- /dev/null +++ b/.github/workflows/vopr.yml.bk @@ -0,0 +1,89 @@ +# INACTIVE PLACEHOLDER — GitHub Actions does NOT run `.bk` files. +# +# This is a ready-to-activate template for FUTURE serf deterministic-simulation +# (VOPR) support. serf has no simulation crate yet; `serf-simulation` below is the +# anticipated equivalent of memberlist's simulation crate. When that crate exists, +# rename this file `vopr.yml.bk` -> `vopr.yml` and adjust the crate name, the test +# target, and the feature flags to match what the crate actually exposes. + +name: vopr + +# Deterministic adversarial SWIM simulation (VOPR). Each run drives a contiguous +# band of seeds; every seed is a pure function of (seed, ticks) and panics with +# its seed on any safety or liveness violation, so a failure replays exactly. +# +# This is a heavy, release-built campaign kept out of the routine test workflow: +# - manual dispatch runs a chosen band (defaults to 0..4096 x 5000 ticks), +# - the nightly schedule rotates to a fresh 4096-seed band for continuous +# coverage (scheduled runs only fire from the default branch), +# - a push touching the simulation or the state machine re-runs the default +# band as a regression gate. +on: + workflow_dispatch: + inputs: + seed_start: + description: First seed (inclusive) + default: "0" + seed_end: + description: Last seed (exclusive) + default: "4096" + ticks: + description: Chaos ticks per seed + default: "5000" + schedule: + - cron: "0 4 * * *" + push: + branches: + - refactor/sansio + - vopr + paths: + - "tests/serf-simulation/**" + - "serf-proto/**" + - ".github/workflows/vopr.yml" + +jobs: + campaign: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: nightly + - uses: Swatinem/rust-cache@v2 + # serf-proto's build.rs runs protoc for the wire codegen. + - name: Install protoc + run: | + sudo apt-get update + sudo apt-get install -y protobuf-compiler + + # Pick the seed band: the dispatch inputs, a schedule-rotated fresh band, + # or the default regression band on push. + - name: Compute seed band + id: band + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then + START="${{ github.event.inputs.seed_start }}" + END="${{ github.event.inputs.seed_end }}" + TICKS="${{ github.event.inputs.ticks }}" + elif [ "${{ github.event_name }}" = "schedule" ]; then + START=$(( ${{ github.run_number }} * 4096 )) + END=$(( START + 4096 )) + TICKS=5000 + else + START=0 + END=4096 + TICKS=5000 + fi + echo "start=$START" >> "$GITHUB_OUTPUT" + echo "end=$END" >> "$GITHUB_OUTPUT" + echo "ticks=$TICKS" >> "$GITHUB_OUTPUT" + echo "VOPR band: seeds [$START, $END) x $TICKS ticks" + + - name: Run the VOPR campaign (release) + env: + VOPR_SEED_START: ${{ steps.band.outputs.start }} + VOPR_SEED_END: ${{ steps.band.outputs.end }} + VOPR_TICKS: ${{ steps.band.outputs.ticks }} + run: > + cargo test --release -p serf-simulation + --test vopr full_campaign -- --ignored --nocapture diff --git a/serf-compio/src/lib.rs b/serf-compio/src/lib.rs index 4a71202a..223ae7ed 100644 --- a/serf-compio/src/lib.rs +++ b/serf-compio/src/lib.rs @@ -32,6 +32,7 @@ #![deny(missing_docs)] #![cfg_attr(docsrs, feature(doc_cfg))] +#![cfg_attr(docsrs, allow(unused_attributes))] #[cfg(feature = "tcp")] mod bridge; diff --git a/serf-driver/src/lib.rs b/serf-driver/src/lib.rs index 8cfe401d..63d8d703 100644 --- a/serf-driver/src/lib.rs +++ b/serf-driver/src/lib.rs @@ -9,6 +9,7 @@ #![deny(missing_docs)] #![cfg_attr(docsrs, feature(doc_cfg))] +#![cfg_attr(docsrs, allow(unused_attributes))] pub mod error; #[cfg(any(feature = "tcp", feature = "quic"))] diff --git a/serf-proto/src/lib.rs b/serf-proto/src/lib.rs index 61c34bf6..0c5ea6d9 100644 --- a/serf-proto/src/lib.rs +++ b/serf-proto/src/lib.rs @@ -4,6 +4,8 @@ //! Depends on `memberlist-proto` for the `Data`/`DataRef` codec primitives; defines serf's //! own message set and framing on top of them. #![deny(missing_docs)] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![cfg_attr(docsrs, allow(unused_attributes))] #[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) use any::{AnyMessage, EncodeError}; diff --git a/serf-reactor/Cargo.toml b/serf-reactor/Cargo.toml index 4b486b68..6945e875 100644 --- a/serf-reactor/Cargo.toml +++ b/serf-reactor/Cargo.toml @@ -103,24 +103,30 @@ serde_json = "1" tokio = { version = "1", features = ["rt-multi-thread", "macros", "time"] } # The real-node TCP test suite (two loopback nodes: join/converge, user event, -# query round-trip, leave/LeftCluster, snapshot forwarders, shutdown). +# query round-trip, leave/LeftCluster, snapshot forwarders, shutdown). The scenario +# bodies are runtime-generic; the file carries a `#[tokio::test]` cell per scenario +# (gated on `tokio`) and a `_smol` cell driven by `SmolRuntime::block_on` (gated on +# `smol`), so `--features smol,tcp --test tcp -- smol` builds and runs the smol cells +# alone. The target requires only the transport feature; a runtime feature (tokio or +# smol) supplies the cells. [[test]] name = "tcp" -required-features = ["tcp", "tokio"] +required-features = ["tcp"] # The real-node TLS test suite (two loopback nodes over a self-signed cert + # accept-any verifier: join/converge, user event, query round-trip, leave/shutdown). +# Tokio + smol cells over the runtime-generic scenarios (see the `tcp` target). [[test]] name = "tls" -required-features = ["tls-rustls-ring", "tokio"] +required-features = ["tls-rustls-ring"] # The real-node QUIC test suite (two loopback nodes over a quinn config bundle: # join/converge, user event, query round-trip, leave/LeftCluster, shutdown, and — # under an encryption backend — encrypted-gossip convergence + mismatched-key -# enforcement). +# enforcement). Tokio + smol cells over the runtime-generic scenarios (see `tcp`). [[test]] name = "quic" -required-features = ["quic-rustls-ring", "tokio"] +required-features = ["quic-rustls-ring"] [package.metadata.docs.rs] # A single coherent crypto-provider set (ring, not aws-lc-rs; tokio, not smol) so diff --git a/serf-reactor/src/driver/quic/mod.rs b/serf-reactor/src/driver/quic/mod.rs index 56bf9d05..d55cbcce 100644 --- a/serf-reactor/src/driver/quic/mod.rs +++ b/serf-reactor/src/driver/quic/mod.rs @@ -328,6 +328,13 @@ where /// quiescent stop). #[cfg(test)] recv_errors_remaining: usize, + /// Test seam: when set, `poll_recv_once` reports a `Poll::Pending` (kernel-empty) + /// quiescent stop WITHOUT reading the real socket. On Windows a UDP `recv_from` + /// after sending to a closed port returns `ConnectionReset` (the ICMP + /// port-unreachable), so a pump test needing a deterministic quiescent stop scripts + /// it here rather than relying on the real socket returning `Pending`. + #[cfg(test)] + recv_force_pending: bool, /// The driver's keyring delegate: applies inbound key-management ops and produces /// the `respond_key` answer. Present only under an encryption backend. #[cfg(encryption)] @@ -378,6 +385,8 @@ where leave_timeout: driver_opts.leave_timeout(), #[cfg(test)] recv_errors_remaining: 0, + #[cfg(test)] + recv_force_pending: false, #[cfg(encryption)] keyring, } @@ -395,6 +404,10 @@ where std::io::ErrorKind::ConnectionRefused, ))); } + #[cfg(test)] + if self.recv_force_pending { + return Poll::Pending; + } let Some(socket) = self.socket.as_ref() else { return Poll::Pending; }; diff --git a/serf-reactor/src/driver/quic/tests.rs b/serf-reactor/src/driver/quic/tests.rs index b2bbda8b..031b4898 100644 --- a/serf-reactor/src/driver/quic/tests.rs +++ b/serf-reactor/src/driver/quic/tests.rs @@ -311,9 +311,12 @@ mod gate { the join, but a completing packet may sit behind the error", ); - // Poll B: the recv loop now stops on a genuine `Poll::Pending` (nothing was ever - // sent to the bound socket). The gate is quiescent, so the past-due reap fires and - // resolves the stuck join `JoinAllFailed`. + // Poll B: the recv loop now stops on a `Poll::Pending` quiescent stop, scripted so + // it is deterministic across platforms. (On Windows the join's QUIC packet to a + // closed port draws an ICMP port-unreachable, so a real `recv_from` would return + // `ConnectionReset` — another non-quiescent error stop — rather than `Pending`.) + // The gate is quiescent, so the past-due reap fires and resolves the stuck join. + driver.recv_force_pending = true; let _ = poll_once(&mut driver); match rx.try_recv().expect("reply channel live") { Some(Err((reached, SerfError::JoinAllFailed(_)))) => { diff --git a/serf-reactor/src/lib.rs b/serf-reactor/src/lib.rs index 23ca0d88..d1b33021 100644 --- a/serf-reactor/src/lib.rs +++ b/serf-reactor/src/lib.rs @@ -43,6 +43,7 @@ #![deny(missing_docs)] #![cfg_attr(docsrs, feature(doc_cfg))] +#![cfg_attr(docsrs, allow(unused_attributes))] #![forbid(unsafe_code)] #[cfg(feature = "tcp")] diff --git a/serf-reactor/tests/quic.rs b/serf-reactor/tests/quic.rs index 9d5ae976..9c56024d 100644 --- a/serf-reactor/tests/quic.rs +++ b/serf-reactor/tests/quic.rs @@ -1,21 +1,26 @@ -//! Real-node QUIC serf tests on tokio: two loopback nodes exercising the reactor -//! QUIC driver end-to-end over a quinn-proto config bundle. Each test spins up +//! Real-node QUIC serf tests: two loopback nodes exercising the reactor QUIC +//! driver end-to-end over a quinn-proto config bundle. Each test spins up //! ephemeral `127.0.0.1:0` nodes via the ergonomic [`Serf::quic`] constructor and //! drives the full pump — QUIC push/pull join over a real quinn handshake, //! coordinator merge, datagram gossip, user events, queries, and graceful //! leave/shutdown — end-to-end proof the reactor QUIC driver works over a concrete //! runtime, meeting the same behaviour bar as the TCP suite. //! +//! The scenario bodies are runtime-generic `async fn ` helpers, so the +//! SAME scenario runs as a `#[tokio::test]` cell over `TokioRuntime` and as a +//! `_smol` cell driven by `SmolRuntime::block_on` — mirroring memberlist-reactor's +//! runtime-parameterized suite. +//! //! Mirrors serf-compio's QUIC smoke tests and the reactor's `tests/tcp.rs`, //! adapted to the reactor's `Send`/`agnostic` model and QUIC's single-socket, //! stream-multiplexed transport. -#![cfg(all(feature = "quic", feature = "tokio"))] +#![cfg(feature = "quic")] use core::time::Duration; use std::{net::SocketAddr, sync::Arc}; -use agnostic::tokio::TokioRuntime; +use agnostic::Runtime; use bytes::Bytes; use futures_util::{StreamExt, future}; use memberlist_proto::UnreliableTransport; @@ -33,8 +38,8 @@ use serf_reactor::{ }; use smol_str::SmolStr; -/// A tokio-backed reactor QUIC node handle. -type Node = Serf; +/// A reactor QUIC node handle over the agnostic runtime `R`. +type Node = Serf; /// A self-signed cert + key for `localhost`, for the test TLS bundle. fn self_signed() -> (Vec>, PrivateKeyDer<'static>) { @@ -137,13 +142,16 @@ fn test_quic_options() -> QuicOptions { /// Build and spawn a reactor QUIC node on an ephemeral loopback port through the /// ergonomic `Serf::quic` constructor. -async fn spawn_node(id: &str) -> Node { +async fn spawn_node(id: &str) -> Node +where + R: Runtime, +{ let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); let opts = QuicTransportOptions::::new() .with_local_id(SmolStr::new(id)) .with_advertise_addr(MaybeResolved::Resolved(bind)) .with_quic_config(test_quic_options()); - Serf::::quic( + Serf::::quic( opts, &SocketAddrResolver, &FirstAddrResolver, @@ -159,13 +167,16 @@ async fn spawn_node(id: &str) -> Node { /// Poll both nodes until each reports the full two-member cluster, or fail on a /// generous timeout so a convergence regression surfaces as a timeout, not a hang. -async fn converge(a: &Node, b: &Node) { - tokio::time::timeout(Duration::from_secs(30), async { +async fn converge(a: &Node, b: &Node) +where + R: Runtime, +{ + R::timeout(Duration::from_secs(30), async { loop { if a.num_members() == 2 && b.num_members() == 2 { break; } - tokio::time::sleep(Duration::from_millis(20)).await; + R::sleep(Duration::from_millis(20)).await; } }) .await @@ -174,10 +185,12 @@ async fn converge(a: &Node, b: &Node) { /// Two nodes on loopback: A joins B (await-result over a real QUIC push/pull), then /// BOTH converge to a two-member cluster and shut down cleanly. -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn two_node_quic_join_converges() { - let b = spawn_node("conv-b").await; - let a = spawn_node("conv-a").await; +async fn two_node_quic_join_converges() +where + R: Runtime, +{ + let b = spawn_node::("conv-b").await; + let a = spawn_node::("conv-a").await; let b_addr = b.advertise_address(); let reached = a @@ -197,10 +210,12 @@ async fn two_node_quic_join_converges() { /// After a two-node QUIC join, a user event broadcast by B is delivered to A's event /// stream carrying the original name and payload (datagram gossip over the shared /// socket). -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn user_event_delivered() { - let b = spawn_node("ue-b").await; - let a = spawn_node("ue-a").await; +async fn user_event_delivered() +where + R: Runtime, +{ + let b = spawn_node::("ue-b").await; + let a = spawn_node::("ue-a").await; let b_addr = b.advertise_address(); // Subscribe before joining so the user event cannot race the subscription. @@ -214,7 +229,7 @@ async fn user_event_delivered() { .await .expect("user event dispatched"); - let got = tokio::time::timeout(Duration::from_secs(30), async { + let got = R::timeout(Duration::from_secs(30), async { loop { match a_events.next().await { Some(Event::User(u)) if u.name.as_str() == "greet" => break Some(u.payload.clone()), @@ -247,10 +262,12 @@ async fn user_event_delivered() { /// delivers the event over UDP and converges, but leaves `datagrams_sent` at `0`. /// `datagrams_sent` advances only on a `DatagramSendStatus::Queued`, so asserting it /// is non-zero fails on that revert while the convergence assertions alone would not. -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn datagram_mode_gossip_rides_quic_datagrams() { - let b = spawn_node("dg-b").await; - let a = spawn_node("dg-a").await; +async fn datagram_mode_gossip_rides_quic_datagrams() +where + R: Runtime, +{ + let b = spawn_node::("dg-b").await; + let a = spawn_node::("dg-a").await; let b_addr = b.advertise_address(); let mut a_events = a.events(); @@ -263,7 +280,7 @@ async fn datagram_mode_gossip_rides_quic_datagrams() { .await .expect("user event dispatched"); - let got = tokio::time::timeout(Duration::from_secs(30), async { + let got = R::timeout(Duration::from_secs(30), async { loop { match a_events.next().await { Some(Event::User(u)) if u.name.as_str() == "greet" => break Some(u.payload.clone()), @@ -284,12 +301,12 @@ async fn datagram_mode_gossip_rides_quic_datagrams() { // fallback. Both nodes hold a warm pooled connection after the join, so their // periodic gossip is queued as datagrams; `datagrams_sent` advances only on a // `DatagramSendStatus::Queued`. - tokio::time::timeout(Duration::from_secs(30), async { + R::timeout(Duration::from_secs(30), async { loop { if b.datagrams_sent() > 0 { break; } - tokio::time::sleep(Duration::from_millis(20)).await; + R::sleep(Duration::from_millis(20)).await; } }) .await @@ -303,10 +320,12 @@ async fn datagram_mode_gossip_rides_quic_datagrams() { /// After a two-node QUIC join, a query issued by A round-trips: B receives the /// `Event::Query`, responds, and A surfaces the matching `Event::QueryResponse`. -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn query_round_trip() { - let b = spawn_node("q-b").await; - let a = spawn_node("q-a").await; +async fn query_round_trip() +where + R: Runtime, +{ + let b = spawn_node::("q-b").await; + let a = spawn_node::("q-a").await; let b_addr = b.advertise_address(); // Subscribe both before the join so neither the query nor its response races ahead @@ -353,7 +372,7 @@ async fn query_round_trip() { } }; - let got = tokio::time::timeout(Duration::from_secs(30), async { + let got = R::timeout(Duration::from_secs(30), async { let (_, got) = future::join(responder, collector).await; got }) @@ -368,10 +387,12 @@ async fn query_round_trip() { /// A graceful leave completes the machine's leave chain: `leave()` resolves only /// once `LeftCluster` fires (the reactor gates the reply on it), that event surfaces /// on the leaver's own stream, and the local endpoint settles at `Left`. -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn leave_emits_left_cluster() { - let b = spawn_node("lv-b").await; - let a = spawn_node("lv-a").await; +async fn leave_emits_left_cluster() +where + R: Runtime, +{ + let b = spawn_node::("lv-b").await; + let a = spawn_node::("lv-a").await; let b_addr = b.advertise_address(); let mut a_events = a.events(); @@ -385,7 +406,7 @@ async fn leave_emits_left_cluster() { a.leave().await.expect("A leaves the cluster"); // `LeftCluster` is also forwarded to A's own subscribers. - let saw = tokio::time::timeout(Duration::from_secs(30), async { + let saw = R::timeout(Duration::from_secs(30), async { loop { match a_events.next().await { Some(Event::LeftCluster) => break true, @@ -400,12 +421,12 @@ async fn leave_emits_left_cluster() { // The local endpoint state settles at `Left` (poll to absorb the snapshot-refresh // race after the leave chain completes). - tokio::time::timeout(Duration::from_secs(5), async { + R::timeout(Duration::from_secs(5), async { loop { if a.state() == SerfState::Left { break; } - tokio::time::sleep(Duration::from_millis(20)).await; + R::sleep(Duration::from_millis(20)).await; } }) .await @@ -419,9 +440,11 @@ async fn leave_emits_left_cluster() { /// acking shutdown, so `shutdown().await` releases the bound port before it /// resolves: a second QUIC node binding the SAME advertise address the instant the /// first shuts down must construct successfully, not fail with `AddrInUse`. -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn quic_shutdown_releases_bound_address_for_rebind() { - let first = spawn_node("rebind-first").await; +async fn quic_shutdown_releases_bound_address_for_rebind() +where + R: Runtime, +{ + let first = spawn_node::("rebind-first").await; let addr = first.advertise_address(); first.shutdown().await.expect("first node shuts down"); @@ -429,7 +452,7 @@ async fn quic_shutdown_releases_bound_address_for_rebind() { .with_local_id(SmolStr::new("rebind-second")) .with_advertise_addr(MaybeResolved::Resolved(addr)) .with_quic_config(test_quic_options()); - let second = Serf::::quic( + let second = Serf::::quic( opts, &SocketAddrResolver, &FirstAddrResolver, @@ -473,7 +496,10 @@ fn test_secret_key(fill: u8) -> SecretKey { /// an incidental push/pull), and the negative test's absence is decisive rather than /// a race against the next scheduled sync. Join-time exchanges are unaffected. #[cfg(encryption)] -async fn spawn_encrypted_node(id: &str, encryption: EncryptionOptions) -> Node { +async fn spawn_encrypted_node(id: &str, encryption: EncryptionOptions) -> Node +where + R: Runtime, +{ let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); let opts = QuicTransportOptions::::new() .with_local_id(SmolStr::new(id)) @@ -481,7 +507,7 @@ async fn spawn_encrypted_node(id: &str, encryption: EncryptionOptions) -> Node { .with_quic_config(test_quic_options()) .with_push_pull_interval(Duration::ZERO) .with_encryption(encryption); - Serf::::quic( + Serf::::quic( opts, &SocketAddrResolver, &FirstAddrResolver, @@ -512,11 +538,13 @@ const GOSSIP_DELIVERY_WINDOW: Duration = Duration::from_secs(10); /// proves `encrypt_gossip`/`decrypt_gossip` round-trip end-to-end rather than /// running as identity transforms. #[cfg(encryption)] -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn two_node_quic_gossip_convergence_encrypted() { +async fn two_node_quic_gossip_convergence_encrypted() +where + R: Runtime, +{ let key = EncryptionOptions::new().with_keyring(Keyring::new(test_secret_key(0x42))); - let b = spawn_encrypted_node("enc-b", key.clone()).await; - let a = spawn_encrypted_node("enc-a", key).await; + let b = spawn_encrypted_node::("enc-b", key.clone()).await; + let a = spawn_encrypted_node::("enc-a", key).await; let b_addr = b.advertise_address(); // Subscribe before joining so the user event cannot race the subscription. @@ -535,7 +563,7 @@ async fn two_node_quic_gossip_convergence_encrypted() { .await .expect("user event dispatched"); - let got = tokio::time::timeout(GOSSIP_DELIVERY_WINDOW, async { + let got = R::timeout(GOSSIP_DELIVERY_WINDOW, async { loop { match a_events.next().await { Some(Event::User(u)) if u.name.as_str() == "greet" => break Some(u.payload.clone()), @@ -575,14 +603,16 @@ async fn two_node_quic_gossip_convergence_encrypted() { /// ride gossip. A matched key WOULD surface the event within this window (the /// positive test proves exactly that), so the absence is not vacuous. #[cfg(encryption)] -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn mismatched_keyring_gossip_does_not_cross() { - let b = spawn_encrypted_node( +async fn mismatched_keyring_gossip_does_not_cross() +where + R: Runtime, +{ + let b = spawn_encrypted_node::( "mis-b", EncryptionOptions::new().with_keyring(Keyring::new(test_secret_key(0x42))), ) .await; - let a = spawn_encrypted_node( + let a = spawn_encrypted_node::( "mis-a", EncryptionOptions::new().with_keyring(Keyring::new(test_secret_key(0x43))), ) @@ -611,7 +641,7 @@ async fn mismatched_keyring_gossip_does_not_cross() { // negation; a CLOSED stream (`None`) is NOT success — it would mean A's driver died // on the bad ciphertext, which cannot prove the event was blocked, so it too is a // failure. Only a clean timeout (the event never surfaces) is the blocked outcome. - let outcome = tokio::time::timeout(GOSSIP_DELIVERY_WINDOW, async { + let outcome = R::timeout(GOSSIP_DELIVERY_WINDOW, async { loop { match a_events.next().await { Some(Event::User(u)) if u.name.as_str() == "secret" => break true, @@ -648,3 +678,110 @@ async fn mismatched_keyring_gossip_does_not_cross() { a.shutdown().await.expect("mis-a shuts down"); b.shutdown().await.expect("mis-b shuts down"); } + +// The tokio cells: the runtime-generic scenarios driven on tokio's multi-thread +// runtime. Gated on the `tokio` feature so the `--test quic -- smol` build (which +// enables only `smol`) can drop the `agnostic/tokio` code path. +#[cfg(feature = "tokio")] +mod tokio_cells { + use agnostic::tokio::TokioRuntime; + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn two_node_quic_join_converges() { + super::two_node_quic_join_converges::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn user_event_delivered() { + super::user_event_delivered::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn datagram_mode_gossip_rides_quic_datagrams() { + super::datagram_mode_gossip_rides_quic_datagrams::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn query_round_trip() { + super::query_round_trip::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn leave_emits_left_cluster() { + super::leave_emits_left_cluster::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn quic_shutdown_releases_bound_address_for_rebind() { + super::quic_shutdown_releases_bound_address_for_rebind::().await; + } + + #[cfg(encryption)] + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn two_node_quic_gossip_convergence_encrypted() { + super::two_node_quic_gossip_convergence_encrypted::().await; + } + + #[cfg(encryption)] + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn mismatched_keyring_gossip_does_not_cross() { + super::mismatched_keyring_gossip_does_not_cross::().await; + } +} + +// The smol cells: the identical scenarios instantiated over `SmolRuntime` and +// driven by smol's `block_on`. The reactor QUIC poll task runs on smol's global +// executor, so the same scenario bodies verify the driver under a second runtime. +// `cargo test --test quic -- smol` selects exactly these. +#[cfg(feature = "smol")] +mod smol_cells { + use agnostic::{RuntimeLite, smol::SmolRuntime}; + + #[test] + fn two_node_quic_join_converges_smol() { + SmolRuntime::block_on(super::two_node_quic_join_converges::()); + } + + #[test] + fn user_event_delivered_smol() { + SmolRuntime::block_on(super::user_event_delivered::()); + } + + #[test] + fn datagram_mode_gossip_rides_quic_datagrams_smol() { + SmolRuntime::block_on(super::datagram_mode_gossip_rides_quic_datagrams::< + SmolRuntime, + >()); + } + + #[test] + fn query_round_trip_smol() { + SmolRuntime::block_on(super::query_round_trip::()); + } + + #[test] + fn leave_emits_left_cluster_smol() { + SmolRuntime::block_on(super::leave_emits_left_cluster::()); + } + + #[test] + fn quic_shutdown_releases_bound_address_for_rebind_smol() { + SmolRuntime::block_on(super::quic_shutdown_releases_bound_address_for_rebind::< + SmolRuntime, + >()); + } + + #[cfg(encryption)] + #[test] + fn two_node_quic_gossip_convergence_encrypted_smol() { + SmolRuntime::block_on(super::two_node_quic_gossip_convergence_encrypted::< + SmolRuntime, + >()); + } + + #[cfg(encryption)] + #[test] + fn mismatched_keyring_gossip_does_not_cross_smol() { + SmolRuntime::block_on(super::mismatched_keyring_gossip_does_not_cross::()); + } +} diff --git a/serf-reactor/tests/tcp.rs b/serf-reactor/tests/tcp.rs index 8963127f..88f45395 100644 --- a/serf-reactor/tests/tcp.rs +++ b/serf-reactor/tests/tcp.rs @@ -1,20 +1,26 @@ -//! Real-node TCP serf tests on tokio: two loopback nodes exercising the reactor -//! stream driver end-to-end. Each test spins up ephemeral `127.0.0.1:0` nodes via -//! the ergonomic [`Serf::tcp`] constructor and drives the full pump — join -//! push/pull, coordinator merge, gossip, user events, queries, and graceful -//! leave/shutdown — end-to-end proof the reactor stream driver works over a -//! concrete runtime. +//! Real-node TCP serf tests: two loopback nodes exercising the reactor stream +//! driver end-to-end. Each test spins up ephemeral `127.0.0.1:0` nodes via the +//! ergonomic [`Serf::tcp`] constructor and drives the full pump — join push/pull, +//! coordinator merge, gossip, user events, queries, and graceful leave/shutdown — +//! end-to-end proof the reactor stream driver works over a concrete runtime. +//! +//! The scenario bodies are runtime-generic `async fn ` helpers, so the +//! SAME scenario runs as a `#[tokio::test]` cell over `TokioRuntime` and as a +//! `_smol` cell driven by `SmolRuntime::block_on` — mirroring memberlist-reactor's +//! runtime-parameterized suite. The `Serf` handle is already runtime- +//! generic; the helpers reach for timers through the runtime (`R::timeout` / +//! `R::sleep`) rather than a concrete runtime's clock. //! //! Mirrors serf-compio's serf behavior tests and memberlist-reactor's real-node //! harness (bind loopback, join, poll-until-converged with a timeout, assert //! membership / events), adapted to the reactor's `Send`/`agnostic` model. -#![cfg(all(feature = "tcp", feature = "tokio"))] +#![cfg(feature = "tcp")] use core::time::Duration; use std::net::SocketAddr; -use agnostic::tokio::TokioRuntime; +use agnostic::Runtime; use bytes::Bytes; use futures_util::{StreamExt, future}; #[cfg(encryption)] @@ -28,17 +34,20 @@ use serf_reactor::{ }; use smol_str::SmolStr; -/// A tokio-backed reactor TCP node handle. -type Node = Serf; +/// A reactor TCP node handle over the agnostic runtime `R`. +type Node = Serf; /// Build and spawn a reactor TCP node on an ephemeral loopback port through the /// ergonomic `Serf::tcp` constructor. -async fn spawn_node(id: &str) -> Node { +async fn spawn_node(id: &str) -> Node +where + R: Runtime, +{ let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); let opts = TcpTransportOptions::::new() .with_local_id(SmolStr::new(id)) .with_advertise_addr(MaybeResolved::Resolved(bind)); - Serf::::tcp( + Serf::::tcp( opts, &SocketAddrResolver, &FirstAddrResolver, @@ -54,13 +63,16 @@ async fn spawn_node(id: &str) -> Node { /// Poll both nodes until each reports the full two-member cluster, or fail on a /// generous timeout so a convergence regression surfaces as a timeout, not a hang. -async fn converge(a: &Node, b: &Node) { - tokio::time::timeout(Duration::from_secs(20), async { +async fn converge(a: &Node, b: &Node) +where + R: Runtime, +{ + R::timeout(Duration::from_secs(20), async { loop { if a.num_members() == 2 && b.num_members() == 2 { break; } - tokio::time::sleep(Duration::from_millis(20)).await; + R::sleep(Duration::from_millis(20)).await; } }) .await @@ -69,10 +81,12 @@ async fn converge(a: &Node, b: &Node) { /// Two nodes on loopback: A joins B (await-result), then BOTH converge to a /// two-member cluster and shut down cleanly. -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn two_node_join_converges() { - let b = spawn_node("conv-b").await; - let a = spawn_node("conv-a").await; +async fn two_node_join_converges() +where + R: Runtime, +{ + let b = spawn_node::("conv-b").await; + let a = spawn_node::("conv-a").await; let b_addr = b.advertise_address(); let reached = a @@ -91,10 +105,12 @@ async fn two_node_join_converges() { /// After a two-node join, a user event broadcast by B is delivered to A's event /// stream carrying the original name and payload. -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn user_event_delivered() { - let b = spawn_node("ue-b").await; - let a = spawn_node("ue-a").await; +async fn user_event_delivered() +where + R: Runtime, +{ + let b = spawn_node::("ue-b").await; + let a = spawn_node::("ue-a").await; let b_addr = b.advertise_address(); // Subscribe before joining so the user event cannot race the subscription. @@ -108,7 +124,7 @@ async fn user_event_delivered() { .await .expect("user event dispatched"); - let got = tokio::time::timeout(Duration::from_secs(20), async { + let got = R::timeout(Duration::from_secs(20), async { loop { match a_events.next().await { Some(Event::User(u)) if u.name.as_str() == "greet" => break Some(u.payload.clone()), @@ -131,10 +147,12 @@ async fn user_event_delivered() { /// After a two-node join, a query issued by A round-trips: B receives the /// `Event::Query`, responds, and A surfaces the matching `Event::QueryResponse`. -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn query_round_trip() { - let b = spawn_node("q-b").await; - let a = spawn_node("q-a").await; +async fn query_round_trip() +where + R: Runtime, +{ + let b = spawn_node::("q-b").await; + let a = spawn_node::("q-a").await; let b_addr = b.advertise_address(); // Subscribe both before the join so neither the query nor its response races @@ -181,7 +199,7 @@ async fn query_round_trip() { } }; - let got = tokio::time::timeout(Duration::from_secs(20), async { + let got = R::timeout(Duration::from_secs(20), async { let (_, got) = future::join(responder, collector).await; got }) @@ -196,10 +214,12 @@ async fn query_round_trip() { /// A graceful leave completes the machine's leave chain: `leave()` resolves only /// once `LeftCluster` fires (the reactor gates the reply on it), that event /// surfaces on the leaver's own stream, and the local endpoint settles at `Left`. -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn leave_emits_left_cluster() { - let b = spawn_node("lv-b").await; - let a = spawn_node("lv-a").await; +async fn leave_emits_left_cluster() +where + R: Runtime, +{ + let b = spawn_node::("lv-b").await; + let a = spawn_node::("lv-a").await; let b_addr = b.advertise_address(); let mut a_events = a.events(); @@ -213,7 +233,7 @@ async fn leave_emits_left_cluster() { a.leave().await.expect("A leaves the cluster"); // `LeftCluster` is also forwarded to A's own subscribers. - let saw = tokio::time::timeout(Duration::from_secs(20), async { + let saw = R::timeout(Duration::from_secs(20), async { loop { match a_events.next().await { Some(Event::LeftCluster) => break true, @@ -228,12 +248,12 @@ async fn leave_emits_left_cluster() { // The local endpoint state settles at `Left` (poll to absorb the snapshot-refresh // race after the leave chain completes). - tokio::time::timeout(Duration::from_secs(5), async { + R::timeout(Duration::from_secs(5), async { loop { if a.state() == SerfState::Left { break; } - tokio::time::sleep(Duration::from_millis(20)).await; + R::sleep(Duration::from_millis(20)).await; } }) .await @@ -247,10 +267,12 @@ async fn leave_emits_left_cluster() { /// the two-member cluster: `members` returns both nodes, `local_member` / `local_id` /// return this node, `state` is `Alive`, `advertise_node` composes id + advertise, /// and `default_query_*` produce a positive, filter-free query default. -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn snapshot_forwarders_reflect_joined_cluster() { - let b = spawn_node("snap-b").await; - let a = spawn_node("snap-a").await; +async fn snapshot_forwarders_reflect_joined_cluster() +where + R: Runtime, +{ + let b = spawn_node::("snap-b").await; + let a = spawn_node::("snap-a").await; let b_addr = b.advertise_address(); let a_id = SmolStr::new("snap-a"); let b_id = SmolStr::new("snap-b"); @@ -304,10 +326,12 @@ async fn snapshot_forwarders_reflect_joined_cluster() { /// `join_many` over two seeds — one reachable (node B), one an unroutable /// blackhole port — returns only the reached seed's address once both exchanges /// terminate. -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn join_many_returns_only_reached_seeds() { - let b = spawn_node("jm-b").await; - let a = spawn_node("jm-a").await; +async fn join_many_returns_only_reached_seeds() +where + R: Runtime, +{ + let b = spawn_node::("jm-b").await; + let a = spawn_node::("jm-a").await; let b_addr = b.advertise_address(); let blackhole: SocketAddr = "127.0.0.1:7219".parse().expect("loopback addr"); @@ -336,10 +360,12 @@ async fn join_many_returns_only_reached_seeds() { /// `remove_failed_node` / `remove_failed_node_prune` are thin `force_leave` /// aliases; calling both on a valid joined node-id completes without error. -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn remove_failed_node_alias_succeeds() { - let b = spawn_node("rfn-b").await; - let a = spawn_node("rfn-a").await; +async fn remove_failed_node_alias_succeeds() +where + R: Runtime, +{ + let b = spawn_node::("rfn-b").await; + let a = spawn_node::("rfn-a").await; let b_addr = b.advertise_address(); let b_id = SmolStr::new("rfn-b"); @@ -373,13 +399,16 @@ fn test_secret_key(fill: u8) -> SecretKey { /// Build and spawn a reactor TCP node on an ephemeral loopback port with /// `encryption` installed as its gossip-and-reliable keyring policy. #[cfg(encryption)] -async fn spawn_encrypted_node(id: &str, encryption: EncryptionOptions) -> Node { +async fn spawn_encrypted_node(id: &str, encryption: EncryptionOptions) -> Node +where + R: Runtime, +{ let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); let opts = TcpTransportOptions::::new() .with_local_id(SmolStr::new(id)) .with_advertise_addr(MaybeResolved::Resolved(bind)) .with_encryption(encryption); - Serf::::tcp( + Serf::::tcp( opts, &SocketAddrResolver, &FirstAddrResolver, @@ -399,11 +428,13 @@ async fn spawn_encrypted_node(id: &str, encryption: EncryptionOptions) -> Node { /// `encrypt_gossip`/`decrypt_gossip` round-trip end-to-end rather than running as /// identity transforms. #[cfg(encryption)] -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn two_node_join_converges_encrypted() { +async fn two_node_join_converges_encrypted() +where + R: Runtime, +{ let key = EncryptionOptions::new().with_keyring(Keyring::new(test_secret_key(0x42))); - let b = spawn_encrypted_node("enc-b", key.clone()).await; - let a = spawn_encrypted_node("enc-a", key).await; + let b = spawn_encrypted_node::("enc-b", key.clone()).await; + let a = spawn_encrypted_node::("enc-a", key).await; let b_addr = b.advertise_address(); let b_id = SmolStr::new("enc-b"); @@ -417,7 +448,7 @@ async fn two_node_join_converges_encrypted() { converge(&a, &b).await; - let observed = tokio::time::timeout(Duration::from_secs(20), async { + let observed = R::timeout(Duration::from_secs(20), async { loop { match a_events.next().await { Some(Event::Member(me)) if me.kind() == MemberEventKind::Join => { @@ -447,14 +478,16 @@ async fn two_node_join_converges_encrypted() { /// not an identity pass-through — without this negative case a passing encrypted /// convergence test could not distinguish real AEAD from an identity transform. #[cfg(encryption)] -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn mismatched_keyring_nodes_do_not_exchange_membership() { - let b = spawn_encrypted_node( +async fn mismatched_keyring_nodes_do_not_exchange_membership() +where + R: Runtime, +{ + let b = spawn_encrypted_node::( "mis-b", EncryptionOptions::new().with_keyring(Keyring::new(test_secret_key(0x42))), ) .await; - let a = spawn_encrypted_node( + let a = spawn_encrypted_node::( "mis-a", EncryptionOptions::new().with_keyring(Keyring::new(test_secret_key(0x43))), ) @@ -476,7 +509,7 @@ async fn mismatched_keyring_nodes_do_not_exchange_membership() { // Absence probe: A must never surface a Join carrying node-b. A short window // covers several gossip / probe / push-pull rounds on loopback — the positive // test forms its cluster within ~1-2s, so a clean 3s window is decisive. - let observed = tokio::time::timeout(Duration::from_secs(3), async { + let observed = R::timeout(Duration::from_secs(3), async { loop { match a_events.next().await { Some(Event::Member(me)) if me.kind() == MemberEventKind::Join => { @@ -498,3 +531,119 @@ async fn mismatched_keyring_nodes_do_not_exchange_membership() { a.shutdown().await.expect("mis-a shuts down"); b.shutdown().await.expect("mis-b shuts down"); } + +// The tokio cells: the runtime-generic scenarios driven on tokio's multi-thread +// runtime. Gated on the `tokio` feature so the `--test tcp -- smol` build (which +// enables only `smol`) can drop the `agnostic/tokio` code path. +#[cfg(feature = "tokio")] +mod tokio_cells { + use agnostic::tokio::TokioRuntime; + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn two_node_join_converges() { + super::two_node_join_converges::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn user_event_delivered() { + super::user_event_delivered::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn query_round_trip() { + super::query_round_trip::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn leave_emits_left_cluster() { + super::leave_emits_left_cluster::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn snapshot_forwarders_reflect_joined_cluster() { + super::snapshot_forwarders_reflect_joined_cluster::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn join_many_returns_only_reached_seeds() { + super::join_many_returns_only_reached_seeds::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn remove_failed_node_alias_succeeds() { + super::remove_failed_node_alias_succeeds::().await; + } + + #[cfg(encryption)] + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn two_node_join_converges_encrypted() { + super::two_node_join_converges_encrypted::().await; + } + + #[cfg(encryption)] + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn mismatched_keyring_nodes_do_not_exchange_membership() { + super::mismatched_keyring_nodes_do_not_exchange_membership::().await; + } +} + +// The smol cells: the identical scenarios instantiated over `SmolRuntime` and +// driven by smol's `block_on`. The reactor poll task runs on smol's global +// executor, so the same scenario bodies verify the driver under a second runtime +// with no per-runtime scenario code. `cargo test --test tcp -- smol` selects +// exactly these. +#[cfg(feature = "smol")] +mod smol_cells { + use agnostic::{RuntimeLite, smol::SmolRuntime}; + + #[test] + fn two_node_join_converges_smol() { + SmolRuntime::block_on(super::two_node_join_converges::()); + } + + #[test] + fn user_event_delivered_smol() { + SmolRuntime::block_on(super::user_event_delivered::()); + } + + #[test] + fn query_round_trip_smol() { + SmolRuntime::block_on(super::query_round_trip::()); + } + + #[test] + fn leave_emits_left_cluster_smol() { + SmolRuntime::block_on(super::leave_emits_left_cluster::()); + } + + #[test] + fn snapshot_forwarders_reflect_joined_cluster_smol() { + SmolRuntime::block_on(super::snapshot_forwarders_reflect_joined_cluster::< + SmolRuntime, + >()); + } + + #[test] + fn join_many_returns_only_reached_seeds_smol() { + SmolRuntime::block_on(super::join_many_returns_only_reached_seeds::()); + } + + #[test] + fn remove_failed_node_alias_succeeds_smol() { + SmolRuntime::block_on(super::remove_failed_node_alias_succeeds::()); + } + + #[cfg(encryption)] + #[test] + fn two_node_join_converges_encrypted_smol() { + SmolRuntime::block_on(super::two_node_join_converges_encrypted::()); + } + + #[cfg(encryption)] + #[test] + fn mismatched_keyring_nodes_do_not_exchange_membership_smol() { + SmolRuntime::block_on( + super::mismatched_keyring_nodes_do_not_exchange_membership::(), + ); + } +} diff --git a/serf-reactor/tests/tls.rs b/serf-reactor/tests/tls.rs index 13431b8c..b21ae281 100644 --- a/serf-reactor/tests/tls.rs +++ b/serf-reactor/tests/tls.rs @@ -1,8 +1,13 @@ -//! Real-node TLS serf tests on tokio: two loopback nodes exercising the reactor -//! stream driver with the rustls record layer end-to-end. Each test spins up -//! ephemeral `127.0.0.1:0` nodes via the ergonomic [`Serf::tls`] constructor and -//! drives the full pump — TLS handshake-on-dial, join push/pull, coordinator merge, -//! gossip, user events, queries, and graceful leave/shutdown. +//! Real-node TLS serf tests: two loopback nodes exercising the reactor stream +//! driver with the rustls record layer end-to-end. Each test spins up ephemeral +//! `127.0.0.1:0` nodes via the ergonomic [`Serf::tls`] constructor and drives the +//! full pump — TLS handshake-on-dial, join push/pull, coordinator merge, gossip, +//! user events, queries, and graceful leave/shutdown. +//! +//! The scenario bodies are runtime-generic `async fn ` helpers, so the +//! SAME scenario runs as a `#[tokio::test]` cell over `TokioRuntime` and as a +//! `_smol` cell driven by `SmolRuntime::block_on` — mirroring memberlist-reactor's +//! runtime-parameterized suite. //! //! Mirrors `tests/tcp.rs` (TLS rides the same stream driver as plain TCP, differing //! only in the record layer) and serf-compio's / memberlist-reactor's TLS harness: @@ -11,12 +16,12 @@ //! trust anchor. The default SNI provider (`Some("localhost")`) matches the cert //! SAN. -#![cfg(all(feature = "tls", feature = "tokio"))] +#![cfg(feature = "tls")] use core::time::Duration; use std::{net::SocketAddr, sync::Arc}; -use agnostic::tokio::TokioRuntime; +use agnostic::Runtime; use bytes::Bytes; use futures_util::{StreamExt, future}; use rustls::{ @@ -32,8 +37,8 @@ use serf_reactor::{ }; use smol_str::SmolStr; -/// A tokio-backed reactor TLS node handle. -type Node = Serf; +/// A reactor TLS node handle over the agnostic runtime `R`. +type Node = Serf; /// Accept-any server-cert verifier for the loopback tests. /// @@ -111,13 +116,16 @@ fn test_tls_options() -> TlsOptions { /// Build and spawn a reactor TLS node on an ephemeral loopback port through the /// ergonomic `Serf::tls` constructor. The default SNI provider (`Some("localhost")`) /// matches the self-signed cert SAN. -async fn spawn_node(id: &str) -> Node { +async fn spawn_node(id: &str) -> Node +where + R: Runtime, +{ let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); let opts = TlsTransportOptions::::new() .with_local_id(SmolStr::new(id)) .with_advertise_addr(MaybeResolved::Resolved(bind)) .with_tls_options(test_tls_options()); - Serf::::tls( + Serf::::tls( opts, &SocketAddrResolver, &FirstAddrResolver, @@ -133,13 +141,16 @@ async fn spawn_node(id: &str) -> Node { /// Poll both nodes until each reports the full two-member cluster, or fail on a /// generous timeout so a convergence regression surfaces as a timeout, not a hang. -async fn converge(a: &Node, b: &Node) { - tokio::time::timeout(Duration::from_secs(20), async { +async fn converge(a: &Node, b: &Node) +where + R: Runtime, +{ + R::timeout(Duration::from_secs(20), async { loop { if a.num_members() == 2 && b.num_members() == 2 { break; } - tokio::time::sleep(Duration::from_millis(20)).await; + R::sleep(Duration::from_millis(20)).await; } }) .await @@ -148,10 +159,12 @@ async fn converge(a: &Node, b: &Node) { /// Two nodes on loopback: A joins B over a real TLS push-pull exchange, then BOTH /// converge to a two-member cluster and shut down cleanly. -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn two_node_tls_join_converges() { - let b = spawn_node("conv-b").await; - let a = spawn_node("conv-a").await; +async fn two_node_tls_join_converges() +where + R: Runtime, +{ + let b = spawn_node::("conv-b").await; + let a = spawn_node::("conv-a").await; let b_addr = b.advertise_address(); let reached = a @@ -170,10 +183,12 @@ async fn two_node_tls_join_converges() { /// After a two-node TLS join, a user event broadcast by B is delivered to A's event /// stream carrying the original name and payload. -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn user_event_delivered() { - let b = spawn_node("ue-b").await; - let a = spawn_node("ue-a").await; +async fn user_event_delivered() +where + R: Runtime, +{ + let b = spawn_node::("ue-b").await; + let a = spawn_node::("ue-a").await; let b_addr = b.advertise_address(); // Subscribe before joining so the user event cannot race the subscription. @@ -187,7 +202,7 @@ async fn user_event_delivered() { .await .expect("user event dispatched"); - let got = tokio::time::timeout(Duration::from_secs(20), async { + let got = R::timeout(Duration::from_secs(20), async { loop { match a_events.next().await { Some(Event::User(u)) if u.name.as_str() == "greet" => break Some(u.payload.clone()), @@ -210,10 +225,12 @@ async fn user_event_delivered() { /// After a two-node TLS join, a query issued by A round-trips: B receives the /// `Event::Query`, responds, and A surfaces the matching `Event::QueryResponse`. -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn query_round_trip() { - let b = spawn_node("q-b").await; - let a = spawn_node("q-a").await; +async fn query_round_trip() +where + R: Runtime, +{ + let b = spawn_node::("q-b").await; + let a = spawn_node::("q-a").await; let b_addr = b.advertise_address(); // Subscribe both before the join so neither the query nor its response races ahead @@ -260,7 +277,7 @@ async fn query_round_trip() { } }; - let got = tokio::time::timeout(Duration::from_secs(20), async { + let got = R::timeout(Duration::from_secs(20), async { let (_, got) = future::join(responder, collector).await; got }) @@ -275,10 +292,12 @@ async fn query_round_trip() { /// A graceful leave completes the machine's leave chain: `leave()` resolves only /// once `LeftCluster` fires (the reactor gates the reply on it), that event surfaces /// on the leaver's own stream, and the local endpoint settles at `Left`. -#[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn leave_emits_left_cluster() { - let b = spawn_node("lv-b").await; - let a = spawn_node("lv-a").await; +async fn leave_emits_left_cluster() +where + R: Runtime, +{ + let b = spawn_node::("lv-b").await; + let a = spawn_node::("lv-a").await; let b_addr = b.advertise_address(); let mut a_events = a.events(); @@ -292,7 +311,7 @@ async fn leave_emits_left_cluster() { a.leave().await.expect("A leaves the cluster"); // `LeftCluster` is also forwarded to A's own subscribers. - let saw = tokio::time::timeout(Duration::from_secs(20), async { + let saw = R::timeout(Duration::from_secs(20), async { loop { match a_events.next().await { Some(Event::LeftCluster) => break true, @@ -307,12 +326,12 @@ async fn leave_emits_left_cluster() { // The local endpoint state settles at `Left` (poll to absorb the snapshot-refresh // race after the leave chain completes). - tokio::time::timeout(Duration::from_secs(5), async { + R::timeout(Duration::from_secs(5), async { loop { if a.state() == SerfState::Left { break; } - tokio::time::sleep(Duration::from_millis(20)).await; + R::sleep(Duration::from_millis(20)).await; } }) .await @@ -321,3 +340,59 @@ async fn leave_emits_left_cluster() { a.shutdown().await.expect("lv-a shuts down"); b.shutdown().await.expect("lv-b shuts down"); } + +// The tokio cells: the runtime-generic scenarios driven on tokio's multi-thread +// runtime. Gated on the `tokio` feature so the `--test tls -- smol` build (which +// enables only `smol`) can drop the `agnostic/tokio` code path. +#[cfg(feature = "tokio")] +mod tokio_cells { + use agnostic::tokio::TokioRuntime; + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn two_node_tls_join_converges() { + super::two_node_tls_join_converges::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn user_event_delivered() { + super::user_event_delivered::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn query_round_trip() { + super::query_round_trip::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn leave_emits_left_cluster() { + super::leave_emits_left_cluster::().await; + } +} + +// The smol cells: the identical scenarios instantiated over `SmolRuntime` and +// driven by smol's `block_on`. `cargo test --test tls -- smol` selects exactly +// these. +#[cfg(feature = "smol")] +mod smol_cells { + use agnostic::{RuntimeLite, smol::SmolRuntime}; + + #[test] + fn two_node_tls_join_converges_smol() { + SmolRuntime::block_on(super::two_node_tls_join_converges::()); + } + + #[test] + fn user_event_delivered_smol() { + SmolRuntime::block_on(super::user_event_delivered::()); + } + + #[test] + fn query_round_trip_smol() { + SmolRuntime::block_on(super::query_round_trip::()); + } + + #[test] + fn leave_emits_left_cluster_smol() { + SmolRuntime::block_on(super::leave_emits_left_cluster::()); + } +} From 93812e838ae4548269b88dbbb1e2842661ed18ca Mon Sep 17 00:00:00 2001 From: Al Liu Date: Fri, 3 Jul 2026 18:47:44 +1200 Subject: [PATCH 042/128] feat(serf-proto): no_std+alloc support (#72) --- serf-proto/Cargo.toml | 83 ++-- serf-proto/src/any/tests.rs | 8 +- serf-proto/src/bridge/mod.rs | 9 +- serf-proto/src/coordinate_client/mod.rs | 3 +- serf-proto/src/coordinate_client/tests.rs | 2 +- serf-proto/src/endpoint/mod.rs | 110 ++--- serf-proto/src/endpoint/serf_parity_tests.rs | 14 +- serf-proto/src/endpoint/tests.rs | 402 +++++++++---------- serf-proto/src/event/mod.rs | 8 +- serf-proto/src/event/tests.rs | 2 +- serf-proto/src/framing/mod.rs | 2 + serf-proto/src/lib.rs | 24 ++ serf-proto/src/mathf.rs | 26 ++ serf-proto/src/members/mod.rs | 12 +- serf-proto/src/members/tests.rs | 6 +- serf-proto/src/messages/tests.rs | 9 +- serf-proto/src/options/mod.rs | 2 +- serf-proto/src/options/tests.rs | 2 +- serf-proto/src/quic_endpoint/mod.rs | 2 +- serf-proto/src/snapshot/mod.rs | 5 +- serf-proto/src/snapshot/tests.rs | 2 +- serf-proto/src/stream_endpoint/mod.rs | 22 +- serf-proto/src/typed/mod.rs | 14 +- 23 files changed, 435 insertions(+), 334 deletions(-) create mode 100644 serf-proto/src/mathf.rs diff --git a/serf-proto/Cargo.toml b/serf-proto/Cargo.toml index 547c59f1..dddb6305 100644 --- a/serf-proto/Cargo.toml +++ b/serf-proto/Cargo.toml @@ -7,52 +7,89 @@ repository.workspace = true rust-version.workspace = true [features] -default = ["tag-regex"] +default = ["std", "tag-regex"] + +# `std` and `alloc` are independent capability tiers (std does NOT pull alloc). +# `alloc` is the minimal heap tier for no_std targets; the core machine + `tcp` +# are no_std+alloc, while `coordinates` (Vivaldi float math), `tag-regex` +# (regex), `tls` (rustls) and `quic` (quinn-proto) require `std`. +std = [ + "memberlist-proto/std", + "bytes/default", + "smol_str/default", + "thiserror/default", + "rustc-hash/default", + "rand/default", + "buffa/default", + "derive_more/std", + "zeroize?/std", +] +alloc = ["memberlist-proto/alloc", "rand/alloc"] + # Encryption key management messages (KeyRequestMessage / KeyResponseMessage). # Pulls in the memberlist-proto encryption backend (at least one AEAD cipher # must be selected; both are forwarded here so the caller can pick). # Activating this feature makes the key message types available and ensures # key material is held in memberlist_proto::SecretKey (typed, cipher-tagged) -# rather than raw bytes. -aes-gcm = ["memberlist-proto/aes-gcm", "dep:zeroize"] -chacha20-poly1305 = ["memberlist-proto/chacha20-poly1305", "dep:zeroize"] +# rather than raw bytes. no_std+alloc (memberlist's ciphers are no_std+alloc). +aes-gcm = ["alloc", "memberlist-proto/aes-gcm", "dep:zeroize"] +chacha20-poly1305 = ["alloc", "memberlist-proto/chacha20-poly1305", "dep:zeroize"] # Umbrella that activates both backends (convenient for tests / CI). encryption = ["aes-gcm", "chacha20-poly1305"] -# Vivaldi network coordinate estimation. Enabled by default under std. -# Disable for embedded / no_std targets or when the coordinate overhead is -# not desired. -coordinates = [] +# Vivaldi network coordinate estimation. Uses f64 transcendental math (ln/sqrt) +# with no libm-based no_std fallback yet, so it requires std. A libm-backed +# no_std coordinates is a future follow-up. +coordinates = ["std"] # Regex-backed tag-filter matching (mirrors Go serf `regexp.MatchString`). # When enabled (default), a `Filter::Tag` expression is compiled as a regex # and matched with partial (anywhere-in-value) semantics. When disabled # (`--no-default-features`), tag matching falls back to exact string equality. -# This crate currently requires std regardless of this flag; disabling -# `tag-regex` is the forward-compatible preparation for a future no_std port, -# not current no_std support. -tag-regex = ["dep:regex"] +# regex is std-only, so this feature pulls `std`; the no_std path is the +# exact-equality fallback. +tag-regex = ["std", "dep:regex"] # Plain-TCP reliable coordinator — enables `memberlist_proto::streams::StreamEndpoint` # (record-layer-shaped, label-prefixed stream transport). Activating this feature # makes `Reliable` implementable for `memberlist_proto::TcpEndpoint` and friends. -tcp = ["memberlist-proto/tcp"] +# The plain-TCP `RawRecords` path is no_std+alloc (the stream codec buffers into +# `Vec`). +tcp = ["alloc", "dep:libm", "memberlist-proto/tcp"] # TLS-over-TCP reliable coordinator — enables the TLS record layer on top of the -# plain-TCP stream transport. Implies `tcp`. -tls = ["tcp", "memberlist-proto/tls"] +# plain-TCP stream transport. Implies `tcp`. rustls is std-only. +tls = ["std", "tcp", "memberlist-proto/tls"] # QUIC coordinator — enables `memberlist_proto::QuicEndpoint` (unreliable datagram + # reliable stream over a single QUIC connection per peer). Requires a TLS backend; # at least one of `quic-rustls-ring` / `quic-rustls-aws-lc-rs` must be selected. -quic = ["memberlist-proto/quic"] +# quinn-proto is std-only. +quic = ["std", "dep:libm", "memberlist-proto/quic"] quic-rustls-ring = ["quic", "memberlist-proto/quic-rustls-ring"] [dependencies] -memberlist-proto = { git = "https://github.com/al8n/memberlist", branch = "main", default-features = false, features = ["std"] } -bytes = { workspace = true } -derive_more = { workspace = true } -rand = { workspace = true } +memberlist-proto = { git = "https://github.com/al8n/memberlist", branch = "main", default-features = false } +# no_std+alloc-capable deps: pinned `default-features = false` so a no_std build +# does not silently pull each dep's `std`; the `std` feature layers it back. +bytes = { version = "1", default-features = false } +derive_more = { version = "2", default-features = false, features = [ + "display", + "is_variant", +] } +rand = { version = "0.10", default-features = false } thiserror.workspace = true -bitflags = "2" +bitflags = { version = "2", default-features = false } buffa = { version = "0.8", default-features = false } -smol_str = { workspace = true } -zeroize = { version = "1", optional = true } +smol_str = { version = "0.3", default-features = false } +# hashbrown + rustc-hash back the `FxHashMap`/`FxHashSet` aliases (std's +# `HashMap`/`HashSet` are not in `alloc`). +hashbrown = { version = "0.17", default-features = false } +rustc-hash = { version = "2", default-features = false } +# f64 `log10`/`ceil` for the query-timeout fanout factor on no_std (core has no +# float transcendentals); pulled only by `tcp`/`quic` (the endpoint's gate). +libm = { version = "0.2", optional = true, default-features = false } +# Zeroizes the transient plaintext key buffer; only pulled by the AEAD features +# (which are the `alloc` tier), so `alloc` covers it. +zeroize = { version = "1", optional = true, default-features = false, features = [ + "alloc", +] } +# regex is std-only; gated behind `tag-regex` (which pulls `std`). regex = { version = "1", optional = true } [build-dependencies] diff --git a/serf-proto/src/any/tests.rs b/serf-proto/src/any/tests.rs index af8fef6e..ef3e39cb 100644 --- a/serf-proto/src/any/tests.rs +++ b/serf-proto/src/any/tests.rs @@ -1,6 +1,6 @@ //! Round-trip tests for AnyMessage encode and decode. -use std::net::SocketAddr; +use core::net::SocketAddr; use bytes::Bytes; use memberlist_proto::Node; @@ -146,7 +146,7 @@ fn any_message_query_round_trip() { filters: vec![], flags: QueryFlag::ACK, relay_factor: 1, - timeout: std::time::Duration::from_secs(1), + timeout: core::time::Duration::from_secs(1), name: SmolStr::from("my-query"), payload: Bytes::from_static(b"qp"), }; @@ -362,7 +362,7 @@ fn any_message_encode_query_round_trip() { filters: vec![], flags: QueryFlag::ACK, relay_factor: 0, - timeout: std::time::Duration::from_secs(1), + timeout: core::time::Duration::from_secs(1), name: SmolStr::from("qname"), payload: Bytes::new(), }); @@ -464,7 +464,7 @@ fn sealed_types_are_crate_internal() { // In-crate: the sealed types still EXIST after the seal (resolve here), // while no longer being `pub` to dependents. fn _assert_crate_visible( - _: core::marker::PhantomData>, + _: core::marker::PhantomData>, ) { } fn _assert_public(_: core::marker::PhantomData) {} diff --git a/serf-proto/src/bridge/mod.rs b/serf-proto/src/bridge/mod.rs index 6e71cfa2..18f90abf 100644 --- a/serf-proto/src/bridge/mod.rs +++ b/serf-proto/src/bridge/mod.rs @@ -4,7 +4,12 @@ //! the buffa side stores them as primitive protobuf types. These functions are //! the single boundary where those conversions happen. -use std::borrow::Cow; +use std::{ + borrow::{Cow, ToOwned}, + boxed::Box, + string::ToString, + vec::Vec, +}; use bytes::Bytes; #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] @@ -409,7 +414,7 @@ where ))?) .map_err(|_| BridgeError::InvalidValue("QueryMessage.relay_factor exceeds u8::MAX".into()))?; // Safe: query timeouts are measured in seconds to minutes, well within u64::MAX nanoseconds. - let timeout = std::time::Duration::from_nanos(b.timeout_nanos.ok_or( + let timeout = core::time::Duration::from_nanos(b.timeout_nanos.ok_or( BridgeError::MissingField("QueryMessage.timeout_nanos".into()), )?); diff --git a/serf-proto/src/coordinate_client/mod.rs b/serf-proto/src/coordinate_client/mod.rs index 7120f6f4..af98d294 100644 --- a/serf-proto/src/coordinate_client/mod.rs +++ b/serf-proto/src/coordinate_client/mod.rs @@ -15,7 +15,8 @@ //! host-based network coordinate systems." IEEE/ACM Transactions on //! Networking, 2010. -use std::{collections::HashMap, time::Duration}; +use core::time::Duration; +use std::collections::HashMap; use memberlist_proto::Rng; use rand::RngExt; diff --git a/serf-proto/src/coordinate_client/tests.rs b/serf-proto/src/coordinate_client/tests.rs index b3d025ce..00fc0aa0 100644 --- a/serf-proto/src/coordinate_client/tests.rs +++ b/serf-proto/src/coordinate_client/tests.rs @@ -1,6 +1,6 @@ //! Unit tests for the pure Vivaldi `CoordinateClient` engine. -use std::time::Duration; +use core::time::Duration; use memberlist_proto::SmallRng; use rand::SeedableRng; diff --git a/serf-proto/src/endpoint/mod.rs b/serf-proto/src/endpoint/mod.rs index 000a309d..25c5e5ed 100644 --- a/serf-proto/src/endpoint/mod.rs +++ b/serf-proto/src/endpoint/mod.rs @@ -56,7 +56,7 @@ //! Sans-I/O contract. Per-source rate-limiting of a flooding peer is likewise //! driver-side responsibility. -use std::collections::VecDeque; +use std::{collections::VecDeque, vec::Vec}; use bytes::Bytes; use memberlist_proto::{ @@ -229,7 +229,7 @@ const MAX_RECEIVED_QUERIES: usize = 2048; /// the response deadline, preventing a flooder from pinning `received_queries` /// entries open for an arbitrarily long time. The bound mirrors the originator's /// own default-timeout heuristic ceiling (several minutes at cluster scale). -const MAX_QUERY_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(600); +const MAX_QUERY_TIMEOUT: core::time::Duration = core::time::Duration::from_secs(600); /// Distinguishes the two call sites of `handle_query`. /// @@ -383,7 +383,7 @@ pub struct QueryParams { /// /// A value of `Duration::ZERO` causes the machine to substitute /// `gossip_interval * query_timeout_mult * log10(n_members + 1)`. - pub timeout: std::time::Duration, + pub timeout: core::time::Duration, } // ── PendingQuery ────────────────────────────────────────────────────────────── @@ -415,11 +415,11 @@ pub(crate) struct KeyResponseTally { /// Number of nodes that responded with `result = false`. pub(crate) num_err: usize, /// Key → count of nodes reporting that key. - pub(crate) keys: std::collections::HashMap, + pub(crate) keys: crate::FxHashMap, /// Primary key → count of nodes reporting it as primary. - pub(crate) primary_keys: std::collections::HashMap, + pub(crate) primary_keys: crate::FxHashMap, /// Per-node error message for nodes that reported result=false. - pub(crate) messages: std::collections::HashMap, + pub(crate) messages: crate::FxHashMap, } /// Bookkeeping for an in-flight query this node originated. @@ -437,13 +437,13 @@ pub(crate) struct PendingQuery { /// Tracks which responders have already replied (dedup by id). /// /// The value is `()` — membership is the only information we need. - pub(crate) responses: std::collections::HashMap, + pub(crate) responses: crate::FxHashMap, /// Tracks which peers have already acknowledged (dedup by id). /// /// Kept separate from `responses` because a single peer may send both an ack /// (on receipt) and a later response; deduping them in the same set would /// drop the response after the ack. Non-empty only for `request_ack` queries. - pub(crate) acks: std::collections::HashMap, + pub(crate) acks: crate::FxHashMap, /// The composite `(ltime, id)` key of the query. pub(crate) query_id: QueryId, /// Whether the originating `query()` requested acks from responders. @@ -725,7 +725,7 @@ where /// a query passes filters and is emitted as `Event::Query`. `respond()` looks /// up the entry to enforce the three guards (size, once-only, deadline) and /// to obtain the originator address for the directed send. - received_queries: std::collections::HashMap>, + received_queries: crate::FxHashMap>, /// serf-level events queued for the driver to drain via `poll_event`. pending_events: VecDeque>, /// The most recent directed-send (address, bytes) produced by @@ -754,7 +754,7 @@ where /// Updated on every successful `PingCompleted` RTT feed. Keyed by node id. /// Removed when a node is reaped from membership (G13). #[cfg(feature = "coordinates")] - coord_cache: std::collections::HashMap, + coord_cache: crate::FxHashMap, /// The `now` instant threaded into the most recent poll/handle call. /// /// The inner `poll_event` loop (drain_inner) fires synchronously from @@ -863,7 +863,7 @@ where event_buffer: EventBuffer::new(event_buf_size), query_buffer: QueryBuffer::new(query_buf_size), pending_queries: Vec::new(), - received_queries: std::collections::HashMap::new(), + received_queries: crate::FxHashMap::default(), pending_events: VecDeque::new(), drain_now: Instant::ORIGIN, // The snapshot starts dirty so the first push-pull always ships a fresh @@ -873,7 +873,7 @@ where #[cfg(feature = "coordinates")] coord_client, #[cfg(feature = "coordinates")] - coord_cache: std::collections::HashMap::new(), + coord_cache: crate::FxHashMap::default(), #[cfg(test)] last_dial_addr: None, #[cfg(test)] @@ -1562,7 +1562,7 @@ where // Entries whose ltime is not acceptable are silently excluded — they would // write a permanent status_time tombstone that no finite join intent can // ever outrank. - let status_map: std::collections::HashMap<&I, LamportTime> = pp + let status_map: crate::FxHashMap<&I, LamportTime> = pp .status_ltimes .iter() .filter(|(_, lt)| ltime_is_acceptable(lt.0)) @@ -1570,7 +1570,7 @@ where .collect(); // Build a fast lookup set for left_members so the join pass can skip them. - let left_set: std::collections::HashSet<&I> = pp.left_members.iter().collect(); + let left_set: crate::FxHashSet<&I> = pp.left_members.iter().collect(); let now = self.drain_now; @@ -2789,8 +2789,8 @@ where let timeout = if params.timeout.is_zero() { let n = self.members.states.len(); let mult = self.opts.query_timeout_mult(); - let log_factor = ((n as f64 + 1.0).log10().ceil() as u32).max(1); - std::time::Duration::from_millis(200) * mult as u32 * log_factor + let log_factor = (crate::mathf::ceil(crate::mathf::log10(n as f64 + 1.0)) as u32).max(1); + core::time::Duration::from_millis(200) * mult as u32 * log_factor } else { params.timeout }; @@ -2837,8 +2837,8 @@ where self.pending_queries.push(PendingQuery { kind: QueryPurpose::App, deadline, - responses: std::collections::HashMap::new(), - acks: std::collections::HashMap::new(), + responses: crate::FxHashMap::default(), + acks: crate::FxHashMap::default(), query_id, request_ack: params.request_ack, conflict_matching: 0, @@ -3771,9 +3771,9 @@ where pub(crate) fn test_seed_member(&mut self, id: I, status: MemberStatus, status_time: LamportTime) where I: Clone, - A: Clone + From, + A: Clone + From, { - let addr: A = std::net::SocketAddr::from(([127, 0, 0, 1], 0u16)).into(); + let addr: A = core::net::SocketAddr::from(([127, 0, 0, 1], 0u16)).into(); let node = memberlist_proto::Node::new(id.clone(), addr); let member = Member::new(node, Tags::new(), status); self @@ -3796,9 +3796,9 @@ where status_time: LamportTime, ) where I: Clone, - A: Clone + From, + A: Clone + From, { - let addr: A = std::net::SocketAddr::from(([127, 0, 0, 1], 0u16)).into(); + let addr: A = core::net::SocketAddr::from(([127, 0, 0, 1], 0u16)).into(); let node = memberlist_proto::Node::new(id.clone(), addr); let member = Member::new(node, tags, status); self @@ -3816,9 +3816,9 @@ where now: Instant, ) where I: Clone, - A: Clone + From, + A: Clone + From, { - let addr: A = std::net::SocketAddr::from(([127, 0, 0, 1], 0u16)).into(); + let addr: A = core::net::SocketAddr::from(([127, 0, 0, 1], 0u16)).into(); let node = memberlist_proto::Node::new(id.clone(), addr); let member = Member::new(node, Tags::new(), MemberStatus::Failed); self @@ -3837,9 +3837,9 @@ where now: Instant, ) where I: Clone, - A: Clone + From, + A: Clone + From, { - let addr: A = std::net::SocketAddr::from(([127, 0, 0, 1], 0u16)).into(); + let addr: A = core::net::SocketAddr::from(([127, 0, 0, 1], 0u16)).into(); let node = memberlist_proto::Node::new(id.clone(), addr); let member = Member::new(node, Tags::new(), MemberStatus::Left); self @@ -3880,10 +3880,10 @@ where pub(crate) fn test_inner_node_joined(&mut self, id: I, now: Instant) where I: Clone, - A: Clone + From, + A: Clone + From, { use std::sync::Arc; - let addr: A = std::net::SocketAddr::from(([127, 0, 0, 1], 0u16)).into(); + let addr: A = core::net::SocketAddr::from(([127, 0, 0, 1], 0u16)).into(); let ns = Arc::new(memberlist_proto::typed::NodeState::new( id, addr, @@ -3898,10 +3898,10 @@ where pub(crate) fn test_inner_node_left(&mut self, id: I, now: Instant) where I: Clone, - A: Clone + From, + A: Clone + From, { use std::sync::Arc; - let addr: A = std::net::SocketAddr::from(([127, 0, 0, 1], 0u16)).into(); + let addr: A = core::net::SocketAddr::from(([127, 0, 0, 1], 0u16)).into(); let ns = Arc::new(memberlist_proto::typed::NodeState::new( id, addr, @@ -3916,10 +3916,10 @@ where pub(crate) fn test_inner_node_updated(&mut self, id: I, now: Instant) where I: Clone, - A: Clone + From, + A: Clone + From, { use std::sync::Arc; - let addr: A = std::net::SocketAddr::from(([127, 0, 0, 1], 0u16)).into(); + let addr: A = core::net::SocketAddr::from(([127, 0, 0, 1], 0u16)).into(); let ns = Arc::new(memberlist_proto::typed::NodeState::new( id, addr, @@ -4099,9 +4099,9 @@ where pub(crate) fn test_seed_left_member(&mut self, id: I, status_time: LamportTime) where I: Clone, - A: Clone + From, + A: Clone + From, { - let addr: A = std::net::SocketAddr::from(([127, 0, 0, 1], 0u16)).into(); + let addr: A = core::net::SocketAddr::from(([127, 0, 0, 1], 0u16)).into(); let node = memberlist_proto::Node::new(id.clone(), addr); let member = Member::new(node, Tags::new(), MemberStatus::Left); self.members.states.insert( @@ -4501,8 +4501,8 @@ where let n = self.members.states.len(); let mult = self.opts.query_timeout_mult(); - let log_factor = ((n as f64 + 1.0).log10().ceil() as u32).max(1); - let timeout = std::time::Duration::from_millis(200) * mult as u32 * log_factor; + let log_factor = (crate::mathf::ceil(crate::mathf::log10(n as f64 + 1.0)) as u32).max(1); + let timeout = core::time::Duration::from_millis(200) * mult as u32 * log_factor; let q = QueryMessage { ltime, @@ -4543,9 +4543,9 @@ where let key_tally = if purpose == QueryPurpose::Key { Some(KeyResponseTally { num_err: 0, - keys: std::collections::HashMap::new(), - primary_keys: std::collections::HashMap::new(), - messages: std::collections::HashMap::new(), + keys: crate::FxHashMap::default(), + primary_keys: crate::FxHashMap::default(), + messages: crate::FxHashMap::default(), }) } else { None @@ -4554,8 +4554,8 @@ where self.pending_queries.push(PendingQuery { kind: purpose, deadline, - responses: std::collections::HashMap::new(), - acks: std::collections::HashMap::new(), + responses: crate::FxHashMap::default(), + acks: crate::FxHashMap::default(), query_id, request_ack: false, // Internal queries never request per-hop acks. conflict_matching: 0, @@ -4812,9 +4812,9 @@ where use crate::event::KeyResponse; let tally = pq.key_tally.unwrap_or_else(|| KeyResponseTally { num_err: 0, - keys: std::collections::HashMap::new(), - primary_keys: std::collections::HashMap::new(), - messages: std::collections::HashMap::new(), + keys: crate::FxHashMap::default(), + primary_keys: crate::FxHashMap::default(), + messages: crate::FxHashMap::default(), }); let num_resp = pq.responses.len(); // num_nodes was captured at query-issue time from members.states.len() @@ -4971,8 +4971,8 @@ where self.pending_queries.push(PendingQuery { kind: QueryPurpose::Conflict, deadline, - responses: std::collections::HashMap::new(), - acks: std::collections::HashMap::new(), + responses: crate::FxHashMap::default(), + acks: crate::FxHashMap::default(), query_id, request_ack: false, conflict_matching: 0, @@ -5001,17 +5001,17 @@ where self.pending_queries.push(PendingQuery { kind: QueryPurpose::Key, deadline, - responses: std::collections::HashMap::new(), - acks: std::collections::HashMap::new(), + responses: crate::FxHashMap::default(), + acks: crate::FxHashMap::default(), query_id, request_ack: false, conflict_matching: 0, num_nodes, key_tally: Some(KeyResponseTally { num_err: 0, - keys: std::collections::HashMap::new(), - primary_keys: std::collections::HashMap::new(), - messages: std::collections::HashMap::new(), + keys: crate::FxHashMap::default(), + primary_keys: crate::FxHashMap::default(), + messages: crate::FxHashMap::default(), }), }); query_id @@ -5089,10 +5089,10 @@ where pub(crate) fn test_inject_inner_joined(&mut self, id: I, now: Instant) where I: Clone, - A: Clone + From, + A: Clone + From, { use std::sync::Arc; - let addr: A = std::net::SocketAddr::from(([127, 0, 0, 1], 0u16)).into(); + let addr: A = core::net::SocketAddr::from(([127, 0, 0, 1], 0u16)).into(); let ns = Arc::new(memberlist_proto::typed::NodeState::new( id, addr, @@ -5233,7 +5233,7 @@ where &mut self, t: &mut T, node_id: &I, - rtt: std::time::Duration, + rtt: core::time::Duration, payload: &Bytes, ) where T: Reliable, @@ -5300,7 +5300,7 @@ where &mut self, t: &mut T, node_id: I, - rtt: std::time::Duration, + rtt: core::time::Duration, payload: Bytes, ) where T: Reliable, diff --git a/serf-proto/src/endpoint/serf_parity_tests.rs b/serf-proto/src/endpoint/serf_parity_tests.rs index 0d038f58..8af97966 100644 --- a/serf-proto/src/endpoint/serf_parity_tests.rs +++ b/serf-proto/src/endpoint/serf_parity_tests.rs @@ -19,7 +19,7 @@ use crate::{ typed::{PushPullMessage, UserEvent, UserEvents}, }; -fn ep() -> StreamEndpoint { +fn ep() -> StreamEndpoint { let inner_opts = EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); let inner = @@ -27,8 +27,8 @@ fn ep() -> StreamEndpoint { let coord = memberlist_proto::streams::StreamEndpoint::<_, _, RawRecords>::new( inner, LabelOptions::new_in(Some(b"serf-test".to_vec()), ()), - Box::new(|_addr: &std::net::SocketAddr| None), - Box::new(|addr: &std::net::SocketAddr| *addr), + Box::new(|_addr: &core::net::SocketAddr| None), + Box::new(|addr: &core::net::SocketAddr| *addr), ); let mut e = StreamEndpoint::new(coord, Options::new()); let _ = e.poll_event(); @@ -36,8 +36,8 @@ fn ep() -> StreamEndpoint { } /// A distinct loopback peer address for the per-peer ignore-join tests. -fn peer(port: u16) -> std::net::SocketAddr { - std::net::SocketAddr::from(([127, 0, 0, 1], port)) +fn peer(port: u16) -> core::net::SocketAddr { + core::net::SocketAddr::from(([127, 0, 0, 1], port)) } // ── base.rs handle_node_join invariants ─────────────────────────────────────── @@ -242,7 +242,7 @@ fn push_pull_body( events, LamportTime::new(query_ltime), ); - AnyMessage::::PushPull(pp) + AnyMessage::::PushPull(pp) .encode() .expect("encode must succeed in test") } @@ -556,7 +556,7 @@ fn tick_drains_inner_before_firing_reap() { // Advance to a time far past reconnect_timeout (24h + 1h) so the reaper // would normally remove member 2. - let past_timeout = t0 + std::time::Duration::from_secs(3600 * 25); + let past_timeout = t0 + core::time::Duration::from_secs(3600 * 25); // Inject a NodeJoined event for member 2 directly through the sieve // (simulates an inner event produced during inner.handle_timeout but before diff --git a/serf-proto/src/endpoint/tests.rs b/serf-proto/src/endpoint/tests.rs index 16866acb..303bb52a 100644 --- a/serf-proto/src/endpoint/tests.rs +++ b/serf-proto/src/endpoint/tests.rs @@ -20,13 +20,13 @@ type TestTransport = RawRecords; /// ignores it). A fixed cluster label keeps the handshake well-formed for the /// loopback tests that complete a real exchange. fn coord( - inner: memberlist_proto::Endpoint, -) -> memberlist_proto::streams::StreamEndpoint { + inner: memberlist_proto::Endpoint, +) -> memberlist_proto::streams::StreamEndpoint { memberlist_proto::streams::StreamEndpoint::new( inner, LabelOptions::new_in(Some(b"serf-test".to_vec()), ()), - Box::new(|_addr: &std::net::SocketAddr| None), - Box::new(|addr: &std::net::SocketAddr| *addr), + Box::new(|_addr: &core::net::SocketAddr| None), + Box::new(|addr: &core::net::SocketAddr| *addr), ) } @@ -34,7 +34,7 @@ fn coord( /// /// Uses `u32` node ids and `SocketAddr` addresses with a deterministically /// seeded `SmallRng` so tests are reproducible. -fn ep() -> StreamEndpoint { +fn ep() -> StreamEndpoint { let inner_opts = EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); let inner = memberlist_proto::Endpoint::new_at( @@ -51,7 +51,7 @@ fn ep() -> StreamEndpoint { /// Build a serf `Endpoint` with coordinates enabled (for coordinate-gated tests). #[cfg(feature = "coordinates")] -fn ep_with_coords() -> StreamEndpoint { +fn ep_with_coords() -> StreamEndpoint { let inner_opts = EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); let inner = memberlist_proto::Endpoint::new_at( @@ -464,7 +464,7 @@ fn leave_from_already_left_is_idempotent() { e.leave(memberlist_proto::Instant::ORIGIN).unwrap(); // Simulate the leave chain completing: inner LeftCluster + delay elapses. e.test_inner_left_cluster(); - let delay = std::time::Duration::from_secs(2); // > leave_propagate_delay (1s) + let delay = core::time::Duration::from_secs(2); // > leave_propagate_delay (1s) e.handle_timeout(memberlist_proto::Instant::ORIGIN + delay); assert!(e.state().is_left(), "should have transitioned to Left"); // A second leave from Left is Ok(()). @@ -518,7 +518,7 @@ fn inner_left_cluster_drives_serf_to_left_and_emits_left_cluster_event() { // Simulate the inner emitting LeftCluster (no live peers → immediate). e.test_inner_left_cluster(); // Advance time past leave_propagate_delay (default 1s). - let after_delay = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(2); + let after_delay = memberlist_proto::Instant::ORIGIN + core::time::Duration::from_secs(2); e.handle_timeout(after_delay); // State must be Left. assert!( @@ -542,7 +542,7 @@ fn leave_complete_deadline_not_fired_before_delay() { e.leave(memberlist_proto::Instant::ORIGIN).unwrap(); e.test_inner_left_cluster(); // Tick to just before the propagation deadline (< 1s). - let before_delay = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_millis(500); + let before_delay = memberlist_proto::Instant::ORIGIN + core::time::Duration::from_millis(500); e.handle_timeout(before_delay); // SerfState must still be Leaving (the deadline hasn't fired yet). assert!(e.state().is_leaving(), "must still be Leaving before delay"); @@ -565,7 +565,7 @@ fn shutdown_prevents_leaving_to_left_transition() { e.test_inner_left_cluster(); // Force Shutdown before the deadline fires. e.core_mut().state = SerfState::Shutdown; - let after_delay = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(2); + let after_delay = memberlist_proto::Instant::ORIGIN + core::time::Duration::from_secs(2); e.handle_timeout(after_delay); // Must remain Shutdown, not Left. assert!( @@ -593,7 +593,7 @@ fn leave_arms_broadcast_deadline() { // Default broadcast_timeout is 5s; deadline = ORIGIN + 5s. assert_eq!( dl, - memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(5) + memberlist_proto::Instant::ORIGIN + core::time::Duration::from_secs(5) ); } @@ -701,7 +701,7 @@ fn poll_timeout_includes_leave_deadlines_when_armed() { let timeout = e .poll_timeout() .expect("must have a deadline after leave()"); - let expected = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(5); + let expected = memberlist_proto::Instant::ORIGIN + core::time::Duration::from_secs(5); assert!( timeout <= expected, "poll_timeout must be ≤ leave_broadcast_deadline ({expected:?}), got {timeout:?}" @@ -713,9 +713,9 @@ fn poll_timeout_includes_leave_deadlines_when_armed() { // Helper: seed a failed member with an explicit address so we can assert what // addr is dialled by the reconnector. fn seed_failed( - e: &mut StreamEndpoint, + e: &mut StreamEndpoint, id: u32, - addr: std::net::SocketAddr, + addr: core::net::SocketAddr, leave_time: memberlist_proto::Instant, ) { e.test_seed_failed_member(id, addr, leave_time); @@ -723,7 +723,7 @@ fn seed_failed( // Helper: seed the endpoint's one alive member (the local node) explicitly so // the probability computation has a stable num_alive value. -fn seed_alive(e: &mut StreamEndpoint, id: u32) { +fn seed_alive(e: &mut StreamEndpoint, id: u32) { e.test_seed_member(id, MemberStatus::Alive, LamportTime::new(0)); } @@ -733,14 +733,14 @@ fn reap_failed_removes_after_reconnect_timeout() { let t0 = memberlist_proto::Instant::ORIGIN; seed_failed(&mut e, 2, "127.0.0.1:1002".parse().unwrap(), t0); // 25 hours > reconnect_timeout (24h) - let past_timeout = t0 + std::time::Duration::from_secs(3600 * 25); + let past_timeout = t0 + core::time::Duration::from_secs(3600 * 25); e.test_fire_reap(past_timeout); assert_eq!( e.test_member_status(2), None, "failed member should be reaped after reconnect_timeout" ); - let reaped = std::iter::from_fn(|| e.poll_event()) + let reaped = core::iter::from_fn(|| e.poll_event()) .any(|ev| matches!(ev, Event::Member(ref me) if me.kind() == MemberEventKind::Reap)); assert!(reaped, "a Member(Reap) event should have been emitted"); } @@ -751,7 +751,7 @@ fn reap_failed_keeps_member_before_reconnect_timeout() { let t0 = memberlist_proto::Instant::ORIGIN; seed_failed(&mut e, 2, "127.0.0.1:1002".parse().unwrap(), t0); // 1 hour < reconnect_timeout (24h) — should NOT reap - let before_timeout = t0 + std::time::Duration::from_secs(3600); + let before_timeout = t0 + core::time::Duration::from_secs(3600); e.test_fire_reap(before_timeout); assert_eq!( e.test_member_status(2), @@ -767,14 +767,14 @@ fn reap_left_removes_after_tombstone_timeout() { let t0 = memberlist_proto::Instant::ORIGIN; e.test_seed_left_member_by_status(2, LamportTime::new(3), t0); // 25 hours > tombstone_timeout (24h) - let past_timeout = t0 + std::time::Duration::from_secs(3600 * 25); + let past_timeout = t0 + core::time::Duration::from_secs(3600 * 25); e.test_fire_reap(past_timeout); assert_eq!( e.test_member_status(2), None, "left member should be reaped after tombstone_timeout" ); - let reaped = std::iter::from_fn(|| e.poll_event()) + let reaped = core::iter::from_fn(|| e.poll_event()) .any(|ev| matches!(ev, Event::Member(ref me) if me.kind() == MemberEventKind::Reap)); assert!(reaped, "a Member(Reap) event should have been emitted"); } @@ -786,7 +786,7 @@ fn reap_intents_removes_stale_intents() { // Buffer a leave intent for an unknown node at t0. e.test_handle_leave_intent(99, LamportTime::new(3), t0); // recent_intent_timeout is 600s (10 min). At t0 + 700s the intent is stale. - let past_intent_timeout = t0 + std::time::Duration::from_secs(700); + let past_intent_timeout = t0 + core::time::Duration::from_secs(700); e.test_fire_reap(past_intent_timeout); // The intent buffer should be empty now (no state was created, the node was unknown). // Verify by checking that the intent is no longer present — seeded as Leave intent for 99. @@ -844,7 +844,7 @@ fn reconnect_deadline_armed_and_polls_in_poll_timeout() { // Before arming: next_reconnect is None internally but poll_timeout may still // return Some from the inner. After handle_timeout fires and re-arms: // drive handle_timeout past the first reconnect_interval (30s). - let after_interval = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(31); + let after_interval = memberlist_proto::Instant::ORIGIN + core::time::Duration::from_secs(31); e.handle_timeout(after_interval); // After the tick the reconnect deadline is re-armed, so poll_timeout is Some. let _ = e.poll_timeout(); // must not panic @@ -856,7 +856,7 @@ fn reap_deadline_fires_via_handle_timeout() { let t0 = memberlist_proto::Instant::ORIGIN; seed_failed(&mut e, 2, "127.0.0.1:1002".parse().unwrap(), t0); // Drive handle_timeout well past reconnect_timeout (24h) + reap_interval (15s). - let far_future = t0 + std::time::Duration::from_secs(3600 * 25 + 16); + let far_future = t0 + core::time::Duration::from_secs(3600 * 25 + 16); e.handle_timeout(far_future); // Member should be reaped. assert_eq!( @@ -983,7 +983,7 @@ fn user_event_arrives_over_user_packet_and_surfaces() { // Event::User, and re-queue the ORIGINAL bytes on the event broadcast // tier (relay-retain: no re-encode). let mut e = ep(); - let serf_bytes = AnyMessage::::UserEvent(UserEventMessage { + let serf_bytes = AnyMessage::::UserEvent(UserEventMessage { ltime: 1.into(), cc: false, name: "deploy".into(), @@ -991,7 +991,7 @@ fn user_event_arrives_over_user_packet_and_surfaces() { }) .encode() .unwrap(); - let from: std::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); + let from: core::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); e.test_inject_user_packet(from, serf_bytes, memberlist_proto::Instant::ORIGIN); let ev = e.poll_event().expect("user event must surface"); @@ -1007,7 +1007,7 @@ fn duplicate_user_event_over_user_packet_is_deduped() { // The relay-retain property: first-sight re-queues original bytes; second // sight is dropped without touching the broadcast queue. let mut e = ep(); - let serf_bytes = AnyMessage::::UserEvent(UserEventMessage { + let serf_bytes = AnyMessage::::UserEvent(UserEventMessage { ltime: 3.into(), cc: false, name: "once".into(), @@ -1016,7 +1016,7 @@ fn duplicate_user_event_over_user_packet_is_deduped() { .encode() .unwrap(); - let from: std::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); + let from: core::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); e.test_inject_user_packet(from, serf_bytes.clone(), memberlist_proto::Instant::ORIGIN); let first = e.poll_event(); assert!( @@ -1039,11 +1039,11 @@ fn join_intent_over_user_packet_buffers_and_requeues_on_intent_tier() { // broadcast tier (rank 0 = highest priority). let mut e = ep(); let serf_bytes = - AnyMessage::::Join(JoinMessage::new(LamportTime::new(7), 2u32)) + AnyMessage::::Join(JoinMessage::new(LamportTime::new(7), 2u32)) .encode() .unwrap(); let before = e.user_broadcast_queue_len(); - let from: std::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); + let from: core::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); e.test_inject_user_packet(from, serf_bytes, memberlist_proto::Instant::ORIGIN); assert!( @@ -1059,7 +1059,7 @@ fn leave_intent_over_user_packet_dispatches_and_requeues() { // intent broadcast tier. let mut e = ep(); e.test_seed_member(2u32, MemberStatus::Alive, LamportTime::new(3)); - let serf_bytes = AnyMessage::::Leave(LeaveMessage::new( + let serf_bytes = AnyMessage::::Leave(LeaveMessage::new( LamportTime::new(8), 2u32, false, @@ -1067,7 +1067,7 @@ fn leave_intent_over_user_packet_dispatches_and_requeues() { .encode() .unwrap(); let before = e.user_broadcast_queue_len(); - let from: std::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); + let from: core::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); e.test_inject_user_packet(from, serf_bytes, memberlist_proto::Instant::ORIGIN); assert_eq!( @@ -1088,11 +1088,11 @@ fn stale_join_intent_over_user_packet_is_not_requeued() { let mut e = ep(); e.test_seed_member(2u32, MemberStatus::Alive, LamportTime::new(10)); let serf_bytes = - AnyMessage::::Join(JoinMessage::new(LamportTime::new(3), 2u32)) + AnyMessage::::Join(JoinMessage::new(LamportTime::new(3), 2u32)) .encode() .unwrap(); let before = e.user_broadcast_queue_len(); - let from: std::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); + let from: core::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); e.test_inject_user_packet(from, serf_bytes, memberlist_proto::Instant::ORIGIN); assert_eq!( @@ -1108,7 +1108,7 @@ fn user_event_over_user_packet_witnesses_event_clock() { // UserPacket. Both Reliable and Unreliable paths run through the same // handler; this test verifies the clock-witness side-effect. let mut e = ep(); - let serf_bytes = AnyMessage::::UserEvent(UserEventMessage { + let serf_bytes = AnyMessage::::UserEvent(UserEventMessage { ltime: 5.into(), cc: false, name: "ping".into(), @@ -1116,7 +1116,7 @@ fn user_event_over_user_packet_witnesses_event_clock() { }) .encode() .unwrap(); - let from: std::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); + let from: core::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); e.test_inject_user_packet(from, serf_bytes, memberlist_proto::Instant::ORIGIN); let ev = e.poll_event().expect("event must surface"); @@ -1135,7 +1135,7 @@ fn user_event_over_user_packet_witnesses_event_clock() { fn malformed_bytes_in_user_packet_are_silently_dropped() { // A UserPacket carrying garbage bytes must not panic or emit events. let mut e = ep(); - let from: std::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); + let from: core::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); e.test_inject_user_packet( from, bytes::Bytes::from_static(b"\xff\xff\xfe"), @@ -1154,7 +1154,7 @@ fn user_event_rebroadcast_uses_original_bytes() { // not re-encoded). We verify this by checking that the queue grows by // exactly the original encoding. let mut e = ep(); - let original_bytes = AnyMessage::::UserEvent(UserEventMessage { + let original_bytes = AnyMessage::::UserEvent(UserEventMessage { ltime: 2.into(), cc: false, name: "ship".into(), @@ -1164,7 +1164,7 @@ fn user_event_rebroadcast_uses_original_bytes() { .unwrap(); let queue_before = e.user_broadcast_queue_len(); - let from: std::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); + let from: core::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); e.test_inject_user_packet( from, original_bytes.clone(), @@ -1267,7 +1267,7 @@ fn ignore_join_stream_recorded_and_consumed_one_shot() { // (idempotently); the matching merge consumes the one-shot entry, and an // unrecorded stream is never present. let mut e = ep(); - let p = std::net::SocketAddr::from(([127, 0, 0, 1], 6100)); + let p = core::net::SocketAddr::from(([127, 0, 0, 1], 6100)); let s = e.start_join_push_pull( p, /*ignore_old*/ true, @@ -1314,7 +1314,7 @@ fn push_pull_local_state_bytes_deterministic() { // produce byte-identical push-pull wire output after resync_local_state. // This verifies that HashMap iteration order in `members.states` does NOT // leak into the encoded PushPullMessage. - fn build_ep_asc() -> StreamEndpoint { + fn build_ep_asc() -> StreamEndpoint { let mut e = ep(); e.test_set_clocks(5, 10, 15); // Insert members in ascending id order: 1, 2, 3, 4, 5. @@ -1324,7 +1324,7 @@ fn push_pull_local_state_bytes_deterministic() { e.resync_local_state(); e } - fn build_ep_desc() -> StreamEndpoint { + fn build_ep_desc() -> StreamEndpoint { let mut e = ep(); e.test_set_clocks(5, 10, 15); // Insert members in descending id order: 5, 4, 3, 2, 1. @@ -1354,7 +1354,7 @@ fn push_pull_local_state_bytes_deterministic() { /// /// Uses `ltime`, `id`, no filters, no flags, no relay, 5 s timeout, /// and a sentinel from-address. -fn test_query(ltime: LamportTime, id: u32) -> QueryMessage { +fn test_query(ltime: LamportTime, id: u32) -> QueryMessage { QueryMessage { ltime, id, @@ -1362,7 +1362,7 @@ fn test_query(ltime: LamportTime, id: u32) -> QueryMessage().unwrap(), + "127.0.0.1:7946".parse::().unwrap(), ) .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); let inner = memberlist_proto::Endpoint::new_at( @@ -1686,19 +1686,19 @@ fn invalid_tag_regex_does_not_advance_rng() { // ── Task 4.2: respond() three guards + query-response fold ─────────────────── -fn addr(port: u16) -> std::net::SocketAddr { +fn addr(port: u16) -> core::net::SocketAddr { format!("127.0.0.1:{port}").parse().unwrap() } fn t_secs(s: u64) -> memberlist_proto::Instant { - memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(s) + memberlist_proto::Instant::ORIGIN + core::time::Duration::from_secs(s) } fn qresp( ltime: LamportTime, id: u32, from_port: u16, -) -> crate::typed::QueryResponseMessage { +) -> crate::typed::QueryResponseMessage { crate::typed::QueryResponseMessage { ltime, id, @@ -1962,14 +1962,14 @@ fn query_response_via_user_packet_wire_path_surfaces_event() { // Encode a QueryResponseMessage as serf-level bytes and inject via the // UserPacket path (mimicking the gossip plane delivery). - let resp = crate::typed::QueryResponseMessage:: { + let resp = crate::typed::QueryResponseMessage:: { ltime: id.ltime, id: id.id, from: memberlist_proto::Node::new(2u32, addr(1002)), flags: QueryFlag::empty(), payload: bytes::Bytes::new(), }; - let serf_bytes = AnyMessage::::QueryResponse(resp) + let serf_bytes = AnyMessage::::QueryResponse(resp) .encode() .unwrap(); e.test_inject_user_packet(addr(1002), serf_bytes, memberlist_proto::Instant::ORIGIN); @@ -1986,7 +1986,7 @@ fn query_response_via_user_packet_wire_path_surfaces_event() { // ── Task 4.3: responder-side relay ─────────────────────────────────────────── /// Build a `Node` at `127.0.0.1:` with `id = port as u32`. -fn relay_node(port: u16) -> memberlist_proto::Node { +fn relay_node(port: u16) -> memberlist_proto::Node { memberlist_proto::Node::new(port as u32, addr(port)) } @@ -2106,7 +2106,7 @@ fn relay_sieve_arm_decodes_relay_message_from_user_packet() { let mut e = ep(); let inner_payload = bytes::Bytes::from_static(b"\x06fake-resp"); let relay = RelayMessage::new(relay_node(1002), inner_payload.clone()); - let serf_bytes = AnyMessage::::Relay(relay) + let serf_bytes = AnyMessage::::Relay(relay) .encode() .unwrap(); @@ -2123,7 +2123,7 @@ fn relay_sieve_arm_decodes_relay_message_from_user_packet() { /// Build a serf `StreamEndpoint` with an explicit RNG seed for the /// serf-level RNG (the relay/reconnect draws). The inner Endpoint uses a fixed /// seed 0; the serf-level seed is the caller-supplied `serf_seed`. -fn ep_with_serf_seed(serf_seed: u64) -> StreamEndpoint { +fn ep_with_serf_seed(serf_seed: u64) -> StreamEndpoint { let inner_opts = memberlist_proto::EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); let inner = memberlist_proto::Endpoint::new_at( @@ -2197,8 +2197,8 @@ fn relay_response_candidate_selection_is_deterministic() { // The selected peer addresses must be identical in both order and identity. // Any divergence here indicates the candidate ordering was not stabilised // before the Fisher-Yates shuffle. - let addrs_a: Vec = sends_a.iter().map(|(a, _)| *a).collect(); - let addrs_b: Vec = sends_b.iter().map(|(a, _)| *a).collect(); + let addrs_a: Vec = sends_a.iter().map(|(a, _)| *a).collect(); + let addrs_b: Vec = sends_b.iter().map(|(a, _)| *a).collect(); assert_eq!( addrs_a, addrs_b, "relay peer selection must be identical across endpoints with the same RNG seed \ @@ -2209,7 +2209,7 @@ fn relay_response_candidate_selection_is_deterministic() { // ── Task 4.4: conflict-resolution and key-management queries ───────────────── fn far_future() -> memberlist_proto::Instant { - memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(3600) + memberlist_proto::Instant::ORIGIN + core::time::Duration::from_secs(3600) } #[test] @@ -2224,7 +2224,7 @@ fn conflict_win_does_not_shut_down() { e.test_fold_conflict_response(qid, 101u32, true); e.test_fold_conflict_response(qid, 102u32, false); - let past = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(3601); + let past = memberlist_proto::Instant::ORIGIN + core::time::Duration::from_secs(3601); e.test_fire_due_query_closes(past); assert!( @@ -2245,7 +2245,7 @@ fn conflict_loss_emits_shutdown() { e.test_fold_conflict_response(qid, 201u32, false); e.test_fold_conflict_response(qid, 202u32, false); - let past = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(3601); + let past = memberlist_proto::Instant::ORIGIN + core::time::Duration::from_secs(3601); e.test_fire_due_query_closes(past); let ev = e.poll_event().expect("conflict loss must emit an event"); @@ -2262,7 +2262,7 @@ fn app_query_close_is_silent() { let mut e = ep(); let now = memberlist_proto::Instant::ORIGIN; let params = QueryParams { - timeout: std::time::Duration::from_millis(1), + timeout: core::time::Duration::from_millis(1), ..Default::default() }; let _ = e.query("test", Bytes::new(), params, now); @@ -2270,7 +2270,7 @@ fn app_query_close_is_silent() { while e.poll_event().is_some() {} // Advance time past the deadline. - let past = now + std::time::Duration::from_secs(1); + let past = now + core::time::Duration::from_secs(1); e.test_fire_due_query_closes(past); // No serf event should be emitted (App queries close silently). @@ -2308,7 +2308,7 @@ fn conflict_query_with_trailing_junk_is_dropped_entirely() { filters: vec![], flags: QueryFlag::empty(), relay_factor: 0, - timeout: std::time::Duration::from_secs(5), + timeout: core::time::Duration::from_secs(5), name: "_serf_conflict".into(), payload: Bytes::from(payload_with_junk), }; @@ -2368,7 +2368,7 @@ fn conflict_query_with_exact_payload_processes_normally() { filters: vec![], flags: QueryFlag::empty(), relay_factor: 0, - timeout: std::time::Duration::from_secs(5), + timeout: core::time::Duration::from_secs(5), name: "_serf_conflict".into(), payload: Bytes::from(id_bytes.to_vec()), }; @@ -2392,7 +2392,7 @@ fn conflict_query_with_exact_payload_processes_normally() { use crate::snapshot::ReplayResult; -fn snapshot_node(id: u32, port: u16) -> memberlist_proto::Node { +fn snapshot_node(id: u32, port: u16) -> memberlist_proto::Node { memberlist_proto::Node::new(id, format!("127.0.0.1:{port}").parse().unwrap()) } @@ -2474,7 +2474,7 @@ fn load_snapshot_skips_self_on_rejoin() { ); assert_eq!( dialled[0], - "127.0.0.1:1002".parse::().unwrap(), + "127.0.0.1:1002".parse::().unwrap(), "rejoin dial must target node 2's address" ); } @@ -2560,7 +2560,7 @@ fn ping_completed_updates_local_coordinate_and_caches_remote() { }; let payload = make_coord_payload(&peer_coord); - e.test_ping_completed(2u32, std::time::Duration::from_millis(40), payload); + e.test_ping_completed(2u32, core::time::Duration::from_millis(40), payload); // Half 1: remote coord is cached under node id 2. assert!( @@ -2584,7 +2584,7 @@ fn ping_completed_is_noop_when_coordinates_disabled() { let payload = Bytes::from_static(b"\x01garbage"); // Must not panic; test_ping_completed is a no-op without the feature or when disabled. #[cfg(feature = "coordinates")] - e.test_ping_completed(2u32, std::time::Duration::from_millis(40), payload); + e.test_ping_completed(2u32, core::time::Duration::from_millis(40), payload); #[cfg(not(feature = "coordinates"))] let _ = payload; // consume without calling the cfg-gated adapter assert!( @@ -2599,7 +2599,7 @@ fn ping_completed_bad_version_is_noop() { // A PingCompleted payload with a wrong version byte must be silently dropped. let mut e = ep_with_coords(); let payload = Bytes::from_static(b"\x02garbage"); // version 2, not 1 - e.test_ping_completed(2u32, std::time::Duration::from_millis(10), payload); + e.test_ping_completed(2u32, core::time::Duration::from_millis(10), payload); assert!( e.cached_coordinate(&2u32).is_none(), "bad version byte must not update coord_cache" @@ -2611,7 +2611,7 @@ fn ping_completed_bad_version_is_noop() { fn ping_completed_empty_payload_is_noop() { // An empty PingCompleted payload must be silently dropped. let mut e = ep_with_coords(); - e.test_ping_completed(2u32, std::time::Duration::from_millis(10), Bytes::new()); + e.test_ping_completed(2u32, core::time::Duration::from_millis(10), Bytes::new()); assert!( e.cached_coordinate(&2u32).is_none(), "empty payload must not update coord_cache" @@ -2632,7 +2632,7 @@ fn reap_forgets_coordinate() { height: 0.0, }; let payload = make_coord_payload(&peer_coord); - e.test_ping_completed(42u32, std::time::Duration::from_millis(20), payload); + e.test_ping_completed(42u32, core::time::Duration::from_millis(20), payload); assert!( e.cached_coordinate(&42u32).is_some(), "coordinate should be cached before reap" @@ -2643,7 +2643,7 @@ fn reap_forgets_coordinate() { e.test_seed_failed_member(42u32, "127.0.0.1:7947".parse().unwrap(), Instant::ORIGIN); // Fire the reaper at a time past reconnect_timeout (default = 24 h = 86 400 s). - let past = Instant::ORIGIN + std::time::Duration::from_secs(90_000); + let past = Instant::ORIGIN + core::time::Duration::from_secs(90_000); e.test_fire_reap(past); // The coordinate cache entry for 42 must be gone. @@ -2659,7 +2659,7 @@ fn reap_forgets_coordinate() { // // Use a tiny buffer (size=4) so ltime=1 and ltime=5 map to the same ring index // without needing a high clock that would make ltime=1 "too old". -fn ep_tiny_event_buf() -> StreamEndpoint { +fn ep_tiny_event_buf() -> StreamEndpoint { let inner_opts = memberlist_proto::EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); let inner = memberlist_proto::Endpoint::new_at( @@ -2724,12 +2724,12 @@ fn ack_query_produces_immediate_ack_directed_send() { // The destination must be the querier's address (127.0.0.1:9999 from test_query). assert_eq!( dest_addr, - "127.0.0.1:9999".parse::().unwrap(), + "127.0.0.1:9999".parse::().unwrap(), "ACK must be directed to the querier" ); // Decode the sent bytes and verify it is a QueryResponse with ACK flag. - let decoded = AnyMessage::::decode(&sent_bytes) + let decoded = AnyMessage::::decode(&sent_bytes) .expect("ACK bytes must decode as AnyMessage"); match decoded { AnyMessage::QueryResponse(resp) => { @@ -2744,7 +2744,7 @@ fn ack_query_produces_immediate_ack_directed_send() { } // Bug 3: Oversized query responses silently consumed. -fn ep_small_resp_limit() -> StreamEndpoint { +fn ep_small_resp_limit() -> StreamEndpoint { let inner_opts = memberlist_proto::EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); let inner = memberlist_proto::Endpoint::new_at( @@ -2825,7 +2825,7 @@ fn leave_broadcast_deadline_is_cleared_after_expiry() { ); // Tick past the deadline. - let past = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(6); + let past = memberlist_proto::Instant::ORIGIN + core::time::Duration::from_secs(6); e.handle_timeout(past); // The deadline must be cleared. @@ -2918,14 +2918,14 @@ fn malformed_conflict_response_does_not_inflate_denominator() { let qid = e.test_register_conflict_query(deadline); // Build a QueryResponseMessage carrying the wrong inner type (UserEvent bytes). - let bad_inner = AnyMessage::::UserEvent(UserEventMessage { + let bad_inner = AnyMessage::::UserEvent(UserEventMessage { ltime: 1.into(), cc: false, name: "bad".into(), payload: bytes::Bytes::new(), }); let bad_payload = bad_inner.encode().expect("encode must succeed"); - let bad_resp = crate::typed::QueryResponseMessage:: { + let bad_resp = crate::typed::QueryResponseMessage:: { ltime: qid.ltime, id: qid.id, from: memberlist_proto::Node::new(200u32, addr(2000)), @@ -2938,7 +2938,7 @@ fn malformed_conflict_response_does_not_inflate_denominator() { e.test_fold_conflict_response(qid, 201u32, true); // Fire query close at a time past the deadline. - let past = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(3601); + let past = memberlist_proto::Instant::ORIGIN + core::time::Duration::from_secs(3601); e.test_fire_due_query_closes(past); // Fix: malformed not counted → num_resp=1, majority=1, matching=1 → WIN → no Shutdown. @@ -2958,7 +2958,7 @@ fn respond_send_failure_returns_err_and_leaves_responded_false() { // pushing the total past 512. The serf query_response_size_limit is raised // to 50_000 so our guard passes and the inner's MTU check is the one that fires. let inner_opts = - EndpointOptions::::new(1u32, "127.0.0.1:7946".parse().unwrap()) + EndpointOptions::::new(1u32, "127.0.0.1:7946".parse().unwrap()) .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()) .with_gossip_mtu(512); // minimum MTU: a 490-byte payload + framing exceeds it. let inner = memberlist_proto::Endpoint::new_at( @@ -2967,7 +2967,7 @@ fn respond_send_failure_returns_err_and_leaves_responded_false() { SmallRng::seed_from_u64(0), ); let opts = Options::new().with_query_response_size_limit(50_000); - let mut e: StreamEndpoint = + let mut e: StreamEndpoint = StreamEndpoint::new(coord(inner), opts); let qid = QueryId { @@ -3103,7 +3103,7 @@ fn zero_conflict_responses_does_not_shut_down() { // increment responses; we just confirm close_conflict_query also handles // the num_resp==0 case correctly when even no malformed response is present. // Fire query close: no responses at all (responses is empty). - let past = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(3601); + let past = memberlist_proto::Instant::ORIGIN + core::time::Duration::from_secs(3601); e.test_fire_due_query_closes(past); assert!( @@ -3122,14 +3122,14 @@ fn zero_valid_conflict_responses_all_malformed_does_not_shut_down() { let qid = e.test_register_conflict_query(deadline); // Inject a QueryResponseMessage carrying wrong inner type (dropped by Bug 2 fix). - let bad_inner = AnyMessage::::UserEvent(UserEventMessage { + let bad_inner = AnyMessage::::UserEvent(UserEventMessage { ltime: 1.into(), cc: false, name: "bad".into(), payload: bytes::Bytes::new(), }); let bad_payload = bad_inner.encode().expect("encode must succeed"); - let bad_resp = crate::typed::QueryResponseMessage:: { + let bad_resp = crate::typed::QueryResponseMessage:: { ltime: qid.ltime, id: qid.id, from: memberlist_proto::Node::new(300u32, addr(3000)), @@ -3145,7 +3145,7 @@ fn zero_valid_conflict_responses_all_malformed_does_not_shut_down() { "malformed conflict response must not be counted" ); - let past = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(3601); + let past = memberlist_proto::Instant::ORIGIN + core::time::Duration::from_secs(3601); e.test_fire_due_query_closes(past); assert!( @@ -3169,14 +3169,14 @@ fn malformed_key_response_does_not_inflate_num_resp() { let qid = e.test_register_key_query(deadline); // Build a QueryResponseMessage carrying wrong inner type. - let bad_inner = AnyMessage::::UserEvent(UserEventMessage { + let bad_inner = AnyMessage::::UserEvent(UserEventMessage { ltime: 1.into(), cc: false, name: "bad".into(), payload: bytes::Bytes::new(), }); let bad_payload = bad_inner.encode().expect("encode must succeed"); - let bad_resp = crate::typed::QueryResponseMessage:: { + let bad_resp = crate::typed::QueryResponseMessage:: { ltime: qid.ltime, id: qid.id, from: memberlist_proto::Node::new(400u32, addr(4000)), @@ -3248,14 +3248,14 @@ fn ingress_max_ltime_query_does_not_regress_clock() { use crate::typed::{QueryFlag, QueryMessage}; let mut e = ep(); e.test_set_clocks(0, 0, 20); - let msg = QueryMessage:: { + let msg = QueryMessage:: { ltime: LamportTime::new(u64::MAX), id: 42, from: memberlist_proto::Node::new(1u32, addr(9000)), filters: vec![], flags: QueryFlag::empty(), relay_factor: 0, - timeout: std::time::Duration::from_secs(1), + timeout: core::time::Duration::from_secs(1), name: "flood".into(), payload: bytes::Bytes::new(), }; @@ -3339,14 +3339,14 @@ fn endpoint_query_buffer_slot_capped_via_adapter() { // Push unique ids up to the cap through handle_query. for id in 0..MAX_QUERY_IDS_PER_LTIME as u32 { - let msg = QueryMessage:: { + let msg = QueryMessage:: { ltime: LamportTime::new(ltime), id, from: memberlist_proto::Node::new(id + 10, addr(9000)), filters: vec![], flags: QueryFlag::NO_BROADCAST, relay_factor: 0, - timeout: std::time::Duration::from_secs(1), + timeout: core::time::Duration::from_secs(1), name: "flood".into(), payload: bytes::Bytes::new(), }; @@ -3360,14 +3360,14 @@ fn endpoint_query_buffer_slot_capped_via_adapter() { ); // One more unique id: must not grow the slot. - let overflow_msg = QueryMessage:: { + let overflow_msg = QueryMessage:: { ltime: LamportTime::new(ltime), id: MAX_QUERY_IDS_PER_LTIME as u32, from: memberlist_proto::Node::new(9999u32, addr(9999)), filters: vec![], flags: QueryFlag::NO_BROADCAST, relay_factor: 0, - timeout: std::time::Duration::from_secs(1), + timeout: core::time::Duration::from_secs(1), name: "overflow".into(), payload: bytes::Bytes::new(), }; @@ -3393,7 +3393,7 @@ fn resync_keeps_dirty_when_inner_snapshot_rejects() { // (1 MiB) which saturating_sub-underflows to 0, so any non-empty serf // PushPull (encoded_len > 0) exceeds the budget and is rejected. let inner_opts = - EndpointOptions::::new(1u32, "127.0.0.1:7946".parse().unwrap()) + EndpointOptions::::new(1u32, "127.0.0.1:7946".parse().unwrap()) .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()) .with_max_stream_frame_size(547); // passes construction but rejects serf PushPull. let inner = memberlist_proto::Endpoint::new_at( @@ -3401,7 +3401,7 @@ fn resync_keeps_dirty_when_inner_snapshot_rejects() { memberlist_proto::Instant::ORIGIN, SmallRng::seed_from_u64(0), ); - let mut e: StreamEndpoint = + let mut e: StreamEndpoint = StreamEndpoint::new(coord(inner), Options::new()); // Force dirty and call resync. @@ -3417,7 +3417,7 @@ fn resync_keeps_dirty_when_inner_snapshot_rejects() { // ── Bug 1: class sweep — reject u64::MAX Lamport times at all ingress sites ── -fn sa(port: u16) -> std::net::SocketAddr { +fn sa(port: u16) -> core::net::SocketAddr { format!("127.0.0.1:{port}").parse().unwrap() } @@ -3526,14 +3526,14 @@ fn query_max_ltime_is_dropped_no_state_mutation() { let mut e = ep(); let clock_before = e.query_time(); - let msg = QueryMessage:: { + let msg = QueryMessage:: { ltime: LamportTime::new(u64::MAX), id: 1, from: memberlist_proto::Node::new(99u32, sa(9001)), filters: vec![], flags: QueryFlag::NO_BROADCAST, relay_factor: 0, - timeout: std::time::Duration::from_secs(1), + timeout: core::time::Duration::from_secs(1), name: "test".into(), payload: bytes::Bytes::new(), }; @@ -3576,7 +3576,7 @@ fn merge_remote_state_max_member_clock_is_ignored() { left_members: vec![], events: vec![], }; - let encoded = AnyMessage::::PushPull(pp) + let encoded = AnyMessage::::PushPull(pp) .encode() .expect("encode must succeed"); e.test_merge_remote_state(encoded); @@ -3737,7 +3737,7 @@ fn key_query_num_nodes_equals_member_count_at_issue_time() { } assert_eq!(e.num_members(), 4); - let far_future = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(9999); + let far_future = memberlist_proto::Instant::ORIGIN + core::time::Duration::from_secs(9999); let _query_id = e.test_register_key_query(far_future); // num_nodes is captured at registration time. @@ -3750,7 +3750,7 @@ fn key_query_num_nodes_equals_member_count_at_issue_time() { ); // Fire the timeout: close_key_query emits Event::KeyResponse. - let after_deadline = far_future + std::time::Duration::from_nanos(1); + let after_deadline = far_future + core::time::Duration::from_nanos(1); e.test_fire_due_query_closes(after_deadline); let ev = e @@ -3778,7 +3778,7 @@ fn key_query_num_nodes_is_captured_at_issue_not_at_close() { e.test_seed_member(10u32, MemberStatus::Alive, LamportTime::new(1)); assert_eq!(e.num_members(), 2); - let far_future = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(9999); + let far_future = memberlist_proto::Instant::ORIGIN + core::time::Duration::from_secs(9999); let _query_id = e.test_register_key_query(far_future); // Now add 2 more members after the query was issued. @@ -3787,7 +3787,7 @@ fn key_query_num_nodes_is_captured_at_issue_not_at_close() { assert_eq!(e.num_members(), 4); // Close the query: num_nodes must reflect membership AT ISSUE TIME (2), not now (4). - let after_deadline = far_future + std::time::Duration::from_nanos(1); + let after_deadline = far_future + core::time::Duration::from_nanos(1); e.test_fire_due_query_closes(after_deadline); let ev = e.poll_event().expect("Event::KeyResponse must be emitted"); @@ -3912,14 +3912,14 @@ fn query_max_minus_one_ltime_is_dropped() { let mut e = ep(); let clock_before = e.query_time(); - let msg = crate::typed::QueryMessage:: { + let msg = crate::typed::QueryMessage:: { ltime: LamportTime::new(u64::MAX - 1), id: 1, from: memberlist_proto::Node::new(99u32, sa(9001)), filters: vec![], flags: crate::typed::QueryFlag::NO_BROADCAST, relay_factor: 0, - timeout: std::time::Duration::from_secs(1), + timeout: core::time::Duration::from_secs(1), name: "bad".into(), payload: bytes::Bytes::new(), }; @@ -3969,7 +3969,7 @@ fn merge_remote_state_max_minus_one_member_clock_drops_entire_message() { left_members: vec![], events: vec![], }; - let encoded = AnyMessage::::PushPull(pp) + let encoded = AnyMessage::::PushPull(pp) .encode() .expect("encode must succeed"); e.test_merge_remote_state(encoded); @@ -4011,7 +4011,7 @@ fn merge_remote_state_max_minus_one_event_clock_drops_entire_message() { left_members: vec![], events: vec![], }; - let encoded = AnyMessage::::PushPull(pp) + let encoded = AnyMessage::::PushPull(pp) .encode() .expect("encode must succeed"); e.test_merge_remote_state(encoded); @@ -4046,7 +4046,7 @@ fn conflict_response_from_unknown_responder_is_not_counted() { let qid = e.test_register_conflict_query(far_future); // Send a response claiming to be from id=999 — unknown, not in members.states. - let forged = crate::typed::QueryResponseMessage:: { + let forged = crate::typed::QueryResponseMessage:: { ltime: qid.ltime, id: qid.id, from: memberlist_proto::Node::new(999u32, addr(9999)), @@ -4082,7 +4082,7 @@ fn conflict_response_flood_of_unknown_ids_does_not_inflate_denominator() { // Send 1000 responses each from a different unknown id. for forger_id in 1000u32..2000 { - let resp = crate::typed::QueryResponseMessage:: { + let resp = crate::typed::QueryResponseMessage:: { ltime: qid.ltime, id: qid.id, from: memberlist_proto::Node::new(forger_id, addr(9000)), @@ -4110,9 +4110,9 @@ fn conflict_response_from_known_member_is_counted() { let qid = e.test_register_conflict_query(far_future); // Build a valid ConflictResponseMessage payload so the payload check passes. - let member_node = memberlist_proto::Node::::new(10u32, addr(7946)); + let member_node = memberlist_proto::Node::::new(10u32, addr(7946)); let conflict_resp = ConflictResponseMessage::new(member_node); - let payload = AnyMessage::::ConflictResponse(conflict_resp) + let payload = AnyMessage::::ConflictResponse(conflict_resp) .encode() .expect("encode conflict response"); @@ -4181,7 +4181,7 @@ fn recent_intents_capped_on_leave_intent_flood() { #[test] fn zero_event_buffer_size_does_not_panic_on_first_event() { let inner_opts = - EndpointOptions::::new(1u32, "127.0.0.1:7946".parse().unwrap()) + EndpointOptions::::new(1u32, "127.0.0.1:7946".parse().unwrap()) .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); let inner = memberlist_proto::Endpoint::new_at( inner_opts, @@ -4190,7 +4190,7 @@ fn zero_event_buffer_size_does_not_panic_on_first_event() { ); // event_buffer_size = 0 should be clamped to 1 internally. let opts = crate::options::Options::new().with_event_buffer_size(0); - let mut e: StreamEndpoint = + let mut e: StreamEndpoint = StreamEndpoint::new(coord(inner), opts); // Must NOT panic. @@ -4205,7 +4205,7 @@ fn zero_event_buffer_size_does_not_panic_on_merge_remote_state() { use crate::typed::PushPullMessage; let inner_opts = - EndpointOptions::::new(1u32, "127.0.0.1:7946".parse().unwrap()) + EndpointOptions::::new(1u32, "127.0.0.1:7946".parse().unwrap()) .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); let inner = memberlist_proto::Endpoint::new_at( inner_opts, @@ -4213,7 +4213,7 @@ fn zero_event_buffer_size_does_not_panic_on_merge_remote_state() { SmallRng::seed_from_u64(0), ); let opts = crate::options::Options::new().with_event_buffer_size(0); - let mut e: StreamEndpoint = + let mut e: StreamEndpoint = StreamEndpoint::new(coord(inner), opts); let pp = PushPullMessage:: { @@ -4230,7 +4230,7 @@ fn zero_event_buffer_size_does_not_panic_on_merge_remote_state() { }], }], }; - let encoded = AnyMessage::::PushPull(pp) + let encoded = AnyMessage::::PushPull(pp) .encode() .expect("encode must succeed"); @@ -4338,14 +4338,14 @@ fn received_queries_capped_at_max_via_handle_query() { // Send MAX_RECEIVED_QUERIES + 20 unique (ltime=1, id=i) queries. // Each passes filter (no filters = broadcast all) and unique id = first sight. for i in 0..(MAX_RECEIVED_QUERIES + 20) as u32 { - let msg = crate::typed::QueryMessage:: { + let msg = crate::typed::QueryMessage:: { ltime: LamportTime::new(1), id: i, from: memberlist_proto::Node::new(99u32, sa(9001)), filters: vec![], flags: QueryFlag::NO_BROADCAST, relay_factor: 0, - timeout: std::time::Duration::from_secs(3600), // huge timeout + timeout: core::time::Duration::from_secs(3600), // huge timeout name: "flood".into(), payload: bytes::Bytes::new(), }; @@ -4368,14 +4368,14 @@ fn inbound_query_timeout_clamped() { let now = memberlist_proto::Instant::ORIGIN; // Inject a query with a 1-hour timeout (far exceeds MAX_QUERY_TIMEOUT = 600s). - let msg = crate::typed::QueryMessage:: { + let msg = crate::typed::QueryMessage:: { ltime: LamportTime::new(1), id: 77, from: memberlist_proto::Node::new(99u32, sa(9001)), filters: vec![], flags: QueryFlag::NO_BROADCAST, relay_factor: 0, - timeout: std::time::Duration::from_secs(3600), + timeout: core::time::Duration::from_secs(3600), name: "test".into(), payload: bytes::Bytes::new(), }; @@ -4437,25 +4437,25 @@ fn inbound_query_oversized_is_dropped_before_state_mutation() { memberlist_proto::SmallRng::seed_from_u64(0), ); let opts = crate::options::Options::new().with_query_size_limit(64); - let mut e: StreamEndpoint = + let mut e: StreamEndpoint = StreamEndpoint::new(coord(inner), opts); // Drain the construction self-join so it does not appear as a spurious event. let _ = e.poll_event(); // Encode a QueryMessage whose payload pushes the wire encoding over 64 bytes. let oversized_payload = bytes::Bytes::from(vec![0u8; 100]); - let q = QueryMessage:: { + let q = QueryMessage:: { ltime: LamportTime::new(3), id: 0xabcd_ef01, from: memberlist_proto::Node::new(2u32, sa(9002)), filters: vec![], flags: QueryFlag::empty(), relay_factor: 0, - timeout: std::time::Duration::from_secs(5), + timeout: core::time::Duration::from_secs(5), name: "ping".into(), payload: oversized_payload, }; - let encoded = AnyMessage::::Query(q) + let encoded = AnyMessage::::Query(q) .encode() .unwrap(); assert!( @@ -4503,18 +4503,18 @@ fn inbound_query_oversized_is_dropped_before_state_mutation() { fn inbound_query_within_size_limit_is_accepted() { let mut e = ep(); // query_size_limit = 1024 (default) - let q = QueryMessage:: { + let q = QueryMessage:: { ltime: LamportTime::new(1), id: 0x1234_5678, from: memberlist_proto::Node::new(2u32, sa(9002)), filters: vec![], flags: QueryFlag::empty(), relay_factor: 0, - timeout: std::time::Duration::from_secs(5), + timeout: core::time::Duration::from_secs(5), name: "ok".into(), payload: bytes::Bytes::from_static(b"small"), }; - let encoded = AnyMessage::::Query(q) + let encoded = AnyMessage::::Query(q) .encode() .unwrap(); assert!( @@ -4550,7 +4550,7 @@ fn inbound_query_within_size_limit_is_accepted() { #[test] fn ack_for_non_ack_query_is_dropped() { let mut e = ep(); - let deadline = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(10); + let deadline = memberlist_proto::Instant::ORIGIN + core::time::Duration::from_secs(10); // Register a pending query WITHOUT request_ack. let qid = QueryId { @@ -4560,8 +4560,8 @@ fn ack_for_non_ack_query_is_dropped() { e.core_mut().pending_queries.push(PendingQuery { kind: QueryPurpose::App, deadline, - responses: std::collections::HashMap::new(), - acks: std::collections::HashMap::new(), + responses: crate::FxHashMap::default(), + acks: crate::FxHashMap::default(), query_id: qid, request_ack: false, conflict_matching: 0, @@ -4572,7 +4572,7 @@ fn ack_for_non_ack_query_is_dropped() { }); // Inject an ACK response for that query. - let ack_msg = crate::typed::QueryResponseMessage:: { + let ack_msg = crate::typed::QueryResponseMessage:: { ltime: LamportTime::new(1), id: 42, from: memberlist_proto::Node::new(99u32, sa(9001)), @@ -4592,7 +4592,7 @@ fn ack_for_non_ack_query_is_dropped() { #[test] fn ack_for_ack_query_is_delivered() { let mut e = ep(); - let deadline = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(10); + let deadline = memberlist_proto::Instant::ORIGIN + core::time::Duration::from_secs(10); let qid = QueryId { ltime: LamportTime::new(1), @@ -4601,8 +4601,8 @@ fn ack_for_ack_query_is_delivered() { e.core_mut().pending_queries.push(PendingQuery { kind: QueryPurpose::App, deadline, - responses: std::collections::HashMap::new(), - acks: std::collections::HashMap::new(), + responses: crate::FxHashMap::default(), + acks: crate::FxHashMap::default(), query_id: qid, request_ack: true, // <-- requesting acks conflict_matching: 0, @@ -4613,7 +4613,7 @@ fn ack_for_ack_query_is_delivered() { }); // Inject an ACK response. - let ack_msg = crate::typed::QueryResponseMessage:: { + let ack_msg = crate::typed::QueryResponseMessage:: { ltime: LamportTime::new(1), id: 42, from: memberlist_proto::Node::new(99u32, sa(9001)), @@ -4644,7 +4644,7 @@ fn user_event_with_trailing_junk_is_dropped_no_event_no_clock_advance() { let mut e = ep(); // Build a valid UserEvent frame. - let valid = AnyMessage::::UserEvent(UserEventMessage { + let valid = AnyMessage::::UserEvent(UserEventMessage { ltime: 7.into(), cc: false, name: "op".into(), @@ -4660,7 +4660,7 @@ fn user_event_with_trailing_junk_is_dropped_no_event_no_clock_advance() { let clock_before = e.event_time(); let queue_before = e.user_broadcast_queue_len(); - let from: std::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); + let from: core::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); e.test_inject_user_packet(from, padded, memberlist_proto::Instant::ORIGIN); @@ -4687,7 +4687,7 @@ fn join_intent_with_trailing_junk_is_dropped() { let mut e = ep(); let valid = - AnyMessage::::Join(JoinMessage::new(LamportTime::new(5), 99u32)) + AnyMessage::::Join(JoinMessage::new(LamportTime::new(5), 99u32)) .encode() .unwrap(); let mut padded = valid.to_vec(); @@ -4696,7 +4696,7 @@ fn join_intent_with_trailing_junk_is_dropped() { let clock_before = e.member_time(); let queue_before = e.user_broadcast_queue_len(); - let from: std::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); + let from: core::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); e.test_inject_user_packet(from, padded, memberlist_proto::Instant::ORIGIN); @@ -4739,7 +4739,7 @@ fn merge_remote_state_with_trailing_junk_is_dropped() { events: vec![], query_ltime: LamportTime::new(2), }; - let valid = AnyMessage::::PushPull(pp) + let valid = AnyMessage::::PushPull(pp) .encode() .expect("encode must succeed"); // Append trailing junk. @@ -4776,17 +4776,17 @@ fn conflict_response_with_trailing_junk_is_not_counted() { use memberlist_proto::Node; let mut e = ep(); - let local_addr: std::net::SocketAddr = "127.0.0.1:7946".parse().unwrap(); - let deadline = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(3600); + let local_addr: core::net::SocketAddr = "127.0.0.1:7946".parse().unwrap(); + let deadline = memberlist_proto::Instant::ORIGIN + core::time::Duration::from_secs(3600); let qid = e.test_register_conflict_query(deadline); // Seed the responder as a known Alive member so the membership gate passes. - let responder_addr: std::net::SocketAddr = "127.0.0.1:2000".parse().unwrap(); + let responder_addr: core::net::SocketAddr = "127.0.0.1:2000".parse().unwrap(); e.test_seed_member(200u32, MemberStatus::Alive, LamportTime::new(1)); // Encode a ConflictResponseMessage that points to the local address (would agree if counted). let resp_msg = ConflictResponseMessage::new(Node::new(999u32, local_addr)); - let valid = AnyMessage::::ConflictResponse(resp_msg) + let valid = AnyMessage::::ConflictResponse(resp_msg) .encode() .expect("encode must succeed"); // Append trailing junk — must be dropped. @@ -4795,7 +4795,7 @@ fn conflict_response_with_trailing_junk_is_not_counted() { let padded = bytes::Bytes::from(padded); // Inject via handle_query_response (which routes to handle_conflict_response_fold). - let qresp = crate::typed::QueryResponseMessage:: { + let qresp = crate::typed::QueryResponseMessage:: { ltime: qid.ltime, id: qid.id, from: Node::new(200u32, responder_addr), @@ -4820,7 +4820,7 @@ fn key_response_with_trailing_junk_is_not_tallied() { use memberlist_proto::Node; let mut e = ep(); - let deadline = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(3600); + let deadline = memberlist_proto::Instant::ORIGIN + core::time::Duration::from_secs(3600); let qid = e.test_register_key_query(deadline); // Seed the responder as a known Alive member so the membership gate passes. @@ -4833,7 +4833,7 @@ fn key_response_with_trailing_junk_is_not_tallied() { keys: vec![], primary_key: None, }; - let valid = AnyMessage::::KeyResponse(key_msg) + let valid = AnyMessage::::KeyResponse(key_msg) .encode() .expect("encode must succeed"); // Append trailing junk — must be dropped. @@ -4842,7 +4842,7 @@ fn key_response_with_trailing_junk_is_not_tallied() { padded.push(0xef); let padded = bytes::Bytes::from(padded); - let qresp = crate::typed::QueryResponseMessage:: { + let qresp = crate::typed::QueryResponseMessage:: { ltime: qid.ltime, id: qid.id, from: Node::new(300u32, "127.0.0.1:3000".parse().unwrap()), @@ -4866,19 +4866,19 @@ fn key_response_with_trailing_junk_is_not_tallied() { #[test] fn zero_reap_interval_advances_deadline() { let inner_opts = - EndpointOptions::::new(1u32, "127.0.0.1:7946".parse().unwrap()) + EndpointOptions::::new(1u32, "127.0.0.1:7946".parse().unwrap()) .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); let inner = memberlist_proto::Endpoint::new_at( inner_opts, memberlist_proto::Instant::ORIGIN, SmallRng::seed_from_u64(0), ); - let opts = Options::new().with_reap_interval(std::time::Duration::ZERO); - let mut e: StreamEndpoint = + let opts = Options::new().with_reap_interval(core::time::Duration::ZERO); + let mut e: StreamEndpoint = StreamEndpoint::new(coord(inner), opts); // Advance time past the first reap deadline so handle_timeout fires it. - let now = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(1); + let now = memberlist_proto::Instant::ORIGIN + core::time::Duration::from_secs(1); e.handle_timeout(now); // The deadline must be strictly after `now` — clamp prevents busy-loop. @@ -4897,18 +4897,18 @@ fn zero_reap_interval_advances_deadline() { #[test] fn zero_reconnect_interval_advances_deadline() { let inner_opts = - EndpointOptions::::new(1u32, "127.0.0.1:7946".parse().unwrap()) + EndpointOptions::::new(1u32, "127.0.0.1:7946".parse().unwrap()) .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); let inner = memberlist_proto::Endpoint::new_at( inner_opts, memberlist_proto::Instant::ORIGIN, SmallRng::seed_from_u64(0), ); - let opts = Options::new().with_reconnect_interval(std::time::Duration::ZERO); - let mut e: StreamEndpoint = + let opts = Options::new().with_reconnect_interval(core::time::Duration::ZERO); + let mut e: StreamEndpoint = StreamEndpoint::new(coord(inner), opts); - let now = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(1); + let now = memberlist_proto::Instant::ORIGIN + core::time::Duration::from_secs(1); e.handle_timeout(now); if let Some(dl) = e.poll_timeout() { @@ -4926,18 +4926,18 @@ fn zero_reconnect_interval_advances_deadline() { #[test] fn zero_queue_check_interval_advances_deadline() { let inner_opts = - EndpointOptions::::new(1u32, "127.0.0.1:7946".parse().unwrap()) + EndpointOptions::::new(1u32, "127.0.0.1:7946".parse().unwrap()) .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); let inner = memberlist_proto::Endpoint::new_at( inner_opts, memberlist_proto::Instant::ORIGIN, SmallRng::seed_from_u64(0), ); - let opts = Options::new().with_queue_check_interval(std::time::Duration::ZERO); - let mut e: StreamEndpoint = + let opts = Options::new().with_queue_check_interval(core::time::Duration::ZERO); + let mut e: StreamEndpoint = StreamEndpoint::new(coord(inner), opts); - let now = memberlist_proto::Instant::ORIGIN + std::time::Duration::from_secs(1); + let now = memberlist_proto::Instant::ORIGIN + core::time::Duration::from_secs(1); e.handle_timeout(now); if let Some(dl) = e.poll_timeout() { @@ -4965,13 +4965,13 @@ fn user_event_total_packet_with_junk_exceeding_size_limit_is_dropped() { ); // Set a tight size limit: 32 bytes. let opts = crate::options::Options::new().with_max_user_event_size(32); - let mut e: StreamEndpoint = + let mut e: StreamEndpoint = StreamEndpoint::new(coord(inner), opts); // Drain the construction self-join so it does not appear as a spurious event. let _ = e.poll_event(); // A small valid UserEvent that fits within 32 bytes on its own. - let valid = AnyMessage::::UserEvent(UserEventMessage { + let valid = AnyMessage::::UserEvent(UserEventMessage { ltime: 3.into(), cc: false, name: "x".into(), @@ -4986,7 +4986,7 @@ fn user_event_total_packet_with_junk_exceeding_size_limit_is_dropped() { } let padded = bytes::Bytes::from(padded); - let from: std::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); + let from: core::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); e.test_inject_user_packet(from, padded, memberlist_proto::Instant::ORIGIN); assert!( @@ -5014,24 +5014,24 @@ fn pre_decode_fence_drops_oversized_valid_query_frame() { ); // Small limit (64 bytes) so that a query with a 100-byte payload exceeds it. let opts = crate::options::Options::new().with_query_size_limit(64); - let mut e: StreamEndpoint = + let mut e: StreamEndpoint = StreamEndpoint::new(coord(inner), opts); // Drain the construction self-join so it does not appear as a spurious event. let _ = e.poll_event(); // Construct a syntactically valid Query whose encoded frame exceeds 64 bytes. - let q = QueryMessage:: { + let q = QueryMessage:: { ltime: LamportTime::new(7), id: 0xdead_beef, from: memberlist_proto::Node::new(2u32, sa(9002)), filters: vec![], flags: QueryFlag::empty(), relay_factor: 0, - timeout: std::time::Duration::from_secs(5), + timeout: core::time::Duration::from_secs(5), name: "oversized".into(), payload: bytes::Bytes::from(vec![0xffu8; 100]), }; - let frame = AnyMessage::::Query(q) + let frame = AnyMessage::::Query(q) .encode() .expect("encode must succeed"); assert!( @@ -5082,13 +5082,13 @@ fn pre_decode_fence_drops_oversized_valid_user_event_frame() { ); // Tight limit: 32 bytes. let opts = crate::options::Options::new().with_max_user_event_size(32); - let mut e: StreamEndpoint = + let mut e: StreamEndpoint = StreamEndpoint::new(coord(inner), opts); // Drain the construction self-join so it does not appear as a spurious event. let _ = e.poll_event(); // Construct a syntactically valid UserEvent whose encoded frame exceeds 32 bytes. - let frame = AnyMessage::::UserEvent(UserEventMessage { + let frame = AnyMessage::::UserEvent(UserEventMessage { ltime: 5.into(), cc: false, name: "big-event".into(), @@ -5211,12 +5211,12 @@ fn duplicate_query_does_not_set_local_state_dirty() { id: 0xdead_beef, from: memberlist_proto::Node::new( 42u32, - "127.0.0.1:9999".parse::().unwrap(), + "127.0.0.1:9999".parse::().unwrap(), ), filters: vec![], flags: QueryFlag::empty(), relay_factor: 0, - timeout: std::time::Duration::from_secs(1), + timeout: core::time::Duration::from_secs(1), name: "test-query".into(), payload: bytes::Bytes::new(), }; @@ -5273,17 +5273,17 @@ fn surfaced_token_survives_overflow_and_overflow_never_surfaces() { let n = MAX_RECEIVED_QUERIES + 1; for i in 0..n as u32 { let req = KeyRequestMessage::new(Some(test_key)); - let payload = AnyMessage::::KeyRequest(req) + let payload = AnyMessage::::KeyRequest(req) .encode() .expect("encode KeyRequestMessage"); - let q = crate::typed::QueryMessage:: { + let q = crate::typed::QueryMessage:: { ltime: LamportTime::new(i as u64 + 1), id: i, from: memberlist_proto::Node::new(99u32, sa(9001)), filters: vec![], flags: QueryFlag::NO_BROADCAST, relay_factor: 0, - timeout: std::time::Duration::from_secs(3600), + timeout: core::time::Duration::from_secs(3600), name: "_serf_install_key".into(), payload, }; @@ -5308,7 +5308,7 @@ fn surfaced_token_survives_overflow_and_overflow_never_surfaces() { Event::KeyRequest(kr) => kr, other => panic!( "expected Event::KeyRequest for first query, got {:?}", - std::mem::discriminant(&other) + core::mem::discriminant(&other) ), }; assert!( @@ -5337,7 +5337,7 @@ fn surfaced_token_survives_overflow_and_overflow_never_surfaces() { assert!( matches!(ev, Event::KeyRequest(_)), "all remaining events must be KeyRequest, got {:?}", - std::mem::discriminant(&ev) + core::mem::discriminant(&ev) ); remaining += 1; } @@ -5377,17 +5377,17 @@ fn local_key_op_self_applies_when_inbound_cap_is_full() { // so none expire during the test. for i in 0..MAX_RECEIVED_QUERIES as u32 { let req = KeyRequestMessage::new(Some(test_key)); - let payload = AnyMessage::::KeyRequest(req) + let payload = AnyMessage::::KeyRequest(req) .encode() .expect("encode KeyRequestMessage for inbound flood"); - let q = crate::typed::QueryMessage:: { + let q = crate::typed::QueryMessage:: { ltime: LamportTime::new(i as u64 + 1), id: i, from: memberlist_proto::Node::new(99u32, sa(9001)), filters: vec![], flags: QueryFlag::NO_BROADCAST, relay_factor: 0, - timeout: std::time::Duration::from_secs(3600), + timeout: core::time::Duration::from_secs(3600), name: "_serf_install_key".into(), payload, }; @@ -5422,7 +5422,7 @@ fn local_key_op_self_applies_when_inbound_cap_is_full() { } other => panic!( "expected Event::KeyRequest from local install_key, got {:?}", - std::mem::discriminant(&other) + core::mem::discriminant(&other) ), } } @@ -5441,17 +5441,17 @@ fn key_must_ops_without_key_are_dropped() { let mut e = ep(); // Encode a KeyRequestMessage with key = None (shape mismatch for must-have-key ops). let req = KeyRequestMessage::new(None); - let payload = AnyMessage::::KeyRequest(req) + let payload = AnyMessage::::KeyRequest(req) .encode() .expect("encode KeyRequestMessage"); - let q = crate::typed::QueryMessage:: { + let q = crate::typed::QueryMessage:: { ltime: LamportTime::new(1), id: 42, from: memberlist_proto::Node::new(99u32, sa(9001)), filters: vec![], flags: QueryFlag::NO_BROADCAST, relay_factor: 0, - timeout: std::time::Duration::from_secs(5), + timeout: core::time::Duration::from_secs(5), name: name.into(), payload, }; @@ -5490,17 +5490,17 @@ fn list_keys_with_key_is_dropped() { let mut e = ep(); let req = KeyRequestMessage::new(Some(test_key)); - let payload = AnyMessage::::KeyRequest(req) + let payload = AnyMessage::::KeyRequest(req) .encode() .expect("encode KeyRequestMessage"); - let q = crate::typed::QueryMessage:: { + let q = crate::typed::QueryMessage:: { ltime: LamportTime::new(1), id: 77, from: memberlist_proto::Node::new(99u32, sa(9001)), filters: vec![], flags: QueryFlag::NO_BROADCAST, relay_factor: 0, - timeout: std::time::Duration::from_secs(5), + timeout: core::time::Duration::from_secs(5), name: "_serf_list_keys".into(), payload, }; @@ -5545,7 +5545,7 @@ fn stale_push_pull_does_not_set_dirty() { left_members: vec![], events: vec![], }; - let encoded = AnyMessage::::PushPull(pp) + let encoded = AnyMessage::::PushPull(pp) .encode() .expect("encode PushPull"); e.test_merge_remote_state(encoded); @@ -5574,7 +5574,7 @@ fn clock_advancing_push_pull_sets_dirty() { left_members: vec![], events: vec![], }; - let encoded = AnyMessage::::PushPull(pp) + let encoded = AnyMessage::::PushPull(pp) .encode() .expect("encode PushPull"); e.test_merge_remote_state(encoded); @@ -5600,13 +5600,13 @@ mod tag_filter_regex { /// Seed the local node (id=1) with the given tags and return an endpoint /// ready for tag-filter query tests. - fn ep_with_tags(tags: Tags) -> StreamEndpoint { + fn ep_with_tags(tags: Tags) -> StreamEndpoint { let mut e = ep(); e.test_seed_member_with_tags(1u32, tags, MemberStatus::Alive, LamportTime::new(0)); e } - fn tag_query(tag: &str, expr: Option<&str>) -> QueryMessage { + fn tag_query(tag: &str, expr: Option<&str>) -> QueryMessage { QueryMessage { filters: vec![Filter::Tag(TagFilter { tag: tag.into(), @@ -5860,10 +5860,10 @@ mod key_request_responder { fn make_key_query( name: &str, key: Option, - ) -> crate::typed::QueryMessage { + ) -> crate::typed::QueryMessage { use crate::{AnyMessage, KeyRequestMessage}; let req = KeyRequestMessage::new(key); - let payload = AnyMessage::::KeyRequest(req) + let payload = AnyMessage::::KeyRequest(req) .encode() .expect("encode KeyRequestMessage"); crate::typed::QueryMessage { @@ -5873,7 +5873,7 @@ mod key_request_responder { filters: vec![], flags: crate::typed::QueryFlag::empty(), relay_factor: 0, - timeout: std::time::Duration::from_secs(5), + timeout: core::time::Duration::from_secs(5), name: name.into(), payload, } @@ -5913,7 +5913,7 @@ mod key_request_responder { } other => panic!( "expected Event::KeyRequest, got {:?}", - std::mem::discriminant(&other) + core::mem::discriminant(&other) ), } // Must not also emit Event::Query. @@ -5936,7 +5936,7 @@ mod key_request_responder { } other => panic!( "expected Event::KeyRequest, got {:?}", - std::mem::discriminant(&other) + core::mem::discriminant(&other) ), } } @@ -5990,7 +5990,7 @@ mod key_request_responder { Event::KeyRequest(kr) => kr, other => panic!( "expected KeyRequest, got {:?}", - std::mem::discriminant(&other) + core::mem::discriminant(&other) ), }; // Verify received_queries entry exists before respond_key. @@ -6020,14 +6020,14 @@ mod key_request_responder { .expect("must have directed send"); assert_eq!( dest, - "127.0.0.1:9999".parse::().unwrap() + "127.0.0.1:9999".parse::().unwrap() ); } #[test] fn key_request_event_debug_does_not_leak_key_bytes() { let key = test_key(); - let req = KeyRequest:: { + let req = KeyRequest:: { op: KeyRequestOperation::Install, key: Some(key), id: 1, @@ -6036,7 +6036,7 @@ mod key_request_responder { relay_factor: 0, deadline: memberlist_proto::Instant::ORIGIN, }; - let ev = Event::::KeyRequest(req); + let ev = Event::::KeyRequest(req); let debug_str = format!("{:?}", ev); // SecretKey's Debug impl uses "" — check the raw bytes don't appear. // For Aes128([0u8;16]) the raw bytes would be "0, 0, 0, 0". @@ -6438,7 +6438,7 @@ fn left_member_replay_at_watermark_boundary_skips_leave() { left_members: vec![99u32], events: vec![], }; - let encoded = AnyMessage::::PushPull(pp) + let encoded = AnyMessage::::PushPull(pp) .encode() .expect("encode must succeed"); // Must not panic. @@ -6660,7 +6660,7 @@ fn push_pull_near_watermark_status_time_integrity_floor() { left_members: vec![], events: vec![], }; - let encoded = AnyMessage::::PushPull(pp) + let encoded = AnyMessage::::PushPull(pp) .encode() .expect("encode must succeed"); e.test_merge_remote_state(encoded); @@ -6835,7 +6835,7 @@ fn push_pull_near_watermark_event_floor_integrity_and_delivery() { left_members: vec![], events: vec![], }; - let encoded = AnyMessage::::PushPull(pp) + let encoded = AnyMessage::::PushPull(pp) .encode() .expect("encode must succeed"); // A suppressed (ignore-join) merge triggers the G4 event_buffer.min_time diff --git a/serf-proto/src/event/mod.rs b/serf-proto/src/event/mod.rs index 7f541ca4..a47133f5 100644 --- a/serf-proto/src/event/mod.rs +++ b/serf-proto/src/event/mod.rs @@ -4,7 +4,7 @@ //! control signals through this enum rather than callbacks, mirroring the //! quinn-proto / memberlist-proto pull-style event model. -use std::sync::Arc; +use std::{sync::Arc, vec::Vec}; use bytes::Bytes; #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] @@ -281,11 +281,11 @@ pub struct KeyResponse { /// Number with `result = false` (error) in their response. pub num_err: usize, /// Key → count of nodes that have that key installed. - pub keys: std::collections::HashMap, + pub keys: crate::FxHashMap, /// Primary key → count of nodes using it as primary. - pub primary_keys: std::collections::HashMap, + pub primary_keys: crate::FxHashMap, /// Per-node error messages (only for nodes with `result = false`). - pub messages: std::collections::HashMap, + pub messages: crate::FxHashMap, } // ── KeyRequestOperation ─────────────────────────────────────────────────────── diff --git a/serf-proto/src/event/tests.rs b/serf-proto/src/event/tests.rs index fa13dadd..a82f6090 100644 --- a/serf-proto/src/event/tests.rs +++ b/serf-proto/src/event/tests.rs @@ -24,7 +24,7 @@ fn member_event_kind_display_matches_as_str() { #[test] fn event_is_variant_helpers() { - use std::net::SocketAddr; + use core::net::SocketAddr; let ev: Event = Event::Shutdown; assert!(ev.is_shutdown()); diff --git a/serf-proto/src/framing/mod.rs b/serf-proto/src/framing/mod.rs index 36ed7d29..d0ad3577 100644 --- a/serf-proto/src/framing/mod.rs +++ b/serf-proto/src/framing/mod.rs @@ -15,6 +15,8 @@ //! the memberlist-proto `merge(WireType::LengthDelimited, TAG)` scheme); //! serf-proto forms new-wire-only clusters. +use std::vec::Vec; + use bytes::Bytes; // ── MessageType ────────────────────────────────────────────────────────────── diff --git a/serf-proto/src/lib.rs b/serf-proto/src/lib.rs index 0c5ea6d9..4f10ba22 100644 --- a/serf-proto/src/lib.rs +++ b/serf-proto/src/lib.rs @@ -3,10 +3,27 @@ //! //! Depends on `memberlist-proto` for the `Data`/`DataRef` codec primitives; defines serf's //! own message set and framing on top of them. +#![cfg_attr(not(feature = "std"), no_std)] #![deny(missing_docs)] #![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(docsrs, allow(unused_attributes))] +// Alias `alloc` to the name `std` so genuine-heap `std::` paths compile unchanged +// under no_std+alloc (and `#[macro_use]` brings `vec!`/`format!` crate-wide). +// Core-resident items are imported from `core::` directly, never via this alias. +#[cfg(all(not(feature = "std"), feature = "alloc"))] +#[macro_use] +extern crate alloc as std; + +#[cfg(feature = "std")] +extern crate std; + +// The protocol state is intrinsically heap-backed (Vec/Box/String/maps), so a +// build with neither capability tier is unsupported. Fail with a clear message +// instead of a cascade of "cannot find type `Vec`" errors. +#[cfg(not(any(feature = "std", feature = "alloc")))] +compile_error!("serf-proto requires the `std` or `alloc` feature"); + #[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) use any::{AnyMessage, EncodeError}; #[cfg(any(feature = "tcp", feature = "quic"))] @@ -68,6 +85,8 @@ pub(crate) mod bridge; #[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) mod framing; #[cfg(any(feature = "tcp", feature = "quic"))] +mod mathf; +#[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) mod messages; pub mod typed; @@ -136,3 +155,8 @@ pub use memberlist_proto::SecretKey; #[cfg(any(feature = "tcp", feature = "quic"))] #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] pub use memberlist_proto::event::{ExchangeCompleted, ExchangeId, ExchangeKind, ExchangeStatus}; + +/// `FxHashMap`/`FxHashSet` backed by hashbrown (no_std-capable) with rustc-hash's +/// Fx hasher — rustc-hash's own `Fx*` map aliases are std-only. +pub(crate) type FxHashMap = hashbrown::HashMap; +pub(crate) type FxHashSet = hashbrown::HashSet; diff --git a/serf-proto/src/mathf.rs b/serf-proto/src/mathf.rs new file mode 100644 index 00000000..cd95a0b5 --- /dev/null +++ b/serf-proto/src/mathf.rs @@ -0,0 +1,26 @@ +//! Floating-point transcendentals (`log10`/`ceil`): routed through `std` on std +//! targets and `libm` on no_std, since `core` has no float math. Centralised so +//! std builds keep bit-matching the historical std results the timing tests pin, +//! while no_std gets a pure-Rust implementation. + +#[cfg(feature = "std")] +#[inline] +pub(crate) fn log10(x: f64) -> f64 { + x.log10() +} +#[cfg(not(feature = "std"))] +#[inline] +pub(crate) fn log10(x: f64) -> f64 { + libm::log10(x) +} + +#[cfg(feature = "std")] +#[inline] +pub(crate) fn ceil(x: f64) -> f64 { + x.ceil() +} +#[cfg(not(feature = "std"))] +#[inline] +pub(crate) fn ceil(x: f64) -> f64 { + libm::ceil(x) +} diff --git a/serf-proto/src/members/mod.rs b/serf-proto/src/members/mod.rs index affc11ee..1048e34a 100644 --- a/serf-proto/src/members/mod.rs +++ b/serf-proto/src/members/mod.rs @@ -11,7 +11,9 @@ //! `memberlist_proto::Instant` threaded in from the driver — no wall-clock //! reads occur inside the pure machine. -use std::collections::HashMap; +use std::vec::Vec; + +use crate::FxHashMap; /// Hard cardinality cap on `Members::recent_intents`. /// @@ -293,10 +295,10 @@ where I: Eq + core::hash::Hash, { /// Full state for every known node (alive, leaving, left, or failed). - pub(crate) states: HashMap>, + pub(crate) states: FxHashMap>, /// Buffered join/leave intents whose inner memberlist event has not yet /// arrived. Newest ltime wins (upsert_intent). - pub(crate) recent_intents: HashMap, + pub(crate) recent_intents: FxHashMap, /// Ids of nodes in the `Left` state, for tombstone reaping. pub(crate) left_members: Vec, /// Ids of nodes in the `Failed` state, for reconnect and reaping. @@ -315,8 +317,8 @@ where { fn default() -> Self { Self { - states: HashMap::default(), - recent_intents: HashMap::default(), + states: FxHashMap::default(), + recent_intents: FxHashMap::default(), left_members: Vec::new(), failed_members: Vec::new(), recent_intent_seq: 0, diff --git a/serf-proto/src/members/tests.rs b/serf-proto/src/members/tests.rs index 6ecf9450..4d95c463 100644 --- a/serf-proto/src/members/tests.rs +++ b/serf-proto/src/members/tests.rs @@ -18,7 +18,7 @@ use super::*; fn upsert_intent_cap_eviction_is_deterministic() { use memberlist_proto::Instant; - type TestMembers = Members; + type TestMembers = Members; // Use cap + a surplus so we trigger exactly one eviction per surplus insert. let n = MAX_RECENT_INTENTS + 16; @@ -128,7 +128,7 @@ fn serf_state_display_matches_as_str() { fn upsert_intent_newest_ltime_wins() { use memberlist_proto::Instant; - type TestMembers = Members; + type TestMembers = Members; let mut m = TestMembers::default(); let t0 = Instant::ORIGIN; @@ -162,7 +162,7 @@ fn upsert_intent_newest_ltime_wins() { fn recent_intent_kind_mismatch_returns_none() { use memberlist_proto::Instant; - type TestMembers = Members; + type TestMembers = Members; let mut m = TestMembers::default(); m.upsert_intent( diff --git a/serf-proto/src/messages/tests.rs b/serf-proto/src/messages/tests.rs index b7c8df51..b764a7e3 100644 --- a/serf-proto/src/messages/tests.rs +++ b/serf-proto/src/messages/tests.rs @@ -1,4 +1,4 @@ -use std::net::SocketAddr; +use core::net::SocketAddr; use buffa::Message as _; use memberlist_proto::Node; @@ -347,7 +347,7 @@ fn query_message_roundtrip_pb_no_filters() { filters: vec![], flags: QueryFlag::ACK, relay_factor: 3, - timeout: std::time::Duration::from_millis(500), + timeout: core::time::Duration::from_millis(500), name: smol_str::SmolStr::from("my-query"), payload: bytes::Bytes::from_static(b"query-payload"), }; @@ -385,7 +385,7 @@ fn query_message_roundtrip_pb_with_filters() { ], flags: QueryFlag::ACK | QueryFlag::NO_BROADCAST, relay_factor: 0, - timeout: std::time::Duration::from_secs(2), + timeout: core::time::Duration::from_secs(2), name: smol_str::SmolStr::from("filtered-query"), payload: bytes::Bytes::new(), }; @@ -456,7 +456,8 @@ fn query_to_pb_timeout_overflow_is_error() { // A duration whose nanosecond count exceeds u64::MAX (requires u128) must // produce BridgeError::InvalidValue, not a silent truncation. // u64::MAX nanos ≈ 584 years; add one second to guarantee overflow. - let huge_timeout = std::time::Duration::from_nanos(u64::MAX) + std::time::Duration::from_secs(1); + let huge_timeout = + core::time::Duration::from_nanos(u64::MAX) + core::time::Duration::from_secs(1); let typed: QueryMessage = QueryMessage { ltime: LamportTime::new(1), id: 1, diff --git a/serf-proto/src/options/mod.rs b/serf-proto/src/options/mod.rs index 8b8245c0..4429bf6f 100644 --- a/serf-proto/src/options/mod.rs +++ b/serf-proto/src/options/mod.rs @@ -4,7 +4,7 @@ //! `serf-core/src/options.rs` port. The struct itself is a plain value type //! (no atomics, no locks) consistent with the single-threaded Sans-I/O machine. -use std::time::Duration; +use core::time::Duration; /// Configuration for the serf `Endpoint`. /// diff --git a/serf-proto/src/options/tests.rs b/serf-proto/src/options/tests.rs index 336441c6..2987afca 100644 --- a/serf-proto/src/options/tests.rs +++ b/serf-proto/src/options/tests.rs @@ -1,4 +1,4 @@ -use std::time::Duration; +use core::time::Duration; use super::Options; diff --git a/serf-proto/src/quic_endpoint/mod.rs b/serf-proto/src/quic_endpoint/mod.rs index a176500b..9acc8cb4 100644 --- a/serf-proto/src/quic_endpoint/mod.rs +++ b/serf-proto/src/quic_endpoint/mod.rs @@ -1197,7 +1197,7 @@ where pub(crate) fn test_ping_completed( &mut self, node_id: I, - rtt: std::time::Duration, + rtt: core::time::Duration, payload: Bytes, ) { self diff --git a/serf-proto/src/snapshot/mod.rs b/serf-proto/src/snapshot/mod.rs index 77270227..a712567d 100644 --- a/serf-proto/src/snapshot/mod.rs +++ b/serf-proto/src/snapshot/mod.rs @@ -27,6 +27,8 @@ //! No file handles, no `std::fs`, no `Instant::now()`. The driver owns I/O; //! this module is consumed by the snapshot replay function in the driver. +use std::vec::Vec; + use bytes::{BufMut, Bytes, BytesMut}; use memberlist_proto::{Data, DataRef}; @@ -516,8 +518,7 @@ where // input-record order — two drivers that replay the same snapshot bytes // produce an identical `DialRequested` sequence. let mut alive_vec: Vec> = Vec::new(); - let mut alive_set: std::collections::HashSet> = - std::collections::HashSet::new(); + let mut alive_set: crate::FxHashSet> = crate::FxHashSet::default(); let mut last_clock = LamportTime::ZERO; let mut last_event_clock = LamportTime::ZERO; let mut last_query_clock = LamportTime::ZERO; diff --git a/serf-proto/src/snapshot/tests.rs b/serf-proto/src/snapshot/tests.rs index 3887ce22..017e1e5b 100644 --- a/serf-proto/src/snapshot/tests.rs +++ b/serf-proto/src/snapshot/tests.rs @@ -1,4 +1,4 @@ -use std::net::SocketAddr; +use core::net::SocketAddr; use memberlist_proto::Node; diff --git a/serf-proto/src/stream_endpoint/mod.rs b/serf-proto/src/stream_endpoint/mod.rs index b8449d72..85a5c898 100644 --- a/serf-proto/src/stream_endpoint/mod.rs +++ b/serf-proto/src/stream_endpoint/mod.rs @@ -24,7 +24,7 @@ //! the core's drain over `poll_inner_event`, and never reaches into the stream //! lifecycle itself. -use std::sync::Arc; +use std::{sync::Arc, vec::Vec}; use bytes::Bytes; use memberlist_proto::{ @@ -681,7 +681,7 @@ where pub(crate) fn test_seed_member(&mut self, id: I, status: MemberStatus, status_time: LamportTime) where I: Clone, - A: Clone + From, + A: Clone + From, { self.core.test_seed_member(id, status, status_time) } @@ -696,7 +696,7 @@ where status_time: LamportTime, ) where I: Clone, - A: Clone + From, + A: Clone + From, { self .core @@ -712,7 +712,7 @@ where now: Instant, ) where I: Clone, - A: Clone + From, + A: Clone + From, { self .core @@ -728,7 +728,7 @@ where now: Instant, ) where I: Clone, - A: Clone + From, + A: Clone + From, { self .core @@ -760,7 +760,7 @@ where pub(crate) fn test_inner_node_joined(&mut self, id: I, now: Instant) where I: Clone, - A: Clone + From, + A: Clone + From, { self.core.test_inner_node_joined(id, now) } @@ -770,7 +770,7 @@ where pub(crate) fn test_inner_node_left(&mut self, id: I, now: Instant) where I: Clone, - A: Clone + From, + A: Clone + From, { self.core.test_inner_node_left(id, now) } @@ -780,7 +780,7 @@ where pub(crate) fn test_inner_node_updated(&mut self, id: I, now: Instant) where I: Clone, - A: Clone + From, + A: Clone + From, { self.core.test_inner_node_updated(id, now) } @@ -894,7 +894,7 @@ where pub(crate) fn test_seed_left_member(&mut self, id: I, status_time: LamportTime) where I: Clone, - A: Clone + From, + A: Clone + From, { self.core.test_seed_left_member(id, status_time) } @@ -1218,7 +1218,7 @@ where pub(crate) fn test_inject_inner_joined(&mut self, id: I, now: Instant) where I: Clone, - A: Clone + From, + A: Clone + From, { self.core.test_inject_inner_joined(id, now) } @@ -1261,7 +1261,7 @@ where pub(crate) fn test_ping_completed( &mut self, node_id: I, - rtt: std::time::Duration, + rtt: core::time::Duration, payload: Bytes, ) { self diff --git a/serf-proto/src/typed/mod.rs b/serf-proto/src/typed/mod.rs index 9b513575..c5559aac 100644 --- a/serf-proto/src/typed/mod.rs +++ b/serf-proto/src/typed/mod.rs @@ -3,7 +3,9 @@ //! These types are what the serf state machine works with directly. The //! `bridge` module converts them to/from the buffa-generated codec types. -use std::collections::HashMap; +use std::vec::Vec; + +use crate::FxHashMap; use bytes::Bytes; use smol_str::SmolStr; @@ -82,22 +84,22 @@ impl Default for Coordinate { /// Node metadata: a string→string map gossiped via node meta. /// -/// Thin newtype over [`HashMap`] so the rest of the crate +/// Thin newtype over a `FxHashMap` so the rest of the crate /// can name the concept without spelling out the full map type. /// Proto3 `map` wire encoding does not guarantee key order, so encoded bytes /// are not canonical for a given set of tags. #[derive(Debug, Default, Clone, PartialEq, Eq)] -pub struct Tags(pub HashMap); +pub struct Tags(pub FxHashMap); impl Tags { /// Creates an empty `Tags` map. pub fn new() -> Self { - Self(HashMap::new()) + Self(FxHashMap::default()) } /// Creates a `Tags` map with the given initial capacity. pub fn with_capacity(cap: usize) -> Self { - Self(HashMap::with_capacity(cap)) + Self(FxHashMap::with_capacity_and_hasher(cap, Default::default())) } /// Returns the number of tag entries. @@ -182,7 +184,7 @@ pub struct QueryMessage { /// Number of relayed duplicate responses requested. pub relay_factor: u8, /// Maximum time allowed between delivery and response. - pub timeout: std::time::Duration, + pub timeout: core::time::Duration, /// Query name. pub name: SmolStr, /// Query payload. From 79de6e49455fcae41eadc3cccebfb060fbface69 Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 3 Jul 2026 19:21:58 +1200 Subject: [PATCH 043/128] feat(serf-embedded): shared no_std driving core (SerfEngine) The no_std+alloc shared driving core for serf's embedded drivers, reusing memberlist-embedded's ReliablePlane / IO-seams / transform / resolver glue and porting its Engine to SerfEngine over serf's super-machine (serf_proto::StreamEndpoint on the plain-TCP RawRecords path) with serf's query / user-event / keyring events folded into the pump; builds no_std on thumbv7em. --- Cargo.toml | 1 + serf-embedded/Cargo.toml | 73 ++ serf-embedded/README.md | 34 + serf-embedded/build.rs | 27 + serf-embedded/src/cidr.rs | 39 + serf-embedded/src/engine/mod.rs | 1352 +++++++++++++++++++++++++++++ serf-embedded/src/engine/tests.rs | 278 ++++++ serf-embedded/src/lib.rs | 84 ++ 8 files changed, 1888 insertions(+) create mode 100644 serf-embedded/Cargo.toml create mode 100644 serf-embedded/README.md create mode 100644 serf-embedded/build.rs create mode 100644 serf-embedded/src/cidr.rs create mode 100644 serf-embedded/src/engine/mod.rs create mode 100644 serf-embedded/src/engine/tests.rs create mode 100644 serf-embedded/src/lib.rs diff --git a/Cargo.toml b/Cargo.toml index 20087146..14042e85 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "serf-proto", + "serf-embedded", "serf-driver", "serf-compio", "serf-reactor", diff --git a/serf-embedded/Cargo.toml b/serf-embedded/Cargo.toml new file mode 100644 index 00000000..f0b2c4fb --- /dev/null +++ b/serf-embedded/Cargo.toml @@ -0,0 +1,73 @@ +[package] +name = "serf-embedded" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +readme = "README.md" +rust-version = "1.96" +description = "Transport-agnostic no_std driving core shared by the embedded serf drivers." + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] + +[features] +default = ["std"] +# `std` and `alloc` are INDEPENDENT capability tiers (std does NOT pull alloc): +# under `std` the real `std` crate supplies the heap; under `alloc` the +# no_std+alloc path aliases `alloc`. Each layers the matching feature on every +# dependency that has one. +std = ["serf-proto/std", "memberlist-embedded/std", "memberlist-proto/std"] +alloc = ["serf-proto/alloc", "memberlist-embedded/alloc", "memberlist-proto/alloc"] + +# CIDR peer-admission allow-list. A `CidrPolicy` set via `Options::with_cidr_policy` +# filters inbound gossip by datagram source and inbound reliable connections by +# peer address at the transport boundary, AND inbound alives by the peer's +# self-advertised address at membership admission. no_std+alloc (the policy holds +# a set of `ipnet` networks), so it pulls the heap tier. +cidr = ["alloc", "memberlist-embedded/cidr", "memberlist-proto/cidr"] + +# Gossip + reliable-plane AEAD encryption backends. Each pass-through enables the +# matching serf-proto, memberlist-embedded, and machine backend so a +# `TransformOptions` carrying a `Keyring` for that cipher can actually encrypt, +# and serf's key-management queries (install/use/remove/list) become available. +# Both AEAD backends are no_std+alloc. +aes-gcm = [ + "alloc", + "serf-proto/aes-gcm", + "memberlist-embedded/aes-gcm", + "memberlist-proto/aes-gcm", +] +chacha20-poly1305 = [ + "alloc", + "serf-proto/chacha20-poly1305", + "memberlist-embedded/chacha20-poly1305", + "memberlist-proto/chacha20-poly1305", +] +# Umbrella that activates both AEAD backends (convenient for tests / CI). +encryption = ["aes-gcm", "chacha20-poly1305"] + +[dependencies] +# The alloc-tier plain-TCP (`RawRecords`) path of serf's super-machine. +serf-proto = { path = "../serf-proto", default-features = false, features = [ + "tcp", +] } +# The payload-agnostic driving glue (ReliablePlane / IO seams / transform / +# resolver) serf-embedded reuses directly; and the shared machine types. +memberlist-embedded = { git = "https://github.com/al8n/memberlist", branch = "main", default-features = false } +memberlist-proto = { git = "https://github.com/al8n/memberlist", branch = "main", default-features = false, features = [ + "tcp", +] } +getrandom = "0.4" +bytes = { version = "1", default-features = false } +# Named in the `user_event` / `query` forwarder signatures (`impl Into`). +smol_str = { version = "0.3", default-features = false } +smallvec-wrapper = { version = "0.4", default-features = false } +hashbrown = { version = "0.17", default-features = false, features = [ + "default-hasher", +] } +# The fixed-capacity, no-heap `ResolvedAddrs` resolver result shares the single +# version smoltcp already pulls into the graph (no second heapless major). +heapless = { version = "0.9", default-features = false } diff --git a/serf-embedded/README.md b/serf-embedded/README.md new file mode 100644 index 00000000..9acd1818 --- /dev/null +++ b/serf-embedded/README.md @@ -0,0 +1,34 @@ +
+

serf-embedded

+
+
+ +The transport-agnostic `no_std` driving core shared by serf's embedded drivers. + +
+ +`serf-embedded` is the link-layer-independent core the future serf smoltcp / +embassy-net drivers are built on. It owns serf's super-machine +(`serf_proto::StreamEndpoint` over the plain-TCP `RawRecords` path) and the +pooled-stream reliable plane, driving both through the [`GossipIo`] / [`StreamIo`] +seams a driver supplies to [`SerfEngine::pump`]. The engine performs **no** socket +I/O: a driver owns the UDP gossip socket and the reliable-stream socket pool, ticks +its own link-layer stack, then calls [`SerfEngine::pump`] to advance the machine +over them. + +It reuses the payload-agnostic glue from +[`memberlist-embedded`](https://docs.rs/memberlist-embedded) directly — the +[`ReliablePlane`], the [`GossipIo`] / [`StreamIo`] seams, the cross-transport +transform pipeline, the bounded resolver result, and the engine sizing / error +types — and adds only [`SerfEngine`], the port of memberlist-embedded's `Engine` +that drives serf's richer super-machine (folding serf's query, user-event, and +key-management events into the pump on top of membership). + +This crate is `no_std` with `alloc`: the two feature tiers `std` and `alloc` are +independent, and the core builds on bare-metal targets such as `thumbv7em-none-eabihf`. + +[`GossipIo`]: https://docs.rs/memberlist-embedded/latest/memberlist_embedded/trait.GossipIo.html +[`StreamIo`]: https://docs.rs/memberlist-embedded/latest/memberlist_embedded/trait.StreamIo.html +[`ReliablePlane`]: https://docs.rs/memberlist-embedded/latest/memberlist_embedded/reliable/struct.ReliablePlane.html +[`SerfEngine`]: https://docs.rs/serf-embedded/latest/serf_embedded/struct.SerfEngine.html +[`SerfEngine::pump`]: https://docs.rs/serf-embedded/latest/serf_embedded/struct.SerfEngine.html#method.pump diff --git a/serf-embedded/build.rs b/serf-embedded/build.rs new file mode 100644 index 00000000..ef7b0465 --- /dev/null +++ b/serf-embedded/build.rs @@ -0,0 +1,27 @@ +//! Emits the aggregate transform cfgs (`compression` / `encryption` / `checksum`), +//! each set when any of its backend features is enabled, so transform code gates +//! on `#[cfg(encryption)]` etc. instead of repeating the full backend list. serf's +//! gossip plane carries only encryption (no compression / checksum backends), so +//! in practice only `encryption` is ever set here; the other two are declared for +//! parity with the reused memberlist-embedded transform surface. + +fn any_feature(names: &[&str]) -> bool { + names + .iter() + .any(|name| std::env::var_os(format!("CARGO_FEATURE_{name}")).is_some()) +} + +fn main() { + println!("cargo::rustc-check-cfg=cfg(compression)"); + println!("cargo::rustc-check-cfg=cfg(encryption)"); + println!("cargo::rustc-check-cfg=cfg(checksum)"); + if any_feature(&["LZ4", "SNAPPY", "ZSTD", "BROTLI"]) { + println!("cargo::rustc-cfg=compression"); + } + if any_feature(&["AES_GCM", "CHACHA20_POLY1305"]) { + println!("cargo::rustc-cfg=encryption"); + } + if any_feature(&["CRC32", "XXHASH32", "XXHASH64", "XXHASH3", "MURMUR3"]) { + println!("cargo::rustc-cfg=checksum"); + } +} diff --git a/serf-embedded/src/cidr.rs b/serf-embedded/src/cidr.rs new file mode 100644 index 00000000..98a71965 --- /dev/null +++ b/serf-embedded/src/cidr.rs @@ -0,0 +1,39 @@ +//! CIDR peer-admission glue for the serf embedded engine. +//! +//! One [`with_cidr_policy`](memberlist_embedded::Options::with_cidr_policy) +//! setting enforces a [`CidrPolicy`](memberlist_proto::CidrPolicy) at the three +//! IP-observable points of a node: the gossip datagram source (engine recv) and +//! the reliable peer (engine accept), both via [`cidr_blocks`] at the transport +//! boundary; and the self-advertised address (membership admission), via the +//! engine installing the policy as the inner of its built-in routable-address +//! alive filter. [`CidrFilter`] carries the policy through the engine field +//! without a `cfg` gate. +//! +//! This mirrors memberlist-embedded's (crate-private) CIDR glue, re-implemented +//! here because serf-embedded reuses memberlist-embedded's `Options` — whose +//! `cidr_policy` field is public — but not its private filter helpers. + +use core::net::IpAddr; + +/// The CIDR policy carried by the engine: the real +/// [`CidrPolicy`](memberlist_proto::CidrPolicy) when the `cidr` feature is on, the +/// zero-sized `()` otherwise — so the engine field needs no `cfg` gate. +#[cfg(feature = "cidr")] +pub(crate) type CidrFilter = Option; +#[cfg(not(feature = "cidr"))] +pub(crate) type CidrFilter = (); + +/// Whether `ip` is blocked by the policy — the transport-boundary half of the +/// filter, applied to a gossip datagram source (recv) and a reliable peer +/// (accept). Always `false` when the `cidr` feature is off, so the guards compile +/// away to nothing. +#[cfg(feature = "cidr")] +#[inline] +pub(crate) fn cidr_blocks(filter: &CidrFilter, ip: IpAddr) -> bool { + filter.as_ref().is_some_and(|policy| policy.is_blocked(&ip)) +} +#[cfg(not(feature = "cidr"))] +#[inline] +pub(crate) fn cidr_blocks(_filter: &CidrFilter, _ip: IpAddr) -> bool { + false +} diff --git a/serf-embedded/src/engine/mod.rs b/serf-embedded/src/engine/mod.rs new file mode 100644 index 00000000..f8f31c5f --- /dev/null +++ b/serf-embedded/src/engine/mod.rs @@ -0,0 +1,1352 @@ +//! The transport-agnostic serf driving core: construction, accessors, the serf +//! command API, and the link-layer-independent `pump`. +//! +//! [`SerfEngine`] owns serf's super-machine ([`StreamEndpoint`]) — serf logic +//! composed over the memberlist reliable stream coordinator on the plain-TCP +//! [`RawRecords`] path — and the reused reliable-plane connection state machine +//! ([`ReliablePlane`]), the gossip scratch buffer, and the join-seed queue. It +//! performs NO socket I/O: a driver supplies the link-layer stack tick plus a +//! [`GossipIo`] and a [`StreamIo`], and [`SerfEngine::pump`] drives the machine +//! over them. A driver wraps the engine, owning the actual sockets/interface; the +//! future `serf-smoltcp` / `serf-embassy` drivers are built on it. +//! +//! This is the serf port of memberlist-embedded's `Engine`: the reliable-plane +//! glue, the [`GossipIo`] / [`StreamIo`] seams, and the transform pipeline are +//! reused from [`memberlist-embedded`](https://docs.rs/memberlist-embedded) +//! directly; only the [`SerfEngine`] here differs, driving serf's richer machine +//! (folding serf's query / user-event / key-management events into the pump on +//! top of membership) instead of the membership-only memberlist machine. + +use core::{hash::Hash, net::SocketAddr, num::NonZeroU8}; + +// Under `no_std + alloc` the prelude does not bring `Box` / `Vec` / `VecDeque` +// into scope; import them explicitly from the aliased `std` (which is `alloc` in +// that build). +#[cfg(feature = "std")] +use std::collections::VecDeque; +#[cfg(not(feature = "std"))] +use std::{boxed::Box, collections::VecDeque, vec::Vec}; + +use std::sync::Arc; + +use bytes::Bytes; +use memberlist_proto::{ + AliveDelegate, Endpoint, EndpointOptions, Instant, LabelOptions, PushPullKind, RawRecords, Rng, + Transmit, + codec::{ + DecodeOptions, EncodeOptions, decode_incoming, encode_outgoing, encode_outgoing_compound, + parse_messages, + }, + streams::{ExchangeId, StreamAction, StreamEndpoint as Coordinator}, + typed::NodeState, +}; +use smallvec_wrapper::MediumVec; + +use serf_proto::{ + StreamEndpoint, + endpoint::{Error as SerfError, QueryId, QueryParams}, + event::{Event, QueryEvent}, + members::{Member, SerfState}, + options::Options as SerfOptions, + typed::Tags, +}; + +#[cfg(encryption)] +use serf_proto::{SecretKey, event::KeyRequest, event::KeyResponseArgs}; + +use memberlist_embedded::{ + GossipIo, InitError, Options, StreamIo, TransformOptions, + reliable::{ConnState, Connection, ReliablePlane}, + socket_addr_is_routable, validate_runtime_config, +}; + +use crate::cidr::{CidrFilter, cidr_blocks}; + +/// The largest the encrypted wrapper can inflate a gossip datagram, or `0` when +/// no encryption backend is built in. serf's gossip plane carries only the +/// encryption wrapper (no checksum / compression), so this is the whole on-wire +/// inflation the receive scratch must accommodate. +#[cfg(encryption)] +const ENCRYPTED_WRAPPER_OVERHEAD: usize = memberlist_proto::ENCRYPTED_WRAPPER_OVERHEAD; +#[cfg(not(encryption))] +const ENCRYPTED_WRAPPER_OVERHEAD: usize = 0; + +/// Size the inbound-gossip receive scratch from the effective gossip MTU. +/// +/// The machine caps an outbound gossip datagram's PLAINTEXT at the configured +/// [`EndpointOptions`] `gossip_mtu`; the on-wire datagram can then exceed that by +/// up to `ENCRYPTED_WRAPPER_OVERHEAD` (the AEAD wrapper header, nonce, and tag) +/// when encryption is enabled. The buffer must hold the largest such datagram, so +/// it is sized to `gossip_mtu + ENCRYPTED_WRAPPER_OVERHEAD`, floored at 1500 (the +/// common Ethernet payload) so a sub-1500 MTU never under-sizes it. A driver's +/// datagram receive may POP a datagram before checking the caller's slice length, +/// so a datagram larger than this buffer is consumed and lost; sizing from the +/// same knob the machine bounds outbound gossip with means a correctly-configured +/// cluster never truncates an in-budget datagram. +fn gossip_recv_buf_size(gossip_mtu: usize) -> usize { + (gossip_mtu + ENCRYPTED_WRAPPER_OVERHEAD).max(1500) +} + +/// An [`AliveDelegate`] that admits a peer only when its advertised address is a +/// routable destination ([`socket_addr_is_routable`]). +/// +/// The machine consults `notify_alive` inline for EVERY admitted Alive — gossip +/// and join push/pull alike — so this one filter drops a non-routable address at +/// admission on both planes. The bad address is never stored as a member and so +/// is never re-gossiped, stopping cluster-wide propagation of a member address no +/// node could ever send a useful packet to. +struct RoutableAddrFilter; + +impl AliveDelegate for RoutableAddrFilter +where + I: memberlist_proto::Id, +{ + fn notify_alive(&self, peer: &NodeState) -> bool { + socket_addr_is_routable(peer.address_ref()) + } +} + +/// An [`AliveDelegate`] that admits a peer only when BOTH the built-in routable +/// filter and an inner delegate accept it. +/// +/// The routable filter is load-bearing on the no_std core, so a configured CIDR +/// policy composes with it (logical AND) rather than replacing it: a peer must +/// pass routable AND the policy. +#[cfg(feature = "cidr")] +struct RoutableAnd(D); + +#[cfg(feature = "cidr")] +impl AliveDelegate for RoutableAnd +where + I: memberlist_proto::Id, + D: AliveDelegate, +{ + fn notify_alive(&self, peer: &NodeState) -> bool { + socket_addr_is_routable(peer.address_ref()) && self.0.notify_alive(peer) + } +} + +/// The transport-agnostic serf driving core. +/// +/// Composes serf's super-machine with the reused pooled-stream reliable plane, +/// driving both through the [`GossipIo`] / [`StreamIo`] traits a driver supplies +/// to [`pump`](SerfEngine::pump). The engine holds NO sockets — the driver owns +/// the link-layer stack and its UDP/stream sockets — so the same core runs under +/// a caller-driven poll loop (smoltcp) or an async executor (embassy-net). +/// +/// `I` is the node identifier type (e.g. `SmolStr`); the address is pinned to +/// [`core::net::SocketAddr`] and the record layer to the plain-TCP +/// [`RawRecords`]. `C` is the driver's opaque connection handle +/// ([`StreamIo::Conn`]). `R` is the memberlist gossip RNG the driver injects at +/// construction; serf's own core RNG (query IDs, relay selection) is a +/// deterministically-seeded `SmallRng` — a production driver that needs +/// per-node-distinct query IDs can seed it via a future two-RNG constructor. +pub struct SerfEngine +where + // Mandated by serf's `StreamEndpoint` field, which keys its membership store by + // `I`. Every impl bounds `I: Id`, which implies these, so no impl restates them. + I: Eq + Hash, +{ + /// serf's super-machine: serf logic over the memberlist reliable coordinator + /// on the plain-TCP `RawRecords` path. `G = R` (the injected gossip RNG); the + /// serf core RNG defaults to `SmallRng`. + endpoint: StreamEndpoint, + /// Sizing / port configuration; retained for the reliable-plane paths. + cfg: Options, + /// Reused pooled connection handles and the exchange-to-handle map for the + /// reliable plane. + plane: ReliablePlane, + /// Heap scratch for one inbound gossip datagram, sized once at construction + /// from the configured gossip MTU (see [`gossip_recv_buf_size`]) and reused + /// every pump. Heap-resident so a large MTU does not blow a constrained stack + /// and the allocation happens exactly once. + gossip_recv: std::vec::Vec, + /// Seed addresses queued by [`join`](Self::join) that have not yet been handed + /// to the machine. Drained in the machine-pump phase of each `pump` tick: one + /// `start_push_pull(seed, Join, now)` per entry, which queues a `Connect` the + /// machine services into a dial consumed later that same tick. + pending_seeds: VecDeque, + /// Cluster label applied to the gossip codec on both encode and decode. When + /// `Some`, the gossip codec stamps a label prefix onto every outbound datagram + /// and rejects any inbound datagram whose label does not match. `None` disables + /// labeling. + label: Option, + /// CIDR transport filter: a gossip datagram from a blocked source IP (recv) or + /// a reliable connection from a blocked peer IP (accept/dial) is dropped before + /// the machine sees it. `()` when the `cidr` feature is off. + cidr_policy: CidrFilter, +} + +// Construction and pure reliable-plane accessors — needing node identity but +// neither the connection-handle key nor the RNG. +impl SerfEngine +where + I: memberlist_proto::Id + Clone, +{ + /// Construct an engine, panicking on a misconfiguration. + /// + /// The convenience wrapper over [`try_new_at`](Self::try_new_at); use it only + /// when the configuration is a static constant known to be valid. + /// + /// # Panics + /// + /// Panics if [`try_new_at`](Self::try_new_at) returns an [`InitError`]. + pub fn new_at( + cfg: Options, + transform: TransformOptions, + ep_cfg: EndpointOptions, + serf_opts: SerfOptions, + now: Instant, + rng: R, + ) -> Self { + Self::try_new_at(cfg, transform, ep_cfg, serf_opts, now, rng) + .expect("SerfEngine::new_at: invalid configuration; use try_new_at to handle") + } + + /// Fallibly construct an engine. + /// + /// Wires serf's super-machine over the memberlist coordinator and sizes the + /// gossip receive scratch. No sockets are bound — the driver owns the gossip + /// and reliable-stream sockets — and no I/O occurs here. + /// + /// # Parameters + /// + /// - `cfg`: engine port / timeout configuration. + /// - `transform`: cross-transport gossip + reliable-plane encryption plus the + /// cluster label. serf's gossip plane carries no compression / checksum, so + /// only the encryption and label fields of `transform` take effect. A + /// configured encryption keyring is probed here (see Errors). + /// - `ep_cfg`: memberlist machine identity (`id`, `advertise`, timing knobs). + /// The user-broadcast tier count is forced to 3 — serf ranks its intent / + /// event / query broadcasts on three tiers. + /// - `serf_opts`: serf-level configuration (reap / reconnect / coalescing / + /// query timing). + /// - `now`: the driver's clock reading at construction. + /// - `rng`: the memberlist gossip RNG, already seeded by the driver from its + /// entropy source. + /// + /// # Errors + /// + /// Returns [`InitError`] instead of panicking when the configuration is + /// invalid: a zero/over-ceiling gossip MTU, a zero port or close timeout, a + /// non-routable or port-mismatched advertise address, a machine-endpoint init + /// failure, or (with an encryption backend built in) an unusable keyring. + pub fn try_new_at( + cfg: Options, + transform: TransformOptions, + ep_cfg: EndpointOptions, + serf_opts: SerfOptions, + now: Instant, + rng: R, + ) -> Result { + // Validate every advertise-independent config field (port, gossip-MTU + // ceiling, close timeout, and the encryption keyring) up front, sharing the + // reused preflight so the deterministic checks live in ONE place. + validate_runtime_config(&cfg, &transform, ep_cfg.gossip_mtu())?; + + // Capture the advertise-dependent values before `ep_cfg` is moved. + let gossip_mtu = ep_cfg.gossip_mtu(); + let advertise = *ep_cfg.advertise_addr_ref(); + + // Reject a non-routable advertise address before the endpoint exists: a node + // must advertise an address its peers can route a reply to. + if !socket_addr_is_routable(&advertise) { + return Err(InitError::NonRoutableAdvertiseAddr(advertise)); + } + // The advertised port must match the single bound port (one port serves both + // the gossip and reliable planes; a direct embedded interface has no NAT). + if advertise.port() != cfg.port { + return Err(InitError::AdvertisePortMismatch); + } + + // Size the inbound-gossip scratch from the configured gossip MTU, keeping the + // driver's ingress in lockstep with the machine's egress bound. + let gossip_recv = std::vec![0u8; gossip_recv_buf_size(gossip_mtu)]; + + // serf ranks its user broadcasts on three tiers (intent / event / query → + // ranks 0 / 1 / 2), so the inner memberlist endpoint needs at least three + // broadcast tiers. + let ep_cfg = ep_cfg.with_user_broadcast_tiers(NonZeroU8::new(3).expect("3 is nonzero")); + + // The CIDR policy gates the alive delegate (composed below) and the + // transport-boundary recv/accept guards (stored on the engine). + #[cfg(feature = "cidr")] + let cidr_policy: CidrFilter = cfg.cidr_policy.clone(); + #[cfg(not(feature = "cidr"))] + let cidr_policy: CidrFilter = (); + + // Build the inner memberlist `Endpoint` (the SWIM machine serf sits on) with + // the injected gossip RNG. `try_new_at` maps a machine init failure to + // `InitError::Endpoint` and starts its timers from a consistent origin. + let mut ep = Endpoint::try_new_at(ep_cfg, now, rng).map_err(InitError::Endpoint)?; + + // Install the routable-address admission filter on the raw `Endpoint` BEFORE + // it is wrapped: the machine consults it inline for every inbound Alive, so a + // peer advertising a non-routable address is dropped at admission. When a CIDR + // policy is set, the routable filter wraps it (routable AND in-policy). + #[cfg(feature = "cidr")] + match cidr_policy.clone() { + Some(policy) => ep.set_alive_delegate(RoutableAnd(policy)), + None => ep.set_alive_delegate(RoutableAddrFilter), + } + #[cfg(not(feature = "cidr"))] + ep.set_alive_delegate(RoutableAddrFilter); + + // Build the reliable-plane label options from the single validated source + // (already validated at the `TransformOptions` setter, so `new_in` is + // infallible here). Plain TCP has no SNI (`|_| None`) and a membership + // address that IS the transport socket (`|addr| *addr`). + let mut label_opts = LabelOptions::new_in(transform.label().map(|b| b.to_vec()), ()); + if transform.skip_inbound_label_check() { + label_opts = label_opts.skip_inbound_label_check(); + } + // Retain the validated label for the gossip codec (same source, both planes + // share one label so they cannot diverge). + let label = transform.label().map(Bytes::copy_from_slice); + + #[allow(unused_mut)] + let mut coord = Coordinator::new( + ep, + label_opts, + Box::new(|_: &SocketAddr| -> Option { None }), + Box::new(|addr: &SocketAddr| *addr), + ); + // Install the gossip-and-reliable encryption keyring; a no-keyring policy is + // the identity transform, so an unencrypted node is unaffected. serf's gossip + // plane applies only this transform (no compression / checksum on gossip). + #[cfg(encryption)] + coord.set_encryption_options(transform.encryption); + + // Wrap the coordinator in serf's super-machine. `new` seeds serf's core RNG + // (query IDs / relay selection) deterministically; it is independent of the + // injected memberlist gossip RNG `R`. + let endpoint = StreamEndpoint::::new(coord, serf_opts); + + Ok(Self { + endpoint, + cfg, + plane: ReliablePlane::new(), + gossip_recv, + pending_seeds: VecDeque::new(), + label, + cidr_policy, + }) + } + + /// Mutable access to the reliable plane's pool, for a driver to push its + /// pre-created connection handles and install the initial listener at + /// construction. + #[inline] + pub fn plane_mut(&mut self) -> &mut ReliablePlane { + &mut self.plane + } + + /// Install the initial passive-open listener handle, set by the driver after it + /// has `listen`ed on that connection slot at construction. + #[inline] + pub fn set_listener(&mut self, c: C) { + self.plane.listener = Some(c); + } + + /// The configured local port (gossip + reliable listener both bind it). + #[inline] + pub fn port(&self) -> u16 { + self.cfg.port + } + + /// Number of inbound reliable connections accepted on the listener since + /// construction. + #[inline] + pub fn accepted_inbound_count(&self) -> u64 { + self.plane.accepted_inbound + } + + /// Number of pooled connection slots currently free. + #[inline] + pub fn pool_free_count(&self) -> usize { + self.plane.pool.free_len() + } + + /// Number of connection slots currently parked mid-close. + #[inline] + pub fn closing_count(&self) -> usize { + self.plane.closing.len() + } + + /// Whether a passive-open listener slot is currently installed. + #[inline] + pub fn listener_present(&self) -> bool { + self.plane.listener.is_some() + } + + /// Number of reliable exchanges currently half-closed (local FIN emitted, still + /// mapped awaiting the peer's reply and/or FIN). + #[inline] + pub fn half_closed_count(&self) -> usize { + self.plane.half_closed_count() + } + + /// Number of reliable exchanges still in `PendingDial` (dial requested, pool + /// exhausted, no slot assigned yet). + #[inline] + pub fn pending_dial_count(&self) -> usize { + self.plane.pending_dial_count() + } +} + +// serf-command and read forwarders — reach serf's super-machine only (not the +// reliable plane), so they need node identity and the gossip RNG but no +// connection-handle key. +impl SerfEngine +where + I: memberlist_proto::Id + Clone, + R: Rng, +{ + /// Arm serf's periodic probe / gossip / push-pull schedulers. Call once before + /// the first `pump`; without it failure detection, dissemination, and + /// anti-entropy never run. + pub fn start(&mut self, now: Instant) { + self.endpoint.start_scheduling(now); + } + + /// `Ok` only while the node is running (serf state `Alive`). After `leave()` + /// the schedulers stop and the machine merges no remote state, so the + /// operations that gate on this reject rather than queue work no peer would + /// observe. + pub fn ensure_running(&self) -> Result<(), SerfError> { + if self.is_running() { + Ok(()) + } else { + Err(SerfError::BadJoinState(self.endpoint.state())) + } + } + + /// Whether serf's endpoint is in the running (`Alive`) state. + #[inline] + fn is_running(&self) -> bool { + self.endpoint.state() == SerfState::Alive + } + + /// serf's current lifecycle state. + #[inline] + pub fn state(&self) -> SerfState { + self.endpoint.state() + } + + /// Number of serf members currently tracked. + #[inline] + pub fn num_members(&self) -> usize { + self.endpoint.num_members() + } + + /// The local node's serf member Lamport clock. + #[inline] + pub fn member_time(&self) -> u64 { + self.endpoint.member_time() + } + + /// The local node's serf event Lamport clock. + #[inline] + pub fn event_time(&self) -> u64 { + self.endpoint.event_time() + } + + /// The local node's serf query Lamport clock. + #[inline] + pub fn query_time(&self) -> u64 { + self.endpoint.query_time() + } + + /// The local node's id. + #[inline] + pub fn local_id(&self) -> &I { + self.endpoint.local_id() + } + + /// A snapshot of every serf member currently tracked (alive, leaving, left, or + /// failed within the reap window), for the observable membership view a driver + /// publishes after each membership change. + #[inline] + pub fn members_snapshot(&self) -> Vec>> { + self.endpoint.members_snapshot() + } + + /// Drain one application-visible serf event, if any. + /// + /// Returns events emitted by the machine during the last `pump` tick — the full + /// serf surface: membership changes, user events, queries, query responses / + /// acks, key-management requests / responses, reliable-exchange completions, + /// and the lifecycle signals (`LeftCluster`, conflict `Shutdown`). Returns + /// `None` when the event queue is empty; call again after the next `pump` tick. + #[inline] + pub fn poll_event(&mut self) -> Option> { + self.endpoint.poll_event() + } + + /// Announce the local node's join intent and record intent to contact these + /// seed addresses. + /// + /// Returns immediately; the pump loop initiates a push/pull state exchange to + /// each routable seed on the next tick. serf's own `join()` rejects a non-Alive + /// endpoint ([`SerfError::BadJoinState`]); on that rejection no seed is queued. + pub fn join(&mut self, seeds: &[SocketAddr]) -> Result<(), SerfError> { + // Announce the serf-level join intent first (this is where the running-state + // gate lives); only queue seeds once it is accepted. + self.endpoint.join()?; + for s in seeds { + // Drop a non-routable seed: it could only produce a doomed dial. Queue only + // seeds a dial can actually complete. + if socket_addr_is_routable(s) { + self.pending_seeds.push_back(*s); + } + } + Ok(()) + } + + /// Begin leaving the cluster. + /// + /// Forwards to serf's graceful-leave path, which gossips the departure and + /// ultimately emits [`Event::LeftCluster`] via [`poll_event`](Self::poll_event). + /// Any seeds still queued from a pre-leave join are dropped: the pump initiates + /// no new push/pull once leaving. + pub fn leave(&mut self, now: Instant) -> Result<(), SerfError> { + self.pending_seeds.clear(); + self.endpoint.leave(now) + } + + /// Force a named node out of the cluster (an operator-driven removal). + pub fn force_leave(&mut self, id: I, prune: bool, now: Instant) -> Result<(), SerfError> { + self.endpoint.force_leave(id, prune, now) + } + + /// Broadcast an application user event to the cluster. + /// + /// `coalesce` requests that identical events be coalesced by name over the + /// user-coalesce window. Peers observe it as [`Event::User`] via `poll_event`. + pub fn user_event( + &mut self, + name: impl Into, + payload: Bytes, + coalesce: bool, + ) -> Result<(), SerfError> { + self.endpoint.user_event(name, payload, coalesce) + } + + /// Issue a cluster-wide query, returning its [`QueryId`]. + /// + /// Responders observe the query as [`Event::Query`] and answer via + /// [`respond`](Self::respond); responses surface on this node as + /// [`Event::QueryResponse`]. + pub fn query( + &mut self, + name: impl Into, + payload: Bytes, + params: QueryParams, + now: Instant, + ) -> Result { + self.endpoint.query(name, payload, params, now) + } + + /// Answer a received query. `token` is the [`QueryEvent`] delivered via + /// [`Event::Query`]. + pub fn respond( + &mut self, + token: &QueryEvent, + payload: Bytes, + now: Instant, + ) -> Result<(), SerfError> { + self.endpoint.respond(token, payload, now) + } + + /// Replace the local node's tags, re-advertising them via the coordinator and + /// refreshing the local member in the membership store. + pub fn set_tags(&mut self, tags: Tags) -> Result<(), SerfError> { + self.endpoint.set_tags(tags) + } + + /// Issue a cluster-wide `install_key` query to add `key` to every node's + /// keyring. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn install_key(&mut self, key: SecretKey, now: Instant) -> Result { + self.endpoint.install_key(key, now) + } + + /// Issue a cluster-wide `use_key` query to promote `key` to primary. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn use_key(&mut self, key: SecretKey, now: Instant) -> Result { + self.endpoint.use_key(key, now) + } + + /// Issue a cluster-wide `remove_key` query to remove `key` from all nodes. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn remove_key(&mut self, key: SecretKey, now: Instant) -> Result { + self.endpoint.remove_key(key, now) + } + + /// Issue a cluster-wide `list_keys` query to enumerate installed keys. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn list_keys(&mut self, now: Instant) -> Result { + self.endpoint.list_keys(now) + } + + /// Answer an inbound key-management request. `req` is the [`KeyRequest`] + /// delivered via [`Event::KeyRequest`]; the driver applies the requested op to + /// its keyring and passes the outcome as `resp`. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn respond_key( + &mut self, + req: &KeyRequest, + resp: KeyResponseArgs, + now: Instant, + ) -> Result<(), SerfError> { + self.endpoint.respond_key(req, resp, now) + } +} + +// Reliable-plane lifecycle helpers that move the connection handle `C` by value +// (into the pool, the listener slot, and the `StreamIo` socket calls) and reach +// serf's machine, so they need `C: Copy + Eq + Hash` and the gossip RNG. +impl SerfEngine +where + I: memberlist_proto::Id + Clone, + C: Copy + Eq + Hash, + R: Rng, +{ + /// Advance serf's state machine once over the driver's already-ticked sockets. + /// Returns the next wakeup deadline: the minimum of the machine's next timer + /// AND the soonest gracefully-closing connection's force-abort instant. The + /// driver folds in its own link-layer next-event deadline. + /// + /// The driver owns the super-loop: it ticks its link-layer stack, calls `pump`, + /// then sleeps until `min(driver_stack_next, pump_result)`, advances the clock, + /// and loops. + /// + /// # Order + /// + /// `pump` runs the same ordered phases as memberlist-embedded's `Engine::pump`, + /// driving serf's super-machine (which threads its own serf pre/post-tick logic + /// through the composed `handle_timeout`): + /// + /// 1a. Reap gracefully-closed (or close-timed-out) connections. + /// 1b. Accept an inbound connection completed on the listener; replenish it. + /// 1c. Rebalance: self-heal a missing listener, then assign free slots to + /// deferred dials (listener-first). + /// 2a/2b. Gossip ingress: drain datagrams into `handle_gossip`, then decrypt + + /// label-strip + decode each buffered frame and feed the typed messages + /// back via `handle_message`. + /// 3. Reliable ingress: drain each connection's rx into `handle_transport_data`; + /// deliver a one-shot EOF on peer FIN. + /// 5. Join-seed drain: `start_push_pull(seed, Join, now)` per queued seed. + /// 6. Machine tick: `handle_timeout` fires due serf + coordinator timers. + /// 7a–7e. Drain `poll_action`, promote, pump outbound, flush deferred FINs, + /// complete `Closing` drains, re-rebalance, then drain + send outbound + /// gossip. + /// 8. Deadline: `min(machine_next, closing_next)`. + pub fn pump(&mut self, now: Instant, gossip: &mut G, stream: &mut S) -> Option + where + G: GossipIo, + S: StreamIo, + { + // 1a. Reap gracefully-closing connections the driver's stack tick advanced to + // completion, so the freed handles back new dials/accepts this same tick. + self.reap_closing(now, stream); + + // 1b/1c. Accept-and-replenish first (listener-first), then self-heal a missing + // listener and assign remaining free slots to deferred dials. Running the + // rebalance BEFORE the machine tick lets a prior-tick `PendingDial` be dialed + // before its bridge can time out. + self.check_listener(now, stream); + self.rebalance_pool(now, stream); + + // 2a. Drain inbound gossip datagrams into the machine's raw ingress buffer. + { + let buf = self.gossip_recv.as_mut_slice(); + let endpoint = &mut self.endpoint; + let cidr_policy = &self.cidr_policy; + while let Some((src, n)) = gossip.recv(buf) { + // Drop a gossip datagram from a CIDR-blocked source before the machine + // sees it. + if !cidr_blocks(cidr_policy, src.ip()) { + endpoint.handle_gossip(src, &buf[..n], now); + } + } + } + + // 2b. Decrypt + label-strip + decode each raw gossip frame and feed typed + // messages back. serf's gossip plane carries only the encryption wrapper (no + // checksum / compression), so a plaintext build feeds the raw bytes straight + // to the label check. + while let Some((src, raw)) = self.endpoint.poll_memberlist_ingress() { + // With an encryption backend built in, strip (and authenticate) the wrapper + // first — identity when no keyring is configured, an `Err` (dropped) for a + // frame the keyring cannot decrypt. Gossip is lossy and self-healing. + #[cfg(encryption)] + let plain = match self.endpoint.decrypt_gossip(&raw) { + Ok(p) => Bytes::from(p), + Err(_) => continue, + }; + #[cfg(not(encryption))] + let plain = raw; + let opts = DecodeOptions::new(self.label.clone()); + // Drop malformed inbound datagrams silently — bad network input must not + // panic the node; SWIM is self-healing. + if let Ok(inner) = decode_incoming(plain, &opts) { + if let Ok(msgs) = parse_messages::(inner) { + for msg in msgs { + self.endpoint.handle_message(src, msg, now); + } + } + } + } + + // 3. Reliable ingress pump: drain each active exchange's socket rx into the + // machine (including the peer-FIN EOF) before the machine tick. + self.pump_inbound_reliable(now, stream); + + // 5. Drain join seeds: each queued seed gets a push/pull exchange initiated + // now. Skipped once leaving/left — a left node initiates no join push/pull. + if self.is_running() { + while let Some(seed) = self.pending_seeds.pop_front() { + // The StreamId is the machine's correlation token; the dial is correlated + // via the ExchangeId carried in the resulting Connect action, so the + // driver does not need to retain it here. + let _sid = self.endpoint.start_push_pull(seed, PushPullKind::Join, now); + } + } + + // 6. Machine tick: fire due serf + coordinator timers. + self.endpoint.handle_timeout(now); + + // 7a. Drain stream actions: open dials, half-close, or tear down exchanges. + self.drain_stream_actions(now, stream); + // 7b. Promote dialing connections whose handshake completed this tick. + self.promote_established(stream); + // 7c. Reliable egress pump: append new transmits, then flush each queue. + self.pump_outbound_reliable(stream); + // 7d. Emit deferred graceful write-half FINs now fully drained. + self.flush_pending_shutdowns(stream); + // 7d'. Complete deferred terminal closes of connections draining in `Closing`. + self.flush_closing(now, stream); + // 7d''. Re-run the listener/dial rebalance over every slot the machine tick + // and teardown just freed back to the pool THIS tick. + self.rebalance_pool(now, stream); + + debug_assert!( + !self.plane.pool.any_where(|&c| stream.reuse_ready(c)) + || (self.plane.listener.is_some() && self.plane.pending_dial_count() == 0), + "end-of-tick: a reuse-ready reliable slot left a listener missing or a PendingDial unserviced" + ); + + // 7e. Egress: drain outbound gossip transmits, encode + encrypt, and send. + self.drain_gossip_transmits(gossip); + + // 8. Next deadline = min(machine, closing). + let machine = self.endpoint.poll_timeout(); + let closing = self + .plane + .closing + .values() + .chain( + self + .plane + .connections + .values() + .filter_map(|c| c.close_deadline.as_ref()), + ) + .min() + .copied(); + min_opt(machine, closing) + } + + /// Reclaim gracefully-closing connections that have finished closing or whose + /// close has exceeded `cfg.close_timeout`. + fn reap_closing(&mut self, now: Instant, stream: &mut S) + where + S: StreamIo, + { + let pool = &mut self.plane.pool; + self.plane.closing.retain(|&c, &mut deadline| { + if !stream.is_open(c) { + pool.give(c); + return false; + } + if now >= deadline { + // Peer vanished mid-FIN: force the socket Closed and reclaim so the pool + // (and the listener replenished from it) recover. + stream.abort(c); + pool.give(c); + return false; + } + true + }); + } + + /// Consume an accept-ready listener, hand its exchange to the machine, and + /// replenish a fresh listener from the pool. + /// + /// The accept gate is `accepted_peer(c).is_some()`, which a driver reports only + /// once the socket is at/after Established with a known remote — never a + /// not-yet-established handshake an RST could revert. + fn check_listener(&mut self, now: Instant, stream: &mut S) + where + S: StreamIo, + { + let c = match self.plane.listener { + Some(c) => c, + None => return, + }; + let Some(peer) = stream.accepted_peer(c) else { + return; + }; + + // Reject a reliable connection from a CIDR-blocked peer at the transport + // boundary: abort the connected socket and reclaim it WITHOUT registering the + // exchange, then re-arm a fresh listener. + if cidr_blocks(&self.cidr_policy, peer.ip()) { + stream.abort(c); + self.plane.pool.give(c); + self.plane.listener = None; + self.ensure_listener(stream); + return; + } + + match self.endpoint.accept_connection(peer, now) { + Some(eid) => { + self + .plane + .connections + .insert(eid, Connection::accepted(peer, c)); + self.plane.accepted_inbound += 1; + } + // Not admitted (leaving, the inbound-stream cap, or a record-layer config + // error): abort the socket AND return its handle to the pool so a rejection + // does not shrink the finite pool one slot at a time. + None => { + stream.abort(c); + self.plane.pool.give(c); + } + } + + self.plane.listener = None; + // Replenish immediately if a slot is free — the same self-heal the poll-phase + // rebalance uses, giving the listener first claim on a free slot. + self.ensure_listener(stream); + } + + /// Re-establish the passive-open listener if it is missing and the pool can + /// supply a reuse-ready slot. A no-op when a listener already exists or the pool + /// has no reuse-ready slot. + fn ensure_listener(&mut self, stream: &mut S) + where + S: StreamIo, + { + if self.plane.listener.is_some() { + return; + } + if let Some(c) = self.plane.pool.take_where(|&c| stream.reuse_ready(c)) { + // `listen()` only fails on port 0 or an already-open socket; a pooled socket + // is Closed and `cfg.port` is the user-supplied non-zero port. + // Ignoring Err: both failure modes are unreachable for a pooled slot here. + let _ = stream.listen(c, self.cfg.port); + self.plane.listener = Some(c); + } + } + + /// Give the listener and any deferred dials first claim on whatever is currently + /// in the pool: self-heal a missing listener, then assign the rest to + /// `PendingDial` connections oldest-first (listener-first). + fn rebalance_pool(&mut self, now: Instant, stream: &mut S) + where + S: StreamIo, + { + self.ensure_listener(stream); + self.drain_pending_dials(now, stream); + } + + /// Assign a freed slot to each connection still waiting in `PendingDial`, + /// oldest-first by ascending `ExchangeId`, and dial it. Stops the moment the + /// pool empties again so the rest stay parked for a later tick. + fn drain_pending_dials(&mut self, now: Instant, stream: &mut S) + where + S: StreamIo, + { + let mut waiting: MediumVec<(ExchangeId, SocketAddr)> = self + .plane + .connections + .iter() + .filter(|(_, c)| c.state == ConnState::PendingDial) + .map(|(&eid, c)| (eid, c.peer)) + .collect(); + if waiting.is_empty() { + return; + } + waiting.sort_by_key(|(eid, _)| eid.get()); + + for (eid, peer) in waiting { + let Some(c) = self.plane.pool.take_where(|&c| stream.reuse_ready(c)) else { + break; + }; + if let Some(conn) = self.plane.connections.get_mut(&eid) { + conn.assign_socket(c); + } + self.dial(eid, peer, c, now, stream); + } + } + + /// Open a TCP dial for the `Dialing` connection `eid` on its assigned slot `c`. + /// + /// A CIDR-blocked or non-routable peer, or a `connect` rejection, reclaims the + /// socket and terminalizes the exchange as a dial FAILURE via + /// `handle_dial_failed` — never a benign EOF that a one-way exchange would read + /// as success. + fn dial(&mut self, eid: ExchangeId, peer: SocketAddr, c: C, now: Instant, stream: &mut S) + where + S: StreamIo, + { + if cidr_blocks(&self.cidr_policy, peer.ip()) || !socket_addr_is_routable(&peer) { + stream.abort(c); + self.plane.pool.give(c); + self.plane.connections.remove(&eid); + self.endpoint.handle_dial_failed(eid, now); + return; + } + + // Derive an ephemeral local port from the ExchangeId so each dial uses a + // distinct port within the IANA ephemeral range (49152–65535). + let local_port = 49152u16 + (eid.get() as u16 % 16384); + if stream.connect(c, peer, local_port).is_err() { + stream.abort(c); + self.plane.pool.give(c); + self.plane.connections.remove(&eid); + self.endpoint.handle_dial_failed(eid, now); + } + } + + /// Drain all `StreamAction`s emitted by the machine this tick: open dials + /// (`Connect`), defer a graceful write-half FIN (`Shutdown`), tear down + /// gracefully (`Close`), or hard-abort a FAILED exchange (`Abort`). + fn drain_stream_actions(&mut self, now: Instant, stream: &mut S) + where + S: StreamIo, + { + while let Some(action) = self.endpoint.poll_action() { + match action { + StreamAction::Connect(info) => { + let eid = info.id(); + let peer = info.peer(); + // Only a reset, reuse-ready slot may back a fresh dial; a freed-but- + // still-resetting slot defers to `PendingDial` until its worker resets. + match self.plane.pool.take_where(|&c| stream.reuse_ready(c)) { + Some(c) => { + self + .plane + .connections + .insert(eid, Connection::dialing(peer, c)); + self.dial(eid, peer, c, now, stream); + } + // Pool exhausted: record a `PendingDial` (no slot) so the dial intent + // is not lost; `drain_pending_dials` assigns a slot once one frees. + None => { + self + .plane + .connections + .insert(eid, Connection::pending_dial(peer)); + } + } + } + StreamAction::Shutdown(r) => { + // Deferred write-half FIN: set the flag; `flush_pending_shutdowns` + // emits it once the socket is Established and its tx ring has drained. + if let Some(conn) = self.plane.connections.get_mut(&r.id()) { + conn.fin_pending = true; + } + } + StreamAction::Close(r) => { + self.teardown(r.id(), now, stream); + } + StreamAction::Abort(r) => { + self.abort_exchange(r.id(), stream); + } + } + } + } + + /// Abort a FAILED exchange (dial failure, label/encryption rejection, or an + /// elapsed deadline): discard its buffered `out`, hard-reset the socket, and + /// reclaim the slot straight to the pool. + fn abort_exchange(&mut self, eid: ExchangeId, stream: &mut S) + where + S: StreamIo, + { + let Some(conn) = self.plane.connections.remove(&eid) else { + return; + }; + if let Some(c) = conn.socket { + stream.abort(c); + self.plane.pool.give(c); + } + } + + /// Tear down a GRACEFULLY completed exchange (`StreamAction::Close`), draining + /// any undelivered outbound bytes before the terminal FIN and reclaiming the + /// slot by socket state. + /// + /// The one case that does NOT remove the connection on the spot is a graceful + /// close whose send-capable socket still holds undelivered bytes: it parks in + /// [`ConnState::Closing`] so the egress pump keeps flushing them, and + /// `flush_closing` FINs + detaches once they are delivered (or the close + /// deadline forces an abort). A graceful close never discards undelivered bytes. + fn teardown(&mut self, eid: ExchangeId, now: Instant, stream: &mut S) + where + S: StreamIo, + { + let Some(conn) = self.plane.connections.get(&eid) else { + return; + }; + let Some(c) = conn.socket else { + // PendingDial: no socket, nothing to reclaim. Removing it is the whole + // teardown, so a retired exchange is never later dialed. + self.plane.connections.remove(&eid); + return; + }; + + let was_half_closed = conn.state == ConnState::HalfClosed; + let out_pending = !conn.out_is_empty(); + let is_open = stream.is_open(c); + let may_send = stream.may_send(c); + let tx_unacked = stream.send_queue(c); + + if !is_open { + // `Closed | TimeWait`: both FINs already exchanged. Reclaim directly. + self.plane.connections.remove(&eid); + self.plane.pool.give(c); + } else if was_half_closed { + // Our FIN is in flight and the tx half is closed, so any `out` remainder is + // undeliverable. Park for the reap backstop. + self.plane.connections.remove(&eid); + self.plane.closing.insert(c, now + self.cfg.close_timeout); + } else if may_send && (out_pending || tx_unacked != 0) { + // Send-capable with outbound bytes the peer has NOT received. FIN-ing now + // would truncate the reply; defer via `Closing` so the egress pump keeps + // draining `out` into the tx ring. + if let Some(conn) = self.plane.connections.get_mut(&eid) { + conn.state = ConnState::Closing; + conn.close_deadline = Some(now + self.cfg.close_timeout); + conn.close_drain_mark = conn.out_bytes() + tx_unacked; + conn.fin_pending = false; + } + } else if may_send { + // Send-capable with nothing left to deliver: emit the graceful FIN now and + // park the handle for the reap backstop. + self.plane.connections.remove(&eid); + stream.close(c); + self.plane.closing.insert(c, now + self.cfg.close_timeout); + } else { + // Abrupt teardown of a socket the peer never established: RST and reclaim. + self.plane.connections.remove(&eid); + stream.abort(c); + self.plane.pool.give(c); + } + } + + /// Complete the deferred terminal close of every connection draining in + /// [`ConnState::Closing`]: FIN once `out` and the tx ring are fully drained, or + /// force-abort one past its (no-progress) close deadline. + fn flush_closing(&mut self, now: Instant, stream: &mut S) + where + S: StreamIo, + { + enum ClosingAction { + Fin(C), + Abort(C), + Progress(usize), + } + + let mut actions: MediumVec<(ExchangeId, ClosingAction)> = MediumVec::new(); + for (&eid, conn) in self.plane.connections.iter() { + if conn.state != ConnState::Closing { + continue; + } + let Some(c) = conn.socket else { continue }; + // Undelivered shrinks ONLY when the peer acks, so a shrink is the + // peer-liveness signal: `close_timeout` bounds a STALL, not the total drain. + let undelivered = conn.out_bytes() + stream.send_queue(c); + if undelivered == 0 { + actions.push((eid, ClosingAction::Fin(c))); + } else if undelivered < conn.close_drain_mark { + actions.push((eid, ClosingAction::Progress(undelivered))); + } else if conn.close_deadline.is_some_and(|d| now >= d) { + actions.push((eid, ClosingAction::Abort(c))); + } + } + + for (eid, outcome) in actions { + match outcome { + ClosingAction::Fin(c) => { + self.plane.connections.remove(&eid); + stream.close(c); + self.plane.closing.insert(c, now + self.cfg.close_timeout); + } + ClosingAction::Abort(c) => { + self.plane.connections.remove(&eid); + stream.abort(c); + self.plane.pool.give(c); + } + ClosingAction::Progress(mark) => { + if let Some(conn) = self.plane.connections.get_mut(&eid) { + conn.close_drain_mark = mark; + conn.close_deadline = Some(now + self.cfg.close_timeout); + } + } + } + } + } + + /// Flush partially-written outbound bytes and drain new transport transmits from + /// the machine into each connection's tx ring, preserving per-connection byte + /// order under partial-write backpressure. + fn pump_outbound_reliable(&mut self, stream: &mut S) + where + S: StreamIo, + { + // Pass 1: append new transmits to their connection's out queue (in order, + // regardless of state — a Dialing / PendingDial connection holds them until + // its socket is writable). Bytes for a torn-down exchange are dropped. + while let Some((eid, _peer, bytes)) = self.endpoint.poll_transport_transmit() { + if let Some(conn) = self.plane.connections.get_mut(&eid) { + conn.out.push_back(bytes); + } + } + + // Pass 2: flush each connection's out queue to its socket. + let pairs: MediumVec<_> = self + .plane + .connections + .iter() + .filter_map(|(&eid, c)| { + if c.out.is_empty() { + return None; + } + c.socket.map(|h| (eid, h)) + }) + .collect(); + + for (eid, c) in pairs { + // A still-handshaking socket is `!may_send`; leave the queue parked and + // retry once Established, so a push/pull half is never dropped mid-open. + if !stream.may_send(c) { + continue; + } + while let Some(front) = self + .plane + .connections + .get(&eid) + .and_then(|conn| conn.out.front().cloned()) + { + let sent = stream.send(c, &front); + if sent >= front.len() { + if let Some(conn) = self.plane.connections.get_mut(&eid) { + conn.out.pop_front(); + } + } else { + // Partial write: replace the front with its unsent tail and stop, so the + // tail stays at the front and later entries are not reordered. + if let Some(conn) = self.plane.connections.get_mut(&eid) { + if let Some(slot) = conn.out.front_mut() { + *slot = front.slice(sent..); + } + } + break; + } + } + } + } + + /// Promote each `Dialing` connection whose TCP handshake has completed to + /// `Established`. + fn promote_established(&mut self, stream: &mut S) + where + S: StreamIo, + { + let promote: MediumVec = self + .plane + .connections + .iter() + .filter(|(_, c)| c.state == ConnState::Dialing) + .filter_map(|(&eid, c)| c.socket.map(|h| (eid, h))) + .filter(|&(_, h)| stream.may_send(h)) + .map(|(eid, _)| eid) + .collect(); + for eid in promote { + if let Some(conn) = self.plane.connections.get_mut(&eid) { + conn.state = ConnState::Established; + } + } + } + + /// Emit deferred graceful write-half FINs for connections whose socket can now + /// carry one losslessly — KEEPING the connection mapped so its inbound reply + /// still pumps. The socket is reclaimed only later, by the machine's `Close`. + fn flush_pending_shutdowns(&mut self, stream: &mut S) + where + S: StreamIo, + { + let ready: MediumVec<_> = self + .plane + .connections + .iter() + .filter(|(_, c)| c.fin_pending && c.state == ConnState::Established && c.out_is_empty()) + .filter_map(|(&eid, c)| c.socket.map(|h| (eid, h))) + // …and the tx ring is fully drained and acknowledged, so every byte reached + // the peer before the FIN. + .filter(|&(_, h)| stream.may_send(h) && stream.send_queue(h) == 0) + .collect(); + + for (eid, c) in ready { + stream.close(c); + if let Some(conn) = self.plane.connections.get_mut(&eid) { + conn.fin_pending = false; + conn.state = ConnState::HalfClosed; + } + } + } + + /// Drain each active connection's socket rx into the machine, delivering a + /// one-shot EOF once the peer's FIN has been received AND the rx buffer is fully + /// drained ([`StreamIo::recv_finished`]). + fn pump_inbound_reliable(&mut self, now: Instant, stream: &mut S) + where + S: StreamIo, + { + const READ_BUF: usize = 4096; + let mut buf = [0u8; READ_BUF]; + + let pairs: MediumVec<_> = self + .plane + .connections + .iter() + .filter_map(|(&eid, c)| c.socket.map(|h| (eid, h))) + .collect(); + + for (eid, c) in pairs { + loop { + match stream.recv(c, &mut buf) { + Some(n) if n > 0 => { + self + .endpoint + .handle_transport_data(eid, &buf[..n], false, now); + } + _ => { + // No data this tick. Deliver the peer FIN exactly once when the receive + // half is gracefully closed and drained. + if stream.recv_finished(c) { + if let Some(conn) = self.plane.connections.get_mut(&eid) { + if !conn.eof_delivered { + conn.eof_delivered = true; + self.endpoint.handle_transport_data(eid, &[], true, now); + } + } + } + break; + } + } + } + } + } + + /// Drain all outbound gossip transmits from the machine, encode each, apply the + /// encryption wrapper (when a backend is built in), and write it to the gossip + /// socket. + /// + /// serf's gossip plane carries no compression / checksum wrappers — only the + /// label frame and, under an encryption backend, the AEAD wrapper. Encoding + /// errors and a full tx ring both silently drop the datagram; gossip is + /// best-effort and SWIM recovers on the next round. + fn drain_gossip_transmits(&mut self, gossip: &mut G) + where + G: GossipIo, + { + let enc = EncodeOptions::new(self.label.clone()); + while let Some(transmit) = self.endpoint.poll_memberlist_transmit() { + let (dest, bytes) = match encode_transmit::(transmit, &enc) { + Some(pair) => pair, + None => continue, + }; + // Apply the encryption wrapper before the wire (identity when no keyring is + // configured). Drop rather than emit plaintext on an encrypted-cluster path + // if the backend rejects the request. + #[allow(unused_mut)] + let mut on_wire: Vec = bytes.to_vec(); + #[cfg(encryption)] + { + on_wire = match self.endpoint.encrypt_gossip(&on_wire) { + Ok(b) => b, + Err(_) => continue, + }; + } + // Last-line egress screens: never emit to a non-routable destination or one + // our own CIDR policy excludes. + if !socket_addr_is_routable(&dest) || cidr_blocks(&self.cidr_policy, dest.ip()) { + continue; + } + gossip.send(&on_wire, dest); + } + } +} + +/// Returns the earlier of two optional deadlines. If only one is `Some`, that +/// deadline wins; if both are `None` the result is `None`. +fn min_opt(a: Option, b: Option) -> Option { + match (a, b) { + (Some(x), Some(y)) => Some(core::cmp::min(x, y)), + (x, y) => x.or(y), + } +} + +/// Encode one outbound gossip transmit using the shared no-std codec. +/// +/// Returns `(dest, encoded_bytes)` on success, or `None` if encoding fails (the +/// caller silently skips the datagram — gossip is lossy). +fn encode_transmit( + t: Transmit, + enc: &EncodeOptions, +) -> Option<(SocketAddr, Bytes)> +where + I: memberlist_proto::Data, +{ + match t { + Transmit::Packet(pkt) => { + let (to, msg) = pkt.into_parts(); + let bytes = encode_outgoing(&msg, enc).ok()?; + Some((to, bytes)) + } + Transmit::Compound(cmp) => { + let (to, msgs) = cmp.into_parts(); + let bytes = encode_outgoing_compound(&msgs, enc).ok()?; + Some((to, bytes)) + } + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-embedded/src/engine/tests.rs b/serf-embedded/src/engine/tests.rs new file mode 100644 index 00000000..c459470c --- /dev/null +++ b/serf-embedded/src/engine/tests.rs @@ -0,0 +1,278 @@ +use super::*; + +use core::{ + net::{IpAddr, Ipv4Addr}, + time::Duration, +}; + +use memberlist_proto::{SeedableRng, SmallRng}; +use smol_str::SmolStr; + +/// A fixed-seed gossip RNG for the engine constructors. These are single-node +/// state tests; a deterministic seed keeps them reproducible. +fn test_rng() -> SmallRng { + SmallRng::seed_from_u64(42) +} + +/// A [`GossipIo`] that never receives and discards every send — for tests that +/// only exercise construction and the machine tick, not the wire. +struct NoGossip; + +impl GossipIo for NoGossip { + fn recv(&mut self, _buf: &mut [u8]) -> Option<(SocketAddr, usize)> { + None + } + + fn send(&mut self, _bytes: &[u8], _dest: SocketAddr) {} +} + +/// A pooled [`StreamIo`] whose sockets never establish — enough to drive the +/// reliable-plane bookkeeping (dial defer, listener replenish) without a fabric. +struct NoStream { + free: std::vec::Vec, +} + +impl NoStream { + fn with_pool(size: u32) -> Self { + Self { + free: (0..size).collect(), + } + } +} + +impl StreamIo for NoStream { + type Conn = u32; + + fn take_free(&mut self) -> Option { + self.free.pop() + } + + fn give(&mut self, c: u32) { + self.free.push(c); + } + + fn free_count(&self) -> usize { + self.free.len() + } + + fn listen(&mut self, _c: u32, _port: u16) -> Result<(), crate::StreamIoError> { + Ok(()) + } + + fn accepted_peer(&self, _c: u32) -> Option { + None + } + + fn connect( + &mut self, + _c: u32, + _remote: SocketAddr, + _local_port: u16, + ) -> Result<(), crate::StreamIoError> { + Err(crate::StreamIoError::Busy) + } + + fn may_send(&self, _c: u32) -> bool { + false + } + + fn may_recv(&self, _c: u32) -> bool { + false + } + + fn is_open(&self, _c: u32) -> bool { + false + } + + fn is_established(&self, _c: u32) -> bool { + false + } + + fn recv(&mut self, _c: u32, _buf: &mut [u8]) -> Option { + None + } + + fn recv_finished(&self, _c: u32) -> bool { + false + } + + fn send(&mut self, _c: u32, _bytes: &[u8]) -> usize { + 0 + } + + fn send_queue(&self, _c: u32) -> usize { + 0 + } + + fn close(&mut self, _c: u32) {} + + fn abort(&mut self, _c: u32) {} +} + +fn node_addr(port: u16) -> SocketAddr { + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)), port) +} + +fn make_engine() -> SerfEngine { + let cfg = Options::new() + .with_port(7946) + .with_close_timeout(Duration::from_secs(10)); + let ep_cfg = EndpointOptions::new(SmolStr::new("test"), node_addr(7946)); + let now = Instant::from_origin(Duration::from_secs(86_400)); + SerfEngine::try_new_at( + cfg, + TransformOptions::default(), + ep_cfg, + SerfOptions::new(), + now, + test_rng(), + ) + .expect("valid configuration must construct without error") +} + +/// A valid configuration constructs, and a freshly built endpoint is `Alive` +/// (running). +#[test] +fn construction_succeeds_and_is_alive() { + let engine = make_engine(); + assert_eq!(engine.state(), SerfState::Alive); + assert!( + engine.ensure_running().is_ok(), + "a freshly constructed endpoint must be running" + ); +} + +/// A non-routable advertise address is rejected at construction rather than +/// gossiped cluster-wide (the reused advertise-routability guard). +#[test] +fn non_routable_advertise_is_rejected() { + let cfg = Options::new() + .with_port(7946) + .with_close_timeout(Duration::from_secs(10)); + // The unspecified address is non-routable. + let bad = SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 7946); + let ep_cfg = EndpointOptions::new(SmolStr::new("test"), bad); + let now = Instant::from_origin(Duration::from_secs(86_400)); + let result = SerfEngine::::try_new_at( + cfg, + TransformOptions::default(), + ep_cfg, + SerfOptions::new(), + now, + test_rng(), + ); + assert!( + matches!(result, Err(InitError::NonRoutableAdvertiseAddr(_))), + "a non-routable advertise address must fail construction" + ); +} + +/// `start` then a single `pump` advances the machine without panicking and +/// returns a wakeup deadline; the single-node engine tracks no remote members. +#[test] +fn start_then_single_pump_does_not_panic() { + let mut engine = make_engine(); + let now = Instant::from_origin(Duration::from_secs(86_400)); + engine.start(now); + + let mut gossip = NoGossip; + let mut stream = NoStream::with_pool(2); + // Must not panic, and the SWIM schedulers armed by `start` yield a finite next + // deadline. + let deadline = engine.pump(now, &mut gossip, &mut stream); + assert!( + deadline.is_some(), + "an armed engine must return a next wakeup deadline" + ); + // Draining events must not panic on a quiescent single node. + while engine.poll_event().is_some() {} +} + +/// `join` announces the serf join intent and queues each routable seed; the pump +/// initiates a push/pull per seed. With an exhausted dial pool the exchange parks +/// as `PendingDial` rather than dropping the dial intent. +#[test] +fn join_queues_seed_then_pump_parks_pending_dial() { + let mut engine = make_engine(); + let now = Instant::from_origin(Duration::from_secs(86_400)); + engine.start(now); + + // A listener is present but the dial pool is empty, so the Connect finds no free + // slot and defers to PendingDial. + engine.set_listener(9); + engine + .join(&[node_addr(7002)]) + .expect("join announces intent and queues the routable seed"); + + let mut gossip = NoGossip; + let mut stream = NoStream::with_pool(0); + engine.pump(now, &mut gossip, &mut stream); + assert_eq!( + engine.pending_dial_count(), + 1, + "an exhausted-pool join seed must park as PendingDial, not drop the dial" + ); +} + +/// A non-routable join seed is dropped rather than queued for a doomed dial; a +/// routable one is queued. +#[test] +fn join_drops_non_routable_seed() { + let mut engine = make_engine(); + let now = Instant::from_origin(Duration::from_secs(86_400)); + engine.start(now); + engine.set_listener(9); + + // Port 0 is non-routable and must not be queued. + let dead = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 2)), 0); + engine.join(&[dead]).expect("join succeeds"); + + let mut gossip = NoGossip; + let mut stream = NoStream::with_pool(0); + engine.pump(now, &mut gossip, &mut stream); + assert_eq!( + engine.pending_dial_count(), + 0, + "a non-routable seed must be dropped, never dialed" + ); +} + +/// A user event is accepted while running. +#[test] +fn user_event_accepted_while_running() { + let mut engine = make_engine(); + let now = Instant::from_origin(Duration::from_secs(86_400)); + engine.start(now); + engine + .user_event("deploy", Bytes::from_static(b"v2"), false) + .expect("a user event is accepted while the node is running"); +} + +/// `leave` transitions the endpoint out of `Alive`, and a subsequent `join` is +/// rejected (serf announces its own join intent only from `Alive`). +#[test] +fn leave_transitions_state_and_blocks_further_join() { + let mut engine = make_engine(); + let now = Instant::from_origin(Duration::from_secs(86_400)); + engine.start(now); + // Let the construction self-join sieve settle before leaving. + let mut gossip = NoGossip; + let mut stream = NoStream::with_pool(2); + engine.pump(now, &mut gossip, &mut stream); + + engine + .leave(now) + .expect("leave from a running node succeeds"); + assert_ne!( + engine.state(), + SerfState::Alive, + "leave must transition the endpoint out of Alive" + ); + assert!( + matches!( + engine.join(&[node_addr(7003)]), + Err(SerfError::BadJoinState(_)) + ), + "a join after leave must be rejected with BadJoinState" + ); +} diff --git a/serf-embedded/src/lib.rs b/serf-embedded/src/lib.rs new file mode 100644 index 00000000..ac910422 --- /dev/null +++ b/serf-embedded/src/lib.rs @@ -0,0 +1,84 @@ +#![doc = include_str!("../README.md")] +#![cfg_attr(not(feature = "std"), no_std)] +#![forbid(unsafe_code)] +#![deny(missing_docs)] +// `collapsible_if`: the nested `if cond { if let ... }` form is kept deliberately — +// flattening multi-level guards into one long let-chain reads worse here. +#![allow(clippy::collapsible_if, clippy::type_complexity, unexpected_cfgs)] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![cfg_attr(docsrs, allow(unused_attributes))] + +// Alias `alloc` to the name `std` so genuine-heap `std::` paths compile unchanged +// under no_std+alloc. Core-resident items are imported from `core::` directly, +// never via this alias; heap macros are written path-qualified (`std::vec!`), so +// no crate-wide `#[macro_use]` is needed. +#[cfg(all(not(feature = "std"), feature = "alloc"))] +extern crate alloc as std; + +#[cfg(feature = "std")] +extern crate std; + +#[cfg(not(any(feature = "std", feature = "alloc")))] +compile_error!("serf-embedded requires the `std` or `alloc` feature"); + +mod cidr; +pub mod engine; + +pub use engine::SerfEngine; + +// ── Reused memberlist-embedded glue ────────────────────────────────────────── +// +// serf-embedded reuses the payload-agnostic driving glue from +// `memberlist-embedded` directly and re-exports it here so a serf embedded +// driver has a single import surface for the seams, the reliable plane, the +// resolver result, the transform pipeline, and the engine sizing / error types. + +pub use memberlist_embedded::{ + // Engine sizing + construction-time validation. + DEFAULT_CLOSE_TIMEOUT, + // The datagram + pooled-stream I/O seams a driver supplies to `SerfEngine::pump`. + GossipIo, + GossipMtuTooLarge, + InitError, + // Engine sizing (ports / close timeout / CIDR policy). + Options, + StreamIo, + StreamIoError, + // Construction-time preflight (advertise-independent config). + validate_runtime_config, +}; +// The pooled-stream reliable plane serf-embedded drives directly. +pub use memberlist_embedded::reliable::{ConnState, Connection, Pool, ReliablePlane}; +// The engine sizing module (for `config::Options` / `config::DEFAULT_CLOSE_TIMEOUT`). +pub use memberlist_embedded::config; +// The bounded, no-heap resolver result shared by the embedded drivers. +pub use memberlist_embedded::resolver::{MAX_RESOLVED_ADDRS_PER_SEED, ResolvedAddrs}; +// The last-line routable-address screen the engine and every egress chokepoint apply. +pub use memberlist_embedded::socket_addr_is_routable; +// The cross-transport transform configuration (label + AEAD encryption; serf's +// gossip plane carries no compression / checksum, so only those are surfaced). +pub use memberlist_embedded::{LabelError, TransformOptions}; +// Admission predicates a caller installs at construction. +pub use memberlist_embedded::{AliveDelegate, MaybeOwned, MaybeResolved, MergeDelegate}; +// CIDR peer-admission policy, installed via `Options::with_cidr_policy`. +#[cfg(feature = "cidr")] +#[cfg_attr(docsrs, doc(cfg(feature = "cidr")))] +pub use memberlist_embedded::{AddrParseError, CidrPolicy, IpNet}; +// AEAD keyring types, for a driver assembling a `TransformOptions` encryption +// policy. +#[cfg(encryption)] +#[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) +)] +pub use memberlist_embedded::{EncryptionOptions, Keyring, SecretKey}; + +// ── serf construction / event surface ──────────────────────────────────────── +// +// The serf config a `SerfEngine` is built from, and the event set its +// `poll_event` surfaces, re-exported so a driver imports them from one place. + +/// serf's own [`Endpoint`](serf_proto::endpoint::Endpoint) configuration, distinct +/// from the memberlist-layer engine [`Options`]. +pub use serf_proto::options::Options as SerfOptions; +pub use serf_proto::{endpoint::Error as SerfError, event::Event}; From bbbbd84fcbc86ccad6ae79c34f37bca96f2ad683 Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 3 Jul 2026 20:08:06 +1200 Subject: [PATCH 044/128] fix(serf-embedded): two-RNG constructor + core-owned join fan-out/correlation SerfEngine now carries both the memberlist gossip RNG and serf's core RNG and takes them via try_new_at_with_rng (was zero-seeding the core RNG -> identical query-id sequences across embedded nodes); and it OWNS the join fan-out -- capturing each start_join_push_pull StreamId, recording ignore_old streams, folding ExchangeCompleted into a reached-set accumulation, clearing ignore streams on terminal, and exposing a driver-pollable poll_join outcome -- so the smoltcp/embassy drivers get await-result join + ignore_old cleanup without a core change; RNG-distinctness + join-correlation tests. --- serf-embedded/src/engine/mod.rs | 552 ++++++++++++++++++++++---- serf-embedded/src/engine/tests.rs | 618 +++++++++++++++++++++++++++++- serf-embedded/src/lib.rs | 2 +- 3 files changed, 1096 insertions(+), 76 deletions(-) diff --git a/serf-embedded/src/engine/mod.rs b/serf-embedded/src/engine/mod.rs index f8f31c5f..12b0d740 100644 --- a/serf-embedded/src/engine/mod.rs +++ b/serf-embedded/src/engine/mod.rs @@ -30,9 +30,10 @@ use std::{boxed::Box, collections::VecDeque, vec::Vec}; use std::sync::Arc; use bytes::Bytes; +use hashbrown::{HashMap, HashSet}; use memberlist_proto::{ - AliveDelegate, Endpoint, EndpointOptions, Instant, LabelOptions, PushPullKind, RawRecords, Rng, - Transmit, + AliveDelegate, Endpoint, EndpointOptions, Instant, LabelOptions, RawRecords, Rng, SeedableRng, + StreamId, Transmit, codec::{ DecodeOptions, EncodeOptions, decode_incoming, encode_outgoing, encode_outgoing_compound, parse_messages, @@ -40,10 +41,10 @@ use memberlist_proto::{ streams::{ExchangeId, StreamAction, StreamEndpoint as Coordinator}, typed::NodeState, }; -use smallvec_wrapper::MediumVec; +use smallvec_wrapper::{MediumVec, OneOrMore}; use serf_proto::{ - StreamEndpoint, + ExchangeKind, ExchangeStatus, StreamEndpoint, endpoint::{Error as SerfError, QueryId, QueryParams}, event::{Event, QueryEvent}, members::{Member, SerfState}, @@ -126,6 +127,161 @@ where } } +/// Opaque handle for one in-flight await-result [`join`](SerfEngine::join), +/// returned by `join` and polled via [`poll_join`](SerfEngine::poll_join). +/// +/// A driver keys its own per-join waiter (a smoltcp poll flag, an embassy signal) +/// on this handle. Two concurrent joins mint distinct handles, so their outcomes +/// never cross-resolve. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct JoinId(u64); + +impl JoinId { + /// The underlying monotonically-allocated sequence number. + #[inline] + pub const fn get(&self) -> u64 { + self.0 + } +} + +/// The address set an await-result [`join`](SerfEngine::join) reached: one entry +/// per outbound push/pull exchange that completed [`ExchangeStatus::Succeeded`]. +/// Duplicate seeds contribute one entry per successful exchange. +pub type ReachedSet = OneOrMore; + +/// The terminal outcome of a fully-resolved await-result join that reached no +/// seed: it dispatched a push/pull to one or more routable seeds but none +/// completed `Succeeded` before every exchange terminated. +/// +/// `contacted` is always `0` for this payload — a non-zero contact count resolves +/// the join `Ok(ReachedSet)` instead. Mirrors serf's `JoinFailed` shape (a no_std +/// twin of `serf-driver`'s, so the embedded core need not pull the std-only +/// driver error surface). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct JoinFailed { + requested: usize, + contacted: usize, +} + +impl JoinFailed { + /// Build a payload from the requested-seed count and the contacted count. + #[inline] + pub const fn new(requested: usize, contacted: usize) -> Self { + Self { + requested, + contacted, + } + } + + /// The number of routable seed addresses the join dispatched a push/pull to. + #[inline] + pub const fn requested(&self) -> usize { + self.requested + } + + /// The number of seeds actually contacted before the join resolved. Always `0` + /// for this payload — a non-zero contact count resolves the join `Ok`. + #[inline] + pub const fn contacted(&self) -> usize { + self.contacted + } +} + +impl core::fmt::Display for JoinFailed { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!( + f, + "join reached {} of {} seed(s)", + self.contacted, self.requested + ) + } +} + +impl core::error::Error for JoinFailed {} + +/// One seed queued by [`join`](SerfEngine::join), awaiting its per-tick +/// `start_join_push_pull` dispatch in the pump. Carries the owning join handle so +/// the resulting exchange folds back into the right waiter, and the join's +/// `ignore_old` flag so the dispatch records the ignore-join stream when set. +struct QueuedSeed { + join: JoinId, + seed: SocketAddr, + ignore_old: bool, +} + +/// Core-owned state for one in-flight await-result join. +/// +/// The sync analogue of serf-reactor's `PendingJoin`, adapted to the driver-poll +/// core: contact accounting is strictly per-OUTBOUND-EXCHANGE, observed via the +/// machine's `Event::ExchangeCompleted` filtered to [`ExchangeKind::PushPull`]. +/// The correlation token that spans dispatch → completion is the START +/// [`StreamId`] captured at [`join`](SerfEngine::join)'s per-seed dispatch and +/// bound to the exchange's [`ExchangeId`] when its `Connect` action surfaces; the +/// completion then matches by that bound `ExchangeId`. +struct PendingJoin { + /// `StreamId`s this join's `start_join_push_pull` calls returned. Matched + /// against each surfaced `Connect`'s `stream_id()` to bind the exchange, and — + /// for an `ignore_old` join — cleared from the machine's ignore set on terminal. + started: HashSet, + /// Number of this join's seeds still queued in `pending_seeds`, undispatched. + /// Decremented as each seed's push/pull is started in the pump; the join is + /// "fully dispatched" (eligible to resolve) once this reaches zero. + unstarted: usize, + /// Outbound exchange ids bound at `Connect` and still awaiting a terminal + /// `ExchangeCompleted`. + pending: HashSet, + /// Peer addresses of the dispatched exchanges that terminated `Succeeded`. + contacted: ReachedSet, + /// Total routable-seed count this join dispatched — the `JoinFailed` denominator. + requested: usize, + /// Whether this is an `ignore_old` join (its `started` streams are recorded in + /// the machine's ignore set and must be cleared on terminal). + ignore_old: bool, + /// The resolved outcome, set once every dispatched exchange has terminated (or + /// on `leave`). `poll_join` returns and removes the waiter once this is `Some`. + resolved: Option>, +} + +impl PendingJoin { + /// Compute the terminal outcome from the accumulated contact set. + fn outcome(&self) -> Result { + if self.contacted.is_empty() { + Err(JoinFailed::new(self.requested, 0)) + } else { + Ok(self.contacted.clone()) + } + } +} + +/// Resolve `pj` if every dispatched exchange has terminated (`unstarted == 0` and +/// `pending` empty) and it is not already resolved: compute its outcome, clear any +/// still-recorded `ignore_old` streams from the machine's ignore set, and stash +/// the outcome for [`poll_join`](SerfEngine::poll_join). +/// +/// Driven on both the completion path (an `ExchangeCompleted` emptied `pending`) +/// and the end-of-pump sweep (a join whose seeds all retired before a `Connect`, +/// or an empty/all-non-routable seed set, never accumulates any `pending`). Clearing +/// the ignore streams HERE — on the exchange terminal, not lazily in `poll_join` — +/// means the machine's ignore set never leaks even if the driver drops the handle. +fn try_resolve_join( + endpoint: &mut StreamEndpoint, + pj: &mut PendingJoin, +) where + I: memberlist_proto::Id + Clone, + G: Rng, + SR: Rng + SeedableRng, +{ + if pj.resolved.is_some() || pj.unstarted != 0 || !pj.pending.is_empty() { + return; + } + if pj.ignore_old { + for sid in &pj.started { + endpoint.clear_ignore_join_stream(*sid); + } + } + pj.resolved = Some(pj.outcome()); +} + /// The transport-agnostic serf driving core. /// /// Composes serf's super-machine with the reused pooled-stream reliable plane, @@ -137,20 +293,22 @@ where /// `I` is the node identifier type (e.g. `SmolStr`); the address is pinned to /// [`core::net::SocketAddr`] and the record layer to the plain-TCP /// [`RawRecords`]. `C` is the driver's opaque connection handle -/// ([`StreamIo::Conn`]). `R` is the memberlist gossip RNG the driver injects at -/// construction; serf's own core RNG (query IDs, relay selection) is a -/// deterministically-seeded `SmallRng` — a production driver that needs -/// per-node-distinct query IDs can seed it via a future two-RNG constructor. -pub struct SerfEngine +/// ([`StreamIo::Conn`]). `G` is the memberlist gossip RNG (peer selection, timing +/// jitter) and `SR` is serf's OWN core RNG (query IDs, relay/reconnect selection); +/// the two are seeded independently, mirroring serf's `StreamEndpoint<.., G, SR>`. +/// A production driver seeds BOTH from its entropy source via +/// [`try_new_at_with_rng`](Self::try_new_at_with_rng) so that fresh nodes do not +/// share a `(ltime, id)` query-id sequence. +pub struct SerfEngine where // Mandated by serf's `StreamEndpoint` field, which keys its membership store by // `I`. Every impl bounds `I: Id`, which implies these, so no impl restates them. I: Eq + Hash, { - /// serf's super-machine: serf logic over the memberlist reliable coordinator - /// on the plain-TCP `RawRecords` path. `G = R` (the injected gossip RNG); the - /// serf core RNG defaults to `SmallRng`. - endpoint: StreamEndpoint, + /// serf's super-machine: serf logic over the memberlist reliable coordinator on + /// the plain-TCP `RawRecords` path, carrying BOTH the injected gossip RNG `G` + /// and serf's own injected core RNG `SR`. + endpoint: StreamEndpoint, /// Sizing / port configuration; retained for the reliable-plane paths. cfg: Options, /// Reused pooled connection handles and the exchange-to-handle map for the @@ -161,11 +319,19 @@ where /// every pump. Heap-resident so a large MTU does not blow a constrained stack /// and the allocation happens exactly once. gossip_recv: std::vec::Vec, - /// Seed addresses queued by [`join`](Self::join) that have not yet been handed - /// to the machine. Drained in the machine-pump phase of each `pump` tick: one - /// `start_push_pull(seed, Join, now)` per entry, which queues a `Connect` the - /// machine services into a dial consumed later that same tick. - pending_seeds: VecDeque, + /// Seeds queued by [`join`](Self::join) that have not yet been handed to the + /// machine. Drained in the machine-pump phase of each `pump` tick: one + /// `start_join_push_pull(seed, ignore_old, now)` per entry, which queues a + /// `Connect` the machine services into a dial consumed later that same tick. Each + /// entry carries its owning [`JoinId`] so the resulting exchange folds back into + /// the right await-result waiter. + pending_seeds: VecDeque, + /// In-flight await-result joins keyed by handle. Each accumulates its reached + /// set from the terminal `ExchangeCompleted` of the push/pulls it dispatched; + /// [`poll_join`](Self::poll_join) drains a resolved one. + pending_joins: HashMap, + /// Monotonic allocator for [`JoinId`]s, so two concurrent joins never collide. + next_join_id: u64, /// Cluster label applied to the gossip codec on both encode and decode. When /// `Some`, the gossip codec stamps a label prefix onto every outbound datagram /// and rejects any inbound datagram whose label does not match. `None` disables @@ -177,37 +343,51 @@ where cidr_policy: CidrFilter, } -// Construction and pure reliable-plane accessors — needing node identity but -// neither the connection-handle key nor the RNG. -impl SerfEngine +// Construction — needing node identity and serf's core RNG being seedable (both +// the two-RNG production path and the deterministic single-RNG convenience wrap +// `StreamEndpoint::new_with_rng`, which bounds serf's `SR: SeedableRng`). +impl SerfEngine where I: memberlist_proto::Id + Clone, + SR: SeedableRng, { - /// Construct an engine, panicking on a misconfiguration. + /// Construct an engine seeding BOTH RNGs, panicking on a misconfiguration. /// - /// The convenience wrapper over [`try_new_at`](Self::try_new_at); use it only - /// when the configuration is a static constant known to be valid. + /// The convenience wrapper over + /// [`try_new_at_with_rng`](Self::try_new_at_with_rng); use it only when the + /// configuration is a static constant known to be valid. /// /// # Panics /// - /// Panics if [`try_new_at`](Self::try_new_at) returns an [`InitError`]. - pub fn new_at( + /// Panics if [`try_new_at_with_rng`](Self::try_new_at_with_rng) returns an + /// [`InitError`]. + pub fn new_at_with_rng( cfg: Options, transform: TransformOptions, ep_cfg: EndpointOptions, serf_opts: SerfOptions, now: Instant, - rng: R, + gossip_rng: G, + serf_rng: SR, ) -> Self { - Self::try_new_at(cfg, transform, ep_cfg, serf_opts, now, rng) - .expect("SerfEngine::new_at: invalid configuration; use try_new_at to handle") + Self::try_new_at_with_rng(cfg, transform, ep_cfg, serf_opts, now, gossip_rng, serf_rng).expect( + "SerfEngine::new_at_with_rng: invalid configuration; use try_new_at_with_rng to handle", + ) } - /// Fallibly construct an engine. + /// Fallibly construct an engine, injecting BOTH the memberlist gossip RNG `G` + /// and serf's own core RNG `SR`. + /// + /// This is the production constructor: serf's core RNG drives query-id + /// generation and relay/reconnect selection, so a driver MUST seed it from its + /// entropy source (getrandom on embedded, exactly as memberlist-smoltcp / + /// memberlist-embassy seed their gossip RNG). Seeding it distinctly per node is + /// what keeps fresh nodes from emitting identical `(ltime, id)` query-id + /// sequences that a real cluster would drop or mis-correlate. /// /// Wires serf's super-machine over the memberlist coordinator and sizes the - /// gossip receive scratch. No sockets are bound — the driver owns the gossip - /// and reliable-stream sockets — and no I/O occurs here. + /// gossip receive scratch. No sockets are bound — the driver owns the gossip and + /// reliable-stream sockets — and no I/O occurs here. /// /// # Parameters /// @@ -222,8 +402,8 @@ where /// - `serf_opts`: serf-level configuration (reap / reconnect / coalescing / /// query timing). /// - `now`: the driver's clock reading at construction. - /// - `rng`: the memberlist gossip RNG, already seeded by the driver from its - /// entropy source. + /// - `gossip_rng`: the memberlist gossip RNG, already seeded by the driver. + /// - `serf_rng`: serf's own core RNG, seeded distinctly by the driver. /// /// # Errors /// @@ -231,13 +411,14 @@ where /// invalid: a zero/over-ceiling gossip MTU, a zero port or close timeout, a /// non-routable or port-mismatched advertise address, a machine-endpoint init /// failure, or (with an encryption backend built in) an unusable keyring. - pub fn try_new_at( + pub fn try_new_at_with_rng( cfg: Options, transform: TransformOptions, ep_cfg: EndpointOptions, serf_opts: SerfOptions, now: Instant, - rng: R, + gossip_rng: G, + serf_rng: SR, ) -> Result { // Validate every advertise-independent config field (port, gossip-MTU // ceiling, close timeout, and the encryption keyring) up front, sharing the @@ -278,7 +459,7 @@ where // Build the inner memberlist `Endpoint` (the SWIM machine serf sits on) with // the injected gossip RNG. `try_new_at` maps a machine init failure to // `InitError::Endpoint` and starts its timers from a consistent origin. - let mut ep = Endpoint::try_new_at(ep_cfg, now, rng).map_err(InitError::Endpoint)?; + let mut ep = Endpoint::try_new_at(ep_cfg, now, gossip_rng).map_err(InitError::Endpoint)?; // Install the routable-address admission filter on the raw `Endpoint` BEFORE // it is wrapped: the machine consults it inline for every inbound Alive, so a @@ -317,10 +498,12 @@ where #[cfg(encryption)] coord.set_encryption_options(transform.encryption); - // Wrap the coordinator in serf's super-machine. `new` seeds serf's core RNG - // (query IDs / relay selection) deterministically; it is independent of the - // injected memberlist gossip RNG `R`. - let endpoint = StreamEndpoint::::new(coord, serf_opts); + // Wrap the coordinator in serf's super-machine, injecting serf's own core RNG + // via `new_with_rng` — the fix for the zero-seeded-core-RNG footgun: serf's + // query IDs / relay selection now draw from the driver-seeded `serf_rng`, + // independent of the injected memberlist gossip RNG `G`. + let endpoint = + StreamEndpoint::::new_with_rng(coord, serf_opts, serf_rng); Ok(Self { endpoint, @@ -328,11 +511,72 @@ where plane: ReliablePlane::new(), gossip_recv, pending_seeds: VecDeque::new(), + pending_joins: HashMap::new(), + next_join_id: 0, label, cidr_policy, }) } + /// Construct an engine with serf's core RNG ZERO-SEEDED, panicking on a + /// misconfiguration. + /// + /// DETERMINISTIC-ONLY convenience: use it only for tests / reproducible + /// fixtures. See [`try_new_at`](Self::try_new_at) for the zero-seed caveat. + /// + /// # Panics + /// + /// Panics if [`try_new_at`](Self::try_new_at) returns an [`InitError`]. + pub fn new_at( + cfg: Options, + transform: TransformOptions, + ep_cfg: EndpointOptions, + serf_opts: SerfOptions, + now: Instant, + gossip_rng: G, + ) -> Self { + Self::try_new_at(cfg, transform, ep_cfg, serf_opts, now, gossip_rng) + .expect("SerfEngine::new_at: invalid configuration; use try_new_at to handle") + } + + /// Fallibly construct an engine with serf's core RNG ZERO-SEEDED. + /// + /// DETERMINISTIC-ONLY: serf's core RNG is seeded from `0`, so every engine built + /// this way emits the SAME `(ltime, id)` query-id sequence. That is fine for + /// tests and reproducible fixtures but WRONG for a real cluster, where distinct + /// nodes must not collide their query ids — production drivers construct via + /// [`try_new_at_with_rng`](Self::try_new_at_with_rng), seeding serf's RNG from + /// entropy. Takes only the gossip RNG; the serf RNG is `SR::seed_from_u64(0)`. + /// + /// # Errors + /// + /// As [`try_new_at_with_rng`](Self::try_new_at_with_rng). + pub fn try_new_at( + cfg: Options, + transform: TransformOptions, + ep_cfg: EndpointOptions, + serf_opts: SerfOptions, + now: Instant, + gossip_rng: G, + ) -> Result { + Self::try_new_at_with_rng( + cfg, + transform, + ep_cfg, + serf_opts, + now, + gossip_rng, + SR::seed_from_u64(0), + ) + } +} + +// Pure reliable-plane accessors — needing node identity but neither the +// connection-handle key nor either RNG. +impl SerfEngine +where + I: memberlist_proto::Id + Clone, +{ /// Mutable access to the reliable plane's pool, for a driver to push its /// pre-created connection handles and install the initial listener at /// construction. @@ -392,15 +636,25 @@ where pub fn pending_dial_count(&self) -> usize { self.plane.pending_dial_count() } + + /// Number of await-result joins currently tracked (in-flight plus resolved but + /// not yet drained via [`poll_join`](Self::poll_join)) — a diagnostic proving + /// the join table does not leak once every join resolves and is polled. + #[inline] + pub fn pending_join_count(&self) -> usize { + self.pending_joins.len() + } } // serf-command and read forwarders — reach serf's super-machine only (not the -// reliable plane), so they need node identity and the gossip RNG but no +// reliable plane), so they need node identity and BOTH RNGs (the machine driver +// surface bounds the gossip `G: Rng` and serf's `SR: Rng + SeedableRng`) but no // connection-handle key. -impl SerfEngine +impl SerfEngine where I: memberlist_proto::Id + Clone, - R: Rng, + G: Rng, + SR: Rng + SeedableRng, { /// Arm serf's periodic probe / gossip / push-pull schedulers. Call once before /// the first `pump`; without it failure detection, dissemination, and @@ -478,39 +732,155 @@ where /// acks, key-management requests / responses, reliable-exchange completions, /// and the lifecycle signals (`LeftCluster`, conflict `Shutdown`). Returns /// `None` when the event queue is empty; call again after the next `pump` tick. + /// + /// A driver awaiting a [`join`](Self::join) result MUST drain events here before + /// checking [`poll_join`](Self::poll_join): a push/pull `ExchangeCompleted` is + /// folded into its await-result join AS IT surfaces here (the reached-set + /// accumulation), and `poll_join` resolves off that folded state. #[inline] pub fn poll_event(&mut self) -> Option> { - self.endpoint.poll_event() + let ev = self.endpoint.poll_event(); + // Fold a push/pull completion into its await-result join: the terminal + // `ExchangeCompleted`'s `eid` was bound to a join at its `Connect` (via the + // START `StreamId`), so a match here removes it from `pending` and — on + // `Succeeded` — accumulates the peer into `contacted`. Resolving off that + // (clearing any ignore streams) happens the instant `pending` empties. + if let Some(Event::ExchangeCompleted(ec)) = &ev { + if ec.kind() == ExchangeKind::PushPull { + let Self { + endpoint, + pending_joins, + .. + } = self; + if let Some(pj) = pending_joins + .values_mut() + .find(|pj| pj.pending.contains(&ec.eid())) + { + pj.pending.remove(&ec.eid()); + if matches!(ec.outcome(), ExchangeStatus::Succeeded) { + pj.contacted.push(*ec.peer()); + } + try_resolve_join(endpoint, pj); + } + } + } + ev } - /// Announce the local node's join intent and record intent to contact these - /// seed addresses. + /// Announce the local node's join intent and begin an await-result join to + /// these seeds, returning a [`JoinId`] the driver polls via + /// [`poll_join`](Self::poll_join). + /// + /// Returns immediately; the pump initiates a push/pull to each routable seed on + /// the next tick. serf's own `join()` rejects a non-Alive endpoint + /// ([`SerfError::BadJoinState`]); on that rejection nothing is queued and no + /// handle is minted. When `ignore_old` is set, each seed's push/pull records its + /// ignore-join stream so the resulting merge suppresses replay of the peer's + /// pre-join user events; the engine clears any such stream that fails to merge + /// when the join terminates (no machine leak). /// - /// Returns immediately; the pump loop initiates a push/pull state exchange to - /// each routable seed on the next tick. serf's own `join()` rejects a non-Alive - /// endpoint ([`SerfError::BadJoinState`]); on that rejection no seed is queued. - pub fn join(&mut self, seeds: &[SocketAddr]) -> Result<(), SerfError> { + /// The join RESOLVES — `poll_join` yields `Some` — once every dispatched + /// push/pull has terminated (each `ExchangeCompleted` folded in): `Ok` with the + /// reached-address set if any seed was contacted, else `Err(JoinFailed)`. A join + /// to an unreachable seed resolves `Err` after that exchange's own stream + /// timeout; the core imposes no separate caller deadline (a driver that wants an + /// earlier give-up drops the handle). + pub fn join( + &mut self, + seeds: &[SocketAddr], + ignore_old: bool, + now: Instant, + ) -> Result { + // Ignoring `now`: the per-seed push/pulls are dispatched (with the tick's + // `now`) in the pump, not here — `join` only announces intent and queues. The + // parameter is kept for API parity with the reactor / a future synchronous + // dispatch. + let _ = now; // Announce the serf-level join intent first (this is where the running-state - // gate lives); only queue seeds once it is accepted. + // gate lives); only mint the handle and queue seeds once it is accepted. self.endpoint.join()?; + + let id = JoinId(self.next_join_id); + self.next_join_id += 1; + + let mut requested = 0usize; for s in seeds { // Drop a non-routable seed: it could only produce a doomed dial. Queue only - // seeds a dial can actually complete. + // seeds a dial can actually complete, and count them as the `JoinFailed` + // denominator. if socket_addr_is_routable(s) { - self.pending_seeds.push_back(*s); + requested += 1; + self.pending_seeds.push_back(QueuedSeed { + join: id, + seed: *s, + ignore_old, + }); } } - Ok(()) + + self.pending_joins.insert( + id, + PendingJoin { + started: HashSet::new(), + unstarted: requested, + pending: HashSet::new(), + contacted: ReachedSet::new(), + requested, + ignore_old, + resolved: None, + }, + ); + Ok(id) + } + + /// Drain the terminal outcome of an await-result [`join`](Self::join), or `None` + /// while it is still in flight. + /// + /// Returns `Some(Ok(reached))` with the address set the join contacted, or + /// `Some(Err(JoinFailed))` if every dispatched push/pull terminated without + /// contacting a seed. `None` means the join has not yet resolved — poll again + /// after the next `pump` + [`poll_event`](Self::poll_event) drain. Removes the + /// waiter on the resolving call (a sync driver polls this each tick; an async + /// driver awaits a signal the pump fires when it flips to `Some`). An unknown or + /// already-drained handle yields `None`. + pub fn poll_join(&mut self, handle: JoinId) -> Option> { + // Yield `None` for an unknown handle (`get?`) or one still in flight + // (`resolved.as_ref()?`); only a resolved join is drained AND removed. + self.pending_joins.get(&handle)?.resolved.as_ref()?; + self + .pending_joins + .remove(&handle) + .and_then(|pj| pj.resolved) } /// Begin leaving the cluster. /// /// Forwards to serf's graceful-leave path, which gossips the departure and /// ultimately emits [`Event::LeftCluster`] via [`poll_event`](Self::poll_event). - /// Any seeds still queued from a pre-leave join are dropped: the pump initiates - /// no new push/pull once leaving. + /// Any seeds still queued from a pre-leave join are dropped, and every in-flight + /// await-result join is resolved from its current reached set (clearing any + /// still-recorded ignore streams so the machine's ignore set never leaks): the + /// pump initiates no new push/pull once leaving. pub fn leave(&mut self, now: Instant) -> Result<(), SerfError> { self.pending_seeds.clear(); + // Force every in-flight join to its terminal from whatever it accumulated so + // far: leaving means no further push/pull dispatch, so an unresolved join + // would otherwise linger forever (and leak its ignore-join streams). + let Self { + endpoint, + pending_joins, + .. + } = self; + for pj in pending_joins.values_mut() { + if pj.resolved.is_none() { + if pj.ignore_old { + for sid in &pj.started { + endpoint.clear_ignore_join_stream(*sid); + } + } + pj.resolved = Some(pj.outcome()); + } + } self.endpoint.leave(now) } @@ -625,12 +995,13 @@ where // Reliable-plane lifecycle helpers that move the connection handle `C` by value // (into the pool, the listener slot, and the `StreamIo` socket calls) and reach -// serf's machine, so they need `C: Copy + Eq + Hash` and the gossip RNG. -impl SerfEngine +// serf's machine, so they need `C: Copy + Eq + Hash` and both RNGs. +impl SerfEngine where I: memberlist_proto::Id + Clone, C: Copy + Eq + Hash, - R: Rng, + G: Rng, + SR: Rng + SeedableRng, { /// Advance serf's state machine once over the driver's already-ticked sockets. /// Returns the next wakeup deadline: the minimum of the machine's next timer @@ -656,15 +1027,16 @@ where /// back via `handle_message`. /// 3. Reliable ingress: drain each connection's rx into `handle_transport_data`; /// deliver a one-shot EOF on peer FIN. - /// 5. Join-seed drain: `start_push_pull(seed, Join, now)` per queued seed. + /// 5. Join-seed drain: `start_join_push_pull(seed, ignore_old, now)` per queued + /// seed, capturing its `StreamId` for await-result correlation. /// 6. Machine tick: `handle_timeout` fires due serf + coordinator timers. /// 7a–7e. Drain `poll_action`, promote, pump outbound, flush deferred FINs, /// complete `Closing` drains, re-rebalance, then drain + send outbound /// gossip. /// 8. Deadline: `min(machine_next, closing_next)`. - pub fn pump(&mut self, now: Instant, gossip: &mut G, stream: &mut S) -> Option + pub fn pump(&mut self, now: Instant, gossip: &mut GI, stream: &mut S) -> Option where - G: GossipIo, + GI: GossipIo, S: StreamIo, { // 1a. Reap gracefully-closing connections the driver's stack tick advanced to @@ -723,14 +1095,20 @@ where // machine (including the peer-FIN EOF) before the machine tick. self.pump_inbound_reliable(now, stream); - // 5. Drain join seeds: each queued seed gets a push/pull exchange initiated - // now. Skipped once leaving/left — a left node initiates no join push/pull. + // 5. Drain join seeds: each queued seed starts a join push/pull now. Skipped + // once leaving/left — a left node initiates no join push/pull. The returned + // `StreamId` is the await-result correlation token: it is recorded on the + // owning join and matched against the resulting `Connect`'s `stream_id()` + // (phase 7a) to bind that exchange's `ExchangeId` into the join's pending set. if self.is_running() { - while let Some(seed) = self.pending_seeds.pop_front() { - // The StreamId is the machine's correlation token; the dial is correlated - // via the ExchangeId carried in the resulting Connect action, so the - // driver does not need to retain it here. - let _sid = self.endpoint.start_push_pull(seed, PushPullKind::Join, now); + while let Some(qs) = self.pending_seeds.pop_front() { + let sid = self + .endpoint + .start_join_push_pull(qs.seed, qs.ignore_old, now); + if let Some(pj) = self.pending_joins.get_mut(&qs.join) { + pj.started.insert(sid); + pj.unstarted = pj.unstarted.saturating_sub(1); + } } } @@ -760,6 +1138,23 @@ where // 7e. Egress: drain outbound gossip transmits, encode + encrypt, and send. self.drain_gossip_transmits(gossip); + // 7f. Resolve any await-result join whose exchanges all terminated. The normal + // (some seed contacted / failed) path resolves as each `ExchangeCompleted` + // folds in `poll_event`; this sweep catches the joins that never accumulate a + // pending exchange at all — an empty/all-non-routable seed set, or seeds that + // retired before a `Connect` — now that this tick's `Connect`s have been + // captured (phase 7a). + { + let Self { + endpoint, + pending_joins, + .. + } = self; + for pj in pending_joins.values_mut() { + try_resolve_join(endpoint, pj); + } + } + // 8. Next deadline = min(machine, closing). let machine = self.endpoint.poll_timeout(); let closing = self @@ -954,6 +1349,19 @@ where StreamAction::Connect(info) => { let eid = info.id(); let peer = info.peer(); + // Bind this exchange to its await-result join, if any: the `Connect`'s + // `stream_id()` is the START `StreamId` a join's `start_join_push_pull` + // returned (phase 5). Matching it here records the machine-allocated + // `ExchangeId` into that join's pending set, so the terminal + // `ExchangeCompleted` (which reports by `eid`) folds back to the right + // waiter — never by the ambiguous peer address. + let sid = info.stream_id(); + for pj in self.pending_joins.values_mut() { + if pj.started.contains(&sid) { + pj.pending.insert(eid); + break; + } + } // Only a reset, reuse-ready slot may back a fresh dial; a freed-but- // still-resetting slot defers to `PendingDial` until its worker resets. match self.plane.pool.take_where(|&c| stream.reuse_ready(c)) { @@ -1282,9 +1690,9 @@ where /// label frame and, under an encryption backend, the AEAD wrapper. Encoding /// errors and a full tx ring both silently drop the datagram; gossip is /// best-effort and SWIM recovers on the next round. - fn drain_gossip_transmits(&mut self, gossip: &mut G) + fn drain_gossip_transmits(&mut self, gossip: &mut GI) where - G: GossipIo, + GI: GossipIo, { let enc = EncodeOptions::new(self.label.clone()); while let Some(transmit) = self.endpoint.poll_memberlist_transmit() { diff --git a/serf-embedded/src/engine/tests.rs b/serf-embedded/src/engine/tests.rs index c459470c..323821dd 100644 --- a/serf-embedded/src/engine/tests.rs +++ b/serf-embedded/src/engine/tests.rs @@ -5,6 +5,8 @@ use core::{ time::Duration, }; +use std::{cell::RefCell, collections::BTreeMap, rc::Rc}; + use memberlist_proto::{SeedableRng, SmallRng}; use smol_str::SmolStr; @@ -201,7 +203,7 @@ fn join_queues_seed_then_pump_parks_pending_dial() { // slot and defers to PendingDial. engine.set_listener(9); engine - .join(&[node_addr(7002)]) + .join(&[node_addr(7002)], false, now) .expect("join announces intent and queues the routable seed"); let mut gossip = NoGossip; @@ -225,7 +227,7 @@ fn join_drops_non_routable_seed() { // Port 0 is non-routable and must not be queued. let dead = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 2)), 0); - engine.join(&[dead]).expect("join succeeds"); + engine.join(&[dead], false, now).expect("join succeeds"); let mut gossip = NoGossip; let mut stream = NoStream::with_pool(0); @@ -270,9 +272,619 @@ fn leave_transitions_state_and_blocks_further_join() { ); assert!( matches!( - engine.join(&[node_addr(7003)]), + engine.join(&[node_addr(7003)], false, now), Err(SerfError::BadJoinState(_)) ), "a join after leave must be rejected with BadJoinState" ); } + +// ── serf core RNG injection (finding 1) ────────────────────────────────────── + +/// Build a fresh single-node engine seeding serf's core RNG from `serf_seed` (the +/// gossip RNG held fixed), start it, and return the `id` of its first issued +/// query — a `u32` drawn straight from serf's core RNG. The seed is the only +/// input that varies, so the query id is a pure function of the serf RNG seed. +fn first_query_id(serf_seed: u64) -> u32 { + let cfg = Options::new() + .with_port(7946) + .with_close_timeout(Duration::from_secs(10)); + let ep_cfg = EndpointOptions::new(SmolStr::new("q"), node_addr(7946)); + let now = Instant::from_origin(Duration::from_secs(86_400)); + let mut engine: SerfEngine = SerfEngine::try_new_at_with_rng( + cfg, + TransformOptions::default(), + ep_cfg, + SerfOptions::new(), + now, + SmallRng::seed_from_u64(1), + SmallRng::seed_from_u64(serf_seed), + ) + .expect("valid configuration must construct"); + engine.start(now); + engine + .query( + "probe", + Bytes::from_static(b"payload"), + QueryParams::default(), + now, + ) + .expect("a query is issued while running") + .id +} + +/// Two engines seeded with DIFFERENT serf RNGs produce DIFFERENT first query ids, +/// and the SAME serf seed reproduces the SAME id — proving the injected serf RNG +/// (not a zero seed or some other entropy) is what threads through to query-id +/// generation, so fresh embedded nodes no longer share a `(ltime, id)` sequence. +#[test] +fn distinct_serf_rng_seeds_yield_distinct_query_ids() { + assert_ne!( + first_query_id(100), + first_query_id(200), + "distinct serf RNG seeds must produce distinct first query ids" + ); + assert_eq!( + first_query_id(100), + first_query_id(100), + "the same serf RNG seed must reproduce the same first query id" + ); +} + +// ── core-owned join fan-out / correlation (finding 2) ───────────────────────── + +/// A failed-dial `ignore_old` join resolves `Err(JoinFailed)` and is reaped: the +/// dial's `connect` errors, the machine terminalizes the push/pull `Failed`, and +/// `poll_join` folds that completion into the join — clearing the recorded +/// ignore-join stream and dropping the waiter (no leak of either the pending-join +/// entry or its machine ignore stream). +#[test] +fn failed_ignore_old_join_resolves_err_and_clears_ignore_streams() { + let mut engine = make_engine(); + let now = Instant::from_origin(Duration::from_secs(86_400)); + engine.start(now); + + // One dial slot in the pool plus a listener; `NoStream::connect` errors, so the + // dial this slot backs fails. + engine.plane_mut().pool.push(5); + engine.set_listener(9); + + let handle = engine + .join(&[node_addr(7002)], true, now) + .expect("join announces intent and mints a handle"); + assert_eq!( + engine.pending_join_count(), + 1, + "the in-flight join is tracked" + ); + + let mut gossip = NoGossip; + let mut stream = NoStream::with_pool(0); + // One pump dispatches the seed, captures the Connect, attempts the dial (which + // errors), and the machine emits the terminal ExchangeCompleted(Failed). + engine.pump(now, &mut gossip, &mut stream); + + // Draining events folds the failed completion into the join; poll_join then + // resolves it. + let mut outcome = None; + for _ in 0..8 { + while engine.poll_event().is_some() {} + if let Some(res) = engine.poll_join(handle) { + outcome = Some(res); + break; + } + engine.pump(now, &mut gossip, &mut stream); + } + + match outcome { + Some(Err(jf)) => { + assert_eq!(jf.requested(), 1, "one routable seed was dispatched"); + assert_eq!(jf.contacted(), 0, "the failed dial contacted no seed"); + } + other => panic!("expected Some(Err(JoinFailed)), got {other:?}"), + } + assert_eq!( + engine.pending_join_count(), + 0, + "a resolved join must be reaped (its ignore stream cleared, not leaked)" + ); +} + +// ── Two-engine in-memory reliable link ─────────────────────────────────────── +// +// A faithful loopback so a real serf join push/pull runs end-to-end through BOTH +// engines' reliable planes, terminalizing the initiating exchange `Succeeded` — +// the only way to drive an `Event::ExchangeCompleted(PushPull, Succeeded)` the +// engine's core-owned join folds into its reached set. One end's `connect` +// registers a pending SYN on the shared fabric; the destination's listener +// completes the passive open when its `accepted_peer` is polled, after which bytes +// ferry both ways over the matched pipe. The link acks instantly (`send_queue` +// is always 0), so a graceful close FINs without a `Closing` drain. + +/// The two directional byte streams of one established pipe plus its FIN/reset +/// flags. The dialer writes `d2a` (the acceptor reads it) and vice versa. +#[derive(Default)] +struct Pipe { + d2a: VecDeque, + a2d: VecDeque, + d_fin: bool, + a_fin: bool, + reset: bool, + established: bool, +} + +/// A SYN parked on the fabric by a dialer's `connect`, awaiting the destination's +/// listener to complete the passive open. +struct PendingSyn { + dest: SocketAddr, + src: SocketAddr, + pipe: u64, +} + +/// Shared fabric state: every established pipe plus the not-yet-accepted SYNs. +#[derive(Default)] +struct FabricInner { + pipes: BTreeMap, + pending: Vec, + next_pipe: u64, +} + +impl FabricInner { + fn fresh() -> Fabric { + Rc::new(RefCell::new(FabricInner::default())) + } +} + +type Fabric = Rc>; + +/// Which end of a pipe a slot is bound to, so `send`/`recv`/`close` route to the +/// correct buffer. +#[derive(Clone, Copy, PartialEq)] +enum End { + Dialer, + Acceptor, +} + +/// What one of an engine's reliable slots is currently doing. +#[derive(Clone)] +enum SlotRole { + Idle, + Listening, + Bound(u64, End), +} + +/// One engine's reliable I/O over the shared fabric: its own slot pool and +/// per-slot role, plus this engine's advertised address (the SYN destination its +/// listener answers). `role` is a `RefCell` so the `&self` `accepted_peer` can +/// re-bind a Listening slot to the Acceptor end the instant it completes a passive +/// open — the same handle the engine keeps for the exchange. +struct LinkRel { + fabric: Fabric, + me: SocketAddr, + free: std::vec::Vec, + role: RefCell>, +} + +impl LinkRel { + fn new(fabric: Fabric, me: SocketAddr, handles: &[u32]) -> Self { + let mut role = BTreeMap::new(); + for &h in handles { + role.insert(h, SlotRole::Idle); + } + Self { + fabric, + me, + free: handles.to_vec(), + role: RefCell::new(role), + } + } + + fn role_of(&self, c: u32) -> Option { + self.role.borrow().get(&c).cloned() + } +} + +/// Bytes waiting for the end `end` to read (the OTHER end's tx buffer). +fn pipe_inbound(p: &Pipe, end: End) -> &VecDeque { + match end { + End::Dialer => &p.a2d, + End::Acceptor => &p.d2a, + } +} + +/// Whether THIS end emitted its FIN. +fn pipe_end_fin(p: &Pipe, end: End) -> bool { + match end { + End::Dialer => p.d_fin, + End::Acceptor => p.a_fin, + } +} + +/// Whether the PEER end emitted its FIN. +fn pipe_peer_fin(p: &Pipe, end: End) -> bool { + match end { + End::Dialer => p.a_fin, + End::Acceptor => p.d_fin, + } +} + +impl StreamIo for LinkRel { + type Conn = u32; + + fn take_free(&mut self) -> Option { + self.free.pop() + } + + fn give(&mut self, c: u32) { + self.role.borrow_mut().insert(c, SlotRole::Idle); + self.free.push(c); + } + + fn free_count(&self) -> usize { + self.free.len() + } + + fn listen(&mut self, c: u32, _port: u16) -> Result<(), crate::StreamIoError> { + self.role.borrow_mut().insert(c, SlotRole::Listening); + Ok(()) + } + + fn accepted_peer(&self, c: u32) -> Option { + if !matches!(self.role_of(c), Some(SlotRole::Listening)) { + return None; + } + let mut fab = self.fabric.borrow_mut(); + let pos = fab.pending.iter().position(|s| s.dest == self.me)?; + let syn = fab.pending.remove(pos); + fab.pipes.entry(syn.pipe).or_default().established = true; + drop(fab); + self + .role + .borrow_mut() + .insert(c, SlotRole::Bound(syn.pipe, End::Acceptor)); + Some(syn.src) + } + + fn connect( + &mut self, + c: u32, + remote: SocketAddr, + _local_port: u16, + ) -> Result<(), crate::StreamIoError> { + let mut fab = self.fabric.borrow_mut(); + let pipe = fab.next_pipe; + fab.next_pipe += 1; + fab.pipes.insert(pipe, Pipe::default()); + fab.pending.push(PendingSyn { + dest: remote, + src: self.me, + pipe, + }); + drop(fab); + self + .role + .borrow_mut() + .insert(c, SlotRole::Bound(pipe, End::Dialer)); + Ok(()) + } + + fn may_send(&self, c: u32) -> bool { + match self.role_of(c) { + Some(SlotRole::Bound(pipe, end)) => { + let fab = self.fabric.borrow(); + match fab.pipes.get(&pipe) { + Some(p) => p.established && !p.reset && !pipe_end_fin(p, end), + None => false, + } + } + _ => false, + } + } + + fn may_recv(&self, c: u32) -> bool { + match self.role_of(c) { + Some(SlotRole::Bound(pipe, end)) => { + let fab = self.fabric.borrow(); + fab + .pipes + .get(&pipe) + .map(|p| !pipe_inbound(p, end).is_empty()) + .unwrap_or(false) + } + _ => false, + } + } + + fn is_open(&self, c: u32) -> bool { + match self.role_of(c) { + Some(SlotRole::Bound(pipe, _)) => { + let fab = self.fabric.borrow(); + match fab.pipes.get(&pipe) { + Some(p) => !p.reset && !(p.d_fin && p.a_fin), + None => false, + } + } + Some(SlotRole::Listening) => true, + _ => false, + } + } + + fn is_established(&self, c: u32) -> bool { + self.may_send(c) + } + + fn recv(&mut self, c: u32, buf: &mut [u8]) -> Option { + let (pipe, end) = match self.role_of(c) { + Some(SlotRole::Bound(pipe, end)) => (pipe, end), + _ => return None, + }; + let mut fab = self.fabric.borrow_mut(); + let p = fab.pipes.get_mut(&pipe)?; + let q = match end { + End::Dialer => &mut p.a2d, + End::Acceptor => &mut p.d2a, + }; + if q.is_empty() { + return None; + } + let n = q.len().min(buf.len()); + for (i, b) in q.drain(..n).enumerate() { + buf[i] = b; + } + Some(n) + } + + fn recv_finished(&self, c: u32) -> bool { + match self.role_of(c) { + Some(SlotRole::Bound(pipe, end)) => { + let fab = self.fabric.borrow(); + match fab.pipes.get(&pipe) { + Some(p) => !p.reset && pipe_peer_fin(p, end) && pipe_inbound(p, end).is_empty(), + None => false, + } + } + _ => false, + } + } + + fn send(&mut self, c: u32, bytes: &[u8]) -> usize { + let (pipe, end) = match self.role_of(c) { + Some(SlotRole::Bound(pipe, end)) => (pipe, end), + _ => return 0, + }; + let mut fab = self.fabric.borrow_mut(); + let Some(p) = fab.pipes.get_mut(&pipe) else { + return 0; + }; + if p.reset || !p.established { + return 0; + } + // Deliver to the peer's rx immediately (the FSM sees the bytes); the link acks + // instantly so nothing lingers as unacked tx. + match end { + End::Dialer => p.d2a.extend(bytes.iter().copied()), + End::Acceptor => p.a2d.extend(bytes.iter().copied()), + } + bytes.len() + } + + fn send_queue(&self, _c: u32) -> usize { + 0 + } + + fn close(&mut self, c: u32) { + if let Some(SlotRole::Bound(pipe, end)) = self.role_of(c) { + let mut fab = self.fabric.borrow_mut(); + if let Some(p) = fab.pipes.get_mut(&pipe) { + match end { + End::Dialer => p.d_fin = true, + End::Acceptor => p.a_fin = true, + } + } + } + } + + fn abort(&mut self, c: u32) { + if let Some(SlotRole::Bound(pipe, _)) = self.role_of(c) { + let mut fab = self.fabric.borrow_mut(); + if let Some(p) = fab.pipes.get_mut(&pipe) { + p.reset = true; + } + } + } +} + +/// A paired gossip relay: datagrams `send`-emitted toward a peer's address land in +/// that peer's inbound queue (and vice versa), so the two engines also exchange +/// SWIM gossip. Each engine holds one end keyed by its own address. +#[derive(Clone)] +struct GossipWire { + outbound: Rc)>>>, + inbound: Rc)>>>, +} + +impl GossipIo for GossipWire { + fn recv(&mut self, buf: &mut [u8]) -> Option<(SocketAddr, usize)> { + let (src, bytes) = self.inbound.borrow_mut().pop_front()?; + let n = bytes.len().min(buf.len()); + buf[..n].copy_from_slice(&bytes[..n]); + Some((src, n)) + } + + fn send(&mut self, bytes: &[u8], dest: SocketAddr) { + self.outbound.borrow_mut().push((dest, bytes.to_vec())); + } +} + +/// A linked two-engine fixture sharing one reliable fabric and a cross-wired +/// gossip relay. `step` pumps both engines once and then ferries each side's +/// emitted gossip into the other side's inbound queue. +struct LinkPair { + a: SerfEngine, + b: SerfEngine, + a_rel: LinkRel, + b_rel: LinkRel, + a_gossip: GossipWire, + b_gossip: GossipWire, + a_addr: SocketAddr, + b_addr: SocketAddr, +} + +impl LinkPair { + /// Two running serf engines `a` (port 7946) and `b` (port 7947) on a shared + /// fabric, each with `pool` dial slots plus a listener. A short `stream_timeout` + /// keeps a wedged exchange from hanging the test. + fn new(pool_handles_a: &[u32], pool_handles_b: &[u32]) -> Self { + let now = Instant::from_origin(Duration::from_secs(86_400)); + let a_addr = node_addr(7946); + let b_addr = node_addr(7947); + + let mk = |id: &str, port: u16, addr: SocketAddr| -> SerfEngine { + let cfg = Options::new() + .with_port(port) + .with_close_timeout(Duration::from_secs(10)); + let ep_cfg = + EndpointOptions::new(SmolStr::new(id), addr).with_stream_timeout(Duration::from_secs(5)); + let mut e: SerfEngine = SerfEngine::try_new_at( + cfg, + TransformOptions::default(), + ep_cfg, + SerfOptions::new(), + now, + test_rng(), + ) + .expect("construct"); + e.start(now); + e + }; + + let mut a = mk("a", 7946, a_addr); + let mut b = mk("b", 7947, b_addr); + + let fabric = FabricInner::fresh(); + // The listener handle is the last in each pool; the rest are dial slots. + let (a_listener, a_dials) = pool_handles_a.split_last().expect("at least one handle"); + let (b_listener, b_dials) = pool_handles_b.split_last().expect("at least one handle"); + for &h in a_dials { + a.plane_mut().pool.push(h); + } + for &h in b_dials { + b.plane_mut().pool.push(h); + } + a.set_listener(*a_listener); + b.set_listener(*b_listener); + + let mut a_rel = LinkRel::new(fabric.clone(), a_addr, pool_handles_a); + let mut b_rel = LinkRel::new(fabric, b_addr, pool_handles_b); + // The engine already owns each pool/listener; remove the listeners from the + // mock free-lists so a re-listen does not double-hand a listener, and arm them. + a_rel.free.retain(|h| h != a_listener); + b_rel.free.retain(|h| h != b_listener); + a_rel.listen(*a_listener, 7946).expect("listen"); + b_rel.listen(*b_listener, 7947).expect("listen"); + + let a2b: Rc)>>> = + Rc::new(RefCell::new(std::vec::Vec::new())); + let b2a: Rc)>>> = + Rc::new(RefCell::new(std::vec::Vec::new())); + let a_in: Rc)>>> = + Rc::new(RefCell::new(VecDeque::new())); + let b_in: Rc)>>> = + Rc::new(RefCell::new(VecDeque::new())); + let a_gossip = GossipWire { + outbound: a2b, + inbound: a_in, + }; + let b_gossip = GossipWire { + outbound: b2a, + inbound: b_in, + }; + + LinkPair { + a, + b, + a_rel, + b_rel, + a_gossip, + b_gossip, + a_addr, + b_addr, + } + } + + /// Pump both engines once at `now`, then ferry each side's emitted gossip into + /// the peer's inbound queue. + fn step(&mut self, now: Instant) { + self.a.pump(now, &mut self.a_gossip, &mut self.a_rel); + self.b.pump(now, &mut self.b_gossip, &mut self.b_rel); + let a_out: std::vec::Vec<_> = self.a_gossip.outbound.borrow_mut().drain(..).collect(); + for (dest, bytes) in a_out { + if dest == self.b_addr { + self + .b_gossip + .inbound + .borrow_mut() + .push_back((self.a_addr, bytes)); + } + } + let b_out: std::vec::Vec<_> = self.b_gossip.outbound.borrow_mut().drain(..).collect(); + for (dest, bytes) in b_out { + if dest == self.a_addr { + self + .a_gossip + .inbound + .borrow_mut() + .push_back((self.b_addr, bytes)); + } + } + } +} + +/// A full serf join over the in-memory reliable link: `a.join([b])` drives the +/// push/pull to a `Succeeded` `ExchangeCompleted`, which the engine folds into the +/// await-result join — accumulating B's address into the reached set — so +/// `poll_join` resolves `Ok([b_addr])`. The join is then reaped and A has learned +/// B through the merge. +#[test] +fn two_engine_join_folds_reached_set_and_poll_join_resolves_ok() { + let mut link = LinkPair::new(&[10, 11], &[20, 21]); + let now = Instant::from_origin(Duration::from_secs(86_400)); + + let handle = link + .a + .join(&[link.b_addr], false, now) + .expect("join announces intent and mints a handle"); + + let mut outcome = None; + for _ in 0..40 { + link.step(now); + // Drain A's events — this folds the push/pull ExchangeCompleted into the join. + while link.a.poll_event().is_some() {} + // Drain B's events too so its queue cannot stall the bridge. + while link.b.poll_event().is_some() {} + if let Some(res) = link.a.poll_join(handle) { + outcome = Some(res); + break; + } + } + + match outcome { + Some(Ok(reached)) => { + assert!( + reached.contains(&link.b_addr), + "the reached set must contain B, folded from the Succeeded push/pull" + ); + } + other => panic!("expected Some(Ok(reached)), got {other:?}"), + } + assert_eq!( + link.a.pending_join_count(), + 0, + "the resolved join must be reaped" + ); + assert_eq!( + link.a.num_members(), + 2, + "A must have learned B through the push/pull merge" + ); +} diff --git a/serf-embedded/src/lib.rs b/serf-embedded/src/lib.rs index ac910422..c022b62d 100644 --- a/serf-embedded/src/lib.rs +++ b/serf-embedded/src/lib.rs @@ -24,7 +24,7 @@ compile_error!("serf-embedded requires the `std` or `alloc` feature"); mod cidr; pub mod engine; -pub use engine::SerfEngine; +pub use engine::{JoinFailed, JoinId, ReachedSet, SerfEngine}; // ── Reused memberlist-embedded glue ────────────────────────────────────────── // From e38caa77903b3f113db25ffb17bb514a4b44a736 Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 3 Jul 2026 21:02:58 +1200 Subject: [PATCH 045/128] fix(serf-embedded): decouple leave from ignore_old cleanup; guard refused leave leave() cleared ignore_old streams and force-resolved joins BEFORE endpoint.leave was accepted, so a late push/pull could merge past a removed ignore token (replaying the seed's pre-join events) and a refused LeaveClockExhausted leave corrupted join state. leave now calls endpoint.leave first (untouched join state if refused while Alive), and on an accepted leave delivers each caller reply once while the ignore cleanup stays gated on the exchange terminal (pump-driven, pending-set empties), mirroring serf-reactor's reply-vs-cleanup decoupling. Adds two doc(hidden) serf-proto test seams and regressions for the late-merge race and the refused-leave case. --- serf-embedded/src/engine/mod.rs | 196 ++++++++++++++++++-------- serf-embedded/src/engine/tests.rs | 135 ++++++++++++++++++ serf-proto/src/endpoint/mod.rs | 12 +- serf-proto/src/stream_endpoint/mod.rs | 21 ++- 4 files changed, 295 insertions(+), 69 deletions(-) diff --git a/serf-embedded/src/engine/mod.rs b/serf-embedded/src/engine/mod.rs index 12b0d740..9c864374 100644 --- a/serf-embedded/src/engine/mod.rs +++ b/serf-embedded/src/engine/mod.rs @@ -209,6 +209,27 @@ struct QueuedSeed { ignore_old: bool, } +/// The caller-reply lifecycle of an await-result join, kept SEPARATE from the +/// `ignore_old` stream cleanup so the two terminals resolve independently. +/// +/// The reply is resolved once — from all-exchanges-done or from +/// [`leave`](SerfEngine::leave) — and delivered to the caller exactly once via +/// [`poll_join`](SerfEngine::poll_join); the ignore-stream cleanup lives on +/// [`PendingJoin`] and stays gated on the exchange terminal. Decoupling them lets +/// `leave` hand the caller its result while a still-in-flight `ignore_old` +/// push/pull keeps its ignore token until it actually merges — mirroring +/// serf-reactor's `PendingJoin`, whose reply and ignore-cleanup terminals are +/// likewise distinct. +enum JoinReply { + /// Not yet resolved; `poll_join` yields `None`. + Pending, + /// Resolved and awaiting the caller; `poll_join` delivers it once, then flips + /// this to `Delivered`. + Ready(Result), + /// Already delivered to the caller; further `poll_join` calls yield `None`. + Delivered, +} + /// Core-owned state for one in-flight await-result join. /// /// The sync analogue of serf-reactor's `PendingJoin`, adapted to the driver-poll @@ -218,14 +239,23 @@ struct QueuedSeed { /// [`StreamId`] captured at [`join`](SerfEngine::join)'s per-seed dispatch and /// bound to the exchange's [`ExchangeId`] when its `Connect` action surfaces; the /// completion then matches by that bound `ExchangeId`. +/// +/// The caller [`reply`](Self::reply) and the `ignore_old` cleanup are DISTINCT +/// terminals: the reply resolves and is delivered once (on all-exchanges-done or +/// on `leave`), while the ignore streams are cleared only when every dispatched +/// exchange has terminated (`pending` empty). Keeping them separate is what lets a +/// `leave` abandon the join for the caller without pulling the ignore token out +/// from under a push/pull that can still merge. struct PendingJoin { /// `StreamId`s this join's `start_join_push_pull` calls returned. Matched /// against each surfaced `Connect`'s `stream_id()` to bind the exchange, and — - /// for an `ignore_old` join — cleared from the machine's ignore set on terminal. + /// for an `ignore_old` join — cleared from the machine's ignore set on the + /// exchange terminal. started: HashSet, /// Number of this join's seeds still queued in `pending_seeds`, undispatched. /// Decremented as each seed's push/pull is started in the pump; the join is - /// "fully dispatched" (eligible to resolve) once this reaches zero. + /// "fully dispatched" once this reaches zero. `leave` forces it to zero — no + /// seed dispatches once the node is leaving. unstarted: usize, /// Outbound exchange ids bound at `Connect` and still awaiting a terminal /// `ExchangeCompleted`. @@ -235,11 +265,15 @@ struct PendingJoin { /// Total routable-seed count this join dispatched — the `JoinFailed` denominator. requested: usize, /// Whether this is an `ignore_old` join (its `started` streams are recorded in - /// the machine's ignore set and must be cleared on terminal). + /// the machine's ignore set and must be cleared on the exchange terminal). ignore_old: bool, - /// The resolved outcome, set once every dispatched exchange has terminated (or - /// on `leave`). `poll_join` returns and removes the waiter once this is `Some`. - resolved: Option>, + /// Whether the `ignore_old` streams have already been cleared from the machine's + /// ignore set, so the cleanup runs exactly once when the last exchange + /// terminates. + ignore_cleared: bool, + /// The caller-reply lifecycle, resolved and delivered once, independent of the + /// ignore-stream cleanup above. + reply: JoinReply, } impl PendingJoin { @@ -251,18 +285,41 @@ impl PendingJoin { Ok(self.contacted.clone()) } } + + /// Deliver the resolved outcome to the caller exactly once: transition + /// `Ready → Delivered` and hand back the outcome, or `None` while still + /// `Pending` (unresolved) or once already `Delivered`. + fn take_ready_reply(&mut self) -> Option> { + if !matches!(self.reply, JoinReply::Ready(_)) { + return None; + } + let JoinReply::Ready(outcome) = core::mem::replace(&mut self.reply, JoinReply::Delivered) + else { + unreachable!("reply matched Ready immediately above"); + }; + Some(outcome) + } + + /// Both terminals reached: the caller has consumed the reply AND every + /// dispatched exchange has terminated, so the waiter can be reaped. + fn is_done(&self) -> bool { + matches!(self.reply, JoinReply::Delivered) && self.unstarted == 0 && self.pending.is_empty() + } } -/// Resolve `pj` if every dispatched exchange has terminated (`unstarted == 0` and -/// `pending` empty) and it is not already resolved: compute its outcome, clear any -/// still-recorded `ignore_old` streams from the machine's ignore set, and stash -/// the outcome for [`poll_join`](SerfEngine::poll_join). +/// Advance `pj` toward its terminals once every dispatched exchange has terminated +/// (`unstarted == 0` and `pending` empty): clear any still-recorded `ignore_old` +/// streams from the machine's ignore set (once), then resolve the caller reply if +/// it is still pending. /// -/// Driven on both the completion path (an `ExchangeCompleted` emptied `pending`) -/// and the end-of-pump sweep (a join whose seeds all retired before a `Connect`, -/// or an empty/all-non-routable seed set, never accumulates any `pending`). Clearing -/// the ignore streams HERE — on the exchange terminal, not lazily in `poll_join` — -/// means the machine's ignore set never leaks even if the driver drops the handle. +/// This is the SOLE site that clears a join's ignore streams, and it is gated on +/// the exchange terminal — never on `leave`, never lazily in `poll_join` — so a +/// still-in-flight `ignore_old` push/pull keeps its ignore token until it actually +/// merges (or its exchange is torn down), and the machine's ignore set never leaks +/// even if the driver drops the handle. Driven on the completion path (an +/// `ExchangeCompleted` emptied `pending`) and the end-of-pump sweep (a join whose +/// seeds all retired before a `Connect`, or an empty/all-non-routable seed set, +/// never accumulates any `pending`). fn try_resolve_join( endpoint: &mut StreamEndpoint, pj: &mut PendingJoin, @@ -271,15 +328,18 @@ fn try_resolve_join( G: Rng, SR: Rng + SeedableRng, { - if pj.resolved.is_some() || pj.unstarted != 0 || !pj.pending.is_empty() { + if pj.unstarted != 0 || !pj.pending.is_empty() { return; } - if pj.ignore_old { + if pj.ignore_old && !pj.ignore_cleared { for sid in &pj.started { endpoint.clear_ignore_join_stream(*sid); } + pj.ignore_cleared = true; + } + if matches!(pj.reply, JoinReply::Pending) { + pj.reply = JoinReply::Ready(pj.outcome()); } - pj.resolved = Some(pj.outcome()); } /// The transport-agnostic serf driving core. @@ -827,7 +887,8 @@ where contacted: ReachedSet::new(), requested, ignore_old, - resolved: None, + ignore_cleared: false, + reply: JoinReply::Pending, }, ); Ok(id) @@ -839,49 +900,58 @@ where /// Returns `Some(Ok(reached))` with the address set the join contacted, or /// `Some(Err(JoinFailed))` if every dispatched push/pull terminated without /// contacting a seed. `None` means the join has not yet resolved — poll again - /// after the next `pump` + [`poll_event`](Self::poll_event) drain. Removes the - /// waiter on the resolving call (a sync driver polls this each tick; an async - /// driver awaits a signal the pump fires when it flips to `Some`). An unknown or - /// already-drained handle yields `None`. + /// after the next `pump` + [`poll_event`](Self::poll_event) drain. The outcome + /// is delivered EXACTLY ONCE: a second poll of the same handle yields `None`. + /// The waiter is reaped on this call once its `ignore_old` cleanup has ALSO + /// completed (every dispatched exchange terminated); until then it lingers so + /// the pump can finish clearing the ignore streams on the exchange terminal + /// (never premature). An unknown or already-delivered handle yields `None`. pub fn poll_join(&mut self, handle: JoinId) -> Option> { - // Yield `None` for an unknown handle (`get?`) or one still in flight - // (`resolved.as_ref()?`); only a resolved join is drained AND removed. - self.pending_joins.get(&handle)?.resolved.as_ref()?; - self - .pending_joins - .remove(&handle) - .and_then(|pj| pj.resolved) + // Deliver the resolved outcome once (`Ready → Delivered`); yield `None` for an + // unknown handle, one still in flight, or one already delivered. + let pj = self.pending_joins.get_mut(&handle)?; + let outcome = pj.take_ready_reply()?; + // The caller has consumed the reply. Reap the waiter only if its ignore-stream + // cleanup terminal has also been reached; otherwise keep it so the pump can + // still fold the outstanding `ExchangeCompleted`s and clear the ignore streams. + if pj.is_done() { + self.pending_joins.remove(&handle); + } + Some(outcome) } /// Begin leaving the cluster. /// - /// Forwards to serf's graceful-leave path, which gossips the departure and - /// ultimately emits [`Event::LeftCluster`] via [`poll_event`](Self::poll_event). - /// Any seeds still queued from a pre-leave join are dropped, and every in-flight - /// await-result join is resolved from its current reached set (clearing any - /// still-recorded ignore streams so the machine's ignore set never leaks): the - /// pump initiates no new push/pull once leaving. + /// Calls serf's graceful-leave path FIRST, so a refused leave that leaves the + /// node `Alive` (e.g. `LeaveClockExhausted`, the `LTIME_MAX` watermark guard) + /// returns its error with the in-flight join state untouched — `?` returns + /// before any of the abandonment below runs. + /// + /// On an ACCEPTED leave — which gossips the departure and ultimately emits + /// [`Event::LeftCluster`] via [`poll_event`](Self::poll_event) — the pump + /// initiates no further push/pull, so any queued seeds are dropped and every + /// in-flight await-result join is handed its caller reply once from its + /// reached-so-far set. The `ignore_old` cleanup is deliberately NOT run here: it + /// stays gated on the exchange terminal ([`try_resolve_join`], driven by the + /// pump fold), so a late successful push/pull that merges after leave still + /// finds its ignore token in place and suppresses the seed's pre-join user + /// events — the same reply-vs-cleanup decoupling serf-reactor uses. pub fn leave(&mut self, now: Instant) -> Result<(), SerfError> { + // Machine leave FIRST: a refused leave returns without mutating serf state, so + // the join state must stay untouched too. + self.endpoint.leave(now)?; + + // Accepted: no seed dispatches once leaving, so drop the queue and mark every + // join fully dispatched, then deliver each still-pending caller reply once. + // Ignore-stream cleanup stays with `try_resolve_join` on the exchange terminal. self.pending_seeds.clear(); - // Force every in-flight join to its terminal from whatever it accumulated so - // far: leaving means no further push/pull dispatch, so an unresolved join - // would otherwise linger forever (and leak its ignore-join streams). - let Self { - endpoint, - pending_joins, - .. - } = self; - for pj in pending_joins.values_mut() { - if pj.resolved.is_none() { - if pj.ignore_old { - for sid in &pj.started { - endpoint.clear_ignore_join_stream(*sid); - } - } - pj.resolved = Some(pj.outcome()); + for pj in self.pending_joins.values_mut() { + pj.unstarted = 0; + if matches!(pj.reply, JoinReply::Pending) { + pj.reply = JoinReply::Ready(pj.outcome()); } } - self.endpoint.leave(now) + Ok(()) } /// Force a named node out of the cluster (an operator-driven removal). @@ -1138,21 +1208,25 @@ where // 7e. Egress: drain outbound gossip transmits, encode + encrypt, and send. self.drain_gossip_transmits(gossip); - // 7f. Resolve any await-result join whose exchanges all terminated. The normal - // (some seed contacted / failed) path resolves as each `ExchangeCompleted` - // folds in `poll_event`; this sweep catches the joins that never accumulate a - // pending exchange at all — an empty/all-non-routable seed set, or seeds that - // retired before a `Connect` — now that this tick's `Connect`s have been - // captured (phase 7a). + // 7f. Resolve + reap await-result joins whose exchanges all terminated. The + // normal (some seed contacted / failed) path resolves as each + // `ExchangeCompleted` folds in `poll_event`; this sweep catches the joins that + // never accumulate a pending exchange at all — an empty/all-non-routable seed + // set, or seeds that retired before a `Connect` — now that this tick's + // `Connect`s have been captured (phase 7a). It also reaps any waiter that has + // reached BOTH terminals (caller reply delivered AND every exchange + // terminated), which is where a leave-abandoned join whose ignore streams + // clear only once its post-leave exchange finally merges is retired. { let Self { endpoint, pending_joins, .. } = self; - for pj in pending_joins.values_mut() { + pending_joins.retain(|_, pj| { try_resolve_join(endpoint, pj); - } + !pj.is_done() + }); } // 8. Next deadline = min(machine, closing). diff --git a/serf-embedded/src/engine/tests.rs b/serf-embedded/src/engine/tests.rs index 323821dd..fa9ad7e1 100644 --- a/serf-embedded/src/engine/tests.rs +++ b/serf-embedded/src/engine/tests.rs @@ -888,3 +888,138 @@ fn two_engine_join_folds_reached_set_and_poll_join_resolves_ok() { "A must have learned B through the push/pull merge" ); } + +// ── leave / ignore_old decoupling ───────────────────────────────────────────── + +/// An `ignore_old` join whose `leave()` races an in-flight push/pull: `leave` must +/// NOT clear the exchange's ignore token, because the exchange can still merge and +/// a merge past a removed token replays the seed's pre-join user events. The token +/// is instead cleared only on the exchange terminal (the pump fold), so it is +/// retained across leave yet never leaks. Regression for a `leave()` that cleared +/// the ignore streams before the exchange terminal. +#[test] +fn leave_retains_ignore_token_until_exchange_terminal() { + let mut link = LinkPair::new(&[10, 11], &[20, 21]); + let now = Instant::from_origin(Duration::from_secs(86_400)); + + // A starts an `ignore_old` join to B and dispatches the dial with a single step, + // so the push/pull is IN FLIGHT (its ignore token recorded, its exchange bound + // into `pending`) — not yet terminal — when A leaves. + let handle = link + .a + .join(&[link.b_addr], /*ignore_old*/ true, now) + .expect("join announces intent and mints a handle"); + link.step(now); + + // The dispatched seed recorded exactly one ignore-join StreamId, and its exchange + // is still in flight (no terminal `ExchangeCompleted` yet). + let sid = { + let pj = &link.a.pending_joins[&handle]; + assert_eq!( + pj.started.len(), + 1, + "the ignore_old seed recorded its StreamId" + ); + assert!( + !pj.pending.is_empty(), + "its push/pull exchange is bound and in flight" + ); + *pj.started.iter().next().expect("one started stream") + }; + assert!( + link.a.endpoint.test_has_ignore_join_stream(sid), + "the ignore token is recorded while the exchange is in flight" + ); + + // A leaves mid-exchange. The fix RETAINS the token (its exchange can still + // merge); the buggy leave cleared it right here, so a late merge would replay + // the seed's pre-join events. + link + .a + .leave(now) + .expect("leave from a running node succeeds"); + assert!( + link.a.endpoint.test_has_ignore_join_stream(sid), + "leave must NOT clear the ignore token while the exchange can still merge" + ); + + // Driving the exchange to its terminal clears the token via the pump fold — the + // SOLE cleanup site — so the machine's ignore set never leaks. + for _ in 0..40 { + link.step(now); + while link.a.poll_event().is_some() {} + while link.b.poll_event().is_some() {} + if !link.a.endpoint.test_has_ignore_join_stream(sid) { + break; + } + } + assert!( + !link.a.endpoint.test_has_ignore_join_stream(sid), + "the ignore token must be cleared on the exchange terminal (no leak)" + ); +} + +/// A refused `leave` — the `LeaveClockExhausted` watermark guard, which leaves the +/// node `Alive` without mutating serf state — must NOT touch the in-flight join +/// state: the queued seed survives, the pending join is neither force-resolved nor +/// reaped, and the endpoint stays `Alive`. The engine calls `endpoint.leave` +/// BEFORE abandoning any join, so a refusal (`?`) returns with the join +/// bookkeeping untouched. Regression for a `leave()` that cleared seeds and +/// force-resolved joins before the machine leave was accepted. +#[test] +fn refused_leave_leaves_join_state_untouched() { + let mut engine = make_engine(); + let now = Instant::from_origin(Duration::from_secs(86_400)); + engine.start(now); + engine.set_listener(9); + + let handle = engine + .join(&[node_addr(7002)], /*ignore_old*/ true, now) + .expect("join announces intent and mints a handle"); + assert_eq!( + engine.pending_join_count(), + 1, + "the in-flight join is tracked" + ); + assert_eq!(engine.pending_seeds.len(), 1, "the routable seed is queued"); + + // Drive serf's member clock to the LTIME_MAX integrity floor so the next leave + // stamp reaches LTIME_MAX and is refused — the watermark serf-proto's own + // leave-watermark test drives the clock to. LTIME_MAX is serf-proto's private + // integrity floor `1 << 63`. + const LTIME_MAX: u64 = 1u64 << 63; + engine.endpoint.test_set_clocks(LTIME_MAX - 1, 0, 0); + + let err = engine + .leave(now) + .expect_err("a leave whose stamp reaches LTIME_MAX must be refused"); + assert!( + matches!(err, SerfError::LeaveClockExhausted), + "expected LeaveClockExhausted, got {err:?}" + ); + + // The node stays Alive and every scrap of join bookkeeping is untouched. + assert_eq!( + engine.state(), + SerfState::Alive, + "a refused leave must leave the node Alive" + ); + assert_eq!( + engine.pending_seeds.len(), + 1, + "the queued seed must survive a refused leave" + ); + assert_eq!( + engine.pending_join_count(), + 1, + "a refused leave must not reap the pending join" + ); + assert!( + matches!(engine.pending_joins[&handle].reply, JoinReply::Pending), + "a refused leave must not force-resolve the pending join" + ); + assert!( + engine.poll_join(handle).is_none(), + "the refused-leave join is still in flight" + ); +} diff --git a/serf-proto/src/endpoint/mod.rs b/serf-proto/src/endpoint/mod.rs index 25c5e5ed..0ceb8fa5 100644 --- a/serf-proto/src/endpoint/mod.rs +++ b/serf-proto/src/endpoint/mod.rs @@ -4083,7 +4083,11 @@ where // ── Clock + intent test helpers ─────────────────────────────────────────── /// Set all three Lamport clocks in one call (test fixture). - #[cfg(test)] + /// + /// Compiled under `test` or the `tcp` feature: the `StreamEndpoint` test-support + /// seam ([`StreamEndpoint::test_set_clocks`](crate::StreamEndpoint::test_set_clocks)) + /// forwards here so a downstream crate's tests can reach the member clock. + #[cfg(any(test, feature = "tcp"))] pub(crate) fn test_set_clocks(&mut self, member: u64, event: u64, query: u64) { self.clock = member; self.event_clock = event; @@ -4172,7 +4176,11 @@ where /// Whether exchange `id` is currently a recorded `ignore_old` join (test /// adapter for cancellation / one-shot-consume / per-exchange assertions). - #[cfg(test)] + /// + /// Compiled under `test` or the `tcp` feature: the `StreamEndpoint` test-support + /// seam ([`StreamEndpoint::test_has_ignore_join_stream`](crate::StreamEndpoint::test_has_ignore_join_stream)) + /// forwards here so a downstream crate's tests can observe the ignore set. + #[cfg(any(test, feature = "tcp"))] pub(crate) fn test_has_ignore_join_stream(&self, id: StreamId) -> bool { self.ignore_join_streams.contains(&id) } diff --git a/serf-proto/src/stream_endpoint/mod.rs b/serf-proto/src/stream_endpoint/mod.rs index 85a5c898..3fc42ddb 100644 --- a/serf-proto/src/stream_endpoint/mod.rs +++ b/serf-proto/src/stream_endpoint/mod.rs @@ -883,9 +883,14 @@ where .test_inject_user_packet(&mut self.transport, from, data, now) } - /// Forwards to [`Endpoint::test_set_clocks`]. - #[cfg(test)] - pub(crate) fn test_set_clocks(&mut self, member: u64, event: u64, query: u64) { + /// Overwrite all three Lamport clocks in one call. + /// + /// A test-support seam (hidden from the public API) that a downstream crate's + /// tests use to drive the member clock to the `LTIME_MAX` integrity floor and + /// exercise the refused-leave (`LeaveClockExhausted`) path without 2^63 real + /// membership events. Forwards to [`Endpoint::test_set_clocks`]. + #[doc(hidden)] + pub fn test_set_clocks(&mut self, member: u64, event: u64, query: u64) { self.core.test_set_clocks(member, event, query) } @@ -959,9 +964,13 @@ where self.core.test_note_ignore_join_stream(id) } - /// Forwards to [`Endpoint::test_has_ignore_join_stream`]. - #[cfg(test)] - pub(crate) fn test_has_ignore_join_stream(&self, id: StreamId) -> bool { + /// Whether exchange `id` is still recorded as an `ignore_old` join target. + /// + /// A test-support seam (hidden from the public API) that a downstream crate's + /// tests use to assert a join's ignore token survives `leave` until the exchange + /// terminal. Forwards to [`Endpoint::test_has_ignore_join_stream`]. + #[doc(hidden)] + pub fn test_has_ignore_join_stream(&self, id: StreamId) -> bool { self.core.test_has_ignore_join_stream(id) } From 43e635a9427adc14f1f709ebcbbebad47433cf3f Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 3 Jul 2026 21:47:53 +1200 Subject: [PATCH 046/128] fix(serf-embedded): fold-before-leave join resolution, leak-free reap + cancel_join, test-support-gated clock seam Port serf-reactor's join lifecycle: leave() drains and folds already-queued ExchangeCompleted into the reached set before computing its abandonment, so a join that succeeded on the wire but was not yet drained resolves Ok(reached) not a stale JoinFailed. The join entry is reaped the instant its exchanges terminate regardless of caller delivery (+ cancel_join for a driver give-up), so a dropped or never-polled handle leaks neither the join table nor the machine ignore set. serf-proto's clock/ignore test seams move behind a non-default test-support feature (were doc-hidden-pub, publicly mutable in any tcp build). Regressions for queued-success-before-leave, dropped-handle reap, and cancel_join. --- serf-embedded/Cargo.toml | 11 ++ serf-embedded/src/engine/mod.rs | 199 ++++++++++++++++++------- serf-embedded/src/engine/tests.rs | 200 ++++++++++++++++++++++++++ serf-proto/Cargo.toml | 6 + serf-proto/src/endpoint/mod.rs | 20 +-- serf-proto/src/stream_endpoint/mod.rs | 22 +-- 6 files changed, 390 insertions(+), 68 deletions(-) diff --git a/serf-embedded/Cargo.toml b/serf-embedded/Cargo.toml index f0b2c4fb..5922d7dd 100644 --- a/serf-embedded/Cargo.toml +++ b/serf-embedded/Cargo.toml @@ -71,3 +71,14 @@ hashbrown = { version = "0.17", default-features = false, features = [ # The fixed-capacity, no-heap `ResolvedAddrs` resolver result shares the single # version smoltcp already pulls into the graph (no second heapless major). heapless = { version = "0.9", default-features = false } + +[dev-dependencies] +# Re-declares serf-proto solely to layer its non-default `test-support` feature +# on for the engine tests (clock overwrite + ignore-set inspection). With the +# resolver-3 dev/normal split this edge is active ONLY for test/bench builds, so a +# production `serf-embedded` still pulls serf-proto without `test-support` and +# never exposes clock mutation. +serf-proto = { path = "../serf-proto", default-features = false, features = [ + "tcp", + "test-support", +] } diff --git a/serf-embedded/src/engine/mod.rs b/serf-embedded/src/engine/mod.rs index 9c864374..9ab5934c 100644 --- a/serf-embedded/src/engine/mod.rs +++ b/serf-embedded/src/engine/mod.rs @@ -226,7 +226,10 @@ enum JoinReply { /// Resolved and awaiting the caller; `poll_join` delivers it once, then flips /// this to `Delivered`. Ready(Result), - /// Already delivered to the caller; further `poll_join` calls yield `None`. + /// Consumed — either handed to the caller by + /// [`poll_join`](SerfEngine::poll_join) or forgotten via + /// [`cancel_join`](SerfEngine::cancel_join); further `poll_join` calls yield + /// `None`. Delivered, } @@ -300,10 +303,15 @@ impl PendingJoin { Some(outcome) } - /// Both terminals reached: the caller has consumed the reply AND every - /// dispatched exchange has terminated, so the waiter can be reaped. - fn is_done(&self) -> bool { - matches!(self.reply, JoinReply::Delivered) && self.unstarted == 0 && self.pending.is_empty() + /// Every dispatched exchange has terminated: no seed still queued (`unstarted`) + /// and no exchange still in flight (`pending`). + /// + /// This is the reap gate — deliberately INDEPENDENT of whether the caller has + /// consumed the reply — so a waiter is reclaimed the instant its work is done + /// rather than lingering forever on a `poll_join` that may never come. The + /// caller's result is delivered if it polls before the reap, else discarded. + fn exchange_work_done(&self) -> bool { + self.unstarted == 0 && self.pending.is_empty() } } @@ -390,6 +398,11 @@ where /// set from the terminal `ExchangeCompleted` of the push/pulls it dispatched; /// [`poll_join`](Self::poll_join) drains a resolved one. pending_joins: HashMap, + /// Machine events [`leave`](Self::leave) drained (and folded into the pending + /// joins) BEFORE resolving its abandonment, held so [`poll_event`](Self::poll_event) + /// still hands them to the driver in order — already folded, never re-folded. + /// Empty outside a `leave` that raced queued completions. + buffered_events: VecDeque>, /// Monotonic allocator for [`JoinId`]s, so two concurrent joins never collide. next_join_id: u64, /// Cluster label applied to the gossip codec on both encode and decode. When @@ -572,6 +585,7 @@ where gossip_recv, pending_seeds: VecDeque::new(), pending_joins: HashMap::new(), + buffered_events: VecDeque::new(), next_join_id: 0, label, cidr_policy, @@ -797,34 +811,54 @@ where /// checking [`poll_join`](Self::poll_join): a push/pull `ExchangeCompleted` is /// folded into its await-result join AS IT surfaces here (the reached-set /// accumulation), and `poll_join` resolves off that folded state. + /// + /// Events a [`leave`](Self::leave) drained early — to fold already-queued + /// completions before resolving its abandonment — are delivered here first, in + /// order and already folded, ahead of any newer machine event. #[inline] pub fn poll_event(&mut self) -> Option> { - let ev = self.endpoint.poll_event(); - // Fold a push/pull completion into its await-result join: the terminal - // `ExchangeCompleted`'s `eid` was bound to a join at its `Connect` (via the - // START `StreamId`), so a match here removes it from `pending` and — on - // `Succeeded` — accumulates the peer into `contacted`. Resolving off that - // (clearing any ignore streams) happens the instant `pending` empties. - if let Some(Event::ExchangeCompleted(ec)) = &ev { - if ec.kind() == ExchangeKind::PushPull { - let Self { - endpoint, - pending_joins, - .. - } = self; - if let Some(pj) = pending_joins - .values_mut() - .find(|pj| pj.pending.contains(&ec.eid())) - { - pj.pending.remove(&ec.eid()); - if matches!(ec.outcome(), ExchangeStatus::Succeeded) { - pj.contacted.push(*ec.peer()); - } - try_resolve_join(endpoint, pj); - } + // A `leave` that raced queued completions has already drained + folded them + // into the pending joins; hand those buffered events to the driver first + // (NEVER re-folding), ahead of any newer machine event. + if let Some(ev) = self.buffered_events.pop_front() { + return Some(ev); + } + let ev = self.endpoint.poll_event()?; + self.fold_join_completion(&ev); + Some(ev) + } + + /// Fold one machine event into the await-result join it terminates, if any. + /// + /// A push/pull `ExchangeCompleted` whose `eid` was bound to a join at its + /// `Connect` (via the START `StreamId`) is removed from that join's `pending` + /// and — on `Succeeded` — accumulates the peer into `contacted`; then + /// [`try_resolve_join`] resolves the join (clearing any ignore streams) the + /// instant `pending` empties. A no-op for every other event. Shared by + /// [`poll_event`](Self::poll_event) and [`leave`](Self::leave)'s + /// pre-resolution drain so both fold identically. + fn fold_join_completion(&mut self, ev: &Event) { + let Event::ExchangeCompleted(ec) = ev else { + return; + }; + if ec.kind() != ExchangeKind::PushPull { + return; + } + let Self { + endpoint, + pending_joins, + .. + } = self; + if let Some(pj) = pending_joins + .values_mut() + .find(|pj| pj.pending.contains(&ec.eid())) + { + pj.pending.remove(&ec.eid()); + if matches!(ec.outcome(), ExchangeStatus::Succeeded) { + pj.contacted.push(*ec.peer()); } + try_resolve_join(endpoint, pj); } - ev } /// Announce the local node's join intent and begin an await-result join to @@ -902,24 +936,61 @@ where /// contacting a seed. `None` means the join has not yet resolved — poll again /// after the next `pump` + [`poll_event`](Self::poll_event) drain. The outcome /// is delivered EXACTLY ONCE: a second poll of the same handle yields `None`. - /// The waiter is reaped on this call once its `ignore_old` cleanup has ALSO - /// completed (every dispatched exchange terminated); until then it lingers so - /// the pump can finish clearing the ignore streams on the exchange terminal - /// (never premature). An unknown or already-delivered handle yields `None`. + /// The waiter is reaped on this call once every dispatched exchange has + /// terminated; if one is still in flight it lingers so the pump can clear the + /// ignore streams on the exchange terminal (never premature). A caller that + /// never polls does NOT leak the waiter — the pump reaps it once its exchanges + /// terminate, discarding the undelivered result (see + /// [`cancel_join`](Self::cancel_join) for an explicit give-up). An unknown or + /// already-consumed handle yields `None`. pub fn poll_join(&mut self, handle: JoinId) -> Option> { // Deliver the resolved outcome once (`Ready → Delivered`); yield `None` for an - // unknown handle, one still in flight, or one already delivered. + // unknown handle, one still in flight, or one already consumed. let pj = self.pending_joins.get_mut(&handle)?; let outcome = pj.take_ready_reply()?; - // The caller has consumed the reply. Reap the waiter only if its ignore-stream - // cleanup terminal has also been reached; otherwise keep it so the pump can - // still fold the outstanding `ExchangeCompleted`s and clear the ignore streams. - if pj.is_done() { + // The caller has consumed the reply. Reap the waiter now if every exchange has + // terminated; otherwise keep it so the pump can still fold the outstanding + // `ExchangeCompleted`s and clear the ignore streams on the exchange terminal. + if pj.exchange_work_done() { self.pending_joins.remove(&handle); } Some(outcome) } + /// Give up an in-flight await-result [`join`](Self::join), forgetting its caller + /// reply leak-free. + /// + /// A driver's supported give-up path (a dropped handle, or a driver-imposed + /// timeout): the join's undelivered result is discarded and the waiter is + /// reaped the instant its exchanges terminate, so neither the join table nor the + /// machine's ignore set leaks. If every exchange has already terminated the + /// waiter is removed immediately (its ignore streams cleared); if a push/pull is + /// still in flight the reply is forgotten but the ignore token is RETAINED until + /// that exchange's terminal (a late merge still suppresses the seed's pre-join + /// user events), then the pump reaps the entry. An unknown or already-consumed + /// handle is a no-op. + pub fn cancel_join(&mut self, handle: JoinId) { + let Self { + endpoint, + pending_joins, + .. + } = self; + let Some(pj) = pending_joins.get_mut(&handle) else { + return; + }; + // Clear the ignore streams + resolve if the exchanges are already done, so an + // immediate removal never strands an ignore-set entry. + try_resolve_join(endpoint, pj); + if pj.exchange_work_done() { + pending_joins.remove(&handle); + } else { + // Still in flight: forget the caller reply (so `poll_join` yields nothing and + // the pump treats it as resolved) while the ignore token stays until the + // exchange terminal, where the pump's fold clears it and reaps the entry. + pj.reply = JoinReply::Delivered; + } + } + /// Begin leaving the cluster. /// /// Calls serf's graceful-leave path FIRST, so a refused leave that leaves the @@ -930,20 +1001,39 @@ where /// On an ACCEPTED leave — which gossips the departure and ultimately emits /// [`Event::LeftCluster`] via [`poll_event`](Self::poll_event) — the pump /// initiates no further push/pull, so any queued seeds are dropped and every - /// in-flight await-result join is handed its caller reply once from its - /// reached-so-far set. The `ignore_old` cleanup is deliberately NOT run here: it - /// stays gated on the exchange terminal ([`try_resolve_join`], driven by the - /// pump fold), so a late successful push/pull that merges after leave still - /// finds its ignore token in place and suppresses the seed's pre-join user - /// events — the same reply-vs-cleanup decoupling serf-reactor uses. + /// in-flight await-result join is handed its caller reply once from its reached + /// set. Any push/pull `ExchangeCompleted` ALREADY QUEUED in the endpoint (a + /// success the driver has not yet drained via `poll_event`) is folded into that + /// reached set FIRST, so a race between a successful join and `leave` resolves + /// `Ok(reached)` rather than a stale `JoinFailed`; the drained events are + /// buffered and still delivered, in order, by the next `poll_event`. The + /// `ignore_old` cleanup is deliberately NOT run here: it stays gated on the + /// exchange terminal ([`try_resolve_join`], driven by the pump fold), so a late + /// successful push/pull that merges after leave still finds its ignore token in + /// place and suppresses the seed's pre-join user events — the same + /// reply-vs-cleanup decoupling serf-reactor uses. pub fn leave(&mut self, now: Instant) -> Result<(), SerfError> { // Machine leave FIRST: a refused leave returns without mutating serf state, so // the join state must stay untouched too. self.endpoint.leave(now)?; + // Fold every ALREADY-QUEUED machine completion into its await-result join + // BEFORE computing any abandonment outcome, so a push/pull that already + // succeeded — its `ExchangeCompleted` queued but not yet drained by the driver + // — lands in the reached set rather than being frozen out as a stale + // `JoinFailed`. The drained events are buffered for `poll_event` (already + // folded, delivered in order), preserving delivery. Mirrors serf-reactor's + // shutdown/leave, which drains-and-folds before it reaps. + while let Some(ev) = self.endpoint.poll_event() { + self.fold_join_completion(&ev); + self.buffered_events.push_back(ev); + } + // Accepted: no seed dispatches once leaving, so drop the queue and mark every - // join fully dispatched, then deliver each still-pending caller reply once. - // Ignore-stream cleanup stays with `try_resolve_join` on the exchange terminal. + // still-pending join fully dispatched, then deliver each still-pending caller + // reply once from its NOW-folded reached set. Ignore-stream cleanup stays with + // `try_resolve_join` on the exchange terminal (never here), so a late push/pull + // that merges after leave keeps its ignore token. self.pending_seeds.clear(); for pj in self.pending_joins.values_mut() { pj.unstarted = 0; @@ -1213,10 +1303,12 @@ where // `ExchangeCompleted` folds in `poll_event`; this sweep catches the joins that // never accumulate a pending exchange at all — an empty/all-non-routable seed // set, or seeds that retired before a `Connect` — now that this tick's - // `Connect`s have been captured (phase 7a). It also reaps any waiter that has - // reached BOTH terminals (caller reply delivered AND every exchange - // terminated), which is where a leave-abandoned join whose ignore streams - // clear only once its post-leave exchange finally merges is retired. + // `Connect`s have been captured (phase 7a). A waiter is reaped INDEPENDENTLY of + // caller delivery: once its exchanges have terminated it is dropped even if the + // caller never polled (the undelivered result discarded), so a dropped handle + // leaks neither the join table nor the machine's ignore set. A leave-abandoned + // join whose ignore streams clear only once a post-leave exchange finally + // merges is retired here too. { let Self { endpoint, @@ -1224,8 +1316,13 @@ where .. } = self; pending_joins.retain(|_, pj| { + // Resolvable BEFORE this sweep (a prior tick's fold, or an earlier sweep)? + // Only then is it reap-eligible; a join `try_resolve_join` resolves for the + // FIRST time in THIS sweep is held one more tick so a caller polling right + // after this pump still sees its outcome. + let was_resolved = !matches!(pj.reply, JoinReply::Pending); try_resolve_join(endpoint, pj); - !pj.is_done() + !(pj.exchange_work_done() && was_resolved) }); } diff --git a/serf-embedded/src/engine/tests.rs b/serf-embedded/src/engine/tests.rs index fa9ad7e1..59e2e2a5 100644 --- a/serf-embedded/src/engine/tests.rs +++ b/serf-embedded/src/engine/tests.rs @@ -1023,3 +1023,203 @@ fn refused_leave_leaves_join_state_untouched() { "the refused-leave join is still in flight" ); } + +// ── fold-before-leave / leak-free reap / cancel_join ────────────────────────── + +/// `leave()` folds an ALREADY-QUEUED successful push/pull completion into the +/// await-result join BEFORE computing its abandonment, so a join that succeeded on +/// the wire but whose `ExchangeCompleted(Succeeded)` the driver had not yet drained +/// resolves `Ok(reached)` — never a stale `JoinFailed`. Regression for a `leave` +/// that froze the reply from the pre-fold (empty) reached set. +#[test] +fn leave_folds_queued_success_before_resolving_join() { + let mut link = LinkPair::new(&[10, 11], &[20, 21]); + let now = Instant::from_origin(Duration::from_secs(86_400)); + + let handle = link + .a + .join(&[link.b_addr], false, now) + .expect("join announces intent and mints a handle"); + + // Drive ONLY the shared reliable fabric (pumping both engines ferries it + // automatically) and NEVER the gossip relay, so A learns B SOLELY through the + // push/pull — making `num_members()==2` a precise "the push/pull merged" signal. + // A's events are left UNDRAINED, so the terminal `ExchangeCompleted(Succeeded)` + // sits queued-and-unfolded in A's machine when A leaves. + for _ in 0..40 { + link.a.pump(now, &mut link.a_gossip, &mut link.a_rel); + link.b.pump(now, &mut link.b_gossip, &mut link.b_rel); + // Drain B so its machine advances the push/pull response; discard both sides' + // emitted gossip so it never teaches A about B out of band. + while link.b.poll_event().is_some() {} + link.a_gossip.outbound.borrow_mut().clear(); + link.b_gossip.outbound.borrow_mut().clear(); + if link.a.num_members() == 2 { + break; + } + } + assert_eq!( + link.a.num_members(), + 2, + "the push/pull merged B into A (the exchange completed at the machine level)" + ); + { + let pj = &link.a.pending_joins[&handle]; + assert!( + matches!(pj.reply, JoinReply::Pending), + "the completion is queued but NOT yet folded/resolved" + ); + assert!( + pj.contacted.is_empty(), + "contacted is empty until the queued completion folds" + ); + assert!( + !pj.pending.is_empty(), + "the exchange is still in the join's pending set (unfolded)" + ); + } + + // leave folds the queued Succeeded completion BEFORE its abandonment, so the + // caller gets Ok(reached) including B — the buggy leave froze a stale JoinFailed + // from the pre-fold (empty) reached set. + link + .a + .leave(now) + .expect("leave from a running node succeeds"); + + // The buffered (already-folded) events still deliver in order. + while link.a.poll_event().is_some() {} + match link.a.poll_join(handle) { + Some(Ok(reached)) => assert!( + reached.contains(&link.b_addr), + "the folded reached set must include B" + ), + other => panic!("expected Ok(reached) including B (fold-before-leave), got {other:?}"), + } + assert_eq!( + link.a.pending_join_count(), + 0, + "the resolved join is reaped after delivery" + ); +} + +/// A never-polled terminal join does NOT leak: once its (failed) exchange +/// terminates, the pump reaps the waiter and clears its `ignore_old` stream WITHOUT +/// any `poll_join` and without arming a pending-join deadline — the caller simply +/// dropped the handle. Regression for a reap gated on caller delivery +/// (`pending_joins` would linger `Ready` forever). +#[test] +fn dropped_never_polled_join_is_reaped_on_exchange_terminal() { + let mut engine = make_engine(); + let now = Instant::from_origin(Duration::from_secs(86_400)); + engine.start(now); + engine.plane_mut().pool.push(5); + engine.set_listener(9); + + let handle = engine + .join(&[node_addr(7002)], /*ignore_old*/ true, now) + .expect("join announces intent and mints a handle"); + + let mut gossip = NoGossip; + let mut stream = NoStream::with_pool(0); + // One pump dispatches the seed (recording its ignore stream), captures the + // Connect, and fails the dial (`NoStream::connect` errors) — queuing the terminal + // ExchangeCompleted(Failed). + engine.pump(now, &mut gossip, &mut stream); + let sid = { + let pj = &engine.pending_joins[&handle]; + assert_eq!( + pj.started.len(), + 1, + "the ignore_old seed recorded its StreamId" + ); + *pj.started.iter().next().expect("one started stream") + }; + assert!( + engine.endpoint.test_has_ignore_join_stream(sid), + "the ignore token is recorded while the exchange is in flight" + ); + + // Drive to the reap WITHOUT ever polling the join: the driver folds the failed + // completion via poll_event (the caller dropped the handle), then a later pump + // reaps the terminal waiter independent of any delivery. + for _ in 0..8 { + while engine.poll_event().is_some() {} + if engine.pending_join_count() == 0 { + break; + } + engine.pump(now, &mut gossip, &mut stream); + } + assert_eq!( + engine.pending_join_count(), + 0, + "a never-polled terminal join must be reaped (no pending-join leak)" + ); + assert!( + !engine.endpoint.test_has_ignore_join_stream(sid), + "its ignore stream must be cleared on the exchange terminal (no machine leak)" + ); +} + +/// `cancel_join` forgets an in-flight await-result join leak-free: the caller's +/// reply is discarded (a later `poll_join` yields nothing), the ignore token is +/// RETAINED while the push/pull can still merge, and once the exchange terminates +/// the pump clears that token and reaps the waiter — no pending-join or ignore-set +/// leak, and no further poll. +#[test] +fn cancel_join_forgets_in_flight_join_leak_free() { + let mut link = LinkPair::new(&[10, 11], &[20, 21]); + let now = Instant::from_origin(Duration::from_secs(86_400)); + + let handle = link + .a + .join(&[link.b_addr], /*ignore_old*/ true, now) + .expect("join announces intent and mints a handle"); + // One step dispatches the seed and binds the exchange IN FLIGHT (its ignore token + // recorded, not yet terminal). + link.step(now); + let sid = { + let pj = &link.a.pending_joins[&handle]; + assert!( + !pj.pending.is_empty(), + "its push/pull exchange is bound and in flight" + ); + *pj.started.iter().next().expect("one started stream") + }; + assert!( + link.a.endpoint.test_has_ignore_join_stream(sid), + "the ignore token is recorded while the exchange is in flight" + ); + + // The caller gives up: cancel_join forgets the reply but RETAINS the ignore token + // (the exchange can still merge). + link.a.cancel_join(handle); + assert!( + link.a.poll_join(handle).is_none(), + "a cancelled join yields no outcome to the caller" + ); + assert!( + link.a.endpoint.test_has_ignore_join_stream(sid), + "cancel_join must NOT clear the ignore token while the exchange can still merge" + ); + + // Driving the exchange to its terminal clears the ignore token AND reaps the + // forgotten waiter — no poll, no leak. + for _ in 0..40 { + link.step(now); + while link.a.poll_event().is_some() {} + while link.b.poll_event().is_some() {} + if link.a.pending_join_count() == 0 { + break; + } + } + assert_eq!( + link.a.pending_join_count(), + 0, + "a cancelled join is reaped once its exchange terminates (no pending-join leak)" + ); + assert!( + !link.a.endpoint.test_has_ignore_join_stream(sid), + "the ignore token is cleared on the exchange terminal (no machine leak)" + ); +} diff --git a/serf-proto/Cargo.toml b/serf-proto/Cargo.toml index dddb6305..aa7e4083 100644 --- a/serf-proto/Cargo.toml +++ b/serf-proto/Cargo.toml @@ -62,6 +62,12 @@ tls = ["std", "tcp", "memberlist-proto/tls"] # quinn-proto is std-only. quic = ["std", "dep:libm", "memberlist-proto/quic"] quic-rustls-ring = ["quic", "memberlist-proto/quic-rustls-ring"] +# Test-support seams: exposes the `StreamEndpoint` / `Endpoint` helpers that +# overwrite the Lamport clocks and inspect the ignore-join set, for a DOWNSTREAM +# crate's tests only. Non-default and never pulled by a production build, so a +# plain `tcp` / `tls` / `quic` build never exposes clock mutation. Enable it from +# a `[dev-dependencies]` edge (e.g. serf-embedded) so it is active for tests alone. +test-support = [] [dependencies] memberlist-proto = { git = "https://github.com/al8n/memberlist", branch = "main", default-features = false } diff --git a/serf-proto/src/endpoint/mod.rs b/serf-proto/src/endpoint/mod.rs index 0ceb8fa5..1a5e4055 100644 --- a/serf-proto/src/endpoint/mod.rs +++ b/serf-proto/src/endpoint/mod.rs @@ -4084,10 +4084,12 @@ where /// Set all three Lamport clocks in one call (test fixture). /// - /// Compiled under `test` or the `tcp` feature: the `StreamEndpoint` test-support - /// seam ([`StreamEndpoint::test_set_clocks`](crate::StreamEndpoint::test_set_clocks)) - /// forwards here so a downstream crate's tests can reach the member clock. - #[cfg(any(test, feature = "tcp"))] + /// Compiled under `test` or the non-default `test-support` feature: the + /// `StreamEndpoint` test-support seam + /// ([`StreamEndpoint::test_set_clocks`](crate::StreamEndpoint::test_set_clocks)) + /// forwards here so a downstream crate's tests can reach the member clock. Not a + /// production build path — a plain `tcp` build never compiles this. + #[cfg(any(test, feature = "test-support"))] pub(crate) fn test_set_clocks(&mut self, member: u64, event: u64, query: u64) { self.clock = member; self.event_clock = event; @@ -4177,10 +4179,12 @@ where /// Whether exchange `id` is currently a recorded `ignore_old` join (test /// adapter for cancellation / one-shot-consume / per-exchange assertions). /// - /// Compiled under `test` or the `tcp` feature: the `StreamEndpoint` test-support - /// seam ([`StreamEndpoint::test_has_ignore_join_stream`](crate::StreamEndpoint::test_has_ignore_join_stream)) - /// forwards here so a downstream crate's tests can observe the ignore set. - #[cfg(any(test, feature = "tcp"))] + /// Compiled under `test` or the non-default `test-support` feature: the + /// `StreamEndpoint` test-support seam + /// ([`StreamEndpoint::test_has_ignore_join_stream`](crate::StreamEndpoint::test_has_ignore_join_stream)) + /// forwards here so a downstream crate's tests can observe the ignore set. Not a + /// production build path — a plain `tcp` build never compiles this. + #[cfg(any(test, feature = "test-support"))] pub(crate) fn test_has_ignore_join_stream(&self, id: StreamId) -> bool { self.ignore_join_streams.contains(&id) } diff --git a/serf-proto/src/stream_endpoint/mod.rs b/serf-proto/src/stream_endpoint/mod.rs index 3fc42ddb..bec43d76 100644 --- a/serf-proto/src/stream_endpoint/mod.rs +++ b/serf-proto/src/stream_endpoint/mod.rs @@ -885,11 +885,13 @@ where /// Overwrite all three Lamport clocks in one call. /// - /// A test-support seam (hidden from the public API) that a downstream crate's - /// tests use to drive the member clock to the `LTIME_MAX` integrity floor and - /// exercise the refused-leave (`LeaveClockExhausted`) path without 2^63 real - /// membership events. Forwards to [`Endpoint::test_set_clocks`]. - #[doc(hidden)] + /// A test-support seam gated behind the non-default `test-support` feature (NOT + /// exposed by a production `tcp` build) that a downstream crate's tests use to + /// drive the member clock to the `LTIME_MAX` integrity floor and exercise the + /// refused-leave (`LeaveClockExhausted`) path without 2^63 real membership + /// events. Forwards to [`Endpoint::test_set_clocks`]. + #[cfg(any(test, feature = "test-support"))] + #[cfg_attr(docsrs, doc(cfg(feature = "test-support")))] pub fn test_set_clocks(&mut self, member: u64, event: u64, query: u64) { self.core.test_set_clocks(member, event, query) } @@ -966,10 +968,12 @@ where /// Whether exchange `id` is still recorded as an `ignore_old` join target. /// - /// A test-support seam (hidden from the public API) that a downstream crate's - /// tests use to assert a join's ignore token survives `leave` until the exchange - /// terminal. Forwards to [`Endpoint::test_has_ignore_join_stream`]. - #[doc(hidden)] + /// A test-support seam gated behind the non-default `test-support` feature (NOT + /// exposed by a production `tcp` build) that a downstream crate's tests use to + /// assert a join's ignore token survives `leave` until the exchange terminal. + /// Forwards to [`Endpoint::test_has_ignore_join_stream`]. + #[cfg(any(test, feature = "test-support"))] + #[cfg_attr(docsrs, doc(cfg(feature = "test-support")))] pub fn test_has_ignore_join_stream(&self, id: StreamId) -> bool { self.core.test_has_ignore_join_stream(id) } From 84252d6dbbedad94e717c3ffdb05d54dfc505c05 Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 3 Jul 2026 22:08:06 +1200 Subject: [PATCH 047/128] fix(serf-embedded): retain join result until delivery/cancel; cancel_join drops un-dispatched seeds The one-tick reap grace could drop a Ready join result before a slow/async caller polled it. Reap the caller-result entry only on poll_join delivery or cancel_join; the ignore-set/machine cleanup stays terminal-gated (no machine leak), matching serf-reactor. cancel_join now removes the handle's un-dispatched seeds (zeroing unstarted, keeping ignore tokens only for started exchanges) so a cancel-before-pump dispatches no push/pull. Add regressions for pump-after-resolve retrieval and cancel-before-pump; reconcile the never-polled test to the split lifetimes. --- serf-embedded/src/engine/mod.rs | 96 ++++++++++-------- serf-embedded/src/engine/tests.rs | 163 +++++++++++++++++++++++++++--- 2 files changed, 202 insertions(+), 57 deletions(-) diff --git a/serf-embedded/src/engine/mod.rs b/serf-embedded/src/engine/mod.rs index 9ab5934c..7d72497a 100644 --- a/serf-embedded/src/engine/mod.rs +++ b/serf-embedded/src/engine/mod.rs @@ -306,10 +306,12 @@ impl PendingJoin { /// Every dispatched exchange has terminated: no seed still queued (`unstarted`) /// and no exchange still in flight (`pending`). /// - /// This is the reap gate — deliberately INDEPENDENT of whether the caller has - /// consumed the reply — so a waiter is reclaimed the instant its work is done - /// rather than lingering forever on a `poll_join` that may never come. The - /// caller's result is delivered if it polls before the reap, else discarded. + /// This is the EXCHANGE-terminal predicate. It gates the `ignore_old` / + /// machine-ignore cleanup ([`try_resolve_join`]) — run the instant the work is + /// done, INDEPENDENT of caller polling, so the machine's ignore set never leaks. + /// It is NOT sufficient on its own to reap the caller-result entry: that reap + /// additionally requires the reply to have been `Delivered` (polled or + /// cancelled), so a resolved result is retained until the caller retrieves it. fn exchange_work_done(&self) -> bool { self.unstarted == 0 && self.pending.is_empty() } @@ -936,13 +938,15 @@ where /// contacting a seed. `None` means the join has not yet resolved — poll again /// after the next `pump` + [`poll_event`](Self::poll_event) drain. The outcome /// is delivered EXACTLY ONCE: a second poll of the same handle yields `None`. - /// The waiter is reaped on this call once every dispatched exchange has - /// terminated; if one is still in flight it lingers so the pump can clear the - /// ignore streams on the exchange terminal (never premature). A caller that - /// never polls does NOT leak the waiter — the pump reaps it once its exchanges - /// terminate, discarding the undelivered result (see - /// [`cancel_join`](Self::cancel_join) for an explicit give-up). An unknown or - /// already-consumed handle yields `None`. + /// This call is the DELIVERY that lets the entry be reaped: once the caller has + /// retrieved the result AND every dispatched exchange has terminated the entry is + /// dropped (here, or by the next pump's sweep). A resolved result is therefore + /// RETAINED until the caller polls it — a pump between resolution and this call + /// never discards it. The machine's ignore set is cleared on the exchange + /// terminal regardless of polling, so a dropped handle leaks no machine state; it + /// leaks only the small unretrieved result entry until a poll or + /// [`cancel_join`](Self::cancel_join) reaps it, so a driver MUST poll or cancel + /// every join it starts. An unknown or already-consumed handle yields `None`. pub fn poll_join(&mut self, handle: JoinId) -> Option> { // Deliver the resolved outcome once (`Ready → Delivered`); yield `None` for an // unknown handle, one still in flight, or one already consumed. @@ -957,36 +961,45 @@ where Some(outcome) } - /// Give up an in-flight await-result [`join`](Self::join), forgetting its caller - /// reply leak-free. + /// Give up an await-result [`join`](Self::join), dropping any of its seeds still + /// queued for dispatch and forgetting its caller reply, leak-free. /// /// A driver's supported give-up path (a dropped handle, or a driver-imposed - /// timeout): the join's undelivered result is discarded and the waiter is - /// reaped the instant its exchanges terminate, so neither the join table nor the - /// machine's ignore set leaks. If every exchange has already terminated the - /// waiter is removed immediately (its ignore streams cleared); if a push/pull is - /// still in flight the reply is forgotten but the ignore token is RETAINED until - /// that exchange's terminal (a late merge still suppresses the seed's pre-join - /// user events), then the pump reaps the entry. An unknown or already-consumed - /// handle is a no-op. + /// timeout). Every one of this join's seeds still queued for its per-tick + /// dispatch is removed, so the pump initiates NO further push/pull on its behalf — + /// a cancel BEFORE the first pump therefore has zero network side effect. If no + /// exchange ever started, the entry is reaped immediately. If a push/pull is + /// still in flight the reply is forgotten but that exchange's ignore token is + /// RETAINED until its terminal (a late merge still suppresses the seed's pre-join + /// user events), then the pump reaps the entry — so only actually-started + /// exchanges keep an ignore token, and neither the join table nor the machine's + /// ignore set leaks. An unknown or already-consumed handle is a no-op. pub fn cancel_join(&mut self, handle: JoinId) { let Self { endpoint, + pending_seeds, pending_joins, .. } = self; let Some(pj) = pending_joins.get_mut(&handle) else { return; }; - // Clear the ignore streams + resolve if the exchanges are already done, so an - // immediate removal never strands an ignore-set entry. + // Drop this join's still-queued seeds so the next pump dispatches no push/pull + // for a cancelled join, and zero its undispatched count to match. Only seeds + // whose exchange ALREADY started (recorded in `started`, bound into `pending`) + // survive, scoping the ignore-token retention below to in-flight exchanges. + pending_seeds.retain(|qs| qs.join != handle); + pj.unstarted = 0; + // Clear the ignore streams + resolve if every started exchange is already done + // (a cancel-before-pump has none), so an immediate removal never strands an + // ignore-set entry. try_resolve_join(endpoint, pj); if pj.exchange_work_done() { pending_joins.remove(&handle); } else { - // Still in flight: forget the caller reply (so `poll_join` yields nothing and - // the pump treats it as resolved) while the ignore token stays until the - // exchange terminal, where the pump's fold clears it and reaps the entry. + // A started push/pull is still in flight: forget the caller reply (so + // `poll_join` yields nothing) while the ignore token stays until the exchange + // terminal, where the pump's fold clears it and reaps the entry. pj.reply = JoinReply::Delivered; } } @@ -1298,17 +1311,19 @@ where // 7e. Egress: drain outbound gossip transmits, encode + encrypt, and send. self.drain_gossip_transmits(gossip); - // 7f. Resolve + reap await-result joins whose exchanges all terminated. The - // normal (some seed contacted / failed) path resolves as each - // `ExchangeCompleted` folds in `poll_event`; this sweep catches the joins that - // never accumulate a pending exchange at all — an empty/all-non-routable seed - // set, or seeds that retired before a `Connect` — now that this tick's - // `Connect`s have been captured (phase 7a). A waiter is reaped INDEPENDENTLY of - // caller delivery: once its exchanges have terminated it is dropped even if the - // caller never polled (the undelivered result discarded), so a dropped handle - // leaks neither the join table nor the machine's ignore set. A leave-abandoned - // join whose ignore streams clear only once a post-leave exchange finally - // merges is retired here too. + // 7f. Resolve await-result joins whose exchanges all terminated, then reap only + // those whose result the caller has already retrieved. [`try_resolve_join`] + // clears the machine's ignore set and resolves the caller reply on the exchange + // terminal — INDEPENDENT of caller polling, so the machine never leaks — which + // also catches the joins that never accumulate a pending exchange at all (an + // empty/all-non-routable seed set, or seeds that retired before a `Connect`) now + // that this tick's `Connect`s have been captured (phase 7a). The entry itself is + // reaped ONLY once its reply is `Delivered` (the caller polled it, or + // `cancel_join` forgot it) AND every exchange has terminated, so a + // resolved-but-unpolled result is retained until the caller retrieves it rather + // than dropped out from under a slow/async waiter. A dropped-without-cancel + // handle then lingers as the small result entry alone — its ignore set already + // cleared — so a driver must poll or cancel every join it starts. { let Self { endpoint, @@ -1316,13 +1331,8 @@ where .. } = self; pending_joins.retain(|_, pj| { - // Resolvable BEFORE this sweep (a prior tick's fold, or an earlier sweep)? - // Only then is it reap-eligible; a join `try_resolve_join` resolves for the - // FIRST time in THIS sweep is held one more tick so a caller polling right - // after this pump still sees its outcome. - let was_resolved = !matches!(pj.reply, JoinReply::Pending); try_resolve_join(endpoint, pj); - !(pj.exchange_work_done() && was_resolved) + !(matches!(pj.reply, JoinReply::Delivered) && pj.exchange_work_done()) }); } diff --git a/serf-embedded/src/engine/tests.rs b/serf-embedded/src/engine/tests.rs index 59e2e2a5..9a8aa502 100644 --- a/serf-embedded/src/engine/tests.rs +++ b/serf-embedded/src/engine/tests.rs @@ -1103,13 +1103,14 @@ fn leave_folds_queued_success_before_resolving_join() { ); } -/// A never-polled terminal join does NOT leak: once its (failed) exchange -/// terminates, the pump reaps the waiter and clears its `ignore_old` stream WITHOUT -/// any `poll_join` and without arming a pending-join deadline — the caller simply -/// dropped the handle. Regression for a reap gated on caller delivery -/// (`pending_joins` would linger `Ready` forever). +/// A never-polled terminal join clears its MACHINE state on the exchange terminal +/// but retains the small caller-result entry until delivery/cancel. Once its +/// (failed) exchange terminates the pump clears the `ignore_old` stream WITHOUT any +/// `poll_join` — so the machine's ignore set never leaks — while the resolved-but- +/// undelivered result entry lingers (it is NOT dropped out from under a caller that +/// has not yet retrieved it). `cancel_join` is the give-up that reaps that entry. #[test] -fn dropped_never_polled_join_is_reaped_on_exchange_terminal() { +fn dropped_never_polled_join_clears_machine_ignore_on_terminal_then_cancel_reaps() { let mut engine = make_engine(); let now = Instant::from_origin(Duration::from_secs(86_400)); engine.start(now); @@ -1140,24 +1141,42 @@ fn dropped_never_polled_join_is_reaped_on_exchange_terminal() { "the ignore token is recorded while the exchange is in flight" ); - // Drive to the reap WITHOUT ever polling the join: the driver folds the failed - // completion via poll_event (the caller dropped the handle), then a later pump - // reaps the terminal waiter independent of any delivery. + // Drive the failed exchange to its terminal WITHOUT ever polling the join (the + // caller dropped the handle): fold the completion via poll_event, then pump so the + // end-of-pump resolution clears the machine ignore set. for _ in 0..8 { while engine.poll_event().is_some() {} - if engine.pending_join_count() == 0 { + if !engine.endpoint.test_has_ignore_join_stream(sid) { break; } engine.pump(now, &mut gossip, &mut stream); } + // The MACHINE state is cleared on the exchange terminal — no machine-ignore leak — + // regardless of caller polling. + assert!( + !engine.endpoint.test_has_ignore_join_stream(sid), + "its ignore stream must be cleared on the exchange terminal (no machine leak)" + ); + // The small caller-result entry is RETAINED until delivery/cancel: it resolved + // `Ready` but was never delivered, so the result is not lost by a never-polled + // reap. assert_eq!( engine.pending_join_count(), - 0, - "a never-polled terminal join must be reaped (no pending-join leak)" + 1, + "the resolved-but-undelivered entry is retained until poll_join or cancel_join" ); assert!( - !engine.endpoint.test_has_ignore_join_stream(sid), - "its ignore stream must be cleared on the exchange terminal (no machine leak)" + matches!(engine.pending_joins[&handle].reply, JoinReply::Ready(_)), + "its reply resolved Ready, awaiting delivery" + ); + + // cancel_join is the supported give-up for a dropped handle: it reaps the retained + // terminal entry (its exchanges already terminal, ignore set already cleared). + engine.cancel_join(handle); + assert_eq!( + engine.pending_join_count(), + 0, + "cancel_join reaps the retained terminal entry (no pending-join leak)" ); } @@ -1223,3 +1242,119 @@ fn cancel_join_forgets_in_flight_join_leak_free() { "the ignore token is cleared on the exchange terminal (no machine leak)" ); } + +/// A resolved-but-undelivered join result survives a `pump` that runs between its +/// resolution and the caller's `poll_join`. The push/pull succeeds and its +/// completion is FOLDED (via `poll_event`) into a `Ready(Ok(..))` reply without the +/// caller polling; a further `pump` then runs the end-of-pump join sweep, which +/// must RETAIN the undelivered result. Regression for a one-tick / exchange-work- +/// done reap that dropped the result under a slow/async waiter — reverting it makes +/// the poll below observe `None`. +#[test] +fn join_result_survives_pump_after_resolve_until_polled() { + let mut link = LinkPair::new(&[10, 11], &[20, 21]); + let now = Instant::from_origin(Duration::from_secs(86_400)); + + let handle = link + .a + .join(&[link.b_addr], false, now) + .expect("join announces intent and mints a handle"); + + // Drive the push/pull to its terminal and FOLD the Succeeded completion (via + // poll_event) WITHOUT ever calling poll_join, so the reply resolves to + // `Ready(Ok(..))` yet is not delivered to the caller. + let mut resolved = false; + for _ in 0..40 { + link.step(now); + while link.a.poll_event().is_some() {} + while link.b.poll_event().is_some() {} + if matches!( + link.a.pending_joins.get(&handle).map(|pj| &pj.reply), + Some(JoinReply::Ready(_)) + ) { + resolved = true; + break; + } + } + assert!( + resolved, + "the join must resolve Ready off the folded Succeeded push/pull" + ); + + // Pump A ONCE MORE before the caller polls. The end-of-pump join sweep must + // RETAIN a resolved-but-undelivered result (a one-tick / exchange-work-done reap + // would drop it here, so the poll below would see None). + link.a.pump(now, &mut link.a_gossip, &mut link.a_rel); + + // The successful result is STILL retrievable after the extra pump. + match link.a.poll_join(handle) { + Some(Ok(reached)) => assert!( + reached.contains(&link.b_addr), + "the reached set retained across the extra pump must include B" + ), + other => { + panic!("a resolved join result must survive a pump before the caller polls, got {other:?}") + } + } + assert_eq!( + link.a.pending_join_count(), + 0, + "the delivered join is reaped once the caller retrieves its result" + ); +} + +/// A `join` immediately cancelled BEFORE any pump dispatches no push/pull: its +/// queued seed is removed, so the pump initiates nothing for the handle, and the +/// waiter is reaped at once with zero network side effect. Regression for a +/// `cancel_join` that left un-dispatched seeds queued — reverting it lets the pump +/// dial the cancelled seed (it parks as a PendingDial) and the join reappear. +#[test] +fn cancel_before_pump_dispatches_no_seed_and_reaps() { + let mut engine = make_engine(); + let now = Instant::from_origin(Duration::from_secs(86_400)); + engine.start(now); + // A listener but an empty dial pool: a dispatched seed's Connect parks as + // PendingDial, so `pending_dial_count` is a precise "a push/pull was started" + // probe. + engine.set_listener(9); + + let handle = engine + .join(&[node_addr(7002)], /*ignore_old*/ true, now) + .expect("join announces intent and mints a handle"); + assert_eq!(engine.pending_seeds.len(), 1, "the routable seed is queued"); + assert_eq!( + engine.pending_join_count(), + 1, + "the in-flight join is tracked" + ); + + // Cancel BEFORE any pump: the queued seed is removed and — no exchange having + // started — the waiter is reaped immediately. + engine.cancel_join(handle); + assert!( + engine.pending_seeds.is_empty(), + "cancel_join must drop the never-dispatched seed so the pump initiates no push/pull" + ); + assert_eq!( + engine.pending_join_count(), + 0, + "a cancel-before-pump forgets everything and reaps the waiter immediately" + ); + + // The pump now dispatches nothing for that handle: no push/pull is started, so no + // exchange parks as PendingDial (a buggy cancel that left the seed queued would + // dial it here). + let mut gossip = NoGossip; + let mut stream = NoStream::with_pool(0); + engine.pump(now, &mut gossip, &mut stream); + assert_eq!( + engine.pending_dial_count(), + 0, + "no push/pull may be dispatched for a seed cancelled before the pump" + ); + assert_eq!( + engine.pending_join_count(), + 0, + "no join reappears after the pump" + ); +} From 4ab09436219aa74b842381ace3e78f76a28d861b Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 3 Jul 2026 22:44:35 +1200 Subject: [PATCH 048/128] fix(serf-embedded): fold join completions in the pump, not poll_event Join completion folding + ignore cleanup were gated on the app draining poll_event, so poll_join could hang after an exchange completed. The pump now drains+folds every ExchangeCompleted into its join (resolve + clear ignore on the terminal) and buffers all events for poll_event, so resolution/cleanup are pump-driven + terminal-gated like serf-reactor, independent of the app consumer. Audited every path; +3 fail-on-revert tests. --- serf-embedded/src/engine/mod.rs | 135 ++++++++------ serf-embedded/src/engine/tests.rs | 280 ++++++++++++++++++++++++------ 2 files changed, 306 insertions(+), 109 deletions(-) diff --git a/serf-embedded/src/engine/mod.rs b/serf-embedded/src/engine/mod.rs index 7d72497a..7b08bc88 100644 --- a/serf-embedded/src/engine/mod.rs +++ b/serf-embedded/src/engine/mod.rs @@ -400,10 +400,12 @@ where /// set from the terminal `ExchangeCompleted` of the push/pulls it dispatched; /// [`poll_join`](Self::poll_join) drains a resolved one. pending_joins: HashMap, - /// Machine events [`leave`](Self::leave) drained (and folded into the pending - /// joins) BEFORE resolving its abandonment, held so [`poll_event`](Self::poll_event) - /// still hands them to the driver in order — already folded, never re-folded. - /// Empty outside a `leave` that raced queued completions. + /// Machine events the pump (and [`leave`](Self::leave)) drained and folded into + /// the pending joins, held so [`poll_event`](Self::poll_event) hands them to the + /// driver in order — already folded, never re-folded. Every `pump` drains the + /// machine's event queue to here, folding each `ExchangeCompleted` into its join + /// BEFORE buffering the event, so join accounting is pump-driven and never waits + /// on the app draining events; `poll_event` is a pure drain of this queue. buffered_events: VecDeque>, /// Monotonic allocator for [`JoinId`]s, so two concurrent joins never collide. next_join_id: u64, @@ -801,33 +803,24 @@ where self.endpoint.members_snapshot() } - /// Drain one application-visible serf event, if any. + /// Drain one application-visible serf event buffered by the last `pump`, if any. /// - /// Returns events emitted by the machine during the last `pump` tick — the full - /// serf surface: membership changes, user events, queries, query responses / - /// acks, key-management requests / responses, reliable-exchange completions, - /// and the lifecycle signals (`LeftCluster`, conflict `Shutdown`). Returns - /// `None` when the event queue is empty; call again after the next `pump` tick. + /// Each [`pump`](Self::pump) drains the machine's event queue to quiescence, + /// folding every push/pull `ExchangeCompleted` into its await-result join and + /// buffering every event — the full serf surface: membership changes, user + /// events, queries, query responses / acks, key-management requests / responses, + /// reliable-exchange completions, and the lifecycle signals (`LeftCluster`, + /// conflict `Shutdown`). This call hands those buffered events to the driver in + /// order, exactly once; it does NOT re-fold (the pump already folded). Returns + /// `None` when the buffer is empty; call again after the next `pump` tick. /// - /// A driver awaiting a [`join`](Self::join) result MUST drain events here before - /// checking [`poll_join`](Self::poll_join): a push/pull `ExchangeCompleted` is - /// folded into its await-result join AS IT surfaces here (the reached-set - /// accumulation), and `poll_join` resolves off that folded state. - /// - /// Events a [`leave`](Self::leave) drained early — to fold already-queued - /// completions before resolving its abandonment — are delivered here first, in - /// order and already folded, ahead of any newer machine event. + /// Join resolution and ignore cleanup are driven by the pump on the exchange + /// terminal, so a driver need NOT drain events here before + /// [`poll_join`](Self::poll_join) — `poll_join` resolves off the pump-folded + /// state. The contract is the reactor-faithful "pump, then poll_join / poll_event". #[inline] pub fn poll_event(&mut self) -> Option> { - // A `leave` that raced queued completions has already drained + folded them - // into the pending joins; hand those buffered events to the driver first - // (NEVER re-folding), ahead of any newer machine event. - if let Some(ev) = self.buffered_events.pop_front() { - return Some(ev); - } - let ev = self.endpoint.poll_event()?; - self.fold_join_completion(&ev); - Some(ev) + self.buffered_events.pop_front() } /// Fold one machine event into the await-result join it terminates, if any. @@ -836,9 +829,9 @@ where /// `Connect` (via the START `StreamId`) is removed from that join's `pending` /// and — on `Succeeded` — accumulates the peer into `contacted`; then /// [`try_resolve_join`] resolves the join (clearing any ignore streams) the - /// instant `pending` empties. A no-op for every other event. Shared by - /// [`poll_event`](Self::poll_event) and [`leave`](Self::leave)'s - /// pre-resolution drain so both fold identically. + /// instant `pending` empties. A no-op for every other event. Called by + /// [`drain_fold_events`](Self::drain_fold_events) for every drained event, so the + /// pump and [`leave`](Self::leave) fold identically. fn fold_join_completion(&mut self, ev: &Event) { let Event::ExchangeCompleted(ec) = ev else { return; @@ -863,6 +856,24 @@ where } } + /// Drain the machine's event queue to quiescence, folding each event into its + /// await-result join ([`fold_join_completion`](Self::fold_join_completion)) and + /// buffering it for [`poll_event`](Self::poll_event) delivery. + /// + /// The SOLE drain of the endpoint's event queue. The pump calls it every tick, so + /// join accounting is pump-driven and never gated on the app polling events; and + /// [`leave`](Self::leave) calls it before resolving its abandonment, so an + /// already-succeeded push/pull lands in the reached set. Because it empties the + /// endpoint queue, a later call folds only the events enqueued since — never + /// re-folding one already buffered — and `buffered_events` preserves the app's + /// event stream in arrival order. + fn drain_fold_events(&mut self) { + while let Some(ev) = self.endpoint.poll_event() { + self.fold_join_completion(&ev); + self.buffered_events.push_back(ev); + } + } + /// Announce the local node's join intent and begin an await-result join to /// these seeds, returning a [`JoinId`] the driver polls via /// [`poll_join`](Self::poll_join). @@ -936,7 +947,8 @@ where /// Returns `Some(Ok(reached))` with the address set the join contacted, or /// `Some(Err(JoinFailed))` if every dispatched push/pull terminated without /// contacting a seed. `None` means the join has not yet resolved — poll again - /// after the next `pump` + [`poll_event`](Self::poll_event) drain. The outcome + /// after the next `pump` (which folds the terminal `ExchangeCompleted` into the + /// join; no [`poll_event`](Self::poll_event) drain is required first). The outcome /// is delivered EXACTLY ONCE: a second poll of the same handle yields `None`. /// This call is the DELIVERY that lets the entry be reaped: once the caller has /// retrieved the result AND every dispatched exchange has terminated the entry is @@ -971,9 +983,12 @@ where /// exchange ever started, the entry is reaped immediately. If a push/pull is /// still in flight the reply is forgotten but that exchange's ignore token is /// RETAINED until its terminal (a late merge still suppresses the seed's pre-join - /// user events), then the pump reaps the entry — so only actually-started - /// exchanges keep an ignore token, and neither the join table nor the machine's - /// ignore set leaks. An unknown or already-consumed handle is a no-op. + /// user events), then the pump — which folds the terminal `ExchangeCompleted` — + /// reaps the entry and clears the token. So only actually-started exchanges keep + /// an ignore token, and neither the join table nor the machine's ignore set leaks. + /// Like [`poll_join`](Self::poll_join) this reflects the pump-folded state (the + /// reactor-faithful "pump, then cancel_join" contract) and does not itself drain + /// the machine. An unknown or already-consumed handle is a no-op. pub fn cancel_join(&mut self, handle: JoinId) { let Self { endpoint, @@ -1015,11 +1030,12 @@ where /// [`Event::LeftCluster`] via [`poll_event`](Self::poll_event) — the pump /// initiates no further push/pull, so any queued seeds are dropped and every /// in-flight await-result join is handed its caller reply once from its reached - /// set. Any push/pull `ExchangeCompleted` ALREADY QUEUED in the endpoint (a - /// success the driver has not yet drained via `poll_event`) is folded into that - /// reached set FIRST, so a race between a successful join and `leave` resolves - /// `Ok(reached)` rather than a stale `JoinFailed`; the drained events are - /// buffered and still delivered, in order, by the next `poll_event`. The + /// set. Any push/pull `ExchangeCompleted` still queued in the endpoint that the + /// pump has not already folded — a `leave` without a fresh `pump`, or a completion + /// `endpoint.leave()` itself enqueues — is folded into that reached set FIRST (via + /// [`drain_fold_events`](Self::drain_fold_events)), so a successful join resolves + /// `Ok(reached)` rather than a stale `JoinFailed`; the drained events are buffered + /// and still delivered, in order, by the next `poll_event`. The /// `ignore_old` cleanup is deliberately NOT run here: it stays gated on the /// exchange terminal ([`try_resolve_join`], driven by the pump fold), so a late /// successful push/pull that merges after leave still finds its ignore token in @@ -1030,17 +1046,15 @@ where // the join state must stay untouched too. self.endpoint.leave(now)?; - // Fold every ALREADY-QUEUED machine completion into its await-result join - // BEFORE computing any abandonment outcome, so a push/pull that already - // succeeded — its `ExchangeCompleted` queued but not yet drained by the driver - // — lands in the reached set rather than being frozen out as a stale - // `JoinFailed`. The drained events are buffered for `poll_event` (already - // folded, delivered in order), preserving delivery. Mirrors serf-reactor's - // shutdown/leave, which drains-and-folds before it reaps. - while let Some(ev) = self.endpoint.poll_event() { - self.fold_join_completion(&ev); - self.buffered_events.push_back(ev); - } + // Fold every still-queued machine completion into its await-result join BEFORE + // computing any abandonment outcome, so a push/pull that already succeeded — its + // `ExchangeCompleted` not yet folded by a pump — lands in the reached set rather + // than being frozen out as a stale `JoinFailed`. The pump normally folds these + // each tick; this covers a `leave` without a fresh `pump` and any completion + // `endpoint.leave()` just enqueued. The drained events are buffered for + // `poll_event` (already folded, delivered in order), preserving delivery. + // Mirrors serf-reactor's shutdown/leave, which drains-and-folds before it reaps. + self.drain_fold_events(); // Accepted: no seed dispatches once leaving, so drop the queue and mark every // still-pending join fully dispatched, then deliver each still-pending caller @@ -1206,6 +1220,9 @@ where /// 7a–7e. Drain `poll_action`, promote, pump outbound, flush deferred FINs, /// complete `Closing` drains, re-rebalance, then drain + send outbound /// gossip. + /// 7f. Drain + fold machine events: fold every push/pull `ExchangeCompleted` + /// into its await-result join and buffer every event for `poll_event`. + /// 7g. Resolve + reap await-result joins on the exchange terminal. /// 8. Deadline: `min(machine_next, closing_next)`. pub fn pump(&mut self, now: Instant, gossip: &mut GI, stream: &mut S) -> Option where @@ -1311,15 +1328,25 @@ where // 7e. Egress: drain outbound gossip transmits, encode + encrypt, and send. self.drain_gossip_transmits(gossip); - // 7f. Resolve await-result joins whose exchanges all terminated, then reap only + // 7f. Drain the machine's event queue to quiescence, folding every push/pull + // `ExchangeCompleted` into its await-result join (reducing `pending`, + // accumulating the reached peer on `Succeeded`) and buffering every drained + // event for later `poll_event` delivery. Folding HERE — in the pump, after this + // tick's `Connect`s bound their exchanges (phase 7a) — is what makes join + // resolution and ignore cleanup terminal-gated and INDEPENDENT of whether the + // app ever drains `poll_event`, matching serf-reactor (which folds in its poll + // loop before the observation hand-off). + self.drain_fold_events(); + + // 7g. Resolve await-result joins whose exchanges all terminated, then reap only // those whose result the caller has already retrieved. [`try_resolve_join`] // clears the machine's ignore set and resolves the caller reply on the exchange // terminal — INDEPENDENT of caller polling, so the machine never leaks — which // also catches the joins that never accumulate a pending exchange at all (an // empty/all-non-routable seed set, or seeds that retired before a `Connect`) now - // that this tick's `Connect`s have been captured (phase 7a). The entry itself is - // reaped ONLY once its reply is `Delivered` (the caller polled it, or - // `cancel_join` forgot it) AND every exchange has terminated, so a + // that this tick's `Connect`s have been captured (phase 7a) and folded (phase + // 7f). The entry itself is reaped ONLY once its reply is `Delivered` (the caller + // polled it, or `cancel_join` forgot it) AND every exchange has terminated, so a // resolved-but-unpolled result is retained until the caller retrieves it rather // than dropped out from under a slow/async waiter. A dropped-without-cancel // handle then lingers as the small result entry alone — its ignore set already diff --git a/serf-embedded/src/engine/tests.rs b/serf-embedded/src/engine/tests.rs index 9a8aa502..4bfb47c8 100644 --- a/serf-embedded/src/engine/tests.rs +++ b/serf-embedded/src/engine/tests.rs @@ -1026,13 +1026,16 @@ fn refused_leave_leaves_join_state_untouched() { // ── fold-before-leave / leak-free reap / cancel_join ────────────────────────── -/// `leave()` folds an ALREADY-QUEUED successful push/pull completion into the -/// await-result join BEFORE computing its abandonment, so a join that succeeded on -/// the wire but whose `ExchangeCompleted(Succeeded)` the driver had not yet drained -/// resolves `Ok(reached)` — never a stale `JoinFailed`. Regression for a `leave` -/// that froze the reply from the pre-fold (empty) reached set. +/// A push/pull driven to `Succeeded` purely by pumps — never `poll_join`'d, with +/// A's events never drained — is folded and resolved `Ready(Ok)` BY THE PUMP; a +/// subsequent `leave()` then preserves that result (it neither re-folds nor +/// clobbers it) while the buffered events still deliver in order, exactly once. So +/// the caller gets `Ok(reached)` including B across a leave. Regression both for a +/// pump that failed to fold (→ the reply stays Pending and leave would freeze a +/// stale `JoinFailed`) and for a leave that double-folds or loses the buffered +/// events. #[test] -fn leave_folds_queued_success_before_resolving_join() { +fn pump_folds_success_then_leave_preserves_it() { let mut link = LinkPair::new(&[10, 11], &[20, 21]); let now = Instant::from_origin(Duration::from_secs(86_400)); @@ -1041,11 +1044,10 @@ fn leave_folds_queued_success_before_resolving_join() { .join(&[link.b_addr], false, now) .expect("join announces intent and mints a handle"); - // Drive ONLY the shared reliable fabric (pumping both engines ferries it - // automatically) and NEVER the gossip relay, so A learns B SOLELY through the - // push/pull — making `num_members()==2` a precise "the push/pull merged" signal. - // A's events are left UNDRAINED, so the terminal `ExchangeCompleted(Succeeded)` - // sits queued-and-unfolded in A's machine when A leaves. + // Drive ONLY the shared reliable fabric (pumping both engines ferries it) and + // NEVER the gossip relay, so A learns B SOLELY through the push/pull — making + // `num_members()==2` a precise "the push/pull merged" signal. A's events are left + // UNDRAINED, so the PUMP alone folds the terminal `ExchangeCompleted(Succeeded)`. for _ in 0..40 { link.a.pump(now, &mut link.a_gossip, &mut link.a_rel); link.b.pump(now, &mut link.b_gossip, &mut link.b_rel); @@ -1063,38 +1065,46 @@ fn leave_folds_queued_success_before_resolving_join() { 2, "the push/pull merged B into A (the exchange completed at the machine level)" ); + // The PUMP folded the Succeeded completion with no poll_join and no A poll_event: + // the reply resolved Ready with B in the reached set and the exchange is done. { let pj = &link.a.pending_joins[&handle]; assert!( - matches!(pj.reply, JoinReply::Pending), - "the completion is queued but NOT yet folded/resolved" + matches!(pj.reply, JoinReply::Ready(_)), + "the pump folded + resolved the success (no poll_event required)" ); assert!( - pj.contacted.is_empty(), - "contacted is empty until the queued completion folds" + pj.contacted.contains(&link.b_addr), + "the pump-folded reached set includes B" ); assert!( - !pj.pending.is_empty(), - "the exchange is still in the join's pending set (unfolded)" + pj.pending.is_empty(), + "the exchange terminal emptied pending in the pump" ); } - // leave folds the queued Succeeded completion BEFORE its abandonment, so the - // caller gets Ok(reached) including B — the buggy leave froze a stale JoinFailed - // from the pre-fold (empty) reached set. + // leave preserves the pump-resolved reply (it neither re-folds nor clobbers it) + // and never loses the buffered events. link .a .leave(now) .expect("leave from a running node succeeds"); - // The buffered (already-folded) events still deliver in order. + // The buffered (already-folded) events still deliver, in order and exactly once. while link.a.poll_event().is_some() {} match link.a.poll_join(handle) { - Some(Ok(reached)) => assert!( - reached.contains(&link.b_addr), - "the folded reached set must include B" - ), - other => panic!("expected Ok(reached) including B (fold-before-leave), got {other:?}"), + Some(Ok(reached)) => { + assert!( + reached.contains(&link.b_addr), + "the folded reached set must include B" + ); + assert_eq!( + reached.len(), + 1, + "B is folded exactly once (no double-fold across the pump + leave)" + ); + } + other => panic!("expected Ok(reached) including B, got {other:?}"), } assert_eq!( link.a.pending_join_count(), @@ -1103,12 +1113,16 @@ fn leave_folds_queued_success_before_resolving_join() { ); } -/// A never-polled terminal join clears its MACHINE state on the exchange terminal -/// but retains the small caller-result entry until delivery/cancel. Once its -/// (failed) exchange terminates the pump clears the `ignore_old` stream WITHOUT any -/// `poll_join` — so the machine's ignore set never leaks — while the resolved-but- -/// undelivered result entry lingers (it is NOT dropped out from under a caller that -/// has not yet retrieved it). `cancel_join` is the give-up that reaps that entry. +/// A never-polled terminal join clears its MACHINE state IN THE PUMP on the +/// exchange terminal, retaining only the small caller-result entry until +/// delivery/cancel. The seed's dial fails synchronously inside the pump, so that +/// same pump folds the terminal `ExchangeCompleted(Failed)` and clears the recorded +/// `ignore_old` stream — with NO `poll_event` and NO `poll_join` — so the machine's +/// ignore set never leaks, while the resolved-but-undelivered result entry lingers +/// (it is NOT dropped out from under a caller that has not yet retrieved it). +/// `cancel_join` is the give-up that reaps that entry. Reverting the pump fold +/// leaves the token recorded after the pump (nothing drained events to fold it) and +/// the reply never resolves. #[test] fn dropped_never_polled_join_clears_machine_ignore_on_terminal_then_cancel_reaps() { let mut engine = make_engine(); @@ -1124,8 +1138,9 @@ fn dropped_never_polled_join_clears_machine_ignore_on_terminal_then_cancel_reaps let mut gossip = NoGossip; let mut stream = NoStream::with_pool(0); // One pump dispatches the seed (recording its ignore stream), captures the - // Connect, and fails the dial (`NoStream::connect` errors) — queuing the terminal - // ExchangeCompleted(Failed). + // Connect, fails the dial (`NoStream::connect` errors) — queuing the terminal + // ExchangeCompleted(Failed) — AND folds that completion in-pump, clearing the + // ignore stream. No poll_event / poll_join is ever called. engine.pump(now, &mut gossip, &mut stream); let sid = { let pj = &engine.pending_joins[&handle]; @@ -1136,30 +1151,15 @@ fn dropped_never_polled_join_clears_machine_ignore_on_terminal_then_cancel_reaps ); *pj.started.iter().next().expect("one started stream") }; - assert!( - engine.endpoint.test_has_ignore_join_stream(sid), - "the ignore token is recorded while the exchange is in flight" - ); - - // Drive the failed exchange to its terminal WITHOUT ever polling the join (the - // caller dropped the handle): fold the completion via poll_event, then pump so the - // end-of-pump resolution clears the machine ignore set. - for _ in 0..8 { - while engine.poll_event().is_some() {} - if !engine.endpoint.test_has_ignore_join_stream(sid) { - break; - } - engine.pump(now, &mut gossip, &mut stream); - } - // The MACHINE state is cleared on the exchange terminal — no machine-ignore leak — - // regardless of caller polling. + // The MACHINE state is cleared IN THE PUMP on the exchange terminal — no + // machine-ignore leak — with no caller polling at all. assert!( !engine.endpoint.test_has_ignore_join_stream(sid), - "its ignore stream must be cleared on the exchange terminal (no machine leak)" + "the pump must clear the ignore stream on the exchange terminal (no machine leak)" ); // The small caller-result entry is RETAINED until delivery/cancel: it resolved - // `Ready` but was never delivered, so the result is not lost by a never-polled - // reap. + // `Ready` in the pump but was never delivered, so the result is not lost by a + // never-polled reap. assert_eq!( engine.pending_join_count(), 1, @@ -1167,7 +1167,7 @@ fn dropped_never_polled_join_clears_machine_ignore_on_terminal_then_cancel_reaps ); assert!( matches!(engine.pending_joins[&handle].reply, JoinReply::Ready(_)), - "its reply resolved Ready, awaiting delivery" + "its reply resolved Ready in the pump, awaiting delivery" ); // cancel_join is the supported give-up for a dropped handle: it reaps the retained @@ -1358,3 +1358,173 @@ fn cancel_before_pump_dispatches_no_seed_and_reaps() { "no join reappears after the pump" ); } + +// ── pump-driven join resolution (fold in the pump, not poll_event) ──────────── + +/// A successful join resolves via the PUMP alone: drive the push/pull to +/// `ExchangeCompleted(Succeeded)` and poll the join after each pump WITHOUT ever +/// calling A's `poll_event`. The pump folds the completion into the join, so +/// `poll_join` returns `Ok(reached)` including B. This is the core of the fix — +/// reverting the pump fold (folding only in `poll_event`) makes `poll_join` return +/// `None` forever here, since A's events are never drained. +#[test] +fn pump_without_poll_event_resolves_successful_join() { + let mut link = LinkPair::new(&[10, 11], &[20, 21]); + let now = Instant::from_origin(Duration::from_secs(86_400)); + + let handle = link + .a + .join(&[link.b_addr], false, now) + .expect("join announces intent and mints a handle"); + + let mut outcome = None; + for _ in 0..40 { + // Pump both engines and ferry gossip/fabric. Drain ONLY B's events; A's + // `poll_event` is NEVER called, so only the pump can fold A's join. + link.step(now); + while link.b.poll_event().is_some() {} + if let Some(res) = link.a.poll_join(handle) { + outcome = Some(res); + break; + } + } + + match outcome { + Some(Ok(reached)) => assert!( + reached.contains(&link.b_addr), + "the reached set folded by the pump must include B" + ), + other => { + panic!("the pump alone must resolve the join Ok(reached) with no poll_event, got {other:?}") + } + } + assert_eq!( + link.a.pending_join_count(), + 0, + "the resolved + delivered join is reaped" + ); +} + +/// A cancelled in-flight join whose started exchange later reaches +/// `ExchangeCompleted` is reaped — entry AND ignore token — by the PUMP alone, with +/// A's `poll_event` never drained. `cancel_join` forgets the reply while the +/// exchange is in flight; driving it to its terminal via pumps then clears the +/// ignore token and reaps the waiter in-pump. Reverting the pump fold leaks both +/// (the terminal never folds without a `poll_event`). +#[test] +fn cancelled_in_flight_join_reaped_by_pump_without_poll_event() { + let mut link = LinkPair::new(&[10, 11], &[20, 21]); + let now = Instant::from_origin(Duration::from_secs(86_400)); + + let handle = link + .a + .join(&[link.b_addr], /*ignore_old*/ true, now) + .expect("join announces intent and mints a handle"); + // One step dispatches the seed and binds the exchange IN FLIGHT. + link.step(now); + let sid = { + let pj = &link.a.pending_joins[&handle]; + assert!( + !pj.pending.is_empty(), + "its push/pull exchange is bound and in flight" + ); + *pj.started.iter().next().expect("one started stream") + }; + assert!( + link.a.endpoint.test_has_ignore_join_stream(sid), + "the ignore token is recorded while the exchange is in flight" + ); + + // The caller gives up mid-flight: cancel forgets the reply but RETAINS the token. + link.a.cancel_join(handle); + assert!( + link.a.endpoint.test_has_ignore_join_stream(sid), + "cancel_join retains the token while the exchange can still merge" + ); + + // Drive the exchange to its terminal via pumps, NEVER draining A's events. The + // pump folds the terminal completion, clears the token, and reaps the waiter. + for _ in 0..40 { + link.step(now); + while link.b.poll_event().is_some() {} + if link.a.pending_join_count() == 0 { + break; + } + } + assert_eq!( + link.a.pending_join_count(), + 0, + "the cancelled join is reaped by the pump (no pending-join leak, no poll_event)" + ); + assert!( + !link.a.endpoint.test_has_ignore_join_stream(sid), + "the ignore token is cleared by the pump on the exchange terminal (no machine leak)" + ); +} + +/// After the pump folds a join's completion, `poll_event` STILL delivers every +/// event — the push/pull `ExchangeCompleted` and the membership changes — exactly +/// once. Folding in the pump must not consume, drop, or duplicate the app's event +/// stream: the completion is buffered (not swallowed by the fold) and the +/// membership events flow through the same buffer. Regression for a fold that +/// consumed the event or a buffer that dropped/duplicated it. +#[test] +fn poll_event_delivers_all_events_exactly_once_after_pump_fold() { + let mut link = LinkPair::new(&[10, 11], &[20, 21]); + let now = Instant::from_origin(Duration::from_secs(86_400)); + + let handle = link + .a + .join(&[link.b_addr], false, now) + .expect("join announces intent and mints a handle"); + + let mut a_events: std::vec::Vec> = std::vec::Vec::new(); + let mut resolved: Option> = None; + for _ in 0..40 { + link.step(now); + while link.b.poll_event().is_some() {} + // The pump already folded any terminal completion; draining here collects the + // events the app observes — the completion MUST still be among them. + while let Some(ev) = link.a.poll_event() { + a_events.push(ev); + } + if resolved.is_none() + && let Some(res) = link.a.poll_join(handle) + { + resolved = Some(res); + } + // Stop once the join resolved AND its completion has been delivered to the app. + if resolved.is_some() + && a_events + .iter() + .any(|ev| matches!(ev, Event::ExchangeCompleted(ec) if ec.kind() == ExchangeKind::PushPull)) + { + break; + } + } + + // The join resolved Ok off the pump-folded state. + match &resolved { + Some(Ok(reached)) => assert!( + reached.contains(&link.b_addr), + "the join resolved Ok(reached) including B" + ), + other => panic!("the join must resolve Ok(reached), got {other:?}"), + } + + // The push/pull ExchangeCompleted was delivered to the app EXACTLY ONCE, even + // though the pump folded it (the fold buffers, never consumes). + let completions = a_events + .iter() + .filter(|ev| matches!(ev, Event::ExchangeCompleted(ec) if ec.kind() == ExchangeKind::PushPull)) + .count(); + assert_eq!( + completions, 1, + "the push/pull ExchangeCompleted must be delivered via poll_event exactly once" + ); + // Membership changes flow through the same buffer (A learned B). + assert!( + a_events.iter().any(|ev| matches!(ev, Event::Member(_))), + "membership events must also be delivered through the buffer" + ); +} From e66f5d132fe39434021de82b43eec0e717217b2b Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 3 Jul 2026 23:15:46 +1200 Subject: [PATCH 049/128] fix(serf-embedded): bound the pump event buffer with drop+counter The pump buffered every endpoint event for poll_event with no bound, so a pump-then-poll_join driver that never drains poll_event grew buffered_events until OOM on a long-running embedded node. Bound it at DEFAULT_EVENT_BUFFER_CAP (1024) with a best-effort drop-oldest + events_dropped counter (accessor), mirroring memberlist-embassy's app_events cap and the serf drivers' load-shed counters. Every event including ExchangeCompleted stays app-facing to match serf-reactor/serf-compio, which surface it after the internal join fold; join completions are folded before buffering so a shed app-event never affects resolution. Regression asserts the bounded backlog + events_dropped with the join still Ok. --- serf-embedded/src/engine/mod.rs | 76 +++++++++++++++++++++++++++++-- serf-embedded/src/engine/tests.rs | 71 +++++++++++++++++++++++++++++ serf-embedded/src/lib.rs | 2 +- 3 files changed, 145 insertions(+), 4 deletions(-) diff --git a/serf-embedded/src/engine/mod.rs b/serf-embedded/src/engine/mod.rs index 7b08bc88..192c3104 100644 --- a/serf-embedded/src/engine/mod.rs +++ b/serf-embedded/src/engine/mod.rs @@ -88,6 +88,19 @@ fn gossip_recv_buf_size(gossip_mtu: usize) -> usize { (gossip_mtu + ENCRYPTED_WRAPPER_OVERHEAD).max(1500) } +/// Cap on the application-event backlog awaiting +/// [`poll_event`](SerfEngine::poll_event). +/// +/// A driver using the supported pump-then-[`poll_join`](SerfEngine::poll_join) +/// flow that never drains [`poll_event`](SerfEngine::poll_event) must not grow that +/// queue without bound. At the cap the OLDEST buffered event is dropped +/// (best-effort, freshest-wins) and counted in +/// [`events_dropped`](SerfEngine::events_dropped), so app-event delivery is lossy +/// under sustained overload while join accounting — folded BEFORE buffering — is +/// never affected. Mirrors memberlist-embassy's bounded `app_events` queue (same +/// cap) and the serf std drivers' load-shed counters. +pub const DEFAULT_EVENT_BUFFER_CAP: usize = 1024; + /// An [`AliveDelegate`] that admits a peer only when its advertised address is a /// routable destination ([`socket_addr_is_routable`]). /// @@ -406,7 +419,19 @@ where /// machine's event queue to here, folding each `ExchangeCompleted` into its join /// BEFORE buffering the event, so join accounting is pump-driven and never waits /// on the app draining events; `poll_event` is a pure drain of this queue. + /// + /// Bounded at [`DEFAULT_EVENT_BUFFER_CAP`]: at the cap the oldest event is dropped + /// and counted in `events_dropped`, so a driver that never drains + /// [`poll_event`](Self::poll_event) cannot grow this queue without limit (which + /// would exhaust memory on a long-running embedded node). buffered_events: VecDeque>, + /// Count of application events shed from `buffered_events` because the driver + /// never drained [`poll_event`](Self::poll_event) fast enough and the backlog hit + /// [`DEFAULT_EVENT_BUFFER_CAP`]. Surfaced via + /// [`events_dropped`](Self::events_dropped) so the best-effort loss is observable + /// (mirroring the serf std drivers' load-shed counter). Join completions are + /// folded before buffering, so a shed app-event never affects join resolution. + events_dropped: u64, /// Monotonic allocator for [`JoinId`]s, so two concurrent joins never collide. next_join_id: u64, /// Cluster label applied to the gossip codec on both encode and decode. When @@ -590,6 +615,7 @@ where pending_seeds: VecDeque::new(), pending_joins: HashMap::new(), buffered_events: VecDeque::new(), + events_dropped: 0, next_join_id: 0, label, cidr_policy, @@ -818,11 +844,33 @@ where /// terminal, so a driver need NOT drain events here before /// [`poll_join`](Self::poll_join) — `poll_join` resolves off the pump-folded /// state. The contract is the reactor-faithful "pump, then poll_join / poll_event". + /// + /// App-event delivery is BEST-EFFORT: the backlog is bounded at + /// [`DEFAULT_EVENT_BUFFER_CAP`], so a driver that pumps and resolves joins but + /// stops draining `poll_event` sheds the oldest surplus events (counted in + /// [`events_dropped`](Self::events_dropped)) rather than growing memory without + /// bound — matching the serf std drivers' lossy observation channel. Join + /// resolution is unaffected (completions are folded before buffering). #[inline] pub fn poll_event(&mut self) -> Option> { self.buffered_events.pop_front() } + /// The number of application events shed from the `poll_event` backlog because it + /// reached [`DEFAULT_EVENT_BUFFER_CAP`] before the driver drained them. + /// + /// App-event delivery is BEST-EFFORT under sustained overload: a driver that + /// pumps and resolves joins but never drains [`poll_event`](Self::poll_event) + /// sheds the oldest surplus events rather than growing memory without bound, and + /// each shed increments this counter (mirroring the serf std drivers' + /// `events_dropped`). Join resolution is unaffected — completions are folded + /// before buffering — so a nonzero count means only that some observational + /// events were not delivered, never that a join was mis-resolved. + #[inline] + pub fn events_dropped(&self) -> u64 { + self.events_dropped + } + /// Fold one machine event into the await-result join it terminates, if any. /// /// A push/pull `ExchangeCompleted` whose `eid` was bound to a join at its @@ -856,6 +904,24 @@ where } } + /// Buffer one already-folded machine event for [`poll_event`](Self::poll_event), + /// bounding the backlog at [`DEFAULT_EVENT_BUFFER_CAP`]. + /// + /// At the cap the OLDEST buffered event is dropped (freshest-wins, so a + /// never-draining driver keeps the most recent surface) and counted in + /// `events_dropped`, so the queue cannot grow without bound and the loss stays + /// observable. The dropped copy is purely the app-delivery one — the event's join + /// completion was already folded by [`drain_fold_events`](Self::drain_fold_events) + /// BEFORE this call — so a drop never affects join resolution. Mirrors + /// memberlist-embassy's bounded `app_events` (drop-oldest on overflow). + fn push_app_event(&mut self, ev: Event) { + if self.buffered_events.len() >= DEFAULT_EVENT_BUFFER_CAP { + self.buffered_events.pop_front(); + self.events_dropped += 1; + } + self.buffered_events.push_back(ev); + } + /// Drain the machine's event queue to quiescence, folding each event into its /// await-result join ([`fold_join_completion`](Self::fold_join_completion)) and /// buffering it for [`poll_event`](Self::poll_event) delivery. @@ -865,12 +931,16 @@ where /// [`leave`](Self::leave) calls it before resolving its abandonment, so an /// already-succeeded push/pull lands in the reached set. Because it empties the /// endpoint queue, a later call folds only the events enqueued since — never - /// re-folding one already buffered — and `buffered_events` preserves the app's - /// event stream in arrival order. + /// re-folding one already buffered. The join completion is folded FIRST, then the + /// event is buffered via [`push_app_event`](Self::push_app_event), which bounds the + /// backlog at [`DEFAULT_EVENT_BUFFER_CAP`] (dropping + counting the oldest on + /// overflow) so a driver that never drains `poll_event` cannot grow it without + /// bound; `buffered_events` preserves the app's event stream in arrival order up to + /// that cap. fn drain_fold_events(&mut self) { while let Some(ev) = self.endpoint.poll_event() { self.fold_join_completion(&ev); - self.buffered_events.push_back(ev); + self.push_app_event(ev); } } diff --git a/serf-embedded/src/engine/tests.rs b/serf-embedded/src/engine/tests.rs index 4bfb47c8..2394da9b 100644 --- a/serf-embedded/src/engine/tests.rs +++ b/serf-embedded/src/engine/tests.rs @@ -1528,3 +1528,74 @@ fn poll_event_delivers_all_events_exactly_once_after_pump_fold() { "membership events must also be delivered through the buffer" ); } + +// ── bounded app-event backlog (no unbounded growth) ─────────────────────────── + +/// A driver that pumps and resolves joins but NEVER drains `poll_event` must not +/// grow the app-event backlog without bound: `buffered_events` is capped at +/// [`DEFAULT_EVENT_BUFFER_CAP`], surplus events are shed OLDEST-first and counted in +/// `events_dropped()`, and — because join completions are folded BEFORE buffering — +/// the shed app-events never affect join resolution (`poll_join` still returns +/// `Ok(reached)`). Regression for an unbounded `buffered_events` that OOMs a +/// long-running embedded node under the supported pump-then-poll_join flow; reverting +/// the bound lets the backlog grow to `2 * cap` and leaves `events_dropped()` at 0. +#[test] +fn undrained_poll_event_backlog_is_bounded_and_counts_drops() { + let mut link = LinkPair::new(&[10, 11], &[20, 21]); + let now = Instant::from_origin(Duration::from_secs(86_400)); + + let handle = link + .a + .join(&[link.b_addr], false, now) + .expect("join announces intent and mints a handle"); + + // Drive the push/pull to its Succeeded terminal by pumping BOTH engines, draining + // ONLY B's events. A's `poll_event` is NEVER called, so the pump alone folds A's + // join completion and every event A observes piles into `buffered_events`. + let mut folded = false; + for _ in 0..40 { + link.step(now); + while link.b.poll_event().is_some() {} + if matches!( + link.a.pending_joins.get(&handle).map(|pj| &pj.reply), + Some(JoinReply::Ready(_)) + ) { + folded = true; + break; + } + } + assert!( + folded, + "the pump must fold the Succeeded push/pull into the join with no poll_event drain" + ); + + // Flood the app-event backlog far past the cap, still WITHOUT draining `poll_event`. + // This exercises the exact bounded push the pump's `drain_fold_events` uses. + for _ in 0..(DEFAULT_EVENT_BUFFER_CAP * 2) { + link.a.push_app_event(Event::LeftCluster); + } + + // The backlog is BOUNDED — it never exceeds the cap however long the app ignores + // it — and every shed event is counted, so the loss is observable rather than an + // unbounded memory leak. + assert_eq!( + link.a.buffered_events.len(), + DEFAULT_EVENT_BUFFER_CAP, + "the undrained app-event backlog must be bounded at the cap, never growing without limit" + ); + assert!( + link.a.events_dropped() >= DEFAULT_EVENT_BUFFER_CAP as u64, + "every shed event must be counted in events_dropped (got {})", + link.a.events_dropped() + ); + + // Join accounting is folded before buffering, so the shed app-events never affect + // resolution: the join still resolves `Ok(reached)` with B. + match link.a.poll_join(handle) { + Some(Ok(reached)) => assert!( + reached.contains(&link.b_addr), + "the join still resolves Ok(reached) including B despite the app-event drops" + ), + other => panic!("the bounded app-event drop must not affect join resolution, got {other:?}"), + } +} diff --git a/serf-embedded/src/lib.rs b/serf-embedded/src/lib.rs index c022b62d..fb92381f 100644 --- a/serf-embedded/src/lib.rs +++ b/serf-embedded/src/lib.rs @@ -24,7 +24,7 @@ compile_error!("serf-embedded requires the `std` or `alloc` feature"); mod cidr; pub mod engine; -pub use engine::{JoinFailed, JoinId, ReachedSet, SerfEngine}; +pub use engine::{DEFAULT_EVENT_BUFFER_CAP, JoinFailed, JoinId, ReachedSet, SerfEngine}; // ── Reused memberlist-embedded glue ────────────────────────────────────────── // From 9271e063f97be70d4071f6bd2e7a309dd1fc8277 Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 3 Jul 2026 23:44:27 +1200 Subject: [PATCH 050/128] fix(serf-embedded): deliver mandatory control events on a non-lossy path The bounded pump buffer dropped the oldest event regardless of type, so a burst over the cap could evict a mandatory Shutdown, KeyRequest, or DialRequested before any driver acted on it. Split delivery: mandatory driver-actioned events take a non-lossy control queue that poll_event drains first; passive observations stay in the bounded drop-oldest buffer with events_dropped. Mirrors serf-reactor's mandatory-then-lossy model; join completions still fold before buffering. --- serf-embedded/src/engine/mod.rs | 232 ++++++++++++++++++++++++------ serf-embedded/src/engine/tests.rs | 102 +++++++++++++ 2 files changed, 287 insertions(+), 47 deletions(-) diff --git a/serf-embedded/src/engine/mod.rs b/serf-embedded/src/engine/mod.rs index 192c3104..8542ba55 100644 --- a/serf-embedded/src/engine/mod.rs +++ b/serf-embedded/src/engine/mod.rs @@ -413,24 +413,47 @@ where /// set from the terminal `ExchangeCompleted` of the push/pulls it dispatched; /// [`poll_join`](Self::poll_join) drains a resolved one. pending_joins: HashMap, + /// Mandatory, driver-actioned control events the pump (and [`leave`](Self::leave)) + /// drained — the NON-LOSSY delivery path. Holds exactly the events a driver must + /// take a side effect on beyond observing them ([`is_mandatory_event`]): the + /// conflict [`Event::Shutdown`] (the driver must STOP), an [`Event::KeyRequest`] + /// (the driver must apply the op and `respond_key`), and an + /// [`Event::DialRequested`] (the driver must dial and report back). + /// + /// UNLIKE `buffered_events` this queue is never dropped-on-overflow: evicting a + /// mandatory event would leave a conflict loser running or a key op timing out. + /// [`poll_event`](Self::poll_event) drains it FIRST, so a burst of observations can + /// neither evict nor postpone a mandatory action. It cannot itself flood — these + /// events are non-bursty (`Shutdown` is terminal; a `KeyRequest` / `DialRequested` + /// is rare and MTU-bounded per pump) and a driver acting on them drains + /// `poll_event` every pump, so it is emptied each tick rather than accumulating + /// like a Member observation flood. + control_events: VecDeque>, /// Machine events the pump (and [`leave`](Self::leave)) drained and folded into /// the pending joins, held so [`poll_event`](Self::poll_event) hands them to the - /// driver in order — already folded, never re-folded. Every `pump` drains the - /// machine's event queue to here, folding each `ExchangeCompleted` into its join - /// BEFORE buffering the event, so join accounting is pump-driven and never waits - /// on the app draining events; `poll_event` is a pure drain of this queue. + /// driver in order — already folded, never re-folded — the LOSSY, best-effort + /// delivery path for PASSIVE observations only (membership changes, user events, + /// queries and their responses / acks, relay-drop notices, key-query results, + /// exchange completions, and the `LeftCluster` notice). Mandatory driver-actioned + /// events are routed to `control_events` instead; see + /// [`route_drained_event`](Self::route_drained_event). Every `pump` drains the + /// machine's event queue, folding each `ExchangeCompleted` into its join BEFORE + /// buffering the observation, so join accounting is pump-driven and never waits on + /// the app draining events. /// - /// Bounded at [`DEFAULT_EVENT_BUFFER_CAP`]: at the cap the oldest event is dropped - /// and counted in `events_dropped`, so a driver that never drains + /// Bounded at [`DEFAULT_EVENT_BUFFER_CAP`]: at the cap the oldest observation is + /// dropped and counted in `events_dropped`, so a driver that never drains /// [`poll_event`](Self::poll_event) cannot grow this queue without limit (which /// would exhaust memory on a long-running embedded node). buffered_events: VecDeque>, - /// Count of application events shed from `buffered_events` because the driver - /// never drained [`poll_event`](Self::poll_event) fast enough and the backlog hit - /// [`DEFAULT_EVENT_BUFFER_CAP`]. Surfaced via + /// Count of PASSIVE observation events shed from `buffered_events` because the + /// driver never drained [`poll_event`](Self::poll_event) fast enough and the + /// backlog hit [`DEFAULT_EVENT_BUFFER_CAP`]. Surfaced via /// [`events_dropped`](Self::events_dropped) so the best-effort loss is observable - /// (mirroring the serf std drivers' load-shed counter). Join completions are - /// folded before buffering, so a shed app-event never affects join resolution. + /// (mirroring the serf std drivers' load-shed counter). Mandatory control events + /// are never dropped (they take the non-lossy `control_events` path) and so are + /// never counted here. Join completions are folded before buffering, so a shed + /// observation never affects join resolution. events_dropped: u64, /// Monotonic allocator for [`JoinId`]s, so two concurrent joins never collide. next_join_id: u64, @@ -614,6 +637,7 @@ where gossip_recv, pending_seeds: VecDeque::new(), pending_joins: HashMap::new(), + control_events: VecDeque::new(), buffered_events: VecDeque::new(), events_dropped: 0, next_join_id: 0, @@ -829,43 +853,62 @@ where self.endpoint.members_snapshot() } - /// Drain one application-visible serf event buffered by the last `pump`, if any. + /// Drain one serf event the last `pump` delivered, if any — mandatory + /// driver-actioned control events FIRST, then passive observations. /// /// Each [`pump`](Self::pump) drains the machine's event queue to quiescence, /// folding every push/pull `ExchangeCompleted` into its await-result join and - /// buffering every event — the full serf surface: membership changes, user - /// events, queries, query responses / acks, key-management requests / responses, + /// routing each event — the full serf surface: membership changes, user events, + /// queries, query responses / acks, key-management requests / responses, /// reliable-exchange completions, and the lifecycle signals (`LeftCluster`, - /// conflict `Shutdown`). This call hands those buffered events to the driver in - /// order, exactly once; it does NOT re-fold (the pump already folded). Returns - /// `None` when the buffer is empty; call again after the next `pump` tick. + /// conflict `Shutdown`) — by class ([`route_drained_event`](Self::route_drained_event)). + /// This call hands those events to the driver exactly once; it does NOT re-fold + /// (the pump already folded). Returns `None` when both queues are empty; call + /// again after the next `pump` tick. + /// + /// A MANDATORY control event ([`Event::Shutdown`], [`Event::KeyRequest`], + /// [`Event::DialRequested`]) is delivered ahead of any queued observation and is + /// NEVER dropped: promoting it changes no serf semantics — `Shutdown` is terminal, + /// and a `KeyRequest` / `DialRequested` side effect is independent of + /// membership-observation order — while guaranteeing an observation flood can + /// neither evict nor delay a driver action. /// /// Join resolution and ignore cleanup are driven by the pump on the exchange /// terminal, so a driver need NOT drain events here before /// [`poll_join`](Self::poll_join) — `poll_join` resolves off the pump-folded /// state. The contract is the reactor-faithful "pump, then poll_join / poll_event". /// - /// App-event delivery is BEST-EFFORT: the backlog is bounded at + /// PASSIVE observation delivery is BEST-EFFORT: that backlog is bounded at /// [`DEFAULT_EVENT_BUFFER_CAP`], so a driver that pumps and resolves joins but - /// stops draining `poll_event` sheds the oldest surplus events (counted in + /// stops draining `poll_event` sheds the oldest surplus observations (counted in /// [`events_dropped`](Self::events_dropped)) rather than growing memory without - /// bound — matching the serf std drivers' lossy observation channel. Join - /// resolution is unaffected (completions are folded before buffering). + /// bound — matching the serf std drivers' lossy observation channel. Mandatory + /// control events are exempt (non-lossy), and join resolution is unaffected + /// (completions are folded before buffering). #[inline] pub fn poll_event(&mut self) -> Option> { + // Mandatory control signals first: a burst of buffered observations must never + // starve or delay a Shutdown / KeyRequest / DialRequested. They live on a + // separate non-lossy queue, so this ordering also makes eviction impossible. + if let Some(ev) = self.control_events.pop_front() { + return Some(ev); + } self.buffered_events.pop_front() } - /// The number of application events shed from the `poll_event` backlog because it - /// reached [`DEFAULT_EVENT_BUFFER_CAP`] before the driver drained them. + /// The number of PASSIVE observation events shed from the `poll_event` backlog + /// because it reached [`DEFAULT_EVENT_BUFFER_CAP`] before the driver drained them. /// - /// App-event delivery is BEST-EFFORT under sustained overload: a driver that + /// Observation delivery is BEST-EFFORT under sustained overload: a driver that /// pumps and resolves joins but never drains [`poll_event`](Self::poll_event) - /// sheds the oldest surplus events rather than growing memory without bound, and - /// each shed increments this counter (mirroring the serf std drivers' - /// `events_dropped`). Join resolution is unaffected — completions are folded - /// before buffering — so a nonzero count means only that some observational - /// events were not delivered, never that a join was mis-resolved. + /// sheds the oldest surplus observations rather than growing memory without bound, + /// and each shed increments this counter (mirroring the serf std drivers' + /// `events_dropped`). MANDATORY driver-actioned events ([`Event::Shutdown`], + /// [`Event::KeyRequest`], [`Event::DialRequested`]) take the non-lossy control + /// path and are NEVER shed, so they are never counted here. Join resolution is + /// unaffected — completions are folded before buffering — so a nonzero count means + /// only that some observations were not delivered, never that a join was + /// mis-resolved or a mandatory action was lost. #[inline] pub fn events_dropped(&self) -> u64 { self.events_dropped @@ -904,16 +947,18 @@ where } } - /// Buffer one already-folded machine event for [`poll_event`](Self::poll_event), - /// bounding the backlog at [`DEFAULT_EVENT_BUFFER_CAP`]. + /// Buffer one PASSIVE observation for [`poll_event`](Self::poll_event), bounding the + /// backlog at [`DEFAULT_EVENT_BUFFER_CAP`]. /// - /// At the cap the OLDEST buffered event is dropped (freshest-wins, so a + /// At the cap the OLDEST buffered observation is dropped (freshest-wins, so a /// never-draining driver keeps the most recent surface) and counted in /// `events_dropped`, so the queue cannot grow without bound and the loss stays - /// observable. The dropped copy is purely the app-delivery one — the event's join - /// completion was already folded by [`drain_fold_events`](Self::drain_fold_events) - /// BEFORE this call — so a drop never affects join resolution. Mirrors - /// memberlist-embassy's bounded `app_events` (drop-oldest on overflow). + /// observable. Only observations reach here — a mandatory driver-actioned event + /// takes the non-lossy [`push_control_event`](Self::push_control_event) path — and + /// the dropped copy is purely the app-delivery one: the event's join completion + /// was already folded by [`route_drained_event`](Self::route_drained_event) BEFORE + /// this call, so a drop never affects join resolution. Mirrors memberlist-embassy's + /// bounded `app_events` (drop-oldest on overflow). fn push_app_event(&mut self, ev: Event) { if self.buffered_events.len() >= DEFAULT_EVENT_BUFFER_CAP { self.buffered_events.pop_front(); @@ -922,25 +967,64 @@ where self.buffered_events.push_back(ev); } - /// Drain the machine's event queue to quiescence, folding each event into its - /// await-result join ([`fold_join_completion`](Self::fold_join_completion)) and - /// buffering it for [`poll_event`](Self::poll_event) delivery. + /// Enqueue one MANDATORY driver-actioned event on the non-lossy `control_events` + /// queue, deduplicating the terminal [`Event::Shutdown`]. + /// + /// Mandatory events ([`is_mandatory_event`]) carry a side effect the driver MUST + /// perform — stop on `Shutdown`, apply + `respond_key` a `KeyRequest`, dial a + /// `DialRequested` — so they are NEVER dropped (dropping one is the bug this split + /// fixes: a burst over the observation cap could otherwise evict a `Shutdown` + /// before any driver acted on it). This queue carries no drop-bound because it + /// cannot flood: the events are non-bursty (`Shutdown` is terminal; + /// `KeyRequest` / `DialRequested` is rare and MTU-bounded per pump) and a driver + /// acting on them drains [`poll_event`](Self::poll_event) every pump, so it is + /// emptied each tick — unlike the observation backlog a pump-then-`poll_join` + /// driver legitimately never drains. `Shutdown` is idempotent-terminal, so at most + /// one is ever queued. + fn push_control_event(&mut self, ev: Event) { + if matches!(ev, Event::Shutdown) + && self + .control_events + .iter() + .any(|e| matches!(e, Event::Shutdown)) + { + return; + } + self.control_events.push_back(ev); + } + + /// Fold one drained machine event into its await-result join, then route it to the + /// correct delivery queue by class. + /// + /// The SOLE per-event routing site, shared by + /// [`drain_fold_events`](Self::drain_fold_events) (the pump and + /// [`leave`](Self::leave)), so the classification is single-sourced. The join + /// completion is folded FIRST ([`fold_join_completion`](Self::fold_join_completion)), + /// so join accounting is pump-driven and never gated on the app polling events; + /// then a MANDATORY event ([`is_mandatory_event`]) goes to the non-lossy + /// `control_events` and a PASSIVE observation to the bounded, drop-oldest + /// `buffered_events`. + fn route_drained_event(&mut self, ev: Event) { + self.fold_join_completion(&ev); + if is_mandatory_event(&ev) { + self.push_control_event(ev); + } else { + self.push_app_event(ev); + } + } + + /// Drain the machine's event queue to quiescence, folding + routing each event via + /// [`route_drained_event`](Self::route_drained_event). /// /// The SOLE drain of the endpoint's event queue. The pump calls it every tick, so /// join accounting is pump-driven and never gated on the app polling events; and /// [`leave`](Self::leave) calls it before resolving its abandonment, so an /// already-succeeded push/pull lands in the reached set. Because it empties the /// endpoint queue, a later call folds only the events enqueued since — never - /// re-folding one already buffered. The join completion is folded FIRST, then the - /// event is buffered via [`push_app_event`](Self::push_app_event), which bounds the - /// backlog at [`DEFAULT_EVENT_BUFFER_CAP`] (dropping + counting the oldest on - /// overflow) so a driver that never drains `poll_event` cannot grow it without - /// bound; `buffered_events` preserves the app's event stream in arrival order up to - /// that cap. + /// re-folding one already delivered. fn drain_fold_events(&mut self) { while let Some(ev) = self.endpoint.poll_event() { - self.fold_join_completion(&ev); - self.push_app_event(ev); + self.route_drained_event(ev); } } @@ -2000,6 +2084,60 @@ where } } +/// Classify a drained machine [`Event`] as a MANDATORY driver-actioned control +/// signal (the driver must take a side effect beyond observing it) versus a PASSIVE +/// observation (which [`SerfEngine`] has already accounted for, or which is pure +/// app-level information). +/// +/// The MANDATORY set is exactly what serf's reliable-stream drivers act on in their +/// synchronous drain — no more, no less — mirroring serf-reactor's `account_event` +/// and serf-compio's `drain_events`: +/// +/// - [`Event::Shutdown`] — the local node lost an id-conflict vote and the driver +/// MUST stop (serf-reactor flags `begin_shutdown`; serf-compio sets its terminal +/// flag), then still delivers the event to subscribers. +/// - [`Event::KeyRequest`] — the driver MUST apply the key op and answer the +/// originator via `respond_key` (both reference drivers do so ahead of the +/// observation hand-off); without it the inbound key op silently times out. +/// - [`Event::DialRequested`] — the driver MUST dial the peer and report back via +/// `dial_succeeded` / `dial_failed`. On the reliable-stream path the coordinator +/// sieves the inner dial request into its own dial queue (surfaced as a +/// `poll_action` `Connect` the pump already services), so this event never +/// actually reaches the drain here; it is classified mandatory so that, were any +/// transport to surface it, a driver-owned dial could never be silently evicted. +/// +/// Every other variant is a PASSIVE observation, delivered best-effort: +/// [`Event::ExchangeCompleted`] (its await-result join is folded non-lossily by +/// [`fold_join_completion`](SerfEngine::fold_join_completion) BEFORE buffering), +/// [`Event::LeftCluster`] (the engine's [`leave`](SerfEngine::leave) resolves its +/// join replies synchronously, so nothing resolves off the event here), +/// [`Event::Member`] / [`Event::User`] / [`Event::Query`] (app observations, a +/// `Query` response being optional), and [`Event::QueryResponse`] / +/// [`Event::QueryAck`] / [`Event::KeyResponse`] / [`Event::RelayDropped`] +/// (correlated internally by the machine, forwarded to the app only). +fn is_mandatory_event(ev: &Event) -> bool { + match ev { + Event::Shutdown | Event::DialRequested(_) => true, + #[cfg(encryption)] + Event::KeyRequest(_) => true, + Event::Member(_) + | Event::User(_) + | Event::Query(_) + | Event::QueryResponse(_) + | Event::QueryAck(_) + | Event::RelayDropped(_) + | Event::LeftCluster + | Event::ExchangeCompleted(_) => false, + #[cfg(encryption)] + Event::KeyResponse(_) => false, + // `Event` is `#[non_exhaustive]`, so an exhaustive match is impossible from a + // downstream crate: an unknown future variant defaults to a best-effort + // observation. Add it to the mandatory arm above if a new driver-actioned event + // is ever introduced upstream. + _ => false, + } +} + /// Returns the earlier of two optional deadlines. If only one is `Some`, that /// deadline wins; if both are `None` the result is `None`. fn min_opt(a: Option, b: Option) -> Option { diff --git a/serf-embedded/src/engine/tests.rs b/serf-embedded/src/engine/tests.rs index 2394da9b..40449e76 100644 --- a/serf-embedded/src/engine/tests.rs +++ b/serf-embedded/src/engine/tests.rs @@ -1599,3 +1599,105 @@ fn undrained_poll_event_backlog_is_bounded_and_counts_drops() { other => panic!("the bounded app-event drop must not affect join resolution, got {other:?}"), } } + +// ── mandatory control events survive an observation flood (non-lossy path) ──── + +/// A MANDATORY driver-actioned event survives an observation flood far exceeding the +/// passive backlog cap. Routed through the SAME per-event path the pump uses +/// ([`SerfEngine::route_drained_event`]), a `Event::Shutdown` lands on the non-lossy +/// control queue while `> DEFAULT_EVENT_BUFFER_CAP` passive `Event::LeftCluster` +/// observations flood the bounded buffer. `poll_event` must STILL yield the Shutdown +/// (delivered ahead of observations, never evicted), the passive backlog must be +/// bounded at the cap, and `events_dropped()` must count the shed observations. +/// +/// This is the fail-on-revert regression: reverting to a single lossy queue routes +/// the Shutdown through the drop-oldest buffer, where the over-cap flood evicts it +/// before any driver could act on it — so the final assertion fails. `Shutdown` is +/// the only mandatory variant constructible from a downstream crate (`KeyRequest` / +/// `DialRequested` carry `pub(crate)` payloads and, on the stream path, the +/// coordinator sieves the dial into a `poll_action` `Connect`), and all three route +/// through the identical `is_mandatory_event` → non-lossy control path exercised +/// here. +#[test] +fn mandatory_event_survives_observation_flood() { + let mut engine = make_engine(); + + // The mandatory event is enqueued FIRST, then a flood of passive observations far + // past the cap — so a single lossy queue (drop-oldest) would evict the Shutdown. + engine.route_drained_event(Event::Shutdown); + for _ in 0..(DEFAULT_EVENT_BUFFER_CAP * 2) { + engine.route_drained_event(Event::LeftCluster); + } + + // The passive backlog is bounded at the cap and every shed observation is counted. + assert_eq!( + engine.buffered_events.len(), + DEFAULT_EVENT_BUFFER_CAP, + "the passive observation backlog must be bounded at the cap" + ); + assert!( + engine.events_dropped() >= 1, + "the over-cap observation flood must shed and count observations (got {})", + engine.events_dropped() + ); + + // The mandatory Shutdown is delivered FIRST and was never evicted by the flood. + assert!( + matches!(engine.poll_event(), Some(Event::Shutdown)), + "the mandatory Shutdown must survive the flood and lead the observations" + ); + + // Nothing after it is a Shutdown (only one is ever queued), and exactly the bounded + // passive backlog remains. + let mut remaining = 0usize; + while let Some(ev) = engine.poll_event() { + remaining += 1; + assert!( + !matches!(ev, Event::Shutdown), + "only one Shutdown is ever queued; it was already delivered first" + ); + } + assert_eq!( + remaining, DEFAULT_EVENT_BUFFER_CAP, + "after the Shutdown, exactly the bounded passive backlog is delivered" + ); +} + +/// The non-lossy control queue dedupes the idempotent-terminal `Event::Shutdown` — a +/// repeated conflict signal queues at most one — and delivers control events ahead of +/// passive observations regardless of arrival order. Together these keep the +/// unbounded control queue from being grown by a duplicated terminal signal and +/// guarantee a mandatory action is never delayed behind queued observations. +#[test] +fn control_queue_dedupes_shutdown_and_leads_observations() { + let mut engine = make_engine(); + + // Observations arrive first, then several Shutdowns. + engine.route_drained_event(Event::LeftCluster); + engine.route_drained_event(Event::LeftCluster); + engine.route_drained_event(Event::Shutdown); + engine.route_drained_event(Event::Shutdown); + engine.route_drained_event(Event::Shutdown); + + assert_eq!( + engine.control_events.len(), + 1, + "repeated Shutdowns dedupe to a single queued terminal signal" + ); + + // Control-first: the Shutdown leads despite the earlier-enqueued observations, then + // the two observations follow in arrival order. + assert!( + matches!(engine.poll_event(), Some(Event::Shutdown)), + "the mandatory Shutdown is delivered ahead of the earlier observations" + ); + assert!( + matches!(engine.poll_event(), Some(Event::LeftCluster)), + "the buffered observations follow the control event, in arrival order" + ); + assert!(matches!(engine.poll_event(), Some(Event::LeftCluster))); + assert!( + engine.poll_event().is_none(), + "both queues are drained after delivering the control event and observations" + ); +} From 0d2b8890a9edf999deb274439602e980fc4d5309 Mon Sep 17 00:00:00 2001 From: al8n Date: Sat, 4 Jul 2026 00:16:48 +1200 Subject: [PATCH 051/128] fix(serf-embedded): bound the mandatory control queue by liveness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The non-lossy control queue held every KeyRequest with no bound, so an encrypted flooder could grow the heap and pin raw key material. Prune past-deadline (dead, unanswerable) KeyRequests each pump/leave and rely on the endpoint's own received-query cap — the same liveness+cardinality discipline serf-proto applies to received_queries — so the queue is bounded while a live in-window request is never dropped; Shutdown stays deduped. --- serf-embedded/src/engine/mod.rs | 89 ++++++++++++----- serf-embedded/src/engine/tests.rs | 152 ++++++++++++++++++++++++++++++ serf-proto/src/event/mod.rs | 26 +++++ 3 files changed, 246 insertions(+), 21 deletions(-) diff --git a/serf-embedded/src/engine/mod.rs b/serf-embedded/src/engine/mod.rs index 8542ba55..9087ab3d 100644 --- a/serf-embedded/src/engine/mod.rs +++ b/serf-embedded/src/engine/mod.rs @@ -420,14 +420,21 @@ where /// (the driver must apply the op and `respond_key`), and an /// [`Event::DialRequested`] (the driver must dial and report back). /// - /// UNLIKE `buffered_events` this queue is never dropped-on-overflow: evicting a - /// mandatory event would leave a conflict loser running or a key op timing out. - /// [`poll_event`](Self::poll_event) drains it FIRST, so a burst of observations can - /// neither evict nor postpone a mandatory action. It cannot itself flood — these - /// events are non-bursty (`Shutdown` is terminal; a `KeyRequest` / `DialRequested` - /// is rare and MTU-bounded per pump) and a driver acting on them drains - /// `poll_event` every pump, so it is emptied each tick rather than accumulating - /// like a Member observation flood. + /// UNLIKE `buffered_events` this queue is never dropped WHILE LIVE: evicting a + /// live mandatory event would leave a conflict loser running or an answerable key + /// op unhandled. [`poll_event`](Self::poll_event) drains it FIRST, so a burst of + /// observations can neither evict nor postpone a mandatory action. + /// + /// It is still bounded, by LIVENESS rather than drop-oldest. An encrypted peer can + /// flood distinct inbound [`Event::KeyRequest`]s (each pinning raw key material), + /// so every pump prunes the ones past their response deadline — dead and + /// unanswerable, losing nothing real — via + /// [`prune_expired_control_events`](Self::prune_expired_control_events). That + /// mirrors the `now < deadline` retain the serf endpoint applies to its own + /// `received_queries`, and because the endpoint caps the live key queries it emits + /// at a fixed inbound maximum, the live `KeyRequest`s held here are transitively + /// bounded by that same cap. `Shutdown` is idempotent-terminal (deduped to one); + /// `DialRequested` carries no deadline and does not flood. control_events: VecDeque>, /// Machine events the pump (and [`leave`](Self::leave)) drained and folded into /// the pending joins, held so [`poll_event`](Self::poll_event) hands them to the @@ -905,10 +912,13 @@ where /// and each shed increments this counter (mirroring the serf std drivers' /// `events_dropped`). MANDATORY driver-actioned events ([`Event::Shutdown`], /// [`Event::KeyRequest`], [`Event::DialRequested`]) take the non-lossy control - /// path and are NEVER shed, so they are never counted here. Join resolution is - /// unaffected — completions are folded before buffering — so a nonzero count means - /// only that some observations were not delivered, never that a join was - /// mis-resolved or a mandatory action was lost. + /// path and are never shed WHILE LIVE, so they are never counted here — a + /// past-deadline `KeyRequest` pruned by + /// [`prune_expired_control_events`](Self::prune_expired_control_events) is dead, not + /// a dropped live event, and is likewise uncounted. Join resolution is unaffected — + /// completions are folded before buffering — so a nonzero count means only that + /// some observations were not delivered, never that a join was mis-resolved or a + /// mandatory action was lost. #[inline] pub fn events_dropped(&self) -> u64 { self.events_dropped @@ -972,15 +982,16 @@ where /// /// Mandatory events ([`is_mandatory_event`]) carry a side effect the driver MUST /// perform — stop on `Shutdown`, apply + `respond_key` a `KeyRequest`, dial a - /// `DialRequested` — so they are NEVER dropped (dropping one is the bug this split - /// fixes: a burst over the observation cap could otherwise evict a `Shutdown` - /// before any driver acted on it). This queue carries no drop-bound because it - /// cannot flood: the events are non-bursty (`Shutdown` is terminal; - /// `KeyRequest` / `DialRequested` is rare and MTU-bounded per pump) and a driver - /// acting on them drains [`poll_event`](Self::poll_event) every pump, so it is - /// emptied each tick — unlike the observation backlog a pump-then-`poll_join` - /// driver legitimately never drains. `Shutdown` is idempotent-terminal, so at most - /// one is ever queued. + /// `DialRequested` — so a LIVE one is NEVER dropped (dropping one is the bug the + /// control/observation split fixes: a burst over the observation cap could + /// otherwise evict a `Shutdown` before any driver acted on it). + /// + /// The queue is NOT drop-oldest, but it is still bounded — by LIVENESS, not + /// cardinality. An encrypted peer can flood distinct inbound `KeyRequest`s + /// (each pinning raw key material), so + /// [`prune_expired_control_events`](Self::prune_expired_control_events) sheds every + /// past-deadline (dead, unanswerable) `KeyRequest` on each pump; this method only + /// appends. `Shutdown` is idempotent-terminal, so at most one is ever queued. fn push_control_event(&mut self, ev: Event) { if matches!(ev, Event::Shutdown) && self @@ -993,6 +1004,30 @@ where self.control_events.push_back(ev); } + /// Shed every past-deadline `KeyRequest` from the non-lossy `control_events` + /// queue, mirroring the endpoint's own `received_queries` liveness prune. + /// + /// A `KeyRequest` past its response `deadline` is dead: the endpoint rejects a + /// `respond_key` sent after the deadline, so the driver can no longer act on it. + /// Dropping it here loses nothing real, promptly releases the raw key material + /// pinned in its payload, and bounds `control_events` to the LIVE mandatory set — + /// the same `now < deadline` retain the serf endpoint applies to its + /// `received_queries` on every `handle_timeout`. Because the endpoint caps its + /// LIVE `received_queries` at a fixed inbound maximum and emits exactly one + /// `Event::KeyRequest` per kept entry (sharing this deadline), the live + /// `KeyRequest`s retained here are transitively bounded by that same cap — no + /// separate engine-side count cap is needed. `Shutdown` / `DialRequested` carry no + /// deadline and are always kept; a repeated `Shutdown` stays deduped by + /// [`push_control_event`](Self::push_control_event). + #[cfg(encryption)] + fn prune_expired_control_events(&mut self, now: Instant) { + // Keep everything that is NOT a past-deadline KeyRequest — mirroring the + // endpoint's `retain(|_, rq| now < rq.deadline)` over received_queries. + self + .control_events + .retain(|ev| !matches!(ev, Event::KeyRequest(kr) if now >= kr.deadline())); + } + /// Fold one drained machine event into its await-result join, then route it to the /// correct delivery queue by class. /// @@ -1210,6 +1245,12 @@ where // Mirrors serf-reactor's shutdown/leave, which drains-and-folds before it reaps. self.drain_fold_events(); + // Shed any now past-deadline KeyRequests the drain just routed, on the same + // liveness rule the pump applies, so a leave without a following pump still bounds + // the control queue. + #[cfg(encryption)] + self.prune_expired_control_events(now); + // Accepted: no seed dispatches once leaving, so drop the queue and mark every // still-pending join fully dispatched, then deliver each still-pending caller // reply once from its NOW-folded reached set. Ignore-stream cleanup stays with @@ -1492,6 +1533,12 @@ where // loop before the observation hand-off). self.drain_fold_events(); + // 7f'. Shed past-deadline (dead, unanswerable) KeyRequests from the non-lossy + // control queue so an encrypted peer flooding distinct key queries cannot grow it + // without bound or pin their key material — bounding it to the live mandatory set. + #[cfg(encryption)] + self.prune_expired_control_events(now); + // 7g. Resolve await-result joins whose exchanges all terminated, then reap only // those whose result the caller has already retrieved. [`try_resolve_join`] // clears the machine's ignore set and resolves the caller reply on the exchange diff --git a/serf-embedded/src/engine/tests.rs b/serf-embedded/src/engine/tests.rs index 40449e76..658725c5 100644 --- a/serf-embedded/src/engine/tests.rs +++ b/serf-embedded/src/engine/tests.rs @@ -1701,3 +1701,155 @@ fn control_queue_dedupes_shutdown_and_leads_observations() { "both queues are drained after delivering the control event and observations" ); } + +// ── mandatory control queue bounded by KeyRequest liveness (encrypted flood) ── + +/// Build an `Event::KeyRequest` with a distinct `id` and an explicit response +/// `deadline`, carrying real key material so a prune demonstrably releases it. This +/// mirrors the endpoint's own emission — one `Event::KeyRequest` whose `deadline` +/// equals its registered `received_queries` entry — via the serf-proto +/// `test-support` constructor (the wire fields are `pub(crate)`, so a downstream +/// test cannot build one directly). +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +fn key_request(id: u32, deadline: Instant) -> Event { + let from = memberlist_proto::Node::new(SmolStr::new("flooder"), node_addr(6000)); + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes128([0x11u8; 16]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = SecretKey::ChaCha20Poly1305([0x11u8; 32]); + Event::KeyRequest(KeyRequest::test_with_deadline( + id, + from, + Some(key), + deadline, + )) +} + +/// An encrypted peer flooding distinct `KeyRequest`s across successive deadline +/// windows cannot grow the non-lossy control queue without bound, even when the +/// driver pumps and resolves joins but NEVER drains `poll_event`. Each window's +/// batch is routed to `control_events`, then `now` advances past its deadline and a +/// `pump` prunes it (dead + unanswerable) — so the queue holds at most one live +/// window's worth, never `windows * batch`. +/// +/// This is the fail-on-revert regression: without the pump's deadline-prune the +/// queue accumulates every window's batch, so the `<= BATCH` bound fails on the +/// second window (and the heap grows without limit under a sustained flood). +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn key_request_flood_across_deadline_windows_is_bounded_without_poll_event() { + let mut engine = make_engine(); + let base = Instant::from_origin(Duration::from_secs(86_400)); + engine.start(base); + + let mut gossip = NoGossip; + let mut stream = NoStream::with_pool(2); + // Settle the construction self-join; its events are passive (buffered_events), + // never mandatory, so they cannot enter control_events. + engine.pump(base, &mut gossip, &mut stream); + + const BATCH: usize = 64; + const WINDOWS: usize = 8; + const WINDOW: Duration = Duration::from_secs(10); + + let mut id = 0u32; + let mut now = base; + let mut peak = 0usize; + for w in 0..WINDOWS { + // This window's KeyRequests are live until `now + WINDOW`. + let deadline = now + WINDOW; + for _ in 0..BATCH { + engine.route_drained_event(key_request(id, deadline)); + id += 1; + } + peak = peak.max(engine.control_events.len()); + + // Advance PAST this window's deadline and pump — WITHOUT draining poll_event — + // so the pump's prune sheds this now-dead batch. + now = deadline + Duration::from_secs(1); + engine.pump(now, &mut gossip, &mut stream); + + assert!( + engine.control_events.len() <= BATCH, + "control queue must stay bounded to one live window, got {} at window {w}", + engine.control_events.len() + ); + } + + // The queue never accumulated across windows: its peak is one batch, not + // WINDOWS * BATCH, and every window's expired batch is gone. + assert!( + peak <= BATCH, + "control queue peaked at {peak}, exceeding a single {BATCH}-request window" + ); + assert!( + engine.control_events.is_empty(), + "after every window's deadline passed, no KeyRequest remains queued" + ); +} + +/// A LIVE (future-deadline) `KeyRequest` is never pruned: a pump at a `now` before +/// its deadline retains it, and `poll_event` still delivers it (mandatory events +/// lead the observation stream). Guards the prune boundary — `now < deadline` keeps +/// it — so the liveness bound never sheds an answerable request. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn live_key_request_is_never_pruned_and_is_delivered() { + let mut engine = make_engine(); + let now = Instant::from_origin(Duration::from_secs(86_400)); + engine.start(now); + + let mut gossip = NoGossip; + let mut stream = NoStream::with_pool(2); + + // Deadline well in the future; the pump runs at `now` (still in-window). + engine.route_drained_event(key_request(7, now + Duration::from_secs(30))); + engine.pump(now, &mut gossip, &mut stream); + + assert!( + matches!(engine.poll_event(), Some(Event::KeyRequest(_))), + "a live (in-deadline) KeyRequest must survive the pump and be delivered by poll_event" + ); +} + +/// A past-deadline `KeyRequest` is pruned by the pump — releasing the raw key +/// material pinned in its payload — and is never surfaced to the driver. Reverting +/// the prune leaves it queued (payload retained), failing the assertions below. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn past_deadline_key_request_is_pruned_and_payload_released() { + let mut engine = make_engine(); + let now = Instant::from_origin(Duration::from_secs(86_400)); + engine.start(now); + + let mut gossip = NoGossip; + let mut stream = NoStream::with_pool(2); + + let deadline = now + Duration::from_secs(5); + engine.route_drained_event(key_request(9, deadline)); + assert_eq!( + engine.control_events.len(), + 1, + "the KeyRequest (holding its key payload) is queued on the control path" + ); + + // Advance PAST its deadline and pump: the prune drops the dead request, dropping + // the queue's sole reference to its key material. + let later = deadline + Duration::from_secs(1); + engine.pump(later, &mut gossip, &mut stream); + + assert!( + !engine + .control_events + .iter() + .any(|ev| matches!(ev, Event::KeyRequest(_))), + "a past-deadline KeyRequest must be pruned from the control queue (its payload released)" + ); + // It is dropped, not deferred: poll_event never surfaces the pruned request. + while let Some(ev) = engine.poll_event() { + assert!( + !matches!(ev, Event::KeyRequest(_)), + "a pruned past-deadline KeyRequest must never surface via poll_event" + ); + } +} diff --git a/serf-proto/src/event/mod.rs b/serf-proto/src/event/mod.rs index a47133f5..83e9fca1 100644 --- a/serf-proto/src/event/mod.rs +++ b/serf-proto/src/event/mod.rs @@ -392,6 +392,32 @@ impl KeyRequest { pub const fn deadline(&self) -> Instant { self.deadline } + + /// Construct a `KeyRequest` with an explicit response `deadline`, for the + /// downstream driver tests that exercise deadline-based control-queue bounding + /// (a driver holding these on a non-lossy queue must prune the past-deadline, + /// unanswerable ones). Gated behind the non-default `test-support` feature; NOT + /// a production build path and NOT part of the wire contract. The + /// operation-carrying fields are inert placeholders — only `id`, `from`, `key`, + /// and `deadline` are caller-chosen. + #[cfg(any(test, feature = "test-support"))] + #[cfg_attr(docsrs, doc(cfg(feature = "test-support")))] + pub fn test_with_deadline( + id: u32, + from: Node, + key: Option, + deadline: Instant, + ) -> Self { + Self { + op: KeyRequestOperation::Install, + key, + id, + ltime: LamportTime::ZERO, + from, + relay_factor: 0, + deadline, + } + } } // ── KeyResponseArgs ─────────────────────────────────────────────────────────── From 92c18f74d5cf4d158f05dd2fdd77ff55f5403c43 Mon Sep 17 00:00:00 2001 From: al8n Date: Sat, 4 Jul 2026 00:54:49 +1200 Subject: [PATCH 052/128] fix(serf-proto): prune expired received-queries before the inbound cap The inbound received-query hard-cap was checked before the deadline-prune, so under a query/key-request flood stale past-deadline entries pinned the cap and dropped a new live query at ingress (every driver ingests inbound data before handle_timeout). Prune expired entries inline before the cap in handle_query (shared with handle_timeout) so only live entries count; a past-deadline entry is unanswerable (respond/respond_key reject it via the G7 deadline guard) so pruning it early is safe. The regression fills the cap with expired entries and asserts a new live inbound query is still admitted. --- serf-proto/src/endpoint/mod.rs | 56 +++++++++++++---- serf-proto/src/endpoint/tests.rs | 89 +++++++++++++++++++++++++++ serf-proto/src/stream_endpoint/mod.rs | 6 ++ 3 files changed, 139 insertions(+), 12 deletions(-) diff --git a/serf-proto/src/endpoint/mod.rs b/serf-proto/src/endpoint/mod.rs index 1a5e4055..11cd0dc1 100644 --- a/serf-proto/src/endpoint/mod.rs +++ b/serf-proto/src/endpoint/mod.rs @@ -1077,9 +1077,9 @@ where self.fire_due_query_closes(now); // Prune expired received-query tokens. Entries for which respond() succeeded - // are removed there; this loop catches those whose deadline elapsed without a + // are removed there; this catches those whose deadline elapsed without a // respond() call (driver missed the response window). - self.received_queries.retain(|_, rq| now < rq.deadline); + self.prune_expired_received_queries(now); // Leave-complete: Leaving → Left after inner LeftCluster + leave_propagate_delay. if let Some(dl) = self.leave_complete_deadline { @@ -1102,6 +1102,18 @@ where } } + /// Drop received-query tokens whose response deadline has elapsed. + /// + /// A past-deadline entry is unanswerable — `respond` / `respond_key` reject it + /// via the G7 deadline guard — so evicting it loses nothing while freeing an + /// inbound-cap slot. Single-sources the liveness predicate shared by the + /// periodic reclaim in `after_inner_timeout` and the inline reclaim in + /// `handle_query` (which runs before the inbound overflow cap so the cap counts + /// only live entries). + fn prune_expired_received_queries(&mut self, now: Instant) { + self.received_queries.retain(|_, rq| now < rq.deadline); + } + // ── inner-event sieve ───────────────────────────────────────────────────── /// Pump the coordinator `t` to exhaustion, routing each inner event through @@ -2974,20 +2986,29 @@ where } } - // Hard-cap (INBOUND only): when `received_queries` is already at - // MAX_RECEIVED_QUERIES, drop the inbound query BEFORE any state mutation — - // no clock witness, no dedup write, no ACK, no event emission, no - // rebroadcast. Every already-inserted entry was surfaced to the driver - // (Event::Query or Event::KeyRequest) and its token must remain answerable - // via respond / respond_key until the deadline or until the driver responds. - // The deadline-pruning in handle_timeout regularly reclaims expired entries; - // the cap is only hit under peer flood. + // Hard-cap (INBOUND only): when `received_queries` holds MAX_RECEIVED_QUERIES + // LIVE tokens, drop the inbound query BEFORE any state mutation — no clock + // witness, no dedup write, no ACK, no event emission, no rebroadcast. Every + // live entry was surfaced to the driver (Event::Query or Event::KeyRequest) + // and its token must remain answerable via respond / respond_key until its + // deadline or until the driver responds. + // + // Prune expired tokens inline first, using the ingress `now` (drain_now), so + // the cap counts only LIVE entries. Ingress precedes the periodic + // deadline-prune in after_inner_timeout within a tick, so without this a + // flood of stale past-deadline tokens — already unanswerable — could pin the + // cap and drop a new live query. Evicting a past-deadline token here is + // safe: respond / respond_key would reject it via the G7 deadline guard. // // Local queries (QueryOrigin::Local) bypass this cap: the initiating node // MUST always self-process its own query regardless of inbound saturation. // Local query volume is app-controlled and not an adversarial flood vector. - if origin == QueryOrigin::Inbound && self.received_queries.len() >= MAX_RECEIVED_QUERIES { - return false; + if origin == QueryOrigin::Inbound { + let now = self.drain_now; + self.prune_expired_received_queries(now); + if self.received_queries.len() >= MAX_RECEIVED_QUERIES { + return false; + } } // Witness a potentially newer query clock. @@ -4284,6 +4305,17 @@ where self.handle_query(t, msg, QueryOrigin::Inbound) } + /// Overwrite `drain_now` (test adapter). + /// + /// Production ingress entry points latch `drain_now`; tests that drive a + /// handler directly (e.g. `test_handle_query`) use this to advance the + /// endpoint's current-time reference between calls without going through + /// `handle_timeout`. + #[cfg(test)] + pub(crate) fn test_set_drain_now(&mut self, now: Instant) { + self.drain_now = now; + } + /// Return the `QueryId` of the last pending query entry (test adapter). #[cfg(test)] pub(crate) fn test_last_query_id(&self) -> Option { diff --git a/serf-proto/src/endpoint/tests.rs b/serf-proto/src/endpoint/tests.rs index 303bb52a..65d909e6 100644 --- a/serf-proto/src/endpoint/tests.rs +++ b/serf-proto/src/endpoint/tests.rs @@ -3086,6 +3086,95 @@ fn expired_received_queries_are_pruned_in_handle_timeout() { ); } +#[test] +fn expired_received_queries_pruned_before_inbound_cap() { + // A query/key-request flood can leave received_queries full of STALE + // (past-deadline, not-yet-pruned) tokens when a new LIVE inbound query + // arrives: every driver ingests inbound data before it runs the periodic + // deadline-prune in after_inner_timeout. handle_query must therefore prune + // expired tokens inline before the inbound cap, so the cap counts only live + // entries and the live query is admitted. Reverting the inline prune (leaving + // only the after_inner_timeout prune) makes the stale-full cap drop the live + // query and fails this test. + let mut e = ep(); + + // Fill received_queries to the cap with inbound queries carrying a short (1s) + // deadline, at drain_now = ORIGIN. Distinct (ltime = i + 1, id = i) pairs are + // each first-sight, so all MAX_RECEIVED_QUERIES entries are inserted (all live + // at ORIGIN, so the inline prune is a no-op during the fill). + e.test_set_drain_now(t_secs(0)); + for i in 0..MAX_RECEIVED_QUERIES as u32 { + let q = QueryMessage:: { + ltime: LamportTime::new(i as u64 + 1), + id: i, + from: memberlist_proto::Node::new(99u32, addr(9001)), + filters: vec![], + flags: QueryFlag::NO_BROADCAST, + relay_factor: 0, + timeout: core::time::Duration::from_secs(1), + name: "flood".into(), + payload: bytes::Bytes::new(), + }; + let _ = e.test_handle_query(q); + } + assert_eq!( + e.test_received_queries_len(), + MAX_RECEIVED_QUERIES, + "received_queries must be exactly at the cap after the flood" + ); + + // Advance the endpoint's clock PAST the fill deadlines (ORIGIN + 1s) WITHOUT + // calling handle_timeout, so the only thing that can reclaim the now-stale + // tokens is the inline prune in handle_query. + e.test_set_drain_now(t_secs(10)); + + // Drain the queued flood events so the live query's event is observed alone. + while e.poll_event().is_some() {} + + // A new LIVE inbound query (future deadline, distinct ltime/id) must be + // ADMITTED: the inline prune reclaims all the stale slots before the cap check. + let live_ltime = LamportTime::new(MAX_RECEIVED_QUERIES as u64 + 100); + let live_id = MAX_RECEIVED_QUERIES as u32 + 100; + let live = QueryMessage:: { + ltime: live_ltime, + id: live_id, + from: memberlist_proto::Node::new(7u32, addr(7000)), + filters: vec![], + flags: QueryFlag::NO_BROADCAST, + relay_factor: 0, + timeout: core::time::Duration::from_secs(30), + name: "live".into(), + payload: bytes::Bytes::new(), + }; + let _ = e.test_handle_query(live); + + // The live query surfaced as Event::Query (it was not dropped at the cap) ... + let ev = e + .poll_event() + .expect("the live inbound query must be admitted and surfaced as Event::Query"); + match ev { + Event::Query(qe) => { + assert_eq!( + qe.ltime(), + live_ltime, + "surfaced event must be the live query" + ); + assert_eq!(qe.id(), live_id, "surfaced event must be the live query"); + } + other => panic!( + "expected Event::Query for the live query, got {:?}", + core::mem::discriminant(&other) + ), + } + + // ... and its token is the only entry left: the stale tokens were pruned. + assert_eq!( + e.test_received_queries_len(), + 1, + "the stale tokens must be pruned inline, leaving only the live token" + ); +} + // ── Bug 1: zero valid conflict responses must not emit Event::Shutdown ──────── #[test] diff --git a/serf-proto/src/stream_endpoint/mod.rs b/serf-proto/src/stream_endpoint/mod.rs index bec43d76..0fb3aeda 100644 --- a/serf-proto/src/stream_endpoint/mod.rs +++ b/serf-proto/src/stream_endpoint/mod.rs @@ -1049,6 +1049,12 @@ where self.core.test_handle_query(&mut self.transport, msg) } + /// Forwards to [`Endpoint::test_set_drain_now`]. + #[cfg(test)] + pub(crate) fn test_set_drain_now(&mut self, now: Instant) { + self.core.test_set_drain_now(now) + } + /// Forwards to [`Endpoint::test_last_query_id`]. #[cfg(test)] pub(crate) fn test_last_query_id(&self) -> Option { From c2737a633ee7ab081e99bbeb33bd994af6eb935b Mon Sep 17 00:00:00 2001 From: al8n Date: Sat, 4 Jul 2026 01:13:36 +1200 Subject: [PATCH 053/128] fix(serf-proto): align received-query prune with the inclusive respond deadline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The received-query prune dropped a token at now==deadline, but respond/respond_key still accept a response at the exact deadline (they reject only now>deadline), so a still-answerable token — notably a mandatory KeyRequest — could be pruned out from under a pending response (reachable at ingress after the prune-before-cap change). Retain while now<=deadline so an entry is dropped only once strictly past its deadline, matching the response guard; the regression answers a token at exactly its deadline after an inbound prune. --- serf-proto/src/endpoint/mod.rs | 18 +++-- serf-proto/src/endpoint/tests.rs | 121 +++++++++++++++++++++++++++++++ 2 files changed, 132 insertions(+), 7 deletions(-) diff --git a/serf-proto/src/endpoint/mod.rs b/serf-proto/src/endpoint/mod.rs index 11cd0dc1..d5fbc92b 100644 --- a/serf-proto/src/endpoint/mod.rs +++ b/serf-proto/src/endpoint/mod.rs @@ -1102,16 +1102,20 @@ where } } - /// Drop received-query tokens whose response deadline has elapsed. - /// - /// A past-deadline entry is unanswerable — `respond` / `respond_key` reject it - /// via the G7 deadline guard — so evicting it loses nothing while freeing an - /// inbound-cap slot. Single-sources the liveness predicate shared by the + /// Drop received-query tokens that are strictly past their response deadline. + /// + /// An entry is pruned only once `now > deadline` — exactly the point at which + /// `respond` / `respond_key` stop accepting a response for it, since their + /// deadline guard rejects only `now > deadline` and a response sent at the + /// exact instant `now == deadline` is still valid. Retaining while + /// `now <= deadline` therefore never evicts a token a pending response could + /// still answer, while a strictly-past token is unanswerable so freeing its + /// inbound-cap slot loses nothing. Single-sources the predicate shared by the /// periodic reclaim in `after_inner_timeout` and the inline reclaim in /// `handle_query` (which runs before the inbound overflow cap so the cap counts - /// only live entries). + /// only answerable entries). fn prune_expired_received_queries(&mut self, now: Instant) { - self.received_queries.retain(|_, rq| now < rq.deadline); + self.received_queries.retain(|_, rq| now <= rq.deadline); } // ── inner-event sieve ───────────────────────────────────────────────────── diff --git a/serf-proto/src/endpoint/tests.rs b/serf-proto/src/endpoint/tests.rs index 65d909e6..d9469d03 100644 --- a/serf-proto/src/endpoint/tests.rs +++ b/serf-proto/src/endpoint/tests.rs @@ -3175,6 +3175,68 @@ fn expired_received_queries_pruned_before_inbound_cap() { ); } +#[test] +fn received_query_answerable_at_exact_deadline_survives_inbound_prune() { + // A received query is answerable up to and including its deadline: respond + // rejects only now > deadline. The ingress prune in handle_query must not + // drop a token at the exact instant now == deadline, or a still-valid response + // fails with AlreadyResponded. Register an inbound query, advance to exactly + // its deadline, run the ingress prune by handling a second inbound query at + // that instant, then answer the first token at now == deadline: it must + // succeed. Reverting the prune to `now < deadline` drops the token here and + // makes the respond fail. + let mut e = ep(); + + // First inbound query at ORIGIN → deadline D = ORIGIN + timeout. + e.test_set_drain_now(t_secs(0)); + let original = QueryMessage:: { + ltime: LamportTime::new(1), + id: 1, + from: memberlist_proto::Node::new(99u32, addr(9001)), + filters: vec![], + flags: QueryFlag::NO_BROADCAST, + relay_factor: 0, + timeout: core::time::Duration::from_secs(5), + name: "original".into(), + payload: bytes::Bytes::new(), + }; + let _ = e.test_handle_query(original); + + // Capture the original token from the surfaced Event::Query; its deadline is D. + let token = match e + .poll_event() + .expect("the original inbound query must surface as Event::Query") + { + Event::Query(qe) => qe, + other => panic!( + "expected Event::Query, got {:?}", + core::mem::discriminant(&other) + ), + }; + let deadline = token.deadline(); + + // Advance to EXACTLY the deadline and handle a DIFFERENT inbound query, which + // runs the ingress prune at now == deadline. + e.test_set_drain_now(deadline); + let other = QueryMessage:: { + ltime: LamportTime::new(2), + id: 2, + from: memberlist_proto::Node::new(7u32, addr(7000)), + filters: vec![], + flags: QueryFlag::NO_BROADCAST, + relay_factor: 0, + timeout: core::time::Duration::from_secs(30), + name: "other".into(), + payload: bytes::Bytes::new(), + }; + let _ = e.test_handle_query(other); + + // The original token, answerable at now == deadline, must NOT have been pruned: + // respond at exactly the deadline must succeed. + e.respond(&token, bytes::Bytes::new(), deadline) + .expect("respond at exactly the deadline must succeed: the token is still answerable"); +} + // ── Bug 1: zero valid conflict responses must not emit Event::Shutdown ──────── #[test] @@ -6113,6 +6175,65 @@ mod key_request_responder { ); } + #[test] + fn key_request_answerable_at_exact_deadline_survives_inbound_prune() { + // A KeyRequest token is answerable up to and including its deadline: + // respond_key rejects only now > deadline. The ingress prune in handle_query + // must not drop the token at the exact instant now == deadline, or a mandatory + // key response fails with AlreadyResponded. Register a key query, advance to + // exactly its deadline, run the ingress prune by handling a second inbound + // query at that instant, then answer the key token at now == deadline: it must + // succeed. Reverting the prune to `now < deadline` drops the token here and + // makes respond_key fail. + let mut e = ep(); + + // Inbound key query at ORIGIN → deadline D = ORIGIN + timeout. + e.test_set_drain_now(t_secs(0)); + let key = test_key(); + let q = make_key_query("_serf_install_key", Some(key)); + let _ = e.test_handle_query(q); + + // Capture the KeyRequest token; its deadline is D. + let req = match e.poll_event().expect("must emit Event::KeyRequest") { + Event::KeyRequest(kr) => kr, + other => panic!( + "expected Event::KeyRequest, got {:?}", + core::mem::discriminant(&other) + ), + }; + let deadline = req.deadline(); + + // Advance to EXACTLY the deadline and handle a DIFFERENT inbound query, which + // runs the ingress prune at now == deadline. + e.test_set_drain_now(deadline); + let other = QueryMessage:: { + ltime: LamportTime::new(2), + id: 2, + from: memberlist_proto::Node::new(7u32, addr(7000)), + filters: vec![], + flags: QueryFlag::NO_BROADCAST, + relay_factor: 0, + timeout: core::time::Duration::from_secs(30), + name: "other".into(), + payload: bytes::Bytes::new(), + }; + let _ = e.test_handle_query(other); + + // The key token, answerable at now == deadline, must NOT have been pruned: + // respond_key at exactly the deadline must succeed. + e.respond_key( + &req, + KeyResponseArgs { + result: true, + message: smol_str::SmolStr::default(), + keys: vec![key], + primary_key: Some(key), + }, + deadline, + ) + .expect("respond_key at exactly the deadline must succeed: the token is still answerable"); + } + #[test] fn key_request_event_debug_does_not_leak_key_bytes() { let key = test_key(); From c71634ee022580d8d8bc1760a5c68d17e8a4cd90 Mon Sep 17 00:00:00 2001 From: al8n Date: Sat, 4 Jul 2026 01:27:09 +1200 Subject: [PATCH 054/128] fix(serf-embedded): retain a KeyRequest through its exact deadline The mandatory control-queue prune dropped a KeyRequest at now==deadline, but respond_key still accepts a response at the exact deadline (rejects only now>deadline), so a live mandatory key request could be pruned out from under a pending respond_key at the inclusive boundary. Retain while now<=deadline (drop only strictly past), matching respond_key and the serf-proto received-query prune; the regression pumps at exactly the deadline and asserts poll_event still delivers the KeyRequest. --- serf-embedded/src/engine/mod.rs | 28 +++++++++++++++++----------- serf-embedded/src/engine/tests.rs | 31 +++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 11 deletions(-) diff --git a/serf-embedded/src/engine/mod.rs b/serf-embedded/src/engine/mod.rs index 9087ab3d..7dd1586c 100644 --- a/serf-embedded/src/engine/mod.rs +++ b/serf-embedded/src/engine/mod.rs @@ -1004,15 +1004,21 @@ where self.control_events.push_back(ev); } - /// Shed every past-deadline `KeyRequest` from the non-lossy `control_events` - /// queue, mirroring the endpoint's own `received_queries` liveness prune. + /// Shed every strictly-past-deadline `KeyRequest` from the non-lossy + /// `control_events` queue, mirroring the endpoint's own `received_queries` + /// liveness prune. /// - /// A `KeyRequest` past its response `deadline` is dead: the endpoint rejects a - /// `respond_key` sent after the deadline, so the driver can no longer act on it. - /// Dropping it here loses nothing real, promptly releases the raw key material - /// pinned in its payload, and bounds `control_events` to the LIVE mandatory set — - /// the same `now < deadline` retain the serf endpoint applies to its - /// `received_queries` on every `handle_timeout`. Because the endpoint caps its + /// A `KeyRequest` is retained while `now <= deadline` and dropped only once + /// strictly past its deadline (`now > deadline`) — the exact point at which + /// `respond_key` stops accepting a response for it, since its deadline guard + /// rejects only `now > deadline` and a `respond_key` at the exact instant + /// `now == deadline` is still valid. Retaining through the inclusive boundary + /// therefore never sheds a request a pending `respond_key` could still answer, + /// while a strictly-past request is dead: dropping it loses nothing real, + /// promptly releases the raw key material pinned in its payload, and bounds + /// `control_events` to the LIVE mandatory set — the same `now <= deadline` + /// retain the serf endpoint applies to its `received_queries` on every + /// `handle_timeout`. Because the endpoint caps its /// LIVE `received_queries` at a fixed inbound maximum and emits exactly one /// `Event::KeyRequest` per kept entry (sharing this deadline), the live /// `KeyRequest`s retained here are transitively bounded by that same cap — no @@ -1021,11 +1027,11 @@ where /// [`push_control_event`](Self::push_control_event). #[cfg(encryption)] fn prune_expired_control_events(&mut self, now: Instant) { - // Keep everything that is NOT a past-deadline KeyRequest — mirroring the - // endpoint's `retain(|_, rq| now < rq.deadline)` over received_queries. + // Keep everything except a strictly-past-deadline KeyRequest — mirroring the + // endpoint's `retain(|_, rq| now <= rq.deadline)` over received_queries. self .control_events - .retain(|ev| !matches!(ev, Event::KeyRequest(kr) if now >= kr.deadline())); + .retain(|ev| !matches!(ev, Event::KeyRequest(kr) if now > kr.deadline())); } /// Fold one drained machine event into its await-result join, then route it to the diff --git a/serf-embedded/src/engine/tests.rs b/serf-embedded/src/engine/tests.rs index 658725c5..cd387ad0 100644 --- a/serf-embedded/src/engine/tests.rs +++ b/serf-embedded/src/engine/tests.rs @@ -1853,3 +1853,34 @@ fn past_deadline_key_request_is_pruned_and_payload_released() { ); } } + +/// A `KeyRequest` at EXACTLY its response `deadline` is still answerable — +/// `respond_key` rejects only `now > deadline`, so a `respond_key` at the exact +/// instant `now == deadline` succeeds. The pump's prune must therefore RETAIN it +/// at `now == deadline`, and `poll_event` must still deliver it. Guards the +/// inclusive prune boundary: dropping it at `now == deadline` would shed a live +/// mandatory key request out from under a response the originator could still +/// send. Reverting the predicate to `now >= deadline` prunes it here, so +/// `poll_event` no longer delivers the request and this fails. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn key_request_at_exact_deadline_survives_prune_and_is_delivered() { + let mut engine = make_engine(); + let now = Instant::from_origin(Duration::from_secs(86_400)); + engine.start(now); + + let mut gossip = NoGossip; + let mut stream = NoStream::with_pool(2); + + // Deadline equal to the pump instant: the request is answerable at exactly D + // (respond_key rejects only now > D), so the prune must keep it. + let deadline = now + Duration::from_secs(5); + engine.route_drained_event(key_request(11, deadline)); + engine.pump(deadline, &mut gossip, &mut stream); + + assert!( + matches!(engine.poll_event(), Some(Event::KeyRequest(_))), + "a KeyRequest at exactly its deadline is still answerable, so the prune must \ + retain it and poll_event must deliver it" + ); +} From 6ddd35792655606070ef4c753567e8feeb119b55 Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 10 Jul 2026 12:17:13 +1200 Subject: [PATCH 055/128] feat(serf-smoltcp): sync caller-poll serf driver over the smoltcp stack MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit serf-smoltcp mirrors memberlist-smoltcp over serf-embedded's SerfEngine — reusing all the payload-agnostic glue via serf-embedded and reimplementing only the thin smoltcp socket views; the Serf handle owns the smoltcp sockets + interface and its poll(now, device) ticks the stack then pumps the engine; serf's mandatory events (KeyRequest->respond_key, Shutdown->stop) are handled driver-side in the poll cycle; 2-node loopback convergence test green. --- Cargo.toml | 1 + serf-smoltcp/Cargo.toml | 78 ++ serf-smoltcp/README.md | 108 +++ serf-smoltcp/build.rs | 17 + serf-smoltcp/src/addr/mod.rs | 42 + serf-smoltcp/src/addr/tests.rs | 23 + serf-smoltcp/src/config/mod.rs | 115 +++ serf-smoltcp/src/config/tests.rs | 11 + serf-smoltcp/src/error/mod.rs | 451 +++++++++++ serf-smoltcp/src/error/tests.rs | 197 +++++ serf-smoltcp/src/gossip_io.rs | 74 ++ serf-smoltcp/src/interface.rs | 94 +++ serf-smoltcp/src/lib.rs | 80 ++ serf-smoltcp/src/resolver.rs | 55 ++ serf-smoltcp/src/serf/mod.rs | 1124 +++++++++++++++++++++++++++ serf-smoltcp/src/serf/tests.rs | 151 ++++ serf-smoltcp/src/stream_io/mod.rs | 235 ++++++ serf-smoltcp/src/stream_io/tests.rs | 253 ++++++ serf-smoltcp/tests/harness/mod.rs | 157 ++++ serf-smoltcp/tests/join_leave.rs | 122 +++ serf-smoltcp/tests/user_event.rs | 99 +++ 21 files changed, 3487 insertions(+) create mode 100644 serf-smoltcp/Cargo.toml create mode 100644 serf-smoltcp/README.md create mode 100644 serf-smoltcp/build.rs create mode 100644 serf-smoltcp/src/addr/mod.rs create mode 100644 serf-smoltcp/src/addr/tests.rs create mode 100644 serf-smoltcp/src/config/mod.rs create mode 100644 serf-smoltcp/src/config/tests.rs create mode 100644 serf-smoltcp/src/error/mod.rs create mode 100644 serf-smoltcp/src/error/tests.rs create mode 100644 serf-smoltcp/src/gossip_io.rs create mode 100644 serf-smoltcp/src/interface.rs create mode 100644 serf-smoltcp/src/lib.rs create mode 100644 serf-smoltcp/src/resolver.rs create mode 100644 serf-smoltcp/src/serf/mod.rs create mode 100644 serf-smoltcp/src/serf/tests.rs create mode 100644 serf-smoltcp/src/stream_io/mod.rs create mode 100644 serf-smoltcp/src/stream_io/tests.rs create mode 100644 serf-smoltcp/tests/harness/mod.rs create mode 100644 serf-smoltcp/tests/join_leave.rs create mode 100644 serf-smoltcp/tests/user_event.rs diff --git a/Cargo.toml b/Cargo.toml index 14042e85..5e6b2a75 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ members = [ "serf-driver", "serf-compio", "serf-reactor", + "serf-smoltcp", ] exclude = ["legacy"] resolver = "3" diff --git a/serf-smoltcp/Cargo.toml b/serf-smoltcp/Cargo.toml new file mode 100644 index 00000000..f7dee14f --- /dev/null +++ b/serf-smoltcp/Cargo.toml @@ -0,0 +1,78 @@ +[package] +name = "serf-smoltcp" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +readme = "README.md" +rust-version = "1.96" +description = "Executor-free no_std serf driver over the smoltcp TCP/IP stack." + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] + +[features] +default = ["std"] +# `std` and `alloc` are INDEPENDENT capability tiers (std does NOT pull alloc): +# under `std` the real `std` crate supplies the heap; under `alloc` the +# no_std+alloc path aliases `alloc`. Each layers the matching feature on every +# dependency that has one. +std = ["serf-embedded/std", "serf-proto/std", "memberlist-proto/std", "smoltcp/std"] +alloc = ["serf-embedded/alloc", "serf-proto/alloc", "memberlist-proto/alloc", "smoltcp/alloc"] + +# CIDR peer-admission allow-list. A `CidrPolicy` set via `Options::with_cidr_policy` +# filters inbound gossip by datagram source and inbound reliable connections by +# peer address at the transport boundary, AND inbound alives by the peer's +# self-advertised address at membership admission. no_std+alloc (pulls the heap tier). +cidr = ["alloc", "serf-embedded/cidr", "memberlist-proto/cidr"] + +# Gossip + reliable-plane AEAD encryption backends. Each pass-through enables the +# matching serf-embedded and machine backend so a `TransformOptions` carrying a +# `Keyring` for that cipher can actually encrypt, and serf's key-management +# queries (install/use/remove/list) plus the driver-side `KeyRequest` response +# become available. Both AEAD backends are no_std+alloc. +aes-gcm = ["alloc", "serf-embedded/aes-gcm", "serf-proto/aes-gcm", "memberlist-proto/aes-gcm"] +chacha20-poly1305 = [ + "alloc", + "serf-embedded/chacha20-poly1305", + "serf-proto/chacha20-poly1305", + "memberlist-proto/chacha20-poly1305", +] +# Umbrella that activates both AEAD backends (convenient for tests / CI). +encryption = ["aes-gcm", "chacha20-poly1305"] + +[dependencies] +# The transport-agnostic no_std serf driving core (SerfEngine + the reused +# memberlist-embedded glue it re-exports); depended on alone for the driving +# surface, mirroring how memberlist-smoltcp depends on memberlist-embedded. +serf-embedded = { path = "../serf-embedded", default-features = false } +# serf's own protocol types named in `SerfEngine`'s command signatures +# (`QueryParams` / `QueryEvent` / `Tags` / `Member` / the key-management types). +serf-proto = { path = "../serf-proto", default-features = false, features = [ + "tcp", +] } +# The Sans-I/O machine types named in `SerfEngine`'s construction / command +# signatures (`Instant` / `EndpointOptions` / `Node` / `SmallRng` / `StreamId`). +# Pinned to the same git+branch as serf-embedded so the graph holds one copy. +memberlist-proto = { git = "https://github.com/al8n/memberlist", branch = "main", default-features = false, features = [ + "tcp", +] } +smoltcp = { version = "0.13", default-features = false, features = [ + "alloc", + "medium-ip", + "medium-ethernet", + "proto-ipv4", + "proto-ipv6", + "socket-udp", + "socket-tcp", +] } +getrandom = "0.4" +bytes = { version = "1", default-features = false } +# Named in the `user_event` / `query` forwarder signatures (`impl Into`), +# matching serf-embedded's command surface. +smol_str = { version = "0.3", default-features = false } +hashbrown = { version = "0.17", default-features = false, features = [ + "default-hasher", +] } diff --git a/serf-smoltcp/README.md b/serf-smoltcp/README.md new file mode 100644 index 00000000..26580f83 --- /dev/null +++ b/serf-smoltcp/README.md @@ -0,0 +1,108 @@ +
+

serf-smoltcp

+
+
+ +Executor-free `no_std` serf driver over the [smoltcp] TCP/IP stack — no OS and no +async runtime required. + +
+ +## Introduction + +`serf-smoltcp` drives serf's transport-agnostic core ([`serf-embedded`]'s +`SerfEngine`, which composes serf's super-machine over the memberlist reliable +coordinator) on a [smoltcp] TCP/IP stack. It owns no executor and performs no +blocking: you pump it from your own poll loop alongside the smoltcp `Interface`, +and it tells you the next instant it wants to be polled. + +It mirrors [`memberlist-smoltcp`] over `serf-embedded` instead of +`memberlist-embedded`: all the payload-agnostic link-layer glue is reused through +`serf-embedded`, and only the thin smoltcp socket views (`GossipIo` / `StreamIo`) +are reimplemented. The `Serf` handle owns the smoltcp sockets + interface, and its +`poll(now, device)` ticks the stack, pumps the engine, then acts on serf's +mandatory events in the poll cycle (a lost id-conflict `Shutdown` flips a stop +flag; an encryption `KeyRequest` is applied to the local keyring and answered), +buffering every event for the app's own `poll_event`. + +`no_std` + `alloc`: the protocol state lives in slab-backed pools, so there is no +per-packet heap traffic on the hot path. + +## Installation + +```toml +[dependencies] +serf-smoltcp = { version = "0.5", default-features = false, features = ["alloc"] } +``` + +## Example + +```rust,ignore +use serf_smoltcp::{ + EndpointOptions, InterfaceOptions, Options, Serf, SerfOptions, SocketAddrResolver, + TransformOptions, +}; +use smol_str::SmolStr; + +// `device` is your smoltcp `Device`; `now` is a portable `Instant` your firmware +// advances. `advertise` is the local node's `SocketAddr`. +let mut node = Serf::::new( + Options::new(), + InterfaceOptions::new(hardware_addr), // + IP addresses, routes, RNG seed + TransformOptions::default(), + EndpointOptions::new(SmolStr::new("node-a"), advertise), + SerfOptions::new(), + &SocketAddrResolver, + &mut device, + now, +); +node.start(now); + +// Pump from your own loop; `poll` advances serf + the smoltcp interface and returns +// the next instant the driver wants to be polled (or `None`). +loop { + let next = node.poll(now, &mut device); + if node.is_shutdown() { + break; + } + while let Some(event) = node.poll_event() { + // react to serf events (member join/leave/update, user events, queries, …) + } + // ...sleep until `next` or until the device is RX-ready, then advance `now`... +} +``` + +## Feature flags + +| Feature | Description | +|---------|-------------| +| `std` *(default)* | host builds and the test harness | +| `alloc` | `no_std` with a global allocator (bare metal) — build with `--no-default-features --features alloc` | +| `aes-gcm` / `-chacha20-poly1305` | AEAD encryption (gossip + plain-TCP reliable) plus serf's key-management surface | +| `encryption` | umbrella for both AEAD backends | +| `cidr` | IP allow-list admission | + +## Design + +- **`no_std` + `alloc`**, panic-free hot path: slab-backed protocol pools, no + per-packet allocation. +- **Caller-poll**: no executor; you drive `poll(now, device)` from your loop and + honor the returned next-wake instant — the same shape as advancing the smoltcp + `Interface`. +- Built on the transport-agnostic [`serf-embedded`] `SerfEngine`, with smoltcp UDP + (gossip) + a TCP socket pool (reliable plane) behind its I/O seams. +- serf's mandatory driver-actioned events are handled DRIVER-side in the poll + cycle, so correctness does not depend on the application draining events. + +## License + +`serf-smoltcp` is under the terms of the MPL-2.0 license. + +See [LICENSE] for details. + +Copyright (c) 2025 Al Liu. + +[smoltcp]: https://crates.io/crates/smoltcp +[`serf-embedded`]: https://crates.io/crates/serf-embedded +[`memberlist-smoltcp`]: https://crates.io/crates/memberlist-smoltcp +[LICENSE]: https://github.com/al8n/serf/blob/main/LICENSE diff --git a/serf-smoltcp/build.rs b/serf-smoltcp/build.rs new file mode 100644 index 00000000..1af9ef00 --- /dev/null +++ b/serf-smoltcp/build.rs @@ -0,0 +1,17 @@ +//! Emits the aggregate `encryption` cfg, set when any AEAD backend feature is +//! enabled, so encryption code gates on `#[cfg(encryption)]` instead of repeating +//! the backend list. serf's gossip plane carries no compression / checksum, so +//! only the encryption aggregate is emitted (mirroring `serf-embedded`). + +fn any_feature(names: &[&str]) -> bool { + names + .iter() + .any(|name| std::env::var_os(format!("CARGO_FEATURE_{name}")).is_some()) +} + +fn main() { + println!("cargo::rustc-check-cfg=cfg(encryption)"); + if any_feature(&["AES_GCM", "CHACHA20_POLY1305"]) { + println!("cargo::rustc-cfg=encryption"); + } +} diff --git a/serf-smoltcp/src/addr/mod.rs b/serf-smoltcp/src/addr/mod.rs new file mode 100644 index 00000000..31d8659f --- /dev/null +++ b/serf-smoltcp/src/addr/mod.rs @@ -0,0 +1,42 @@ +//! Boundary conversions between the driver's `core::net::SocketAddr` / +//! `memberlist_proto::Instant` and smoltcp's `IpEndpoint` / `Instant`. + +use core::net::SocketAddr; +use memberlist_proto::Instant; +use smoltcp::{time::Instant as SmoltcpInstant, wire::IpEndpoint}; + +/// Convert a `SocketAddr` to a smoltcp `IpEndpoint`. +/// +/// serf addresses are family-agnostic configuration, so this boundary helper +/// accepts both IPv4 and IPv6. smoltcp's `From for Endpoint` requires +/// both `proto-ipv4` and `proto-ipv6`, which this crate enables. +#[inline] +pub(crate) fn to_endpoint(addr: SocketAddr) -> IpEndpoint { + addr.into() +} + +/// Convert a smoltcp `IpEndpoint` back to a `SocketAddr`. +#[inline] +pub(crate) fn from_endpoint(ep: IpEndpoint) -> SocketAddr { + ep.into() +} + +/// Convert a `memberlist_proto::Instant` to a smoltcp `Instant` (millisecond granularity). +#[inline] +pub(crate) fn to_smoltcp_instant(now: Instant) -> SmoltcpInstant { + debug_assert!( + now.since_origin().as_millis() <= i64::MAX as u128, + "instant too far from origin for smoltcp" + ); + SmoltcpInstant::from_millis(now.since_origin().as_millis() as i64) +} + +/// Convert a smoltcp `Instant` back to a `memberlist_proto::Instant`. +#[inline] +pub(crate) fn from_smoltcp_instant(t: SmoltcpInstant) -> Instant { + debug_assert!(t.total_millis() >= 0, "smoltcp instant before origin"); + Instant::from_origin(core::time::Duration::from_millis(t.total_millis() as u64)) +} + +#[cfg(test)] +mod tests; diff --git a/serf-smoltcp/src/addr/tests.rs b/serf-smoltcp/src/addr/tests.rs new file mode 100644 index 00000000..828c5ede --- /dev/null +++ b/serf-smoltcp/src/addr/tests.rs @@ -0,0 +1,23 @@ +use super::*; +use core::net::{IpAddr, Ipv4Addr, SocketAddr}; + +#[test] +fn socket_addr_round_trips_through_ip_endpoint() { + let sa = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 7)), 7946); + let ep = to_endpoint(sa); + assert_eq!(from_endpoint(ep), sa); +} + +#[test] +fn ipv6_socket_addr_round_trips() { + use core::net::Ipv6Addr; + let sa = SocketAddr::new(IpAddr::V6(Ipv6Addr::LOCALHOST), 7946); + assert_eq!(from_endpoint(to_endpoint(sa)), sa); +} + +#[test] +fn machine_instant_round_trips_through_smoltcp_instant() { + let now = memberlist_proto::Instant::from_origin(core::time::Duration::from_millis(123_456)); + let s = to_smoltcp_instant(now); + assert_eq!(from_smoltcp_instant(s), now); +} diff --git a/serf-smoltcp/src/config/mod.rs b/serf-smoltcp/src/config/mod.rs new file mode 100644 index 00000000..3419fa2d --- /dev/null +++ b/serf-smoltcp/src/config/mod.rs @@ -0,0 +1,115 @@ +//! Runtime, alloc-backed sizing for the smoltcp driver. The machine is +//! alloc-backed, so fixed-array const generics would buy nothing. + +use core::time::Duration; + +/// Default [`Options::close_timeout`]: 10 seconds. +/// +/// A graceful TCP close (FIN/ACK exchange) over a healthy link completes in a +/// few round-trips; 10 s is a generous bound that rides out WAN latency while +/// still promptly reclaiming a socket whose peer vanished mid-close. It mirrors +/// the machine's default stream/handshake deadline so a stuck reliable exchange +/// and its closing socket are reclaimed on the same order of timescale. +pub const DEFAULT_CLOSE_TIMEOUT: Duration = Duration::from_secs(10); + +/// Sizing and ports for [`Serf`](crate::Serf). All buffers are fixed-capacity at +/// construction (smoltcp has no growable backing on no_std); gossip overflow +/// drops, reliable overflow backpressures. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct Options { + /// Local port the node binds. The gossip UDP socket and the reliable-plane + /// TCP listener both use it, and it is the port peers reach the node at — the + /// single-port serf model (one advertised `SocketAddr` serves both planes, + /// since a UDP and a TCP socket on the same port number are independent). + pub port: u16, + /// Pooled TCP sockets (max concurrent reliable exchanges + 1 listener). + pub tcp_pool_size: usize, + /// Per-TCP-socket rx ring bytes. + pub tcp_socket_rx_bytes: usize, + /// Per-TCP-socket tx ring bytes. + pub tcp_socket_tx_bytes: usize, + /// UDP rx datagram metadata slots. + pub udp_rx_packets: usize, + /// UDP tx datagram metadata slots. + pub udp_tx_packets: usize, + /// UDP rx payload byte arena. + pub udp_rx_payload_bytes: usize, + /// UDP tx payload byte arena. + pub udp_tx_payload_bytes: usize, + /// Maximum time a gracefully-closing TCP socket may stay parked before it is + /// force-aborted and returned to the pool. + /// + /// smoltcp applies no TCP timeout by default, so a peer that vanishes during + /// the FIN handshake (FinWait/LastAck) keeps the socket open indefinitely and + /// the handle never returns to the free-list — permanently shrinking the pool + /// and the listener replenished from it. Bounding the close guarantees + /// recovery. A healthy close completes well before this and is reclaimed the + /// moment it reaches `Closed`; the timeout only governs the vanished-peer case. + pub close_timeout: Duration, + /// CIDR peer-admission policy. Filters inbound gossip by datagram source and + /// inbound reliable connections by peer address at the transport boundary, AND + /// inbound alives by the peer's self-advertised address at membership + /// admission. `None` (the default) admits every address. Present only with the + /// `cidr` feature; set it via [`with_cidr_policy`](Options::with_cidr_policy). + #[cfg(feature = "cidr")] + #[cfg_attr(docsrs, doc(cfg(feature = "cidr")))] + pub cidr_policy: Option, +} + +impl Default for Options { + fn default() -> Self { + Self { + port: 7946, + tcp_pool_size: 4, + tcp_socket_rx_bytes: 4096, + tcp_socket_tx_bytes: 4096, + udp_rx_packets: 8, + udp_tx_packets: 8, + udp_rx_payload_bytes: 8 * 1500, + udp_tx_payload_bytes: 8 * 1500, + close_timeout: DEFAULT_CLOSE_TIMEOUT, + #[cfg(feature = "cidr")] + cidr_policy: None, + } + } +} + +impl Options { + /// Defaults tuned for a small embedded cluster. + pub fn new() -> Self { + Self::default() + } + + /// Override the pooled TCP socket count. + pub fn with_tcp_pool_size(mut self, n: usize) -> Self { + self.tcp_pool_size = n; + self + } + + /// Override the local port (the gossip UDP socket and the reliable-plane TCP + /// listener both bind it). + pub fn with_port(mut self, p: u16) -> Self { + self.port = p; + self + } + + /// Override the graceful-close timeout (see [`Options::close_timeout`]). + pub fn with_close_timeout(mut self, d: Duration) -> Self { + self.close_timeout = d; + self + } + + /// Install a CIDR peer-admission policy (see [`Options::cidr_policy`]). One + /// policy gates the gossip source and reliable peer at the transport boundary + /// AND the advertised address at membership admission. + #[cfg(feature = "cidr")] + #[cfg_attr(docsrs, doc(cfg(feature = "cidr")))] + pub fn with_cidr_policy(mut self, policy: serf_embedded::CidrPolicy) -> Self { + self.cidr_policy = Some(policy); + self + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-smoltcp/src/config/tests.rs b/serf-smoltcp/src/config/tests.rs new file mode 100644 index 00000000..95e3ca0f --- /dev/null +++ b/serf-smoltcp/src/config/tests.rs @@ -0,0 +1,11 @@ +use super::*; + +#[test] +fn defaults_are_sane_and_overridable() { + let c = Options::new(); + assert!(c.tcp_pool_size >= 1); + assert!(c.udp_rx_payload_bytes > 0); + let c = Options::new().with_tcp_pool_size(8).with_port(1234); + assert_eq!(c.tcp_pool_size, 8); + assert_eq!(c.port, 1234); +} diff --git a/serf-smoltcp/src/error/mod.rs b/serf-smoltcp/src/error/mod.rs new file mode 100644 index 00000000..81e4b523 --- /dev/null +++ b/serf-smoltcp/src/error/mod.rs @@ -0,0 +1,451 @@ +//! Errors from constructing a [`Serf`](crate::Serf). + +use core::fmt; + +use memberlist_proto::EndpointInitError; + +use crate::interface::{HardwareAddress, IpCidr, Medium, Route}; +use core::net::SocketAddr; +use std::boxed::Box; + +/// Why constructing a [`Serf`](crate::Serf) failed. +/// +/// Every variant is a misconfiguration or environment fault reported in place +/// of a panic, so a caller assembling an [`InterfaceOptions`](crate::InterfaceOptions) +/// from untrusted or runtime values can recover. +#[derive(Debug)] +#[non_exhaustive] +pub enum InitError { + /// The configured hardware address's medium does not match the bound device. + /// + /// smoltcp's `Interface::new` would otherwise panic; the driver checks first + /// and returns this instead. + MediumMismatch(MediumMismatch), + /// The configured hardware address selects a medium this driver does not + /// support (smoltcp's `Ieee802154`). + /// + /// The driver enables only `medium-ip` and `medium-ethernet`; an + /// `Ieee802154` hardware address — constructible only when a downstream crate + /// unifies smoltcp's `medium-ieee802154` feature on — would otherwise reach an + /// `unreachable!()` while deriving the medium. The driver rejects it here. + UnsupportedMedium, + /// The configured Ethernet hardware address is not unicast. + /// + /// smoltcp's `Interface::new` stores the configured hardware address without + /// validating it (it never calls `check_hardware_addr`), so a + /// broadcast/multicast MAC would not panic but would install an invalid L2 + /// source/acceptance identity. The driver rejects it instead. The offending + /// address is carried for diagnostics. + NonUnicastHardwareAddress(HardwareAddress), + /// A configured IP CIDR's address is neither unicast nor unspecified. + /// + /// smoltcp's `update_ip_addrs` calls `check_ip_addrs`, which `panic!`s on an + /// address that is not unicast and not unspecified (e.g. multicast or the + /// limited broadcast address). The driver mirrors that exact condition and + /// returns this instead of panicking. The offending CIDR is carried for + /// diagnostics. + NonUnicastIpAddress(IpCidr), + /// The configured advertise address + /// ([`EndpointOptions::advertise_addr_ref`](memberlist_proto::EndpointOptions::advertise_addr_ref)) + /// is not a routable destination. + /// + /// A node must advertise an address its peers can route a reply to. An + /// unspecified/multicast/broadcast IP or port 0 would be gossiped to the + /// cluster and then be useless to every peer that selected it as an egress + /// destination — smoltcp's socket layer rejects the unspecified address and + /// port 0 (`Unaddressable`) and the route / neighbor lookup asserts the address + /// is unicast. The offending address is carried for diagnostics. + NonRoutableAdvertiseAddr(SocketAddr), + /// [`InterfaceOptions::ip_addrs`](crate::InterfaceOptions::ip_addrs) was empty, + /// so the interface would accept no packets. + MissingIpAddress, + /// More IP addresses were configured than smoltcp's interface capacity + /// (`IFACE_MAX_ADDR_COUNT`). + TooManyIpAddresses, + /// More routes were configured than smoltcp's route-table capacity + /// (`IFACE_MAX_ROUTE_COUNT`). + TooManyRoutes, + /// [`Options::port`](crate::Options::port) is zero. + /// + /// smoltcp's `udp::Socket::bind` and `tcp::Socket::listen` reject port 0 + /// (`Unaddressable`), which would otherwise panic inside the fallible + /// constructor; the driver rejects it up front instead. + ZeroPort, + /// The advertised port does not match the bound port. + /// + /// The node binds one [`Options::port`](crate::Options::port) for both the + /// gossip UDP socket and the reliable TCP listener (the single-port serf + /// model). On a direct smoltcp interface (no NAT) a node is reachable only at + /// the port it binds, so its advertised port + /// ([`EndpointOptions::advertise_addr_ref`](memberlist_proto::EndpointOptions::advertise_addr_ref)) + /// must equal it. Otherwise every peer routes to a port nothing is listening on. + AdvertisePortMismatch, + /// The advertised IP is not one assigned to the interface. + /// + /// smoltcp drops any inbound packet whose destination is not an assigned + /// interface address (the `has_ip_addr` gate in `process_ipv4`/`process_ipv6`), + /// so a node advertising an IP absent from + /// [`InterfaceOptions::ip_addrs`](crate::InterfaceOptions::ip_addrs) is + /// unreachable on both planes — peers gossip and dial an address its own + /// interface discards. The advertised IP must exactly match a configured + /// interface address. The offending advertise address is carried for + /// diagnostics. + AdvertiseAddrNotLocal(SocketAddr), + /// A configured route's gateway (`via_router`) is not unicast. + /// + /// On Ethernet egress smoltcp resolves an off-link next hop through its + /// neighbor cache, whose `lookup` asserts the protocol address is unicast (a + /// release-mode `assert!`). A multicast/broadcast/unspecified `via_router` + /// would pass construction and then crash the running node at first off-link + /// egress. The offending route is carried for diagnostics. + NonUnicastRouteGateway(Route), + /// A configured route's prefix and gateway are different IP families. + /// + /// A route to an IPv4 prefix via an IPv6 gateway (or the reverse) can never + /// resolve a next hop; it is rejected here rather than silently failing at + /// egress. The offending route is carried for diagnostics. + RouteFamilyMismatch(Route), + /// Drawing a random seed from the system entropy source failed (only when no + /// seed was pinned via + /// [`InterfaceOptions::with_random_seed`](crate::InterfaceOptions::with_random_seed)). + Entropy, + /// The address resolver failed while resolving the advertise address. + /// + /// The resolver's error type is generic, so it is boxed to preserve the + /// `source()` chain; a caller that knows its concrete resolver can downcast. + /// No `Send`/`Sync` bound — the smoltcp [`Resolver`](crate::Resolver) is + /// single-threaded by design, so its error need not cross threads. + Resolve(Box), + /// The address resolver succeeded but yielded no address for the advertise + /// address, so the node would have nothing to advertise. + NoAddresses, + /// The SWIM machine endpoint failed to initialize. + Endpoint(EndpointInitError), + /// The configured encryption keyring cannot be used by this build. + /// + /// Construction and runtime rotation probe every configured key (primary then + /// secondaries) entropy-free. A key whose AEAD backend was not compiled into + /// this binary surfaces here as + /// [`EncryptionError::UnsupportedAlgorithm`](memberlist_proto::EncryptionError::UnsupportedAlgorithm), + /// and a key whose cipher variant disagrees with its algorithm tag as + /// [`KeyMismatch`](memberlist_proto::EncryptionError::KeyMismatch), turning what + /// would otherwise be a silent runtime drop of every encrypted gossip datagram + /// into a typed construction error. + /// + /// The probe validates only this PERMANENT usability — it does not validate the + /// per-send nonce source. Encryption is cross-transport (gossip datagrams and the + /// plaintext reliable plane), and every encrypted frame draws a fresh nonce from + /// `getrandom` at send time, so on a target whose backend is missing or failing + /// the node still constructs and then cannot encrypt outbound traffic — gossip + /// datagrams and reliable exchanges alike fail as they are sent. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + Encryption(memberlist_proto::EncryptionError), + /// The configured gossip MTU's on-wire datagram cannot fit a UDP packet. + /// + /// The driver sizes its UDP arenas from `gossip_mtu + ENCRYPTED_WRAPPER_OVERHEAD` + /// (the largest on-wire datagram the machine can emit). A `gossip_mtu` whose + /// on-wire size exceeds the 65507-byte UDP payload limit could never be sent, + /// and the unchecked arena arithmetic would overflow (panic in a checked build, + /// wrap to an undersized arena in release). The configured value and the + /// effective ceiling are carried for diagnostics. + GossipMtuTooLarge(GossipMtuTooLarge), + /// A UDP arena byte count (`udp_*_packets × max on-wire datagram`) overflows + /// `usize`. + /// + /// Reachable on a 32-bit target with an astronomically large UDP packet-slot + /// count: the product would wrap to an undersized arena. Rejected here rather + /// than allocating a silently-too-small buffer. + UdpArenaTooLarge, + /// [`Options::tcp_pool_size`](crate::Options::tcp_pool_size) is below 2. + /// + /// Construction dedicates one pooled TCP socket to the inbound listener and + /// uses the rest for dials and accepts. With 0 there is no listener (no + /// reliable plane at all); with 1 the listener consumes the only socket, + /// leaving none to dial — so the node can never dial a seed to join and can + /// accept at most one inbound at a time. The functional minimum is 2 (a + /// listener plus one dial/accept socket). + TcpPoolTooSmall, + /// [`Options::tcp_socket_rx_bytes`](crate::Options::tcp_socket_rx_bytes) or + /// [`Options::tcp_socket_tx_bytes`](crate::Options::tcp_socket_tx_bytes) is zero. + /// + /// smoltcp's `RingBuffer::new` does not panic on zero-length storage — it + /// builds a permanently-empty ring — so a 0-byte rx (or tx) buffer is a socket + /// that can never receive (or send): a silently-dead reliable plane. Both must + /// be non-zero. + ZeroTcpSocketBuffer, + /// [`Options::tcp_socket_rx_bytes`](crate::Options::tcp_socket_rx_bytes) exceeds + /// smoltcp's 1 GiB receive-buffer limit. + /// + /// smoltcp's `tcp::Socket::new` `panic!`s when the receive-buffer capacity + /// exceeds 1 GiB (`> 1 << 30`); a caller-supplied `tcp_socket_rx_bytes` past + /// that limit would panic inside this fallible constructor. The transmit + /// buffer has no such limit. Rejected here instead. + TcpRxBufferTooLarge, + /// [`Options::udp_rx_packets`](crate::Options::udp_rx_packets) or + /// [`Options::udp_tx_packets`](crate::Options::udp_tx_packets) is zero. + /// + /// The gossip `udp::PacketBuffer` is built with that many per-packet metadata + /// slots. Zero slots is a ring that can never enqueue or dequeue a datagram, + /// so gossip can never be received (or sent): a silently-dead gossip plane. + /// Both must be non-zero. + ZeroUdpPackets, + /// [`Options::close_timeout`](crate::Options::close_timeout) is zero. + /// + /// `close_timeout` bounds the graceful reliable-close drain: a connection + /// still `Closing` past `now + close_timeout` is force-aborted. A zero timeout + /// sets that deadline to `now`, so every graceful close is force-aborted + /// immediately — the drain never runs and an in-flight push/pull response is + /// truncated. Must be non-zero. + ZeroCloseTimeout, +} + +/// The configured gossip MTU exceeds the largest plaintext payload whose on-wire +/// datagram still fits a UDP packet. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct GossipMtuTooLarge { + /// The configured `gossip_mtu` that was rejected. + pub gossip_mtu: usize, + /// The largest acceptable `gossip_mtu`: `65507 - ENCRYPTED_WRAPPER_OVERHEAD`. + pub ceiling: usize, +} + +impl fmt::Display for GossipMtuTooLarge { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "gossip_mtu {} exceeds the maximum sendable plaintext payload of {} bytes \ + (the on-wire datagram must fit the 65507-byte UDP payload limit)", + self.gossip_mtu, self.ceiling + ) + } +} + +/// The configured hardware-address medium did not match the bound device's +/// medium. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MediumMismatch { + /// The medium implied by the configured [`HardwareAddress`](crate::HardwareAddress). + pub expected: Medium, + /// The medium the bound device actually reports. + pub actual: Medium, +} + +impl fmt::Display for MediumMismatch { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "hardware address medium {:?} does not match device medium {:?}", + self.expected, self.actual + ) + } +} + +impl fmt::Display for InitError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + InitError::MediumMismatch(m) => write!(f, "{m}"), + InitError::UnsupportedMedium => f.write_str( + "configured hardware address selects a medium this driver does not support \ + (only IP and Ethernet are supported)", + ), + InitError::NonUnicastHardwareAddress(addr) => { + write!(f, "hardware address {addr} is not unicast") + } + InitError::NonUnicastIpAddress(cidr) => { + write!(f, "IP address {} is not unicast", cidr.address()) + } + InitError::NonRoutableAdvertiseAddr(addr) => { + write!(f, "advertise address {addr} is not a routable destination") + } + InitError::MissingIpAddress => { + f.write_str("no IP address configured; the interface would accept no packets") + } + InitError::TooManyIpAddresses => { + f.write_str("more IP addresses configured than the smoltcp interface can hold") + } + InitError::TooManyRoutes => { + f.write_str("more routes configured than the smoltcp route table can hold") + } + InitError::ZeroPort => f.write_str("port is zero (smoltcp rejects port 0)"), + InitError::AdvertisePortMismatch => { + f.write_str("advertised port does not match the bound port") + } + InitError::AdvertiseAddrNotLocal(addr) => { + write!( + f, + "advertised IP {} is not assigned to the interface", + addr.ip() + ) + } + InitError::NonUnicastRouteGateway(route) => { + write!(f, "route gateway {} is not unicast", route.via_router) + } + InitError::RouteFamilyMismatch(route) => write!( + f, + "route prefix {} and gateway {} are different IP families", + route.cidr, route.via_router + ), + InitError::Entropy => f.write_str("system entropy source failed while drawing a random seed"), + InitError::Resolve(e) => write!(f, "advertise address resolution failed: {e}"), + InitError::NoAddresses => f.write_str("advertise address resolution returned no addresses"), + InitError::Endpoint(e) => write!(f, "SWIM endpoint initialization failed: {e}"), + #[cfg(encryption)] + InitError::Encryption(e) => write!(f, "encryption configuration is unusable: {e}"), + InitError::GossipMtuTooLarge(m) => write!(f, "{m}"), + InitError::UdpArenaTooLarge => { + f.write_str("UDP arena byte count (packets × max datagram) overflows usize") + } + InitError::TcpPoolTooSmall => { + f.write_str("tcp_pool_size must be at least 2 (a listener plus one dial/accept socket)") + } + InitError::ZeroTcpSocketBuffer => { + f.write_str("tcp_socket_rx_bytes and tcp_socket_tx_bytes must be non-zero") + } + InitError::TcpRxBufferTooLarge => { + f.write_str("tcp_socket_rx_bytes exceeds smoltcp's 1 GiB receive-buffer limit") + } + InitError::ZeroUdpPackets => { + f.write_str("udp_rx_packets and udp_tx_packets must be non-zero") + } + InitError::ZeroCloseTimeout => f.write_str("close_timeout must be non-zero"), + } + } +} + +impl InitError { + /// Map a [`SerfEngine`](serf_embedded::SerfEngine) construction error into the + /// driver's [`InitError`]. + /// + /// The driver pre-validates the port, gossip MTU, close timeout, and advertise + /// address before building the engine, so in practice the engine fails only with + /// [`Endpoint`](serf_embedded::InitError::Endpoint) (machine init) or + /// [`Encryption`](serf_embedded::InitError::Encryption) (an unusable keyring). + /// The remaining variants are mapped to their driver equivalents anyway so the + /// conversion is total and stays correct if the driver's pre-checks are ever + /// reordered or relaxed. + pub(crate) fn from_embedded(e: serf_embedded::InitError) -> Self { + use serf_embedded::InitError as E; + match e { + E::NonRoutableAdvertiseAddr(addr) => InitError::NonRoutableAdvertiseAddr(addr), + E::AdvertisePortMismatch => InitError::AdvertisePortMismatch, + E::ZeroPort => InitError::ZeroPort, + E::ZeroCloseTimeout => InitError::ZeroCloseTimeout, + E::GossipMtuTooLarge(m) => InitError::GossipMtuTooLarge(GossipMtuTooLarge { + gossip_mtu: m.gossip_mtu, + ceiling: m.ceiling, + }), + E::Endpoint(inner) => InitError::Endpoint(inner), + #[cfg(encryption)] + E::Encryption(inner) => InitError::Encryption(inner), + // `serf_embedded::InitError` is `#[non_exhaustive]`, so a wildcard is + // required even though every variant it defines today is handled above and + // this arm is unreachable. A future engine-only failure mode reaching here + // surfaces as a generic endpoint-init failure and would warrant its own + // driver variant when added. + _ => InitError::Endpoint(EndpointInitError::AwarenessMultiplierZero), + } + } +} + +impl From for InitError { + fn from(e: EndpointInitError) -> Self { + InitError::Endpoint(e) + } +} + +#[cfg(encryption)] +impl From for InitError { + fn from(e: memberlist_proto::EncryptionError) -> Self { + InitError::Encryption(e) + } +} + +#[cfg(feature = "std")] +#[cfg_attr(docsrs, doc(cfg(feature = "std")))] +impl std::error::Error for InitError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + InitError::Endpoint(e) => Some(e), + #[cfg(encryption)] + InitError::Encryption(e) => Some(e), + InitError::Resolve(e) => Some(e.as_ref()), + _ => None, + } + } +} + +/// Why a [`Serf::join`](crate::Serf::join) failed. +/// +/// A join first resolves each seed through the supplied resolver, then records +/// the intent with the engine; either step can fail. +#[derive(Debug)] +#[non_exhaustive] +pub enum JoinError { + /// The address resolver failed while resolving a seed. + /// + /// The resolver's error type is generic, so it is boxed to preserve the + /// `source()` chain; a caller that knows its concrete resolver can downcast. + /// No `Send`/`Sync` bound — the smoltcp [`Resolver`](crate::Resolver) is + /// single-threaded by design, so its error need not cross threads. + Resolve(Box), + /// The engine rejected the join (e.g. the node is not in the running state). + Control(serf_embedded::SerfError), + /// A non-empty seed set resolved to no wire address — a discovery failure + /// rather than a successful no-op join. + NoAddresses, +} + +impl JoinError { + /// Whether this is a resolver failure. + #[inline] + pub const fn is_resolve(&self) -> bool { + matches!(self, JoinError::Resolve(_)) + } + + /// Whether this is an engine-control failure. + #[inline] + pub const fn is_control(&self) -> bool { + matches!(self, JoinError::Control(_)) + } + + /// Whether a non-empty seed set resolved to no wire address. + #[inline] + pub const fn is_no_addresses(&self) -> bool { + matches!(self, JoinError::NoAddresses) + } +} + +impl fmt::Display for JoinError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + JoinError::Resolve(e) => write!(f, "seed address resolution failed: {e}"), + JoinError::Control(e) => write!(f, "join was rejected: {e}"), + JoinError::NoAddresses => write!(f, "no wire address resolved for any seed"), + } + } +} + +impl From for JoinError { + fn from(e: serf_embedded::SerfError) -> Self { + JoinError::Control(e) + } +} + +#[cfg(feature = "std")] +#[cfg_attr(docsrs, doc(cfg(feature = "std")))] +impl std::error::Error for JoinError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + JoinError::Resolve(e) => Some(e.as_ref()), + JoinError::Control(e) => Some(e), + JoinError::NoAddresses => None, + } + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-smoltcp/src/error/tests.rs b/serf-smoltcp/src/error/tests.rs new file mode 100644 index 00000000..34f58d28 --- /dev/null +++ b/serf-smoltcp/src/error/tests.rs @@ -0,0 +1,197 @@ +use super::*; +use crate::interface::{ + EthernetAddress, HardwareAddress, IpAddress, IpCidr, Ipv4Address, Medium, Route, +}; +use core::net::{Ipv4Addr, SocketAddr}; +use std::{format, vec, vec::Vec}; + +fn sample_cidr() -> IpCidr { + IpCidr::new(IpAddress::v4(224, 0, 0, 1), 24) +} + +fn sample_route() -> Route { + Route::new_ipv4_gateway(Ipv4Address::new(10, 0, 0, 1)) +} + +fn sample_socket_addr() -> SocketAddr { + SocketAddr::new(Ipv4Addr::new(10, 0, 0, 1).into(), 7946) +} + +// A non-unicast (multicast) MAC: the low bit of the first octet set. +fn multicast_mac() -> HardwareAddress { + HardwareAddress::Ethernet(EthernetAddress([0x01, 0, 0, 0, 0, 1])) +} + +// Build one representative value of every `InitError` variant so the Display +// and Debug arms are all exercised. +fn all_variants() -> Vec { + vec![ + InitError::MediumMismatch(MediumMismatch { + expected: Medium::Ethernet, + actual: Medium::Ip, + }), + InitError::UnsupportedMedium, + InitError::NonUnicastHardwareAddress(multicast_mac()), + InitError::NonUnicastIpAddress(sample_cidr()), + InitError::NonRoutableAdvertiseAddr(sample_socket_addr()), + InitError::MissingIpAddress, + InitError::TooManyIpAddresses, + InitError::TooManyRoutes, + InitError::ZeroPort, + InitError::AdvertisePortMismatch, + InitError::AdvertiseAddrNotLocal(sample_socket_addr()), + InitError::NonUnicastRouteGateway(sample_route()), + InitError::RouteFamilyMismatch(sample_route()), + InitError::Entropy, + InitError::Endpoint(EndpointInitError::AwarenessMultiplierZero), + #[cfg(encryption)] + InitError::Encryption(memberlist_proto::EncryptionError::AuthFailed), + InitError::GossipMtuTooLarge(GossipMtuTooLarge { + gossip_mtu: 70_000, + ceiling: 65_467, + }), + InitError::UdpArenaTooLarge, + InitError::TcpPoolTooSmall, + InitError::ZeroTcpSocketBuffer, + InitError::TcpRxBufferTooLarge, + InitError::ZeroUdpPackets, + InitError::ZeroCloseTimeout, + ] +} + +#[test] +fn every_variant_displays_and_debugs_non_empty() { + for err in all_variants() { + assert!( + !format!("{err}").is_empty(), + "Display non-empty for {err:?}" + ); + assert!(!format!("{err:?}").is_empty(), "Debug non-empty"); + } +} + +#[test] +fn gossip_mtu_too_large_payload_display() { + let payload = GossipMtuTooLarge { + gossip_mtu: 70_000, + ceiling: 65_467, + }; + let shown = format!("{payload}"); + assert!(shown.contains("70000"), "{shown}"); + assert!(shown.contains("65467"), "{shown}"); + // Copy + PartialEq are derived. + assert_eq!(payload, payload); + assert!(!format!("{payload:?}").is_empty()); +} + +#[test] +fn medium_mismatch_payload_display() { + let payload = MediumMismatch { + expected: Medium::Ethernet, + actual: Medium::Ip, + }; + assert!(!format!("{payload}").is_empty()); + assert_eq!(payload, payload); + assert!(!format!("{payload:?}").is_empty()); +} + +#[test] +fn from_endpoint_init_error() { + let err: InitError = EndpointInitError::AwarenessMultiplierZero.into(); + assert!(matches!( + err, + InitError::Endpoint(EndpointInitError::AwarenessMultiplierZero) + )); +} + +#[cfg(encryption)] +#[test] +fn from_encryption_error() { + let err: InitError = memberlist_proto::EncryptionError::NoMatchingKey.into(); + assert!(matches!(err, InitError::Encryption(_))); +} + +// `from_embedded` maps each embedded failure mode to its driver equivalent, +// and routes any future (non_exhaustive) variant to a generic endpoint error. +#[test] +fn from_embedded_maps_each_mode() { + use serf_embedded::InitError as E; + + assert!(matches!( + InitError::from_embedded(E::ZeroPort), + InitError::ZeroPort + )); + assert!(matches!( + InitError::from_embedded(E::AdvertisePortMismatch), + InitError::AdvertisePortMismatch + )); + assert!(matches!( + InitError::from_embedded(E::ZeroCloseTimeout), + InitError::ZeroCloseTimeout + )); + assert!(matches!( + InitError::from_embedded(E::NonRoutableAdvertiseAddr(sample_socket_addr())), + InitError::NonRoutableAdvertiseAddr(_) + )); + assert!(matches!( + InitError::from_embedded(E::Endpoint(EndpointInitError::AwarenessMultiplierZero)), + InitError::Endpoint(_) + )); + #[cfg(encryption)] + assert!(matches!( + InitError::from_embedded(E::Encryption(memberlist_proto::EncryptionError::AuthFailed)), + InitError::Encryption(_) + )); + + // The carried ceiling/value survive the GossipMtuTooLarge remap. + let mapped = InitError::from_embedded(E::GossipMtuTooLarge(serf_embedded::GossipMtuTooLarge { + gossip_mtu: 99_999, + ceiling: 65_467, + })); + match mapped { + InitError::GossipMtuTooLarge(g) => { + assert_eq!(g.gossip_mtu, 99_999); + assert_eq!(g.ceiling, 65_467); + } + other => panic!("expected GossipMtuTooLarge, got {other:?}"), + } +} + +#[test] +fn join_error_predicates_and_display() { + let e = JoinError::NoAddresses; + assert!(e.is_no_addresses()); + assert!(!e.is_resolve()); + assert!(!e.is_control()); + assert!(!format!("{e}").is_empty()); +} + +// Under `std` the `Error::source` chains only for the wrapping variants. +#[cfg(feature = "std")] +#[test] +fn source_chains_only_for_wrapping_variants() { + use std::error::Error as _; + + assert!( + InitError::Endpoint(EndpointInitError::AwarenessMultiplierZero) + .source() + .is_some() + ); + #[cfg(encryption)] + assert!( + InitError::Encryption(memberlist_proto::EncryptionError::AuthFailed) + .source() + .is_some() + ); + // A leaf variant carries no source. + assert!(InitError::ZeroPort.source().is_none()); + assert!(InitError::Entropy.source().is_none()); + assert!( + InitError::GossipMtuTooLarge(GossipMtuTooLarge { + gossip_mtu: 70_000, + ceiling: 65_467, + }) + .source() + .is_none() + ); +} diff --git a/serf-smoltcp/src/gossip_io.rs b/serf-smoltcp/src/gossip_io.rs new file mode 100644 index 00000000..242327c9 --- /dev/null +++ b/serf-smoltcp/src/gossip_io.rs @@ -0,0 +1,74 @@ +//! The [`GossipIo`](serf_embedded::GossipIo) implementation over the bound smoltcp +//! gossip `udp::Socket`. +//! +//! A short-lived view, rebuilt each [`Serf::poll`](crate::Serf::poll) over the +//! already-ticked socket set and the gossip UDP [`SocketHandle`]. The engine reads +//! inbound gossip and writes outbound gossip through it without touching smoltcp; +//! the actual stack tick (`iface.poll`) the driver performs before handing this +//! view to the engine. +//! +//! smoltcp keeps every socket — the gossip UDP socket and the reliable-plane TCP +//! pool alike — in one [`SocketSet`], so the gossip and stream views must share +//! mutable access to it. They borrow it through a [`RefCell`] the driver holds for +//! the duration of one pump; each trait method takes a brief `borrow_mut`, never +//! holding a socket borrow across a call into the other view, so the borrows never +//! overlap at runtime. + +use core::{cell::RefCell, net::SocketAddr}; + +use serf_embedded::GossipIo; +use smoltcp::{ + iface::{SocketHandle, SocketSet}, + socket::udp, +}; + +use crate::addr::{from_endpoint, to_endpoint}; + +/// A [`GossipIo`] view over the gossip `udp::Socket` in a shared [`SocketSet`]. +/// +/// Resolves the gossip socket by its [`SocketHandle`] on each call, taking a brief +/// `borrow_mut` of the shared set. +pub(crate) struct SmoltcpGossip<'a, 'b> { + sockets: &'a RefCell<&'a mut SocketSet<'b>>, + udp: SocketHandle, +} + +impl<'a, 'b> SmoltcpGossip<'a, 'b> { + /// Build the view over the shared `sockets` for the gossip socket `udp`. + pub(crate) fn new(sockets: &'a RefCell<&'a mut SocketSet<'b>>, udp: SocketHandle) -> Self { + Self { sockets, udp } + } +} + +impl GossipIo for SmoltcpGossip<'_, '_> { + fn recv(&mut self, buf: &mut [u8]) -> Option<(SocketAddr, usize)> { + let mut set = self.sockets.borrow_mut(); + let sock = set.get_mut::(self.udp); + // Pop the next deliverable datagram. `recv_slice` is called only while + // `can_recv()` holds, so an empty rx ring is a clean `None` rather than an + // `Exhausted` error to interpret. + while sock.can_recv() { + match sock.recv_slice(buf) { + Ok((n, meta)) => return Some((from_endpoint(meta.endpoint), n)), + // The datagram exceeded `buf` and was already POPPED by `recv_slice` + // (smoltcp dequeues before the length check), so it is consumed and gone. + // This is an over-budget peer datagram — larger than the configured gossip + // MTU plus encryption overhead the buffer is sized for. Skip it and CONTINUE + // draining the rest of the rx ring rather than returning, so one oversized + // datagram cannot stall delivery of the in-budget datagrams queued behind it. + Err(udp::RecvError::Truncated) => continue, + // The ring is empty (`can_recv()` raced false): nothing more to deliver. + Err(udp::RecvError::Exhausted) => return None, + } + } + None + } + + fn send(&mut self, bytes: &[u8], dest: SocketAddr) { + let mut set = self.sockets.borrow_mut(); + let sock = set.get_mut::(self.udp); + // Ignoring Err: gossip is best-effort — a full or errored UDP tx ring drops + // this datagram and SWIM recovers on the next gossip round. + let _ = sock.send_slice(bytes, to_endpoint(dest)); + } +} diff --git a/serf-smoltcp/src/interface.rs b/serf-smoltcp/src/interface.rs new file mode 100644 index 00000000..c581515c --- /dev/null +++ b/serf-smoltcp/src/interface.rs @@ -0,0 +1,94 @@ +//! Public configuration for the smoltcp [`Interface`](smoltcp::iface::Interface) +//! the driver binds to. +//! +//! The caller assembles an [`InterfaceOptions`] from smoltcp's own wire and +//! routing types — re-exported here so a downstream crate need not depend on +//! smoltcp directly — and the driver applies it verbatim at construction. + +// `Vec` resolves to `std::vec::Vec`, which under `no_std + alloc` is the +// `alloc` crate aliased to `std` in `lib.rs`. +use std::vec::Vec; + +pub use smoltcp::{ + iface::Route, + phy::Medium, + wire::{EthernetAddress, HardwareAddress, IpAddress, IpCidr, Ipv4Address, Ipv6Address}, +}; + +/// How to configure the smoltcp [`Interface`](smoltcp::iface::Interface) the +/// driver binds to. +/// +/// The hardware address implies the medium and is validated against the bound +/// device at construction (a mismatch is a typed +/// [`InitError::MediumMismatch`](crate::InitError::MediumMismatch), not a +/// panic). The IP addresses and routes are applied verbatim. The random seed +/// defaults to system entropy — recommended, so smoltcp's TCP initial sequence +/// number and ephemeral port selection cannot repeat across reboots — and may +/// be pinned to a fixed value for deterministic tests. +/// +/// Build it with [`new`](Self::new) plus the repeatable `with_*` setters: +/// +/// ``` +/// use serf_smoltcp::{HardwareAddress, InterfaceOptions, IpAddress, IpCidr}; +/// +/// let cfg = InterfaceOptions::new(HardwareAddress::Ip) +/// .with_ip_addr(IpCidr::new(IpAddress::v4(10, 0, 0, 1), 24)); +/// ``` +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct InterfaceOptions { + /// The interface's hardware address. Its variant selects the medium: + /// [`HardwareAddress::Ip`] is `Medium::Ip`, [`HardwareAddress::Ethernet`] is + /// `Medium::Ethernet`. An Ethernet address must be unicast (smoltcp rejects a + /// multicast/broadcast source MAC). + pub hardware_addr: HardwareAddress, + /// The unicast IP addresses (with prefix lengths) the interface accepts + /// packets for. Must be non-empty — an interface with no address would + /// silently drop everything. Bounded by smoltcp's `IFACE_MAX_ADDR_COUNT`. + pub ip_addrs: Vec, + /// The interface's routing table (e.g. a default gateway for off-link peers). + /// Empty is fine for an all-on-link cluster. Bounded by smoltcp's + /// `IFACE_MAX_ROUTE_COUNT`. + pub routes: Vec, + /// The seed for smoltcp's interface RNG (TCP ISN and ephemeral port + /// selection). `None` draws a fresh seed from system entropy at construction; + /// `Some(seed)` pins it for deterministic tests. + pub random_seed: Option, +} + +impl InterfaceOptions { + /// Begin a configuration for the given hardware address, with no IP + /// addresses, no routes, and an entropy-drawn random seed. + /// + /// At least one IP address must be added via [`with_ip_addr`](Self::with_ip_addr) + /// before the interface can be constructed. + pub fn new(hardware_addr: HardwareAddress) -> Self { + Self { + hardware_addr, + ip_addrs: Vec::new(), + routes: Vec::new(), + random_seed: None, + } + } + + /// Add one unicast IP address (with prefix length) the interface accepts + /// packets for. Repeatable; addresses accumulate in order. + pub fn with_ip_addr(mut self, cidr: IpCidr) -> Self { + self.ip_addrs.push(cidr); + self + } + + /// Add one route to the interface's routing table. Repeatable; routes + /// accumulate in order. + pub fn with_route(mut self, route: Route) -> Self { + self.routes.push(route); + self + } + + /// Pin the interface RNG seed to a fixed value (for deterministic tests), + /// instead of drawing it from system entropy. + pub fn with_random_seed(mut self, seed: u64) -> Self { + self.random_seed = Some(seed); + self + } +} diff --git a/serf-smoltcp/src/lib.rs b/serf-smoltcp/src/lib.rs new file mode 100644 index 00000000..e8be1ef8 --- /dev/null +++ b/serf-smoltcp/src/lib.rs @@ -0,0 +1,80 @@ +#![doc = include_str!("../README.md")] +#![cfg_attr(not(feature = "std"), no_std)] +#![forbid(unsafe_code)] +#![deny(missing_docs)] +// `collapsible_if`: the nested `if cond { if let ... }` form is kept deliberately — +// flattening multi-level guards into one long let-chain reads worse here. +#![allow(clippy::collapsible_if, clippy::type_complexity, unexpected_cfgs)] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![cfg_attr(docsrs, allow(unused_attributes))] + +// Alias `alloc` to the name `std` so genuine-heap `std::` paths compile unchanged +// under no_std+alloc. Core-resident items are imported from `core::` directly, +// never via this alias; heap macros are written path-qualified (`std::vec!`), so +// no crate-wide `#[macro_use]` is needed. +#[cfg(all(not(feature = "std"), feature = "alloc"))] +extern crate alloc as std; + +#[cfg(feature = "std")] +extern crate std; + +#[cfg(not(any(feature = "std", feature = "alloc")))] +compile_error!("serf-smoltcp requires the `std` or `alloc` feature"); + +pub use bytes::Bytes; +pub use config::Options; +pub use error::{GossipMtuTooLarge, InitError, JoinError, MediumMismatch}; +pub use interface::{ + EthernetAddress, HardwareAddress, InterfaceOptions, IpAddress, IpCidr, Ipv4Address, Ipv6Address, + Medium, Route, +}; +pub use resolver::{Resolver, SocketAddrResolver}; +pub use serf::Serf; + +// The Sans-I/O machine types that appear in the public construction / command +// signatures, re-exported so a caller need not depend on `memberlist-proto` +// directly. +pub use memberlist_proto::{EndpointOptions, Instant}; +// serf's driving-core surface, re-exported from `serf-embedded` so the smoltcp API +// is self-contained: the event set, the resolver result + admission predicates, the +// transform config, the await-result join types, and serf's config / error types. +pub use serf_embedded::{ + AliveDelegate, DEFAULT_EVENT_BUFFER_CAP, Event, JoinFailed, JoinId, LabelError, + MAX_RESOLVED_ADDRS_PER_SEED, MaybeOwned, MaybeResolved, MergeDelegate, ReachedSet, ResolvedAddrs, + SerfError, SerfOptions, TransformOptions, socket_addr_is_routable, +}; +// serf's own protocol types named in the command signatures. +pub use serf_proto::{ + endpoint::{QueryId, QueryParams}, + event::QueryEvent, + members::{Member, MemberStatus, SerfState}, + typed::Tags, +}; + +// CIDR peer-admission policy, installed via `Options::with_cidr_policy`. +#[cfg(feature = "cidr")] +#[cfg_attr(docsrs, doc(cfg(feature = "cidr")))] +pub use serf_embedded::{AddrParseError, CidrPolicy, IpNet}; +// AEAD keyring types (for a caller assembling a `TransformOptions` encryption +// policy) and the inbound key-management request/response types the driver acts on. +#[cfg(encryption)] +#[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) +)] +pub use serf_embedded::{EncryptionOptions, Keyring, SecretKey}; +#[cfg(encryption)] +#[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) +)] +pub use serf_proto::event::{KeyRequest, KeyRequestOperation, KeyResponseArgs}; + +mod addr; +mod config; +mod error; +mod gossip_io; +mod interface; +mod resolver; +mod serf; +mod stream_io; diff --git a/serf-smoltcp/src/resolver.rs b/serf-smoltcp/src/resolver.rs new file mode 100644 index 00000000..8bbebf3f --- /dev/null +++ b/serf-smoltcp/src/resolver.rs @@ -0,0 +1,55 @@ +//! Synchronous address resolution: the [`Resolver`] trait and the trivial +//! [`SocketAddrResolver`]. +//! +//! Resolution happens only at the driver boundary — construction (the local +//! advertise address) and [`join`](crate::Serf::join) (the seeds) — so a resolver +//! is passed to those methods rather than stored on the [`Serf`](crate::Serf). +//! Everything past the boundary is a wire [`SocketAddr`]; the embedded engine only +//! ever sees resolved addresses. + +use core::net::SocketAddr; + +use serf_embedded::ResolvedAddrs; + +/// Synchronously resolves an unresolved address (e.g. a `host:port` name) into +/// candidate wire [`SocketAddr`]s. +/// +/// Synchronous — and free of `Send`/`Sync`/`'static` bounds — because smoltcp is +/// a caller-poll driver with no async runtime and a single-threaded stack. +/// `resolve` returns zero or more candidates, since one name may map to several +/// A/AAAA records. +/// +/// The result is a [`ResolvedAddrs`]: a bounded, no-heap collection capped at +/// [`MAX_RESOLVED_ADDRS_PER_SEED`](serf_embedded::MAX_RESOLVED_ADDRS_PER_SEED). +/// The cap is enforced by the type, so a resolver cannot hand back an unbounded +/// result for the driver to allocate and truncate after the fact. +pub trait Resolver { + /// The unresolved address this resolver accepts. + type Address; + + /// The error returned when resolution fails. `'static` so the driver can box + /// it into a typed [`InitError`](crate::InitError) / [`JoinError`](crate::JoinError) + /// while preserving the `source()` chain. + type Error: core::error::Error + 'static; + + /// Resolves `address` into a bounded set of candidate wire addresses. + fn resolve(&self, address: &Self::Address) -> Result; +} + +/// A [`Resolver`] for callers that already hold wire [`SocketAddr`]s: it passes +/// each address through unchanged and never fails. +#[derive(Debug, Clone, Copy, Default)] +pub struct SocketAddrResolver; + +impl Resolver for SocketAddrResolver { + type Address = SocketAddr; + type Error = core::convert::Infallible; + + fn resolve(&self, address: &SocketAddr) -> Result { + let mut addrs = ResolvedAddrs::new(); + // Ignoring Err: pushing one element onto a freshly-created, empty bounded + // vec whose capacity is MAX_RESOLVED_ADDRS_PER_SEED (8) cannot overflow. + let _ = addrs.push(*address); + Ok(addrs) + } +} diff --git a/serf-smoltcp/src/serf/mod.rs b/serf-smoltcp/src/serf/mod.rs new file mode 100644 index 00000000..f9836af4 --- /dev/null +++ b/serf-smoltcp/src/serf/mod.rs @@ -0,0 +1,1124 @@ +//! serf handle: construction, accessors, the caller-owned poll loop, and the +//! serf command surface. +//! +//! [`Serf`] wraps the transport-agnostic [`SerfEngine`](serf_embedded::SerfEngine) +//! with a smoltcp TCP/IP stack. The caller owns the device (`D`) and drives the +//! node by calling [`poll`](Serf::poll) in a super-loop: it ticks the smoltcp +//! stack, drives the engine over a [`SmoltcpGossip`] + [`SmoltcpStream`] view of +//! the just-ticked sockets, then acts on serf's mandatory driver-actioned events +//! IN the poll cycle (a lost id-conflict [`Event::Shutdown`] flips a stop flag; an +//! encryption [`Event::KeyRequest`] is applied to the local keyring and answered +//! via `respond_key`), buffering every drained event for the app's own +//! [`poll_event`](Serf::poll_event). All protocol work lives in the shared engine; +//! this driver supplies only the link layer plus the mandatory-event side effects. + +use core::{ + cell::RefCell, + hash::Hash, + marker::PhantomData, + net::{IpAddr, SocketAddr}, +}; + +use std::{boxed::Box, collections::VecDeque, sync::Arc, vec::Vec}; + +use memberlist_proto::{EndpointOptions, Instant, Rng, SeedableRng, SmallRng}; +use serf_embedded::{ + DEFAULT_EVENT_BUFFER_CAP, Event, JoinFailed, JoinId, MaybeResolved, ReachedSet, SerfEngine, + SerfError, SerfOptions, validate_runtime_config, +}; +use serf_proto::{ + endpoint::{QueryId, QueryParams}, + event::QueryEvent, + members::{Member, SerfState}, + typed::Tags, +}; +use smoltcp::{ + iface::{Config as IfConfig, Interface, SocketHandle, SocketSet}, + phy::Device, + socket::{tcp, udp}, +}; + +#[cfg(encryption)] +use serf_embedded::{Keyring, SecretKey}; +#[cfg(encryption)] +use serf_proto::event::{KeyRequest, KeyRequestOperation, KeyResponseArgs}; + +use crate::{ + InitError, InterfaceOptions, JoinError, Options, Resolver, TransformOptions, + addr::{from_smoltcp_instant, to_endpoint, to_smoltcp_instant}, + error::{GossipMtuTooLarge, MediumMismatch}, + gossip_io::SmoltcpGossip, + interface::{HardwareAddress, Medium}, + stream_io::SmoltcpStream, +}; + +/// The maximum UDP payload (`u16` length minus the 8-byte UDP header), the +/// hard ceiling for an on-wire gossip datagram. Matches the async drivers. +const UDP_PAYLOAD_MAX: usize = 65507; + +/// The largest the encrypted wrapper can inflate a gossip datagram, or `0` when +/// no encryption backend is built in. serf's gossip plane carries only the +/// encryption wrapper (no checksum / compression), so this is the whole on-wire +/// inflation the arena/ceiling arithmetic must accommodate. +#[cfg(encryption)] +const ENCRYPTED_WRAPPER_OVERHEAD: usize = memberlist_proto::ENCRYPTED_WRAPPER_OVERHEAD; +#[cfg(not(encryption))] +const ENCRYPTED_WRAPPER_OVERHEAD: usize = 0; + +/// The largest per-socket TCP receive-buffer smoltcp accepts: 1 GiB. +/// +/// smoltcp's `tcp::Socket::new` `panic!`s when the receive-buffer capacity +/// exceeds this (`if rx_capacity > (1 << 30)`, socket/tcp.rs), derived from the +/// RFC 1323 window-scale ceiling of 2^30. A caller-supplied +/// [`Options::tcp_socket_rx_bytes`](crate::Options::tcp_socket_rx_bytes) past it +/// would panic inside the fallible constructor, so `try_new` rejects it first. +/// The transmit buffer has no such limit and is not capped. +const TCP_RX_BUFFER_MAX: usize = 1 << 30; + +/// Whether `addr` is a destination the smoltcp stack can actually use. +/// +/// `to_endpoint(*addr).addr.is_unicast()` calls smoltcp's OWN `IpAddress::is_unicast` +/// — the exact function its route/neighbor lookups assert on — so the driver and +/// smoltcp agree byte-for-byte on what "routable" means. This mirrors the engine's +/// transport-neutral `socket_addr_is_routable` (`!(broadcast || multicast || +/// unspecified)` plus `port != 0`) at the smoltcp boundary. +pub(crate) fn endpoint_is_routable(addr: &SocketAddr) -> bool { + to_endpoint(*addr).addr.is_unicast() && addr.port() != 0 +} + +/// Derive a per-node RNG seed for *pinned* (deterministic) mode by FNV-1a hashing +/// a canonical buffer: a per-RNG domain tag, the interface seed, then the full +/// advertise address (IP octets, then port). Folding the interface seed and the +/// address through the hash makes the derived seed distinct from the interface RNG +/// and per-node unique, so two nodes that pin the same interface seed still get +/// divergent schedules. Pinned mode is for REPRODUCIBILITY (deterministic tests), +/// not secrecy. +fn seed_from(domain: u64, interface_seed: u64, advertise: &SocketAddr) -> u64 { + const FNV_PRIME: u64 = 0x0000_0100_0000_01b3; + let mut acc: u64 = 0xcbf2_9ce4_8422_2325; + let mut fold = |bytes: &[u8]| { + for &byte in bytes { + acc ^= u64::from(byte); + acc = acc.wrapping_mul(FNV_PRIME); + } + }; + fold(&domain.to_le_bytes()); + fold(&interface_seed.to_le_bytes()); + match advertise.ip() { + IpAddr::V4(v4) => fold(&v4.octets()), + IpAddr::V6(v6) => fold(&v6.octets()), + } + fold(&advertise.port().to_le_bytes()); + acc +} + +/// FNV domain tag for the memberlist gossip RNG (peer selection, timing jitter). +const GOSSIP_DOMAIN: u64 = 0x9E37_79B9_7F4A_7C15; +/// FNV domain tag for serf's own core RNG (query ids, relay/reconnect selection); +/// distinct from [`GOSSIP_DOMAIN`] so the two schedules never coincide. +const SERF_DOMAIN: u64 = 0x2545_F491_4F6C_DD1D; + +/// Assemble the [`serf_embedded::Options`] the engine reads from the driver's +/// [`crate::Options`]. +/// +/// The `Options` name collision: the driver's `crate::Options` carries link-layer +/// sizing (socket buffers, UDP arenas, `tcp_pool_size`) that stays on the driver, +/// while `serf_embedded::Options` carries only the port and close timeout (plus the +/// CIDR policy) the engine reads directly. +fn embedded_options(cfg: &Options) -> serf_embedded::Options { + let opts = serf_embedded::Options::new() + .with_port(cfg.port) + .with_close_timeout(cfg.close_timeout); + #[cfg(feature = "cidr")] + let opts = match cfg.cidr_policy.clone() { + Some(policy) => opts.with_cidr_policy(policy), + None => opts, + }; + opts +} + +/// Derive the medium a [`HardwareAddress`] selects, or `None` for a medium this +/// driver does not support (smoltcp's `Ieee802154`), so the caller can surface a +/// typed [`InitError::UnsupportedMedium`] instead of reaching an `unreachable!()`. +fn hardware_address_medium(addr: &HardwareAddress) -> Option { + match addr { + HardwareAddress::Ip => Some(Medium::Ip), + HardwareAddress::Ethernet(_) => Some(Medium::Ethernet), + #[allow(unreachable_patterns)] + _ => None, + } +} + +/// Returns the earlier of two optional deadlines. If only one is `Some`, that +/// deadline wins; if both are `None` the result is `None`. +fn min_opt(a: Option, b: Option) -> Option { + match (a, b) { + (Some(x), Some(y)) => Some(core::cmp::min(x, y)), + (x, y) => x.or(y), + } +} + +/// Apply one inbound [`KeyRequest`] to the driver's local keyring, producing the +/// [`KeyResponseArgs`] the poll cycle forwards to `respond_key`. +/// +/// Mirrors serf-reactor's `apply_key_request`: the driver owns its key store and +/// answers the four key-management operations from it. A node with no keyring +/// configured reports failure. The op is applied to the driver-local keyring copy; +/// keeping the live coordinator encryption in lockstep is a deferred refinement +/// (the same shape as serf-reactor, whose keyring delegate is likewise separate +/// from the coordinator's construction-time keyring). +#[cfg(encryption)] +fn apply_key_request( + keyring: &mut Option, + req: &KeyRequest, +) -> KeyResponseArgs { + let Some(kr) = keyring.as_mut() else { + return KeyResponseArgs { + result: false, + message: "no keyring configured on this node".into(), + keys: Vec::new(), + primary_key: None, + }; + }; + match (req.op(), req.key()) { + (KeyRequestOperation::Install, Some(key)) => { + kr.insert_secondary(*key); + KeyResponseArgs { + result: true, + message: "".into(), + keys: Vec::new(), + primary_key: None, + } + } + (KeyRequestOperation::Use, Some(key)) => match kr.promote(key.as_bytes()) { + Ok(()) => KeyResponseArgs { + result: true, + message: "".into(), + keys: Vec::new(), + primary_key: None, + }, + Err(_) => KeyResponseArgs { + result: false, + message: "requested primary key is not installed".into(), + keys: Vec::new(), + primary_key: None, + }, + }, + (KeyRequestOperation::Remove, Some(key)) => match kr.remove_secondary(key.as_bytes()) { + Ok(()) => KeyResponseArgs { + result: true, + message: "".into(), + keys: Vec::new(), + primary_key: None, + }, + Err(_) => KeyResponseArgs { + result: false, + message: "key is not a removable secondary".into(), + keys: Vec::new(), + primary_key: None, + }, + }, + (KeyRequestOperation::List, _) => { + let mut keys: Vec = Vec::with_capacity(1 + kr.secondaries().len()); + keys.push(*kr.primary_ref()); + keys.extend(kr.secondaries().iter().copied()); + KeyResponseArgs { + result: true, + message: "".into(), + keys, + primary_key: Some(*kr.primary_ref()), + } + } + (_, None) => KeyResponseArgs { + result: false, + message: "key-management request missing its required key".into(), + keys: Vec::new(), + primary_key: None, + }, + } +} + +/// An executor-free serf node that composes serf's super-machine (via +/// [`SerfEngine`](serf_embedded::SerfEngine)) with a smoltcp TCP/IP stack. +/// +/// The caller owns the device (`D`) and drives the node by calling [`poll`](Self::poll) +/// in a super-loop. Construction binds the gossip UDP socket, allocates the +/// reliable-plane TCP socket pool, and wires up the engine; no I/O occurs there. +/// +/// `I` is the node identifier type (e.g. `SmolStr`). `A` is the resolver's +/// unresolved address type. `D` is the smoltcp [`Device`]. `G` is the memberlist +/// gossip RNG and `SR` is serf's own core RNG (both default to [`SmallRng`]); the +/// two are seeded independently so fresh nodes never share a query-id schedule. +pub struct Serf +where + I: Eq + Hash, +{ + iface: Interface, + /// The seed handed to smoltcp's interface RNG at construction (TCP ISN / + /// ephemeral port selection). Retained because smoltcp does not expose it; + /// surfaced via [`Serf::interface_random_seed`] for diagnostics. + iface_random_seed: u64, + sockets: SocketSet<'static>, + /// Handle into `sockets` for the gossip UDP socket. + udp: SocketHandle, + /// The transport-agnostic serf driving core: serf's super-machine, the + /// reliable-plane connection state machine and its `SocketHandle` pool, the + /// gossip codec pipeline, and the join/await-result queues. + engine: SerfEngine, + /// The local node's resolved advertise address, retained for + /// [`advertise_address`](Self::advertise_address) (the engine does not surface it). + advertise: SocketAddr, + /// Driver-level application-event queue. Each [`poll`](Self::poll) drains the + /// engine's events (mandatory-first), acts on the mandatory ones, and buffers + /// every event here so the app's own [`poll_event`](Self::poll_event) observes + /// the full serf surface (including the mandatory events) after the driver acted. + /// Bounded at [`DEFAULT_EVENT_BUFFER_CAP`] with drop-oldest so an app that never + /// drains it cannot grow memory without bound on a long-running embedded node. + app_events: VecDeque>, + /// Count of app events shed from `app_events` because the app never drained + /// [`poll_event`](Self::poll_event) fast enough and the backlog hit the cap. + app_events_dropped: u64, + /// Set once the driver observed a lost id-conflict [`Event::Shutdown`]; the + /// caller reads it via [`is_shutdown`](Self::is_shutdown) and stops polling. + shutdown: bool, + /// The driver-local keyring answering inbound [`Event::KeyRequest`]s, seeded from + /// the construction `TransformOptions` encryption keyring (or `None` when the node + /// is unencrypted). + #[cfg(encryption)] + keyring: Option, + // `D` is passed to construction and each `poll`; `PhantomData` makes the struct + // generic over it without holding it. + _device: PhantomData, + // Ties the handle to the resolver's unresolved address type. `fn(A)` keeps the + // marker contravariant in `A` and free of drop/auto-trait obligations. + _a: PhantomData, +} + +// Construction seeding BOTH RNGs from the interface seed / system entropy. +impl Serf +where + I: memberlist_proto::Id + Clone, +{ + /// Construct a node, panicking on a misconfiguration or entropy failure. + /// + /// The convenience wrapper over [`try_new`](Self::try_new); use it only when the + /// configuration is a static constant known to be valid and the host's entropy + /// source cannot fail. + /// + /// # Panics + /// + /// Panics if [`try_new`](Self::try_new) returns an [`InitError`]. + #[allow(clippy::too_many_arguments)] + pub fn new( + cfg: Options, + iface: InterfaceOptions, + transform: TransformOptions, + ep_cfg: EndpointOptions, + serf_opts: SerfOptions, + resolver: &Res, + device: &mut D, + now: Instant, + ) -> Self + where + Res: Resolver
, + D: Device, + { + Self::try_new( + cfg, iface, transform, ep_cfg, serf_opts, resolver, device, now, + ) + .expect("Serf::new: invalid configuration or entropy failure; use try_new to handle") + } + + /// Fallibly construct a node. + /// + /// Builds the smoltcp `Interface`, allocates the gossip UDP socket and the + /// reliable-plane TCP socket pool, and wires up the engine. Both the gossip RNG + /// and serf's core RNG are seeded here — derived from the interface seed and the + /// advertise address when [`InterfaceOptions::random_seed`](crate::InterfaceOptions) + /// is pinned, or from an independent `getrandom` draw when it is not — so a + /// production node never shares a query-id schedule with a peer. + /// + /// # Errors + /// + /// Returns [`InitError`] instead of panicking when the configuration is invalid + /// for the bound device (an unsupported or mismatched medium, a non-unicast + /// hardware or IP address, a missing/over-capacity address or route, a + /// non-routable advertise address, an entropy failure, a resolver failure, an + /// unusable encryption keyring, or a machine-endpoint init failure). + #[allow(clippy::too_many_arguments)] + pub fn try_new( + cfg: Options, + iface: InterfaceOptions, + transform: TransformOptions, + ep_cfg: EndpointOptions, + serf_opts: SerfOptions, + resolver: &Res, + device: &mut D, + now: Instant, + ) -> Result + where + Res: Resolver
, + D: Device, + { + // Advertise-independent config preflight before touching the link layer. + let embedded_cfg = embedded_options(&cfg); + validate_runtime_config(&embedded_cfg, &transform, ep_cfg.gossip_mtu()) + .map_err(InitError::from_embedded)?; + + // Resolve the advertise address, then re-type `ep_cfg` so the rest of + // construction only ever sees the resolved wire `SocketAddr`. + let resolved_advertise = resolver + .resolve(ep_cfg.advertise_addr_ref()) + .map_err(|e| InitError::Resolve(Box::new(e)))? + .into_iter() + .next() + .ok_or(InitError::NoAddresses)?; + let ep_cfg = ep_cfg.map_advertise(|_| resolved_advertise); + + // Interface seed (pinned or system entropy) plus per-RNG derived seeds. In + // entropy mode the gossip and serf seeds are independent system-entropy draws; + // in pinned mode they are deterministically derived from the interface seed and + // advertise address (reproducible, per-node distinct — see `seed_from`). + let advertise = *ep_cfg.advertise_addr_ref(); + let (random_seed, gossip_seed, serf_seed) = match iface.random_seed { + Some(s) => ( + s, + seed_from(GOSSIP_DOMAIN, s, &advertise), + seed_from(SERF_DOMAIN, s, &advertise), + ), + None => { + let mut b = [0u8; 24]; + getrandom::fill(&mut b).map_err(|_| InitError::Entropy)?; + let word = |i: usize| { + u64::from_le_bytes([ + b[i], + b[i + 1], + b[i + 2], + b[i + 3], + b[i + 4], + b[i + 5], + b[i + 6], + b[i + 7], + ]) + }; + (word(0), word(8), word(16)) + } + }; + + Self::assemble( + cfg, + iface, + transform, + ep_cfg, + serf_opts, + device, + now, + random_seed, + SmallRng::seed_from_u64(gossip_seed), + SmallRng::seed_from_u64(serf_seed), + ) + } +} + +// Construction with caller-supplied RNGs, plus the shared assembly path (needs +// serf's core RNG seedable, matching the engine's construction bound). +impl Serf +where + I: memberlist_proto::Id + Clone, + SR: SeedableRng, +{ + /// Like [`new`](Self::new) but with caller-supplied gossip + serf RNGs; the caller + /// owns seeding them. The interface seed is still drawn here (pinned or from + /// `getrandom`) to drive smoltcp's TCP-stack RNG. + /// + /// # Errors + /// + /// Returns [`InitError`] on the same conditions as [`try_new`](Self::try_new). + #[allow(clippy::too_many_arguments)] + pub fn with_rng( + cfg: Options, + iface: InterfaceOptions, + transform: TransformOptions, + ep_cfg: EndpointOptions, + serf_opts: SerfOptions, + resolver: &Res, + device: &mut D, + now: Instant, + gossip_rng: G, + serf_rng: SR, + ) -> Result + where + Res: Resolver
, + D: Device, + { + let embedded_cfg = embedded_options(&cfg); + validate_runtime_config(&embedded_cfg, &transform, ep_cfg.gossip_mtu()) + .map_err(InitError::from_embedded)?; + + let resolved_advertise = resolver + .resolve(ep_cfg.advertise_addr_ref()) + .map_err(|e| InitError::Resolve(Box::new(e)))? + .into_iter() + .next() + .ok_or(InitError::NoAddresses)?; + let ep_cfg = ep_cfg.map_advertise(|_| resolved_advertise); + + // The caller owns both protocol RNGs; only the interface seed is drawn here. + let random_seed = match iface.random_seed { + Some(s) => s, + None => { + let mut b = [0u8; 8]; + getrandom::fill(&mut b).map_err(|_| InitError::Entropy)?; + u64::from_le_bytes(b) + } + }; + + Self::assemble( + cfg, + iface, + transform, + ep_cfg, + serf_opts, + device, + now, + random_seed, + gossip_rng, + serf_rng, + ) + } + + /// The link-layer + engine assembly shared by every constructor: validate the + /// medium/addresses/sizing against smoltcp, build the interface and sockets, run + /// the advertise-address screens, then build the engine over the resolved + /// `ep_cfg` with the supplied (already-seeded) RNGs. + #[allow(clippy::too_many_arguments)] + fn assemble( + cfg: Options, + iface: InterfaceOptions, + transform: TransformOptions, + ep_cfg: EndpointOptions, + serf_opts: SerfOptions, + device: &mut D, + now: Instant, + random_seed: u64, + gossip_rng: G, + serf_rng: SR, + ) -> Result + where + D: Device, + { + // Seed the driver-local keyring from the construction encryption policy BEFORE + // `transform` is moved into the engine, so inbound `KeyRequest`s can be answered + // from the same key material the node was built with. + #[cfg(encryption)] + let keyring = transform.encryption.keyring().cloned(); + + let embedded_cfg = embedded_options(&cfg); + + // 1. Validate the medium up front: smoltcp's `Interface::new` asserts the + // hardware address's medium equals the device's; derive it ourselves and + // reject an unsupported (Ieee802154) or mismatched medium as a typed error. + let expected = + hardware_address_medium(&iface.hardware_addr).ok_or(InitError::UnsupportedMedium)?; + let actual = device.capabilities().medium; + if expected != actual { + return Err(InitError::MediumMismatch(MediumMismatch { + expected, + actual, + })); + } + + // 2. An Ethernet hardware address must be unicast (smoltcp stores it without + // re-checking). The `Ip` variant carries no L2 address and is always fine. + if let HardwareAddress::Ethernet(mac) = &iface.hardware_addr { + if !mac.is_unicast() { + return Err(InitError::NonUnicastHardwareAddress(iface.hardware_addr)); + } + } + + // 3. An interface with no address silently drops every packet. + if iface.ip_addrs.is_empty() { + return Err(InitError::MissingIpAddress); + } + + // 4. Every configured IP must be unicast or unspecified (smoltcp's + // `check_ip_addrs` `panic!`s otherwise). + for cidr in &iface.ip_addrs { + if !cidr.address().is_unicast() && !cidr.address().is_unspecified() { + return Err(InitError::NonUnicastIpAddress(*cidr)); + } + } + + // Every route's gateway must be unicast and share the prefix's IP family, or it + // can never resolve a next hop (a release-mode assert / dead route at egress). + for route in &iface.routes { + if !route.via_router.is_unicast() { + return Err(InitError::NonUnicastRouteGateway(*route)); + } + if route.cidr.address().version() != route.via_router.version() { + return Err(InitError::RouteFamilyMismatch(*route)); + } + } + + // smoltcp rejects port 0 on bind/listen; screen it before allocating sockets. + if cfg.port == 0 { + return Err(InitError::ZeroPort); + } + + // Reject a gossip MTU whose on-wire datagram cannot fit a UDP packet (the UDP + // arenas are sized from it, and the addition must not overflow). + let gossip_mtu_ceiling = UDP_PAYLOAD_MAX - ENCRYPTED_WRAPPER_OVERHEAD; + if ep_cfg.gossip_mtu() > gossip_mtu_ceiling { + return Err(InitError::GossipMtuTooLarge(GossipMtuTooLarge { + gossip_mtu: ep_cfg.gossip_mtu(), + ceiling: gossip_mtu_ceiling, + })); + } + + // A functional reliable plane needs a listener plus one dial/accept socket. + if cfg.tcp_pool_size < 2 { + return Err(InitError::TcpPoolTooSmall); + } + // A zero-length ring is a permanently-dead socket; both halves must be non-zero. + if cfg.tcp_socket_rx_bytes == 0 || cfg.tcp_socket_tx_bytes == 0 { + return Err(InitError::ZeroTcpSocketBuffer); + } + // smoltcp `panic!`s on a receive buffer past 1 GiB. + if cfg.tcp_socket_rx_bytes > TCP_RX_BUFFER_MAX { + return Err(InitError::TcpRxBufferTooLarge); + } + // Zero packet-metadata slots is a gossip ring that can never enqueue/dequeue. + if cfg.udp_rx_packets == 0 || cfg.udp_tx_packets == 0 { + return Err(InitError::ZeroUdpPackets); + } + // A zero close timeout force-aborts every graceful close immediately. + if cfg.close_timeout.is_zero() { + return Err(InitError::ZeroCloseTimeout); + } + + // Build the interface with the resolved interface seed, then apply addresses + // and routes (both bounded by smoltcp's `heapless::Vec` capacities). + let mut ic = IfConfig::new(iface.hardware_addr); + ic.random_seed = random_seed; + let mut iface_obj = Interface::new(ic, device, to_smoltcp_instant(now)); + + let mut overflow = false; + iface_obj.update_ip_addrs(|addrs| { + for cidr in &iface.ip_addrs { + if addrs.push(*cidr).is_err() { + overflow = true; + break; + } + } + }); + if overflow { + return Err(InitError::TooManyIpAddresses); + } + + let mut route_overflow = false; + iface_obj.routes_mut().update(|table| { + for route in &iface.routes { + if table.push(*route).is_err() { + route_overflow = true; + break; + } + } + }); + if route_overflow { + return Err(InitError::TooManyRoutes); + } + + let iface = iface_obj; + + // Allocate the gossip UDP socket over an alloc-backed, growable socket store. + let mut sockets = SocketSet::new(Vec::new()); + + // Floor each UDP payload arena at "configured datagram slots × max on-wire + // datagram" so an in-budget datagram is never rejected by an under-sized arena; + // `checked_mul` guards a 32-bit overflow. + let max_datagram = ep_cfg.gossip_mtu() + ENCRYPTED_WRAPPER_OVERHEAD; + let udp_rx_arena = cfg.udp_rx_payload_bytes.max( + cfg + .udp_rx_packets + .checked_mul(max_datagram) + .ok_or(InitError::UdpArenaTooLarge)?, + ); + let udp_tx_arena = cfg.udp_tx_payload_bytes.max( + cfg + .udp_tx_packets + .checked_mul(max_datagram) + .ok_or(InitError::UdpArenaTooLarge)?, + ); + + let udp_rx = udp::PacketBuffer::new( + std::vec![udp::PacketMetadata::EMPTY; cfg.udp_rx_packets], + std::vec![0u8; udp_rx_arena], + ); + let udp_tx = udp::PacketBuffer::new( + std::vec![udp::PacketMetadata::EMPTY; cfg.udp_tx_packets], + std::vec![0u8; udp_tx_arena], + ); + let mut udp_sock = udp::Socket::new(udp_rx, udp_tx); + // `bind` fails only on port 0 (rejected above) or an already-open socket (fresh + // here); propagate rather than `expect` so no panic escapes the constructor. + udp_sock.bind(cfg.port).map_err(|_| InitError::ZeroPort)?; + let udp = sockets.add(udp_sock); + + // The advertise address must be routable and one the interface actually holds, + // or the node is unreachable on both planes. + if !endpoint_is_routable(ep_cfg.advertise_addr_ref()) { + return Err(InitError::NonRoutableAdvertiseAddr( + *ep_cfg.advertise_addr_ref(), + )); + } + let advertised_ip = to_endpoint(*ep_cfg.advertise_addr_ref()).addr; + if !iface.has_ip_addr(advertised_ip) { + return Err(InitError::AdvertiseAddrNotLocal( + *ep_cfg.advertise_addr_ref(), + )); + } + let advertise = *ep_cfg.advertise_addr_ref(); + + // Build the engine. `try_new_at_with_rng` maps a machine/keyring/advertise + // failure to a typed `InitError`; it installs the routable-address admission + // filter and the label/encryption transforms internally and forwards the CIDR + // policy carried on `embedded_cfg`. + let mut engine = SerfEngine::try_new_at_with_rng( + embedded_cfg, + transform, + ep_cfg, + serf_opts, + now, + gossip_rng, + serf_rng, + ) + .map_err(InitError::from_embedded)?; + + // Allocate pooled TCP sockets and register their handles with the engine's + // reliable plane; dedicate one to the passive-open listener. + for _ in 0..cfg.tcp_pool_size { + let rx = tcp::SocketBuffer::new(std::vec![0u8; cfg.tcp_socket_rx_bytes]); + let tx = tcp::SocketBuffer::new(std::vec![0u8; cfg.tcp_socket_tx_bytes]); + engine + .plane_mut() + .pool + .push(sockets.add(tcp::Socket::new(rx, tx))); + } + if let Some(h) = engine.plane_mut().pool.take() { + // `listen` fails only on port 0 (rejected) or an already-open socket (fresh). + sockets + .get_mut::(h) + .listen(cfg.port) + .map_err(|_| InitError::ZeroPort)?; + engine.set_listener(h); + } + + Ok(Self { + iface, + iface_random_seed: random_seed, + sockets, + udp, + engine, + advertise, + app_events: VecDeque::new(), + app_events_dropped: 0, + shutdown: false, + #[cfg(encryption)] + keyring, + _device: PhantomData, + _a: PhantomData, + }) + } +} + +// Pure reads over the driver and the reliable plane — needing neither RNG. +impl Serf +where + I: memberlist_proto::Id + Clone, +{ + /// The seed handed to smoltcp's interface RNG at construction. + #[doc(hidden)] + #[inline] + pub fn interface_random_seed(&self) -> u64 { + self.iface_random_seed + } + + /// The local node's advertised `SocketAddr`. + #[inline] + pub fn advertise_address(&self) -> SocketAddr { + self.advertise + } + + /// Whether the driver has observed a lost id-conflict [`Event::Shutdown`] and the + /// caller should stop polling. + #[inline] + pub fn is_shutdown(&self) -> bool { + self.shutdown + } + + /// Drain one application-visible serf event the last [`poll`](Self::poll) buffered, + /// mandatory driver-actioned events first (the driver has ALREADY acted on them), + /// then passive observations. `None` when the queue is empty. + #[inline] + pub fn poll_event(&mut self) -> Option> { + self.app_events.pop_front() + } + + /// The number of app events shed from the [`poll_event`](Self::poll_event) backlog + /// because it reached [`DEFAULT_EVENT_BUFFER_CAP`] before the app drained them. + #[inline] + pub fn events_dropped(&self) -> u64 { + self.app_events_dropped + } + + /// Number of inbound reliable connections accepted since construction. + #[doc(hidden)] + #[inline] + pub fn accepted_inbound_count(&self) -> u64 { + self.engine.accepted_inbound_count() + } + + /// Number of pooled TCP sockets currently free. + #[doc(hidden)] + #[inline] + pub fn pool_free_count(&self) -> usize { + self.engine.pool_free_count() + } + + /// Number of TCP sockets currently parked mid-close. + #[doc(hidden)] + #[inline] + pub fn closing_count(&self) -> usize { + self.engine.closing_count() + } + + /// Number of reliable exchanges currently half-closed. + #[doc(hidden)] + #[inline] + pub fn half_closed_count(&self) -> usize { + self.engine.half_closed_count() + } + + /// Whether a passive-open listener socket is currently installed. + #[doc(hidden)] + #[inline] + pub fn listener_present(&self) -> bool { + self.engine.listener_present() + } + + /// Number of reliable exchanges still in `PendingDial`. + #[doc(hidden)] + #[inline] + pub fn pending_dial_count(&self) -> usize { + self.engine.pending_dial_count() + } + + /// Number of await-result joins currently tracked. + #[doc(hidden)] + #[inline] + pub fn pending_join_count(&self) -> usize { + self.engine.pending_join_count() + } +} + +// serf reads + command surface + the poll loop — reach serf's super-machine, so +// they carry the gossip `G: Rng` and serf's `SR: Rng + SeedableRng` bounds. The +// engine's connection handle is the concrete `SocketHandle` (Copy + Eq + Hash), so +// the pump's `C` bound is satisfied without an extra parameter. +impl Serf +where + I: memberlist_proto::Id + Clone, + G: Rng, + SR: Rng + SeedableRng, +{ + /// Arm serf's periodic probe / gossip / push-pull schedulers. Call once before + /// the first [`poll`](Self::poll). + pub fn start(&mut self, now: Instant) { + self.engine.start(now); + } + + /// serf's current lifecycle state. + #[inline] + pub fn state(&self) -> SerfState { + self.engine.state() + } + + /// Number of serf members currently tracked (including the local node). + #[inline] + pub fn num_members(&self) -> usize { + self.engine.num_members() + } + + /// The local node's id. + #[inline] + pub fn local_id(&self) -> I { + self.engine.local_id().clone() + } + + /// A snapshot of every serf member currently tracked (alive, leaving, left, or + /// failed within the reap window). + #[inline] + pub fn members(&self) -> Vec>> { + self.engine.members_snapshot() + } + + /// The local node's serf member Lamport clock. + #[inline] + pub fn member_time(&self) -> u64 { + self.engine.member_time() + } + + /// The local node's serf event Lamport clock. + #[inline] + pub fn event_time(&self) -> u64 { + self.engine.event_time() + } + + /// The local node's serf query Lamport clock. + #[inline] + pub fn query_time(&self) -> u64 { + self.engine.query_time() + } + + /// Announce the local node's join intent and begin an await-result join to these + /// seeds, returning a [`JoinId`] the caller polls via [`poll_join`](Self::poll_join). + /// + /// Each seed is resolved through `resolver` (a [`MaybeResolved::Resolved`] address + /// is used verbatim, a [`MaybeResolved::Unresolved`] one is expanded into the wire + /// addresses the resolver yields). Returns immediately; the poll loop initiates a + /// push/pull to each routable seed on the next tick. When `ignore_old` is set, + /// each seed's push/pull suppresses replay of the peer's pre-join user events. + /// + /// # Errors + /// + /// Returns [`JoinError::Control`] when serf rejects the join (e.g. the node is not + /// running), [`JoinError::Resolve`] on a resolver failure, or + /// [`JoinError::NoAddresses`] when a non-empty seed set resolves to no address. + pub fn join( + &mut self, + resolver: &Res, + seeds: &[MaybeResolved], + ignore_old: bool, + now: Instant, + ) -> Result + where + Res: Resolver
, + { + let mut resolved = Vec::with_capacity(seeds.len()); + for seed in seeds { + match seed { + MaybeResolved::Resolved(s) => resolved.push(*s), + MaybeResolved::Unresolved(a) => resolved.extend( + resolver + .resolve(a) + .map_err(|e| JoinError::Resolve(Box::new(e)))?, + ), + } + } + if !seeds.is_empty() && resolved.is_empty() { + return Err(JoinError::NoAddresses); + } + self + .engine + .join(&resolved, ignore_old, now) + .map_err(JoinError::Control) + } + + /// Drain the terminal outcome of an await-result [`join`](Self::join), or `None` + /// while it is still in flight. Delivered exactly once per handle. + #[inline] + pub fn poll_join(&mut self, handle: JoinId) -> Option> { + self.engine.poll_join(handle) + } + + /// Give up an await-result [`join`](Self::join), dropping any still-queued seeds + /// and forgetting its caller reply, leak-free. + #[inline] + pub fn cancel_join(&mut self, handle: JoinId) { + self.engine.cancel_join(handle); + } + + /// Begin leaving the cluster. Gossips the departure and ultimately emits + /// [`Event::LeftCluster`] via [`poll_event`](Self::poll_event). + pub fn leave(&mut self, now: Instant) -> Result<(), SerfError> { + self.engine.leave(now) + } + + /// Force a named node out of the cluster (an operator-driven removal). + pub fn force_leave(&mut self, id: I, prune: bool, now: Instant) -> Result<(), SerfError> { + self.engine.force_leave(id, prune, now) + } + + /// Broadcast an application user event to the cluster. `coalesce` requests that + /// identical events be coalesced by name. Peers observe it as [`Event::User`]. + pub fn user_event( + &mut self, + name: impl Into, + payload: bytes::Bytes, + coalesce: bool, + ) -> Result<(), SerfError> { + self.engine.user_event(name, payload, coalesce) + } + + /// Issue a cluster-wide query, returning its [`QueryId`]. Responders observe it as + /// [`Event::Query`] and answer via [`respond`](Self::respond). + pub fn query( + &mut self, + name: impl Into, + payload: bytes::Bytes, + params: QueryParams, + now: Instant, + ) -> Result { + self.engine.query(name, payload, params, now) + } + + /// Answer a received query. `token` is the [`QueryEvent`] delivered via + /// [`Event::Query`]. + pub fn respond( + &mut self, + token: &QueryEvent, + payload: bytes::Bytes, + now: Instant, + ) -> Result<(), SerfError> { + self.engine.respond(token, payload, now) + } + + /// Replace the local node's tags, re-advertising them and refreshing the local + /// member in the membership store. + pub fn set_tags(&mut self, tags: Tags) -> Result<(), SerfError> { + self.engine.set_tags(tags) + } + + /// Issue a cluster-wide `install_key` query to add `key` to every node's keyring. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn install_key(&mut self, key: SecretKey, now: Instant) -> Result { + self.engine.install_key(key, now) + } + + /// Issue a cluster-wide `use_key` query to promote `key` to primary. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn use_key(&mut self, key: SecretKey, now: Instant) -> Result { + self.engine.use_key(key, now) + } + + /// Issue a cluster-wide `remove_key` query to remove `key` from all nodes. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn remove_key(&mut self, key: SecretKey, now: Instant) -> Result { + self.engine.remove_key(key, now) + } + + /// Issue a cluster-wide `list_keys` query to enumerate installed keys. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn list_keys(&mut self, now: Instant) -> Result { + self.engine.list_keys(now) + } + + /// Advance both the smoltcp stack and serf's state machine once, then act on + /// serf's mandatory driver-actioned events. Returns the next wakeup deadline: + /// the minimum of the smoltcp stack's next scheduled event, the machine's next + /// timer, and any engine-owned deadline (the soonest closing socket's abort). + /// + /// # Order + /// + /// 1. **Stack tick** — `iface.poll` drains the device and services TCP/UDP. + /// 2. **Engine pump** — the engine runs every protocol phase over a + /// [`SmoltcpGossip`] + [`SmoltcpStream`] view of the just-ticked sockets. + /// 3. **Mandatory events** — drain the engine's events (mandatory-first) and act + /// on the driver-actioned ones: [`Event::Shutdown`] flips the stop flag; an + /// [`Event::KeyRequest`] is applied to the local keyring and answered via + /// `respond_key`. Every drained event is buffered for the app's own + /// [`poll_event`](Self::poll_event), so membership / user / query observations + /// and the mandatory events alike remain visible AFTER the driver acted. A key + /// response issued here is a directed gossip transmit the engine emits on the + /// NEXT `poll`'s pump. + /// 4. **Deadline** — fold the stack's next scheduled event into the engine's + /// returned wakeup. + pub fn poll(&mut self, now: Instant, device: &mut D) -> Option + where + D: Device, + { + let s_now = to_smoltcp_instant(now); + + // 1. Stack tick. + self.iface.poll(s_now, device, &mut self.sockets); + + // 2. Engine pump over a view of the just-ticked sockets. The gossip and stream + // views share mutable access to the one `SocketSet` through a `RefCell` held for + // the pump; each takes a brief borrow and never holds one across a call into the + // other, so the borrows never overlap. + let next = { + let sockets = RefCell::new(&mut self.sockets); + let mut gossip = SmoltcpGossip::new(&sockets, self.udp); + let mut stream = SmoltcpStream::new(&mut self.iface, &sockets); + self.engine.pump(now, &mut gossip, &mut stream) + }; + + // 3. Mandatory-event handling in the poll cycle (driver-owned). + self.drain_engine_events(now); + + // 4. Fold the stack's next scheduled event into the engine's deadline. + let stack = self + .iface + .poll_at(s_now, &self.sockets) + .map(from_smoltcp_instant); + min_opt(stack, next) + } + + /// Drain the engine's event queue (mandatory-first), take the driver-owned side + /// effect on each mandatory event, and buffer every event for the app. + fn drain_engine_events(&mut self, now: Instant) { + // Ignoring: `now` is consumed only by the encryption `KeyRequest` arm below, + // so a build without an AEAD backend does not read it. + #[cfg(not(encryption))] + let _ = now; + while let Some(ev) = self.engine.poll_event() { + match &ev { + // A lost id-conflict vote means the local node MUST stop; flag it. The event + // still reaches the app via `poll_event`. + Event::Shutdown => self.shutdown = true, + // An inbound key-management request: apply the op to the local keyring and + // answer the originator. The response is a directed gossip transmit emitted + // on the next pump. + #[cfg(encryption)] + Event::KeyRequest(req) => { + let resp = apply_key_request(&mut self.keyring, req); + // Ignoring Err: `respond_key` fails only when the response cannot be + // routed; the key op has already applied to the driver keyring. + let _ = self.engine.respond_key(req, resp, now); + } + _ => {} + } + self.push_app_event(ev); + } + } + + /// Buffer one event for [`poll_event`](Self::poll_event), bounding the backlog at + /// [`DEFAULT_EVENT_BUFFER_CAP`] with drop-oldest so a never-draining app cannot + /// grow memory without bound. + fn push_app_event(&mut self, ev: Event) { + if self.app_events.len() >= DEFAULT_EVENT_BUFFER_CAP { + self.app_events.pop_front(); + self.app_events_dropped += 1; + } + self.app_events.push_back(ev); + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-smoltcp/src/serf/tests.rs b/serf-smoltcp/src/serf/tests.rs new file mode 100644 index 00000000..56199d20 --- /dev/null +++ b/serf-smoltcp/src/serf/tests.rs @@ -0,0 +1,151 @@ +use core::net::{IpAddr, Ipv4Addr, SocketAddr}; + +use memberlist_proto::Instant; +use smol_str::SmolStr; +use smoltcp::{ + phy::{ChecksumCapabilities, Device, DeviceCapabilities, Medium, RxToken, TxToken}, + time::Instant as SmolInstant, +}; + +use crate::{ + EndpointOptions, HardwareAddress, InterfaceOptions, IpAddress, IpCidr, Options, Serf, + SerfOptions, SerfState, SocketAddrResolver, TransformOptions, +}; + +/// A `Medium::Ip` device that never delivers a frame — enough to construct a node +/// (construction binds sockets and builds the interface but performs no I/O). +struct NullDevice; + +struct NRx; +struct NTx; + +impl RxToken for NRx { + fn consume R>(self, f: F) -> R { + f(&[]) + } +} + +impl TxToken for NTx { + fn consume R>(self, len: usize, f: F) -> R { + let mut buf = std::vec![0u8; len]; + f(&mut buf) + } +} + +impl Device for NullDevice { + type RxToken<'a> = NRx; + type TxToken<'a> = NTx; + + fn receive(&mut self, _t: SmolInstant) -> Option<(NRx, NTx)> { + None + } + + fn transmit(&mut self, _t: SmolInstant) -> Option { + Some(NTx) + } + + fn capabilities(&self) -> DeviceCapabilities { + let mut caps = DeviceCapabilities::default(); + caps.medium = Medium::Ip; + caps.max_transmission_unit = 1500; + caps.checksum = ChecksumCapabilities::ignored(); + caps + } +} + +fn ip_iface(octet: u8) -> InterfaceOptions { + InterfaceOptions::new(HardwareAddress::Ip) + .with_ip_addr(IpCidr::new(IpAddress::v4(10, 0, 0, octet), 24)) + .with_random_seed(octet as u64) +} + +fn now() -> Instant { + Instant::from_origin(core::time::Duration::from_secs(86_400)) +} + +fn try_build( + advertise: SocketAddr, + device: &mut NullDevice, +) -> Result, crate::InitError> { + Serf::try_new( + Options::new(), + ip_iface(1), + TransformOptions::default(), + EndpointOptions::new(SmolStr::new("a"), advertise), + SerfOptions::new(), + &SocketAddrResolver, + device, + now(), + ) +} + +#[test] +fn construction_succeeds_alive_single_member() { + let mut dev = NullDevice; + let node = try_build( + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)), 7946), + &mut dev, + ) + .expect("a valid configuration constructs"); + assert_eq!(node.state(), SerfState::Alive); + // serf registers its self-member lazily once the node starts operating, so a + // freshly constructed (not-yet-started) node has no members recorded yet. + assert_eq!(node.num_members(), 0); + assert!(!node.is_shutdown()); + assert_eq!(node.local_id(), SmolStr::new("a")); + assert_eq!( + node.advertise_address(), + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)), 7946) + ); + // The pinned interface seed is applied verbatim. + assert_eq!(node.interface_random_seed(), 1); +} + +#[test] +fn non_routable_advertise_is_rejected() { + let mut dev = NullDevice; + // The unspecified address is not a routable advertise destination. + // `Serf` is not `Debug` (it holds a smoltcp `Interface`), so `expect_err` is + // unavailable; match the error out with `let-else`. + let Err(err) = try_build( + SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 7946), + &mut dev, + ) else { + panic!("a non-routable advertise must be rejected"); + }; + assert!(matches!(err, crate::InitError::NonRoutableAdvertiseAddr(_))); +} + +#[test] +fn zero_port_is_rejected() { + let mut dev = NullDevice; + let Err(err) = Serf::::try_new( + Options::new().with_port(0), + ip_iface(1), + TransformOptions::default(), + EndpointOptions::new( + SmolStr::new("a"), + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)), 7946), + ), + SerfOptions::new(), + &SocketAddrResolver, + &mut dev, + now(), + ) else { + panic!("port 0 must be rejected"); + }; + assert!(matches!(err, crate::InitError::ZeroPort)); +} + +#[test] +fn advertise_not_local_is_rejected() { + let mut dev = NullDevice; + // 10.0.0.9 is routable but not assigned to the interface (which holds 10.0.0.1). + let Err(err) = try_build( + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 9)), 7946), + &mut dev, + ) else { + panic!("an advertise IP the interface lacks must be rejected"); + }; + assert!(matches!(err, crate::InitError::AdvertiseAddrNotLocal(_))); +} diff --git a/serf-smoltcp/src/stream_io/mod.rs b/serf-smoltcp/src/stream_io/mod.rs new file mode 100644 index 00000000..1bd61f41 --- /dev/null +++ b/serf-smoltcp/src/stream_io/mod.rs @@ -0,0 +1,235 @@ +//! The [`StreamIo`](serf_embedded::StreamIo) implementation over the pooled smoltcp +//! reliable-plane `tcp::Socket`s, keyed by [`SocketHandle`]. +//! +//! A short-lived view, rebuilt each [`Serf::poll`](crate::Serf::poll) over the +//! already-ticked socket set and the [`Interface`] (the latter needed only by +//! `connect`, which threads the interface `context()` into smoltcp's +//! `tcp::Socket::connect`). The engine owns the reliable-plane state machine and +//! its connection pool ([`ReliablePlane`](serf_embedded::ReliablePlane)) and drives +//! the sockets entirely through this view; the stack tick (`iface.poll`) the driver +//! performs before handing the view to the engine. +//! +//! smoltcp keeps every socket in one [`SocketSet`], so this view shares mutable +//! access to it with the gossip view through a [`RefCell`] the driver holds for +//! one pump (see [`crate::gossip_io`]); each method takes a brief `borrow`/`borrow_mut`, +//! never holding a socket borrow across a call into the other view. +//! +//! # Pool ownership +//! +//! The [`StreamIo`] trait exposes `take_free` / `give` / `free_count` for a driver +//! that backs its connection slots from its OWN free-list. This driver does not: +//! the engine's [`ReliablePlane`](serf_embedded::ReliablePlane) owns the +//! `SocketHandle` free-list (seeded at construction via `plane_mut().pool.push`) +//! and the engine reaches it directly, never through these three trait methods, so +//! for this view they are inert (see their impls). Every socket-touching operation +//! the engine DOES route through the view — `listen` / `connect` / `recv` / `send` +//! / `close` / `abort` and the lifecycle predicates — is implemented faithfully +//! below. + +use core::{cell::RefCell, net::SocketAddr}; + +use serf_embedded::{StreamIo, StreamIoError}; +use smoltcp::{ + iface::{Interface, SocketHandle, SocketSet}, + socket::tcp, +}; + +use crate::addr::{from_endpoint, to_endpoint}; + +/// A [`StreamIo`] view over the pooled reliable-plane `tcp::Socket`s in a shared +/// [`SocketSet`], plus the [`Interface`] needed to dial. +/// +/// Resolves each socket by its [`SocketHandle`], taking a brief `borrow`/`borrow_mut` +/// of the shared set; `connect` additionally borrows the interface for its context. +pub(crate) struct SmoltcpStream<'a, 'b> { + iface: &'a mut Interface, + sockets: &'a RefCell<&'a mut SocketSet<'b>>, +} + +impl<'a, 'b> SmoltcpStream<'a, 'b> { + /// Build the view over `iface` and the shared `sockets`. + pub(crate) fn new(iface: &'a mut Interface, sockets: &'a RefCell<&'a mut SocketSet<'b>>) -> Self { + Self { iface, sockets } + } +} + +impl StreamIo for SmoltcpStream<'_, '_> { + type Conn = SocketHandle; + + // The engine owns the `SocketHandle` free-list inside its `ReliablePlane` and + // reaches it directly, so it never calls these three on the view. They are + // implemented as no-ops that report an empty pool: there is no driver-side + // free-list for this view to draw from, and reporting one would be a lie that a + // hypothetical future caller could misread. The single authority is the engine's + // `ReliablePlane::pool`. + + fn take_free(&mut self) -> Option { + None + } + + fn give(&mut self, _c: Self::Conn) {} + + fn free_count(&self) -> usize { + 0 + } + + fn listen(&mut self, c: Self::Conn, port: u16) -> Result<(), StreamIoError> { + // `listen()` fails on port 0 (`Unaddressable`) or an already-open socket + // (`InvalidState`). Map both to a non-fatal error rather than panicking; the + // engine's listener paths only ever pass a non-zero port and a freshly-reset + // (Closed) socket, so neither fires in practice. + self + .sockets + .borrow_mut() + .get_mut::(c) + .listen(port) + .map_err(|_| StreamIoError::Unaddressable) + } + + fn accepted_peer(&self, c: Self::Conn) -> Option { + let set = self.sockets.borrow(); + let sock = set.get::(c); + // The accept gate is `may_send()`, true only in Established and CloseWait. Both + // mean the handshake settled and our send half is open (CloseWait additionally + // covers a peer that already half-closed after sending its push/pull half — + // still a completed, accept-worthy connection). + // + // `is_active()` MUST NOT be used here: it is also true in SynReceived (the + // half-open state after we send the SYN-ACK but before the remote's final ACK). + // Accepting in SynReceived would move a not-yet-established socket out of the + // listener slot while it still carries the listen endpoint; a retransmit/RST + // during the unfinished handshake then flips that socket back to Listen (smoltcp + // `tcp.rs`: an RST in SynReceived with a non-zero listen_endpoint reverts to + // Listen), silently turning the exchange's socket into a second listener and + // wedging the join. Gating on `may_send()` accepts strictly at/after Established, + // where an RST closes the socket cleanly instead of reverting it. + if !sock.may_send() { + return None; + } + sock.remote_endpoint().map(from_endpoint) + } + + fn connect( + &mut self, + c: Self::Conn, + remote: SocketAddr, + local_port: u16, + ) -> Result<(), StreamIoError> { + // smoltcp's `connect` needs the interface context to resolve the local + // endpoint. The interface is a separate field from the socket set, so the + // `&mut self.iface` context borrow and the socket set's `borrow_mut` do not + // conflict. + let remote_ep = to_endpoint(remote); + let cx = self.iface.context(); + self + .sockets + .borrow_mut() + .get_mut::(c) + .connect(cx, remote_ep, local_port) + .map_err(|_| StreamIoError::Unaddressable) + } + + fn may_send(&self, c: Self::Conn) -> bool { + self.sockets.borrow().get::(c).may_send() + } + + fn may_recv(&self, c: Self::Conn) -> bool { + self.sockets.borrow().get::(c).may_recv() + } + + fn is_open(&self, c: Self::Conn) -> bool { + self.sockets.borrow().get::(c).is_open() + } + + fn is_established(&self, c: Self::Conn) -> bool { + // The engine's `promote_established` check is "send-capable" — Established (and + // also CloseWait if the peer FIN'd before we did). That is exactly smoltcp's + // `may_send()`, so mirror it rather than re-deriving from `state()`. + self.sockets.borrow().get::(c).may_send() + } + + fn recv_finished(&self, c: Self::Conn) -> bool { + // True iff `tcp::Socket::recv_slice` would return `RecvError::Finished`, which + // smoltcp's `recv_error_check` defines as `!may_recv() && rx_fin_received`. A + // graceful peer FIN (a clean EOF) must report `true`; a connection RESET (RST) + // must report `false` — a reset is a failure, not an orderly end-of-stream. + // + // `rx_fin_received` is private, so derive it from the public `state()`. A + // received FIN sets the flag and moves the socket into exactly one of CloseWait + // (from Established/SynReceived), Closing (from FinWait1), LastAck (CloseWait + // after our own `close()`), or TimeWait (from FinWait1/FinWait2); those post-FIN + // states persist until our side FINs and the peer ACKs (or the 2MSL TimeWait + // elapses), so the engine's per-pump inbound drain always observes the EOF in one + // of them BEFORE the socket can reach `Closed`. `Closed` is therefore EXCLUDED: + // smoltcp enters `Closed` on a received RST WITHOUT setting `rx_fin_received` + // (`socket/tcp.rs`: the RST arm and `reset()` both clear it / never set it), so a + // RST-driven `Closed` is `recv_error_check == InvalidState`, NOT `Finished`. + // Mapping `Closed` to `true` would turn a reset into a clean EOF, silently + // completing an exchange the peer actually aborted; excluding it loses no real + // FIN because the graceful EOF was already delivered from a post-FIN state above. + // + // `!may_recv()` is the second term: it is false while the rx ring still holds + // bytes (`may_recv()` is true via `can_recv()` there), so this reports `false` + // until every byte the peer sent before its FIN has been drained — delivering + // the EOF exactly once, after the data, never before. It is also false while + // handshaking and for an open-but-empty Established ring, so no spurious EOF + // reaches the machine. + let set = self.sockets.borrow(); + let sock = set.get::(c); + matches!( + sock.state(), + tcp::State::CloseWait | tcp::State::Closing | tcp::State::LastAck | tcp::State::TimeWait + ) && !sock.may_recv() + } + + fn recv(&mut self, c: Self::Conn, buf: &mut [u8]) -> Option { + match self + .sockets + .borrow_mut() + .get_mut::(c) + .recv_slice(buf) + { + // `Ok(0)` is a momentarily empty Established ring (no data this tick); the + // engine treats `None` and `Some(0)` identically, but reporting `None` keeps + // the contract crisp ("no readable bytes this tick"). + Ok(0) => None, + Ok(n) => Some(n), + // EOF is delivered via `recv_finished`, never via `recv`: map both the drained + // peer-FIN (`Finished`) and the not-yet-receivable handshaking socket + // (`InvalidState`) to `None`. The engine consults `recv_finished` on a `None` + // read to deliver the one-shot EOF. + Err(tcp::RecvError::Finished) | Err(tcp::RecvError::InvalidState) => None, + } + } + + fn send(&mut self, c: Self::Conn, bytes: &[u8]) -> usize { + // `send_slice` accepts as many bytes as the tx ring has room for (a partial + // write on a full ring) and errors only with `InvalidState` when the tx half is + // not open. The engine gates this call on `may_send()` (it skips a + // not-send-capable socket in `pump_outbound_reliable`), so the error path is + // unreachable here; treat any error as "0 bytes accepted" so a torn-down tx half + // simply makes no progress rather than panicking. + // Ignoring Err: the engine only sends on a send-capable socket; 0 = no progress. + self + .sockets + .borrow_mut() + .get_mut::(c) + .send_slice(bytes) + .unwrap_or(0) + } + + fn send_queue(&self, c: Self::Conn) -> usize { + self.sockets.borrow().get::(c).send_queue() + } + + fn close(&mut self, c: Self::Conn) { + self.sockets.borrow_mut().get_mut::(c).close(); + } + + fn abort(&mut self, c: Self::Conn) { + self.sockets.borrow_mut().get_mut::(c).abort(); + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-smoltcp/src/stream_io/tests.rs b/serf-smoltcp/src/stream_io/tests.rs new file mode 100644 index 00000000..3d924fa0 --- /dev/null +++ b/serf-smoltcp/src/stream_io/tests.rs @@ -0,0 +1,253 @@ +use super::*; + +use std::{cell::RefCell, collections::VecDeque, rc::Rc, vec, vec::Vec}; + +use core::net::{IpAddr, Ipv4Addr}; +use smoltcp::{ + iface::{Config as IfConfig, Interface, SocketSet}, + phy::{ChecksumCapabilities, Device, DeviceCapabilities, Medium, RxToken, TxToken}, + time::Instant as SmolInstant, + wire::{HardwareAddress, IpAddress, IpCidr}, +}; + +/// A shared in-memory frame FIFO. +type Wire = Rc>>>; + +/// One end of a loopback `Medium::Ip` link: reads from `rx`, writes to `tx`. +struct LoopDevice { + rx: Wire, + tx: Wire, +} + +/// Cross-wire two `LoopDevice`s so each side's TX is the other's RX. +fn link() -> (LoopDevice, LoopDevice) { + let a2b: Wire = Rc::new(RefCell::new(VecDeque::new())); + let b2a: Wire = Rc::new(RefCell::new(VecDeque::new())); + ( + LoopDevice { + rx: b2a.clone(), + tx: a2b.clone(), + }, + LoopDevice { rx: a2b, tx: b2a }, + ) +} + +struct LRx(Vec); +struct LTx(Wire); + +impl RxToken for LRx { + fn consume R>(self, f: F) -> R { + f(&self.0) + } +} + +impl TxToken for LTx { + fn consume R>(self, len: usize, f: F) -> R { + let mut buf = vec![0u8; len]; + let r = f(&mut buf); + self.0.borrow_mut().push_back(buf); + r + } +} + +impl Device for LoopDevice { + type RxToken<'a> = LRx; + type TxToken<'a> = LTx; + + fn receive(&mut self, _t: SmolInstant) -> Option<(LRx, LTx)> { + let frame = self.rx.borrow_mut().pop_front()?; + Some((LRx(frame), LTx(self.tx.clone()))) + } + + fn transmit(&mut self, _t: SmolInstant) -> Option { + Some(LTx(self.tx.clone())) + } + + fn capabilities(&self) -> DeviceCapabilities { + let mut caps = DeviceCapabilities::default(); + caps.medium = Medium::Ip; + caps.max_transmission_unit = 1500; + caps.checksum = ChecksumCapabilities::ignored(); + caps + } +} + +/// Build a `Medium::Ip` interface at `10.0.0.{octet}/24` over `device`. +fn iface(device: &mut LoopDevice, octet: u8) -> Interface { + let mut cfg = IfConfig::new(HardwareAddress::Ip); + cfg.random_seed = octet as u64; + let mut iface = Interface::new(cfg, device, SmolInstant::from_millis(0)); + iface.update_ip_addrs(|addrs| { + addrs + .push(IpCidr::new(IpAddress::v4(10, 0, 0, octet), 24)) + .expect("push ip"); + }); + iface +} + +/// Establish one TCP connection between two loopback interfaces and return the +/// handles plus the per-node `(iface, socket-set, device)` so a test can drive +/// either side through the [`SmoltcpStream`] view. +/// +/// `a` is the active opener (dials `b`); `b` is the passive listener. Returns +/// `(ha, hb)` — the socket handles — leaving both sockets Established. +#[allow(clippy::type_complexity)] +fn established() -> ( + (Interface, SocketSet<'static>, LoopDevice, SocketHandle), + (Interface, SocketSet<'static>, LoopDevice, SocketHandle), +) { + let (mut dev_a, mut dev_b) = link(); + let mut if_a = iface(&mut dev_a, 1); + let mut if_b = iface(&mut dev_b, 2); + let mut set_a = SocketSet::new(Vec::new()); + let mut set_b = SocketSet::new(Vec::new()); + + let mk = || { + tcp::Socket::new( + tcp::SocketBuffer::new(vec![0u8; 4096]), + tcp::SocketBuffer::new(vec![0u8; 4096]), + ) + }; + let ha = set_a.add(mk()); + let hb = set_b.add(mk()); + + let local_a = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)), 7946); + let remote_b = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 2)), 7946); + + // B listens; A dials. Both views borrow their own set for the call only. + { + let cell_b = RefCell::new(&mut set_b); + let mut sb = SmoltcpStream::new(&mut if_b, &cell_b); + sb.listen(hb, 7946).expect("listen"); + } + { + let cell_a = RefCell::new(&mut set_a); + let mut sa = SmoltcpStream::new(&mut if_a, &cell_a); + sa.connect(ha, remote_b, local_a.port()).expect("connect"); + } + + // Pump both stacks until A is send-capable (the handshake settled). + for t in 0..50u64 { + let now = SmolInstant::from_millis(t as i64); + if_a.poll(now, &mut dev_a, &mut set_a); + if_b.poll(now, &mut dev_b, &mut set_b); + if set_a.get::(ha).may_send() && set_b.get::(hb).may_send() { + break; + } + } + assert!( + set_a.get::(ha).may_send(), + "handshake did not complete (A not send-capable)" + ); + + ((if_a, set_a, dev_a, ha), (if_b, set_b, dev_b, hb)) +} + +/// Pump both stacks a few ticks so in-flight segments (a FIN, an RST, an ACK) +/// are delivered and the receiving socket's state machine advances. +fn settle( + a: &mut (Interface, SocketSet<'static>, LoopDevice, SocketHandle), + b: &mut (Interface, SocketSet<'static>, LoopDevice, SocketHandle), + start_ms: u64, +) { + for t in start_ms..start_ms + 20 { + let now = SmolInstant::from_millis(t as i64); + a.0.poll(now, &mut a.2, &mut a.1); + b.0.poll(now, &mut b.2, &mut b.1); + } +} + +/// `recv_finished` of the `SmoltcpStream` view over `node`'s socket. +fn recv_finished(node: &mut (Interface, SocketSet<'static>, LoopDevice, SocketHandle)) -> bool { + let cell = RefCell::new(&mut node.1); + let view = SmoltcpStream::new(&mut node.0, &cell); + view.recv_finished(node.3) +} + +/// A graceful peer FIN is reported as a clean EOF: after B `close()`s its write +/// half, A drains the bytes and observes `recv_finished == true`. +#[test] +fn graceful_fin_is_reported_as_eof() { + let (mut a, mut b) = established(); + + // B sends a payload then gracefully closes its write half (FIN after the data). + b.1 + .get_mut::(b.3) + .send_slice(b"hello") + .expect("send"); + b.1.get_mut::(b.3).close(); + settle(&mut a, &mut b, 100); + + // Before draining, the rx ring still holds "hello", so no premature EOF. + assert!( + !recv_finished(&mut a), + "EOF must not be reported while buffered bytes remain" + ); + // Drain the payload. + let mut buf = [0u8; 16]; + let n = a + .1 + .get_mut::(a.3) + .recv_slice(&mut buf) + .expect("recv"); + assert_eq!(&buf[..n], b"hello"); + + // With the data drained and the peer FIN received, the EOF is now reported. + // A is in CloseWait (it has not yet closed its own write half). + assert_eq!(a.1.get::(a.3).state(), tcp::State::CloseWait); + assert!( + recv_finished(&mut a), + "a drained graceful FIN must report recv_finished == true (clean EOF)" + ); +} + +/// A connection RESET (RST) is NOT a clean EOF. After B `abort()`s (sending an +/// RST), A's socket reaches `Closed` WITHOUT `rx_fin_received`, so the pre-refactor +/// `recv_slice` would return `InvalidState` — `recv_finished` must therefore report +/// `false`, surfacing the reset as a failure rather than a graceful completion. +#[test] +fn reset_is_not_reported_as_eof() { + let (mut a, mut b) = established(); + + // B aborts: smoltcp emits an RST and moves B's socket straight to Closed. + b.1.get_mut::(b.3).abort(); + settle(&mut a, &mut b, 100); + + // The RST drives A to Closed (no FIN handshake, no rx_fin_received). + assert_eq!( + a.1.get::(a.3).state(), + tcp::State::Closed, + "a received RST must drive the peer socket to Closed" + ); + assert!( + !recv_finished(&mut a), + "a reset Closed socket must report recv_finished == false (a reset is a \ + failure, not a graceful EOF) — mapping it to true would silently complete \ + an aborted exchange" + ); +} + +/// A RST that arrives mid-stream (the peer aborts after sending some bytes, the +/// classic abrupt teardown) is likewise not a clean EOF: the buffered bytes are +/// dropped by the reset and `recv_finished` stays `false`. +#[test] +fn mid_stream_reset_is_not_reported_as_eof() { + let (mut a, mut b) = established(); + + // B sends a payload, then immediately aborts (RST) before A drains it. + b.1 + .get_mut::(b.3) + .send_slice(b"partial") + .expect("send"); + settle(&mut a, &mut b, 100); + b.1.get_mut::(b.3).abort(); + settle(&mut a, &mut b, 200); + + // The RST resets A's socket: it is Closed and its rx buffer was cleared, so + // there is no orderly EOF to report. + assert_eq!(a.1.get::(a.3).state(), tcp::State::Closed); + assert!( + !recv_finished(&mut a), + "a mid-stream reset must not be reported as a graceful EOF" + ); +} diff --git a/serf-smoltcp/tests/harness/mod.rs b/serf-smoltcp/tests/harness/mod.rs new file mode 100644 index 00000000..53d8e56e --- /dev/null +++ b/serf-smoltcp/tests/harness/mod.rs @@ -0,0 +1,157 @@ +//! Deterministic two-node test substrate: a paired in-memory smoltcp `Device` +//! (one node's TX is the other's RX) on `medium::Ip`, plus a virtual clock. +#![allow(dead_code)] + +use std::{cell::RefCell, collections::VecDeque, rc::Rc}; + +use core::net::IpAddr; + +use serf_smoltcp::{HardwareAddress, InterfaceOptions, IpCidr}; +use smoltcp::{ + phy::{ChecksumCapabilities, Device, DeviceCapabilities, Medium, RxToken, TxToken}, + time::Instant, +}; + +/// Build an [`InterfaceOptions`] for the harness's `Medium::Ip` devices: +/// `HardwareAddress::Ip` plus the node's `/24` address. +/// +/// The interface RNG seed is derived deterministically from the node's IP, so a +/// harness node's whole stack — smoltcp ports/ISNs and the gossip / serf RNG +/// schedules the driver derives from this seed — is reproducible across runs. +pub fn ip_iface(ip: IpAddr) -> InterfaceOptions { + let seed = match ip { + IpAddr::V4(v4) => u32::from(v4) as u64, + IpAddr::V6(v6) => { + let o = v6.octets(); + u64::from_be_bytes([o[8], o[9], o[10], o[11], o[12], o[13], o[14], o[15]]) + } + }; + InterfaceOptions::new(HardwareAddress::Ip) + .with_ip_addr(IpCidr::new(ip.into(), 24)) + .with_random_seed(seed) +} + +type Wire = Rc>>>; + +/// One end of a virtual link: reads from `rx`, writes to `tx`. +pub struct PairedDevice { + rx: Wire, + tx: Wire, + mtu: usize, +} + +impl PairedDevice { + /// Whether a frame has been delivered to this node's receive queue but not yet + /// drained by a `poll`. A deadline-driven loop uses this to model "an arriving + /// packet wakes its receiver": while any node has an inbound frame pending, the + /// loop re-polls promptly instead of sleeping the shared clock to a far timer. + pub fn inbound_pending(&self) -> bool { + !self.rx.borrow().is_empty() + } +} + +/// Build the two ends of one virtual link. +/// +/// Frames sent by the `A` end arrive at the `B` end's receive queue, and vice +/// versa — the two FIFOs are cross-wired so `A.tx == B.rx` and `B.tx == A.rx`. +pub fn link(mtu: usize) -> (PairedDevice, PairedDevice) { + let a2b: Wire = Rc::new(RefCell::new(VecDeque::new())); + let b2a: Wire = Rc::new(RefCell::new(VecDeque::new())); + ( + PairedDevice { + rx: b2a.clone(), + tx: a2b.clone(), + mtu, + }, + PairedDevice { + rx: a2b, + tx: b2a, + mtu, + }, + ) +} + +pub struct VRx(Vec); +pub struct VTx(Wire); + +impl RxToken for VRx { + fn consume(self, f: F) -> R + where + F: FnOnce(&[u8]) -> R, + { + f(&self.0) + } +} + +impl TxToken for VTx { + fn consume(self, len: usize, f: F) -> R + where + F: FnOnce(&mut [u8]) -> R, + { + let mut buf = vec![0u8; len]; + let r = f(&mut buf); + self.0.borrow_mut().push_back(buf); + r + } +} + +impl Device for PairedDevice { + type RxToken<'a> + = VRx + where + Self: 'a; + type TxToken<'a> + = VTx + where + Self: 'a; + + fn receive(&mut self, _timestamp: Instant) -> Option<(VRx, VTx)> { + let frame = self.rx.borrow_mut().pop_front()?; + Some((VRx(frame), VTx(self.tx.clone()))) + } + + fn transmit(&mut self, _timestamp: Instant) -> Option { + Some(VTx(self.tx.clone())) + } + + fn capabilities(&self) -> DeviceCapabilities { + let mut caps = DeviceCapabilities::default(); + caps.medium = Medium::Ip; + caps.max_transmission_unit = self.mtu; + caps.checksum = ChecksumCapabilities::ignored(); + caps + } +} + +/// Deterministic virtual clock anchored away from zero. +/// +/// `memberlist_proto::Instant - Duration` saturates at the origin; the +/// 86 400 s offset gives backward-aging headroom for any suspicion or failure +/// timers that need to subtract from `now`. +pub struct Clock { + ms: u64, +} + +impl Clock { + pub fn new() -> Self { + Self { ms: 86_400_000 } + } + + pub fn now(&self) -> serf_smoltcp::Instant { + serf_smoltcp::Instant::from_origin(core::time::Duration::from_millis(self.ms)) + } + + pub fn advance_ms(&mut self, by: u64) { + self.ms += by; + } + + /// Jump the clock forward to `target` (a deadline returned by `poll`), rounding + /// UP to the next whole millisecond. Never moves backwards. + pub fn advance_to(&mut self, target: serf_smoltcp::Instant) { + let ns = target.since_origin().as_nanos(); + let target_ms = ns.div_ceil(1_000_000) as u64; + if target_ms > self.ms { + self.ms = target_ms; + } + } +} diff --git a/serf-smoltcp/tests/join_leave.rs b/serf-smoltcp/tests/join_leave.rs new file mode 100644 index 00000000..1ba4008d --- /dev/null +++ b/serf-smoltcp/tests/join_leave.rs @@ -0,0 +1,122 @@ +//! Reliable-plane end-to-end: a serf node JOINS a seed via a real push/pull state +//! exchange over TCP, both converge on a 2-member view, then the joiner LEAVES +//! gracefully and observes the corresponding lifecycle event. +//! +//! This exercises the full reliable plane over the smoltcp stack: dial +//! (`StreamAction::Connect` → `tcp::Socket::connect`), the TCP three-way handshake +//! over the paired device, the listener accept, the bidirectional push/pull byte +//! pump, and graceful teardown — the proof that the `Serf` handle's dial / accept / +//! pump wiring carries a serf join to completion. + +mod harness; + +use core::net::{IpAddr, Ipv4Addr, SocketAddr}; + +use serf_smoltcp::{ + EndpointOptions, Event, MaybeResolved, Options, Serf, SerfOptions, SocketAddrResolver, + TransformOptions, +}; +use smol_str::SmolStr; + +fn addr(ip: u8, port: u16) -> SocketAddr { + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, ip)), port) +} + +/// A real TCP push/pull serf join, a synced 2-member view, then a clean leave. +/// +/// # Phase 1 — join +/// +/// B dials A as a seed. Both nodes poll each tick so the SYN/SYN-ACK/ACK and the +/// push/pull frames flow through the paired device FIFOs. Convergence is +/// `num_members() == 2` on BOTH nodes — A learns B from the inbound push/pull, B +/// learns A from the reply. The 20 s virtual budget is generous headroom over the +/// zero-latency link, not an expected duration. +/// +/// # Phase 2 — leave +/// +/// B calls `leave`, which gossips the departure; B must observe `Event::LeftCluster`. +/// Both nodes keep polling so the leave broadcast and any reliable teardown frames +/// cross the link. +#[test] +fn join_from_seed_then_clean_leave() { + const BUDGET: u32 = 2000; + + let (mut da, mut db) = harness::link(1500); + let mut clk = harness::Clock::new(); + let now = clk.now(); + + // Node A is the seed; node B is the joiner. + let mut a: Serf = Serf::new( + Options::new(), + harness::ip_iface(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1))), + TransformOptions::default(), + EndpointOptions::new(SmolStr::new("a"), addr(1, 7946)), + SerfOptions::new(), + &SocketAddrResolver, + &mut da, + now, + ); + let mut b: Serf = Serf::new( + Options::new(), + harness::ip_iface(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 2))), + TransformOptions::default(), + EndpointOptions::new(SmolStr::new("b"), addr(2, 7946)), + SerfOptions::new(), + &SocketAddrResolver, + &mut db, + now, + ); + + a.start(now); + b.start(now); + + // B joins via A as a seed: a REAL push/pull over TCP. + let jid = b + .join( + &SocketAddrResolver, + &[MaybeResolved::Resolved(addr(1, 7946))], + false, + now, + ) + .expect("join from a running node"); + + // Drive both until both see 2 members (join push/pull synced state), bounded. + let mut joined = false; + for _ in 0..BUDGET { + let _ = a.poll(clk.now(), &mut da); + let _ = b.poll(clk.now(), &mut db); + // Drain the await-result join so its waiter does not linger once resolved. + let _ = b.poll_join(jid); + if a.num_members() == 2 && b.num_members() == 2 { + joined = true; + break; + } + clk.advance_ms(10); + } + assert!( + joined, + "join push/pull did not converge: a={} b={}", + a.num_members(), + b.num_members() + ); + + // B leaves; B must observe LeftCluster. + b.leave(clk.now()).expect("leave"); + let mut b_left = false; + for _ in 0..BUDGET { + let _ = a.poll(clk.now(), &mut da); + let _ = b.poll(clk.now(), &mut db); + while let Some(ev) = b.poll_event() { + if matches!(ev, Event::LeftCluster) { + b_left = true; + } + } + // Drain A's events so its buffer does not shed under the leave gossip. + while a.poll_event().is_some() {} + if b_left { + break; + } + clk.advance_ms(10); + } + assert!(b_left, "b did not complete leave (LeftCluster)"); +} diff --git a/serf-smoltcp/tests/user_event.rs b/serf-smoltcp/tests/user_event.rs new file mode 100644 index 00000000..f7f2db8a --- /dev/null +++ b/serf-smoltcp/tests/user_event.rs @@ -0,0 +1,99 @@ +//! Gossip-plane end-to-end: two serf nodes converge over a real TCP push/pull +//! join, then one broadcasts a user event that the other observes through its +//! driver-buffered `poll_event`. Exercises the `Serf` handle's outbound gossip +//! path and the mandatory-event drain's app-event buffering. + +mod harness; + +use core::net::{IpAddr, Ipv4Addr, SocketAddr}; + +use serf_smoltcp::{ + Bytes, EndpointOptions, Event, MaybeResolved, Options, Serf, SerfOptions, SocketAddrResolver, + TransformOptions, +}; +use smol_str::SmolStr; + +fn addr(ip: u8, port: u16) -> SocketAddr { + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, ip)), port) +} + +/// After a converged join, A broadcasts a user event; B observes `Event::User`. +#[test] +fn user_event_propagates_across_the_gossip_plane() { + const BUDGET: u32 = 4000; + + let (mut da, mut db) = harness::link(1500); + let mut clk = harness::Clock::new(); + let now = clk.now(); + + let mut a: Serf = Serf::new( + Options::new(), + harness::ip_iface(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1))), + TransformOptions::default(), + EndpointOptions::new(SmolStr::new("a"), addr(1, 7946)), + SerfOptions::new(), + &SocketAddrResolver, + &mut da, + now, + ); + let mut b: Serf = Serf::new( + Options::new(), + harness::ip_iface(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 2))), + TransformOptions::default(), + EndpointOptions::new(SmolStr::new("b"), addr(2, 7946)), + SerfOptions::new(), + &SocketAddrResolver, + &mut db, + now, + ); + + a.start(now); + b.start(now); + + let jid = b + .join( + &SocketAddrResolver, + &[MaybeResolved::Resolved(addr(1, 7946))], + false, + now, + ) + .expect("join from a running node"); + + // Converge on a 2-member view first. + let mut joined = false; + for _ in 0..BUDGET { + let _ = a.poll(clk.now(), &mut da); + let _ = b.poll(clk.now(), &mut db); + let _ = b.poll_join(jid); + // Keep the event backlogs drained so neither sheds while converging. + while a.poll_event().is_some() {} + while b.poll_event().is_some() {} + if a.num_members() == 2 && b.num_members() == 2 { + joined = true; + break; + } + clk.advance_ms(10); + } + assert!(joined, "nodes did not converge before the user event"); + + // A broadcasts a user event; drive gossip until B observes it. + a.user_event("greeting", Bytes::from_static(b"hello"), false) + .expect("queue user event from a running node"); + + let mut b_saw_user = false; + for _ in 0..BUDGET { + let _ = a.poll(clk.now(), &mut da); + let _ = b.poll(clk.now(), &mut db); + while let Some(ev) = b.poll_event() { + if matches!(ev, Event::User(_)) { + b_saw_user = true; + } + } + while a.poll_event().is_some() {} + if b_saw_user { + break; + } + clk.advance_ms(10); + } + assert!(b_saw_user, "b did not observe the user event A broadcast"); +} From 363ab0b51cd0029faa01492f4c76c712d3a86460 Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 10 Jul 2026 15:03:38 +1200 Subject: [PATCH 056/128] fix(serf-smoltcp): loop back self-addressed gossip, wake on queued responses, sum drop counters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit smoltcp does not loop self-addressed datagrams like an OS socket, so a node's response to its own query/key-request was lost — deliver dest==advertise datagrams into the engine's ingress and drive poll to quiescence so the response is collected and any respond_key is egressed within the tick (folding now into the deadline if work remains); events_dropped now sums the engine's and driver's drop counters; regressions cover the query round-trip (local+remote responses), deadline-driven key response, and single-pump over-cap drop counting. --- serf-smoltcp/src/gossip_io.rs | 68 ++++++++- serf-smoltcp/src/serf/mod.rs | 147 +++++++++++++------ serf-smoltcp/src/serf/tests.rs | 110 +++++++++++++- serf-smoltcp/tests/query.rs | 252 +++++++++++++++++++++++++++++++++ 4 files changed, 529 insertions(+), 48 deletions(-) create mode 100644 serf-smoltcp/tests/query.rs diff --git a/serf-smoltcp/src/gossip_io.rs b/serf-smoltcp/src/gossip_io.rs index 242327c9..3dab0ad5 100644 --- a/serf-smoltcp/src/gossip_io.rs +++ b/serf-smoltcp/src/gossip_io.rs @@ -16,6 +16,8 @@ use core::{cell::RefCell, net::SocketAddr}; +use std::{collections::VecDeque, vec::Vec}; + use serf_embedded::GossipIo; use smoltcp::{ iface::{SocketHandle, SocketSet}, @@ -28,20 +30,73 @@ use crate::addr::{from_endpoint, to_endpoint}; /// /// Resolves the gossip socket by its [`SocketHandle`] on each call, taking a brief /// `borrow_mut` of the shared set. +/// +/// # Self-delivery loopback +/// +/// A real OS UDP socket loops a datagram addressed to its own bound address back +/// into `recv`; smoltcp does not — a self-addressed frame goes to the wire and is +/// dropped. serf directs a node's response to its OWN locally-originated query / +/// key-request to the originator's advertise address, i.e. THIS node's address, so +/// on smoltcp that response would be lost and a local op would report +/// `num_resp < num_nodes`. This view emulates the OS self-delivery: [`send`] to the +/// node's own `advertise` address queues the datagram on a driver-owned `loopback` +/// buffer instead of the socket, and [`recv`] returns those queued datagrams FIRST +/// (as if received from `advertise`) so a subsequent pump ingests them exactly as an +/// OS loopback would. The bytes are the already-transformed wire form the engine +/// emitted, so feeding them back through the normal gossip ingress decodes them +/// identically. +/// +/// [`send`]: GossipIo::send +/// [`recv`]: GossipIo::recv pub(crate) struct SmoltcpGossip<'a, 'b> { sockets: &'a RefCell<&'a mut SocketSet<'b>>, udp: SocketHandle, + /// Driver-owned self-delivery buffer: datagrams this node addressed to its own + /// `advertise` address, awaiting loopback ingestion on the next pump. A separate + /// field, never inside the `SocketSet` borrow. + loopback: &'a RefCell<&'a mut VecDeque>>, + /// The local node's resolved advertise address — the destination a self-addressed + /// gossip datagram carries. + advertise: SocketAddr, } impl<'a, 'b> SmoltcpGossip<'a, 'b> { - /// Build the view over the shared `sockets` for the gossip socket `udp`. - pub(crate) fn new(sockets: &'a RefCell<&'a mut SocketSet<'b>>, udp: SocketHandle) -> Self { - Self { sockets, udp } + /// Build the view over the shared `sockets` for the gossip socket `udp`, with the + /// driver's `loopback` self-delivery buffer and the node's `advertise` address. + pub(crate) fn new( + sockets: &'a RefCell<&'a mut SocketSet<'b>>, + udp: SocketHandle, + loopback: &'a RefCell<&'a mut VecDeque>>, + advertise: SocketAddr, + ) -> Self { + Self { + sockets, + udp, + loopback, + advertise, + } } } impl GossipIo for SmoltcpGossip<'_, '_> { fn recv(&mut self, buf: &mut [u8]) -> Option<(SocketAddr, usize)> { + // Self-delivery first: return any datagrams this node addressed to its own + // advertise address, exactly as an OS UDP socket would loop them back into recv, + // before draining the real rx ring. + loop { + let datagram = self.loopback.borrow_mut().pop_front(); + let Some(datagram) = datagram else { break }; + let n = datagram.len(); + if n <= buf.len() { + buf[..n].copy_from_slice(&datagram); + return Some((self.advertise, n)); + } + // An own datagram larger than the ingress buffer cannot be delivered; drop it + // and continue, mirroring the rx-ring `Truncated` skip below. A self datagram + // is bounded by this node's own gossip MTU + encryption overhead, so it fits + // the arena-sized ingress buffer in practice — this is defence-in-depth. + } + let mut set = self.sockets.borrow_mut(); let sock = set.get_mut::(self.udp); // Pop the next deliverable datagram. `recv_slice` is called only while @@ -65,6 +120,13 @@ impl GossipIo for SmoltcpGossip<'_, '_> { } fn send(&mut self, bytes: &[u8], dest: SocketAddr) { + if dest == self.advertise { + // Self-addressed: smoltcp will not loop this back into its own rx ring the way + // an OS UDP socket does, so deliver it into the driver's loopback buffer for + // the next pump's ingress instead of dropping it on the wire. + self.loopback.borrow_mut().push_back(bytes.to_vec()); + return; + } let mut set = self.sockets.borrow_mut(); let sock = set.get_mut::(self.udp); // Ignoring Err: gossip is best-effort — a full or errored UDP tx ring drops diff --git a/serf-smoltcp/src/serf/mod.rs b/serf-smoltcp/src/serf/mod.rs index f9836af4..f4a919dc 100644 --- a/serf-smoltcp/src/serf/mod.rs +++ b/serf-smoltcp/src/serf/mod.rs @@ -75,6 +75,17 @@ const ENCRYPTED_WRAPPER_OVERHEAD: usize = 0; /// The transmit buffer has no such limit and is not capped. const TCP_RX_BUFFER_MAX: usize = 1 << 30; +/// The most pump → drain passes one [`Serf::poll`] makes to reach quiescence. +/// +/// A single `poll` re-pumps while a pass produced new work the current deadline / +/// egress has not yet reflected: a `respond_key` the drain just queued, or a +/// self-addressed datagram the pump's egress just looped back (see +/// [`SmoltcpGossip`]). Each self query / key response settles in a few passes; this +/// caps a pathological self-delivery cycle so a single `poll` cannot spin forever. +/// On hitting the cap with work still pending, `poll` folds `now` into its returned +/// deadline so the caller re-polls at once rather than sleeping past it. +const MAX_SELF_DELIVERY_ITERS: usize = 8; + /// Whether `addr` is a destination the smoltcp stack can actually use. /// /// `to_endpoint(*addr).addr.is_unicast()` calls smoltcp's OWN `IpAddress::is_unicast` @@ -278,6 +289,13 @@ where /// Count of app events shed from `app_events` because the app never drained /// [`poll_event`](Self::poll_event) fast enough and the backlog hit the cap. app_events_dropped: u64, + /// Driver-owned self-delivery buffer for gossip datagrams this node addressed to + /// its OWN advertise address (a response to its own query / key-request). smoltcp + /// does not loop a self-addressed datagram back into recv like an OS UDP socket, + /// so [`SmoltcpGossip`] diverts such datagrams here on send and replays them on + /// recv, and [`poll`](Self::poll) drives the pump to quiescence so the looped-back + /// datagram is ingested and its response collected within the same tick. + loopback: VecDeque>, /// Set once the driver observed a lost id-conflict [`Event::Shutdown`]; the /// caller reads it via [`is_shutdown`](Self::is_shutdown) and stops polling. shutdown: bool, @@ -722,6 +740,7 @@ where advertise, app_events: VecDeque::new(), app_events_dropped: 0, + loopback: VecDeque::new(), shutdown: false, #[cfg(encryption)] keyring, @@ -764,13 +783,6 @@ where self.app_events.pop_front() } - /// The number of app events shed from the [`poll_event`](Self::poll_event) backlog - /// because it reached [`DEFAULT_EVENT_BUFFER_CAP`] before the app drained them. - #[inline] - pub fn events_dropped(&self) -> u64 { - self.app_events_dropped - } - /// Number of inbound reliable connections accepted since construction. #[doc(hidden)] #[inline] @@ -849,6 +861,22 @@ where self.engine.num_members() } + /// The number of app events shed because a consumer did not keep up: the engine's + /// own passive-observation drops plus this driver's + /// [`poll_event`](Self::poll_event) backlog drops. + /// + /// Both stages are bounded at [`DEFAULT_EVENT_BUFFER_CAP`] with drop-oldest, and a + /// single pump can shed observations INSIDE the engine (its bounded queue) before + /// the driver's queue — freshly drained each poll — ever fills, so this sums BOTH + /// counters; reporting only the driver's would under-count real loss. + #[inline] + pub fn events_dropped(&self) -> u64 { + self + .engine + .events_dropped() + .saturating_add(self.app_events_dropped) + } + /// The local node's id. #[inline] pub fn local_id(&self) -> I { @@ -1029,26 +1057,37 @@ where self.engine.list_keys(now) } - /// Advance both the smoltcp stack and serf's state machine once, then act on - /// serf's mandatory driver-actioned events. Returns the next wakeup deadline: - /// the minimum of the smoltcp stack's next scheduled event, the machine's next - /// timer, and any engine-owned deadline (the soonest closing socket's abort). + /// Advance both the smoltcp stack and serf's state machine, act on serf's + /// mandatory driver-actioned events, and drive to quiescence within the tick. + /// Returns the next wakeup deadline: the minimum of the smoltcp stack's next + /// scheduled event, the machine's next timer, and any engine-owned deadline (the + /// soonest closing socket's abort). /// /// # Order /// /// 1. **Stack tick** — `iface.poll` drains the device and services TCP/UDP. - /// 2. **Engine pump** — the engine runs every protocol phase over a - /// [`SmoltcpGossip`] + [`SmoltcpStream`] view of the just-ticked sockets. - /// 3. **Mandatory events** — drain the engine's events (mandatory-first) and act - /// on the driver-actioned ones: [`Event::Shutdown`] flips the stop flag; an - /// [`Event::KeyRequest`] is applied to the local keyring and answered via + /// 2. **Pump → drain, to quiescence** — the engine runs every protocol phase over + /// a [`SmoltcpGossip`] + [`SmoltcpStream`] view of the just-ticked sockets + /// (computing the next deadline and egressing outbound gossip), then the drain + /// acts on the driver-actioned events: [`Event::Shutdown`] flips the stop flag; + /// an [`Event::KeyRequest`] is applied to the local keyring and answered via /// `respond_key`. Every drained event is buffered for the app's own /// [`poll_event`](Self::poll_event), so membership / user / query observations - /// and the mandatory events alike remain visible AFTER the driver acted. A key - /// response issued here is a directed gossip transmit the engine emits on the - /// NEXT `poll`'s pump. - /// 4. **Deadline** — fold the stack's next scheduled event into the engine's - /// returned wakeup. + /// and the mandatory events alike remain visible AFTER the driver acted. + /// + /// The pump computes its deadline and egresses BEFORE the drain runs, so a + /// `respond_key` the drain queues — or a self-addressed datagram the pump's + /// egress diverts into the [`loopback`](SmoltcpGossip) buffer — would not be + /// reflected by that pass. This step therefore re-pumps at the SAME `now` while + /// a pass queued a key response OR left the loopback non-empty (re-pumping at an + /// unchanged `now` is safe: due timers already fired, so the extra pass only + /// egresses the queued send and ingests the looped-back datagram), bounded by + /// [`MAX_SELF_DELIVERY_ITERS`]. A remote `respond_key` is thus egressed within + /// this `poll`, and a self-addressed response is looped back, ingested, and its + /// query response collected — all before `poll` returns. + /// 3. **Deadline** — fold the stack's next scheduled event into the settled + /// engine deadline. If the quiescence loop exhausted its budget with work still + /// pending, fold `now` in too so the caller re-polls immediately. pub fn poll(&mut self, now: Instant, device: &mut D) -> Option where D: Device, @@ -1058,21 +1097,35 @@ where // 1. Stack tick. self.iface.poll(s_now, device, &mut self.sockets); - // 2. Engine pump over a view of the just-ticked sockets. The gossip and stream - // views share mutable access to the one `SocketSet` through a `RefCell` held for - // the pump; each takes a brief borrow and never holds one across a call into the - // other, so the borrows never overlap. - let next = { - let sockets = RefCell::new(&mut self.sockets); - let mut gossip = SmoltcpGossip::new(&sockets, self.udp); - let mut stream = SmoltcpStream::new(&mut self.iface, &sockets); - self.engine.pump(now, &mut gossip, &mut stream) - }; - - // 3. Mandatory-event handling in the poll cycle (driver-owned). - self.drain_engine_events(now); + // 2. Pump → drain, re-running at the same `now` until neither a queued key + // response nor a looped-back self-datagram remains, so both are handled within + // this tick. The gossip and stream views share the one `SocketSet` through a + // `RefCell` held for each pump; each takes a brief borrow and never holds one + // across a call into the other, and the loopback buffer is a separate field + // outside that borrow. + let mut next = None; + let mut settled = false; + for _ in 0..MAX_SELF_DELIVERY_ITERS { + next = { + let sockets = RefCell::new(&mut self.sockets); + let loopback = RefCell::new(&mut self.loopback); + let mut gossip = SmoltcpGossip::new(&sockets, self.udp, &loopback, self.advertise); + let mut stream = SmoltcpStream::new(&mut self.iface, &sockets); + self.engine.pump(now, &mut gossip, &mut stream) + }; + let queued = self.drain_engine_events(now); + if !queued && self.loopback.is_empty() { + settled = true; + break; + } + } - // 4. Fold the stack's next scheduled event into the engine's deadline. + // 3. Fold the stack's next scheduled event into the engine's deadline. On an + // unsettled loop (work still pending at the iteration cap), fold `now` so the + // caller re-polls at once rather than sleeping past the stranded work. + if !settled { + next = min_opt(next, Some(now)); + } let stack = self .iface .poll_at(s_now, &self.sockets) @@ -1082,30 +1135,42 @@ where /// Drain the engine's event queue (mandatory-first), take the driver-owned side /// effect on each mandatory event, and buffer every event for the app. - fn drain_engine_events(&mut self, now: Instant) { - // Ignoring: `now` is consumed only by the encryption `KeyRequest` arm below, - // so a build without an AEAD backend does not read it. + /// + /// Returns whether the drain queued outbound gossip work the current pump's egress + /// did not see — a successful `respond_key` — so [`poll`](Self::poll) knows to + /// re-pump and egress it within the same tick. + fn drain_engine_events(&mut self, now: Instant) -> bool { + // `now` and the queued-outbound signal are consumed only by the encryption + // `KeyRequest` arm; a build without an AEAD backend reads neither and queues no + // key response, so its drain never re-pumps on this account. #[cfg(not(encryption))] let _ = now; + #[cfg(encryption)] + let mut queued = false; + #[cfg(not(encryption))] + let queued = false; while let Some(ev) = self.engine.poll_event() { match &ev { // A lost id-conflict vote means the local node MUST stop; flag it. The event // still reaches the app via `poll_event`. Event::Shutdown => self.shutdown = true, // An inbound key-management request: apply the op to the local keyring and - // answer the originator. The response is a directed gossip transmit emitted - // on the next pump. + // answer the originator. The response is a directed gossip transmit egressed + // on the re-pump [`poll`](Self::poll) runs while `queued` is set. #[cfg(encryption)] Event::KeyRequest(req) => { let resp = apply_key_request(&mut self.keyring, req); // Ignoring Err: `respond_key` fails only when the response cannot be // routed; the key op has already applied to the driver keyring. - let _ = self.engine.respond_key(req, resp, now); + if self.engine.respond_key(req, resp, now).is_ok() { + queued = true; + } } _ => {} } self.push_app_event(ev); } + queued } /// Buffer one event for [`poll_event`](Self::poll_event), bounding the backlog at diff --git a/serf-smoltcp/src/serf/tests.rs b/serf-smoltcp/src/serf/tests.rs index 56199d20..cc4968a6 100644 --- a/serf-smoltcp/src/serf/tests.rs +++ b/serf-smoltcp/src/serf/tests.rs @@ -1,6 +1,14 @@ -use core::net::{IpAddr, Ipv4Addr, SocketAddr}; +use core::{ + cell::RefCell, + net::{IpAddr, Ipv4Addr, SocketAddr}, +}; -use memberlist_proto::Instant; +use memberlist_proto::{ + Instant, Node, + codec::{EncodeOptions, encode_outgoing}, + typed::{Alive, DelegateVersion, Message, Meta, ProtocolVersion}, +}; +use serf_embedded::GossipIo; use smol_str::SmolStr; use smoltcp::{ phy::{ChecksumCapabilities, Device, DeviceCapabilities, Medium, RxToken, TxToken}, @@ -8,8 +16,9 @@ use smoltcp::{ }; use crate::{ - EndpointOptions, HardwareAddress, InterfaceOptions, IpAddress, IpCidr, Options, Serf, - SerfOptions, SerfState, SocketAddrResolver, TransformOptions, + DEFAULT_EVENT_BUFFER_CAP, EndpointOptions, HardwareAddress, InterfaceOptions, IpAddress, IpCidr, + Options, Serf, SerfOptions, SerfState, SocketAddrResolver, TransformOptions, + stream_io::SmoltcpStream, }; /// A `Medium::Ip` device that never delivers a frame — enough to construct a node @@ -137,6 +146,99 @@ fn zero_port_is_rejected() { assert!(matches!(err, crate::InitError::ZeroPort)); } +/// A [`GossipIo`] that replays a fixed list of pre-encoded datagrams once (one per +/// `recv`), draining `send`. Feeds a controlled flood of inbound gossip straight to +/// the engine's pump. +struct FloodGossip { + frames: Vec>, + idx: usize, + src: SocketAddr, +} + +impl GossipIo for FloodGossip { + fn recv(&mut self, buf: &mut [u8]) -> Option<(SocketAddr, usize)> { + let frame = self.frames.get(self.idx)?; + self.idx += 1; + let n = frame.len().min(buf.len()); + buf[..n].copy_from_slice(&frame[..n]); + Some((self.src, n)) + } + + fn send(&mut self, _bytes: &[u8], _dest: SocketAddr) {} +} + +/// Encode a well-formed gossip datagram carrying a single `Alive` for a distinct +/// node id/address, using the default (unlabelled, unencrypted) codec so the engine +/// decodes it exactly as any inbound gossip frame. +fn alive_frame(i: usize) -> Vec { + let ip = Ipv4Addr::new(10, 1, (i / 250) as u8, (i % 250 + 1) as u8); + let node = Node::new( + SmolStr::from(std::format!("flood-{i}")), + SocketAddr::new(IpAddr::V4(ip), 7946), + ); + let alive = Alive::new(1, node) + .with_meta(Meta::empty()) + .with_protocol_version(ProtocolVersion::V1) + .with_delegate_version(DelegateVersion::V1); + let msg: Message = Message::Alive(alive); + encode_outgoing(&msg, &EncodeOptions::new(None)) + .expect("encode alive gossip frame") + .to_vec() +} + +/// A single pump that produces more passive observations than the event buffer cap +/// makes the ENGINE shed the excess into its own `events_dropped` counter, while the +/// cap-sized remainder still fits the driver's empty queue (no driver-side drop). +/// [`Serf::events_dropped`] must sum BOTH counters, so it must report the loss — +/// returning only the driver's count would report 0 despite real drops. +#[test] +fn single_pump_over_cap_drop_is_counted() { + let mut dev = NullDevice; + let mut node = try_build( + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)), 7946), + &mut dev, + ) + .expect("a valid configuration constructs"); + node.start(now()); + + // Flood > cap distinct alives in ONE pump: each is a brand-new member → one + // `Event::Member(Join)` observation, so the pump emits > cap passive events and + // the engine sheds the surplus. + let count = DEFAULT_EVENT_BUFFER_CAP + 200; + let frames: Vec> = (0..count).map(alive_frame).collect(); + let mut flood = FloodGossip { + frames, + idx: 0, + src: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 2)), 7946), + }; + + // Pump directly over the flood gossip and the real reliable-plane view (no + // reliable activity occurs; the alives ride the gossip plane). + { + let sockets = RefCell::new(&mut node.sockets); + let mut stream = SmoltcpStream::new(&mut node.iface, &sockets); + node.engine.pump(now(), &mut flood, &mut stream); + } + + // The engine shed observations this single pump, so the driver's public counter — + // the engine's drops plus its own — must be non-zero. Before the fix it returned + // only the (still-zero) driver count and reported 0 despite the loss. + assert!( + node.events_dropped() > 0, + "events_dropped must reflect the engine's over-cap passive-observation drops, got {}", + node.events_dropped() + ); + + // Draining the buffered survivors into the driver's own (empty, cap-sized) queue + // adds no driver-side drop, so the reported total still reflects the engine loss. + node.drain_engine_events(now()); + assert!( + node.events_dropped() > 0, + "events_dropped must still reflect the engine drops after the driver drains, got {}", + node.events_dropped() + ); +} + #[test] fn advertise_not_local_is_rejected() { let mut dev = NullDevice; diff --git a/serf-smoltcp/tests/query.rs b/serf-smoltcp/tests/query.rs new file mode 100644 index 00000000..d6a9b2e3 --- /dev/null +++ b/serf-smoltcp/tests/query.rs @@ -0,0 +1,252 @@ +//! Query-plane end-to-end over the smoltcp gossip self-delivery loopback. +//! +//! serf directs a node's response to its OWN locally-originated query / key-request +//! to the originator's advertise address = THIS node's address. A real OS UDP socket +//! loops such a self-addressed datagram back into recv; smoltcp does not, so the +//! driver must emulate the OS self-delivery. These tests exercise that path: the +//! query round-trip proves the originator observes its OWN (local, self-addressed) +//! response alongside the remote peer's, and the key-response test proves a +//! locally-originated key op counts the local node in `num_resp` by the query +//! deadline. + +mod harness; + +use core::{ + net::{IpAddr, Ipv4Addr, SocketAddr}, + time::Duration, +}; + +use serf_smoltcp::{ + Bytes, EndpointOptions, Event, Instant, MaybeResolved, Options, QueryEvent, QueryParams, Serf, + SerfOptions, SocketAddrResolver, TransformOptions, +}; +use smol_str::SmolStr; + +fn addr(ip: u8, port: u16) -> SocketAddr { + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, ip)), port) +} + +/// Advance the shared clock the way a real event loop sleeps: a delivered frame +/// wakes its receiver at once, otherwise sleep to the soonest returned deadline. +/// Never stalls — a `now`-valued or absent deadline steps a single millisecond. +fn advance(clk: &mut harness::Clock, targets: &[Option], woke: bool) { + if woke { + clk.advance_ms(1); + return; + } + let target = targets.iter().copied().flatten().min(); + match target { + Some(t) if t > clk.now() => clk.advance_to(t), + _ => clk.advance_ms(1), + } +} + +/// Drain a node's buffered events: answer every observed `Event::Query` (directing +/// the reply to the query's originator), and record every `Event::QueryResponse`'s +/// responder id. Draining also keeps the node's app-event buffer from shedding. +fn service_queries( + node: &mut Serf, + now: Instant, + responders: &mut Vec, +) { + let mut pending: Vec> = Vec::new(); + while let Some(ev) = node.poll_event() { + match ev { + Event::Query(qe) => pending.push(qe), + Event::QueryResponse(qr) => responders.push(qr.from().id_ref().clone()), + _ => {} + } + } + for qe in pending { + // Ignoring Err: a duplicate / past-deadline respond is a no-op the test tolerates. + let _ = node.respond(&qe, Bytes::from_static(b"pong"), now); + } +} + +/// Two converged nodes; one issues a query; the originator must collect responses +/// from BOTH the remote peer AND itself. The local node self-processes its own +/// query and directs its reply to its own advertise address — a self-addressed +/// gossip datagram that smoltcp does not loop back on its own. Without the driver's +/// self-delivery loopback the originator never observes its OWN response, so +/// `a_resp` would hold only `"b"` and this test fails. +#[test] +fn query_collects_local_and_remote_responses() { + const BUDGET: u32 = 4000; + + let (mut da, mut db) = harness::link(1500); + let mut clk = harness::Clock::new(); + let now = clk.now(); + + let mut a: Serf = Serf::new( + Options::new(), + harness::ip_iface(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1))), + TransformOptions::default(), + EndpointOptions::new(SmolStr::new("a"), addr(1, 7946)), + SerfOptions::new(), + &SocketAddrResolver, + &mut da, + now, + ); + let mut b: Serf = Serf::new( + Options::new(), + harness::ip_iface(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 2))), + TransformOptions::default(), + EndpointOptions::new(SmolStr::new("b"), addr(2, 7946)), + SerfOptions::new(), + &SocketAddrResolver, + &mut db, + now, + ); + + a.start(now); + b.start(now); + + let jid = b + .join( + &SocketAddrResolver, + &[MaybeResolved::Resolved(addr(1, 7946))], + false, + now, + ) + .expect("join from a running node"); + + // Converge on a 2-member view first (proven fixed cadence). + let mut joined = false; + for _ in 0..BUDGET { + let _ = a.poll(clk.now(), &mut da); + let _ = b.poll(clk.now(), &mut db); + let _ = b.poll_join(jid); + while a.poll_event().is_some() {} + while b.poll_event().is_some() {} + if a.num_members() == 2 && b.num_members() == 2 { + joined = true; + break; + } + clk.advance_ms(10); + } + assert!(joined, "nodes did not converge before the query"); + + // A issues a query; both nodes answer every Event::Query they observe. + a.query( + "ping", + Bytes::from_static(b"q"), + QueryParams { + timeout: Duration::from_secs(5), + ..Default::default() + }, + clk.now(), + ) + .expect("query from a running node"); + + // Drive to the query deadline via the returned poll deadlines (not a fixed + // cadence). A must collect responses from BOTH itself ("a", via the loopback) and + // B ("b", over the wire). + let mut a_resp: Vec = Vec::new(); + let mut b_resp: Vec = Vec::new(); + let mut done = false; + for _ in 0..BUDGET { + let a_next = a.poll(clk.now(), &mut da); + let b_next = b.poll(clk.now(), &mut db); + service_queries(&mut a, clk.now(), &mut a_resp); + service_queries(&mut b, clk.now(), &mut b_resp); + if a_resp.iter().any(|id| id == "a") && a_resp.iter().any(|id| id == "b") { + done = true; + break; + } + let woke = da.inbound_pending() || db.inbound_pending(); + advance(&mut clk, &[a_next, b_next], woke); + } + + assert!( + done, + "originator did not collect both responses in time: a_resp={a_resp:?}" + ); + assert!( + a_resp.iter().any(|id| id == "a"), + "originator did not observe its OWN self-addressed, looped-back response: a_resp={a_resp:?}" + ); + assert!( + a_resp.iter().any(|id| id == "b"), + "originator did not observe the remote peer's response: a_resp={a_resp:?}" + ); +} + +/// A locally-originated key query self-processes and directs its response to the +/// node's own advertise address. Driven only to the returned poll deadlines, the +/// key op must complete with the local node counted in `num_resp` — proving the +/// self-addressed key response is looped back, ingested, and collected within the +/// tick rather than stranded past the query deadline. Without the loopback the +/// encrypted self-response is dropped on the wire and `num_resp` is 0. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn key_query_collects_local_response_within_deadline() { + use serf_smoltcp::{EncryptionOptions, Keyring, SecretKey}; + + const BUDGET: u32 = 4000; + + let (mut dev, _peer) = harness::link(1500); + let mut clk = harness::Clock::new(); + let now = clk.now(); + + // Either AEAD backend gates this test; pick whichever key variant is compiled. + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes256([0x42; 32]); + #[cfg(all(feature = "chacha20-poly1305", not(feature = "aes-gcm")))] + let key = SecretKey::ChaCha20Poly1305([0x42; 32]); + let transform = TransformOptions::default() + .with_encryption(EncryptionOptions::new().with_keyring(Keyring::new(key))); + + let mut a: Serf = Serf::new( + Options::new(), + harness::ip_iface(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1))), + transform, + EndpointOptions::new(SmolStr::new("a"), addr(1, 7946)), + SerfOptions::new(), + &SocketAddrResolver, + &mut dev, + now, + ); + + a.start(now); + + // Let the local node register its own membership: an internal (key) query counts + // a response only from a known member, and the self-response is from this node. + let mut registered = false; + for _ in 0..200 { + let next = a.poll(clk.now(), &mut dev); + while a.poll_event().is_some() {} + if a.num_members() >= 1 { + registered = true; + break; + } + let woke = dev.inbound_pending(); + advance(&mut clk, &[next], woke); + } + assert!(registered, "local node did not register its own membership"); + + a.list_keys(clk.now()) + .expect("list_keys from a running node"); + + // Drive only to the returned poll deadlines until the key op closes (at the query + // deadline). num_resp must include the local node's own looped-back response. + let mut num_resp = None; + for _ in 0..BUDGET { + let next = a.poll(clk.now(), &mut dev); + while let Some(ev) = a.poll_event() { + if let Event::KeyResponse(kr) = ev { + num_resp = Some(kr.num_resp); + } + } + if num_resp.is_some() { + break; + } + let woke = dev.inbound_pending(); + advance(&mut clk, &[next], woke); + } + + let n = num_resp.expect("the list_keys query must complete with a KeyResponse by its deadline"); + assert!( + n >= 1, + "the local node's own key response must be collected via the self-delivery loopback (num_resp={n})" + ); +} From 314194ea828adc79dd9e6dfe4e2391a099401e83 Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 10 Jul 2026 15:52:45 +1200 Subject: [PATCH 057/128] feat(serf-embedded): apply key-management requests to the live wire keyring MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Key ops previously had no runtime path to the coordinator's wire keyring, so drivers kept success-reporting shadow copies while the wire kept the construction-time keys; add handle_key_request — the single post-construction mutation chokepoint that applies install/use/remove/list to the live keyring (refusing primary removal), pushes it to the coordinator, answers from live state via respond_key, plus a keyring() accessor. serf-proto's StreamEndpoint gains encryption_options/set_encryption_options forwarders to reach it; wire-level tests prove rotated traffic flows and removed-key traffic is rejected. --- serf-embedded/src/engine/mod.rs | 153 +++++++++++++++++++- serf-embedded/src/engine/tests.rs | 192 ++++++++++++++++++++++++++ serf-proto/src/event/mod.rs | 17 +-- serf-proto/src/stream_endpoint/mod.rs | 30 ++++ 4 files changed, 383 insertions(+), 9 deletions(-) diff --git a/serf-embedded/src/engine/mod.rs b/serf-embedded/src/engine/mod.rs index 7dd1586c..fe63342c 100644 --- a/serf-embedded/src/engine/mod.rs +++ b/serf-embedded/src/engine/mod.rs @@ -53,7 +53,12 @@ use serf_proto::{ }; #[cfg(encryption)] -use serf_proto::{SecretKey, event::KeyRequest, event::KeyResponseArgs}; +use memberlist_proto::Keyring; +#[cfg(encryption)] +use serf_proto::{ + SecretKey, + event::{KeyRequest, KeyRequestOperation, KeyResponseArgs}, +}; use memberlist_embedded::{ GossipIo, InitError, Options, StreamIo, TransformOptions, @@ -1379,6 +1384,152 @@ where ) -> Result<(), SerfError> { self.endpoint.respond_key(req, resp, now) } + + /// Apply one inbound key-management [`KeyRequest`] to the engine's LIVE wire + /// keyring and answer the originator in the same call. + /// + /// The live keyring is the coordinator's cross-transport [`EncryptionOptions`] + /// keyring — the single source of truth both the gossip and the reliable plane + /// encrypt under. This method reads it, applies the requested op, and pushes the + /// result back through the coordinator ([`set_encryption_options`]), so a + /// completed rotation actually re-keys the wire instead of updating a driver-held + /// shadow the wire never sees. It is the ONLY post-construction keyring mutation + /// path, so the reported key state and the on-wire AEAD cannot diverge: + /// + /// - `install` inserts the key as a secondary (idempotent), + /// - `use` promotes the key to primary, + /// - `remove` drops a secondary — refusing the current primary, + /// - `list` snapshots the keys and primary from the post-op live state. + /// + /// A node with no keyring configured answers `result = false` and makes no wire + /// change. A failed op (unknown key, or removing the primary) answers + /// `result = false` with a message and leaves the keyring untouched. + /// + /// Apply-then-respond: the op is applied to the wire keyring first; the + /// [`respond_key`](Self::respond_key) that follows is best-effort. The cluster-wide + /// op has already happened on this node even if the response is past its deadline + /// or cannot be routed, so an `Err` here means only that the acknowledgement was + /// not queued — never that the op was skipped. Returns `Ok(())` when a response + /// was queued (a driver re-egresses it within the tick). + /// + /// [`EncryptionOptions`]: memberlist_proto::EncryptionOptions + /// [`set_encryption_options`]: serf_proto::StreamEndpoint::set_encryption_options + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn handle_key_request( + &mut self, + req: &KeyRequest, + now: Instant, + ) -> Result<(), SerfError> { + let resp = self.apply_key_request(req); + self.respond_key(req, resp, now) + } + + /// Read-modify-write the coordinator's live keyring for one [`KeyRequest`], + /// returning the answer built from the post-op live state. The single keyring + /// mutation chokepoint behind [`handle_key_request`](Self::handle_key_request). + #[cfg(encryption)] + fn apply_key_request(&mut self, req: &KeyRequest) -> KeyResponseArgs { + let mut encryption = self.endpoint.encryption_options().clone(); + let Some(current) = encryption.keyring() else { + return KeyResponseArgs { + result: false, + message: "no keyring configured on this node".into(), + ..Default::default() + }; + }; + let mut keyring = current.clone(); + let (resp, mutated) = match (req.op(), req.key()) { + (KeyRequestOperation::Install, Some(key)) => { + keyring.insert_secondary(*key); + ( + KeyResponseArgs { + result: true, + ..Default::default() + }, + true, + ) + } + (KeyRequestOperation::Use, Some(key)) => match keyring.promote(key.as_bytes()) { + Ok(()) => ( + KeyResponseArgs { + result: true, + ..Default::default() + }, + true, + ), + Err(_) => ( + KeyResponseArgs { + result: false, + message: "requested primary key is not installed".into(), + ..Default::default() + }, + false, + ), + }, + (KeyRequestOperation::Remove, Some(key)) => match keyring.remove_secondary(key.as_bytes()) { + Ok(()) => ( + KeyResponseArgs { + result: true, + ..Default::default() + }, + true, + ), + Err(_) => ( + KeyResponseArgs { + result: false, + message: "key is not a removable secondary".into(), + ..Default::default() + }, + false, + ), + }, + (KeyRequestOperation::List, _) => { + let mut keys = Vec::with_capacity(1 + keyring.secondaries().len()); + keys.push(*keyring.primary_ref()); + keys.extend(keyring.secondaries().iter().copied()); + ( + KeyResponseArgs { + result: true, + primary_key: Some(*keyring.primary_ref()), + keys, + ..Default::default() + }, + false, + ) + } + (_, None) => ( + KeyResponseArgs { + result: false, + message: "key-management request missing its required key".into(), + ..Default::default() + }, + false, + ), + }; + // Push the rotated keyring back to the coordinator so the gossip and reliable + // planes re-key in lockstep. A read-only or failed op leaves the wire unchanged. + if mutated { + encryption.set_keyring(keyring); + self.endpoint.set_encryption_options(encryption); + } + resp + } + + /// The engine's LIVE wire keyring — the coordinator's current keyring, the same + /// state [`handle_key_request`](Self::handle_key_request) mutates and the gossip + /// and reliable planes encrypt under. `None` when the node is unencrypted. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn keyring(&self) -> Option<&Keyring> { + self.endpoint.encryption_options().keyring() + } } // Reliable-plane lifecycle helpers that move the connection handle `C` by value diff --git a/serf-embedded/src/engine/tests.rs b/serf-embedded/src/engine/tests.rs index cd387ad0..d38b309f 100644 --- a/serf-embedded/src/engine/tests.rs +++ b/serf-embedded/src/engine/tests.rs @@ -10,6 +10,9 @@ use std::{cell::RefCell, collections::BTreeMap, rc::Rc}; use memberlist_proto::{SeedableRng, SmallRng}; use smol_str::SmolStr; +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +use memberlist_proto::EncryptionOptions; + /// A fixed-seed gossip RNG for the engine constructors. These are single-node /// state tests; a deterministic seed keeps them reproducible. fn test_rng() -> SmallRng { @@ -1718,6 +1721,7 @@ fn key_request(id: u32, deadline: Instant) -> Event { #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] let key = SecretKey::ChaCha20Poly1305([0x11u8; 32]); Event::KeyRequest(KeyRequest::test_with_deadline( + KeyRequestOperation::Install, id, from, Some(key), @@ -1884,3 +1888,191 @@ fn key_request_at_exact_deadline_survives_prune_and_is_delivered() { retain it and poll_event must deliver it" ); } + +// ── key-management applied to the LIVE wire keyring ─────────────────────────── + +/// A fixed AEAD key filled with `fill`, in whichever backend is compiled. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +fn secret_key(fill: u8) -> SecretKey { + #[cfg(feature = "aes-gcm")] + { + SecretKey::Aes256([fill; 32]) + } + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + { + SecretKey::ChaCha20Poly1305([fill; 32]) + } +} + +/// A `SerfEngine` whose gossip and reliable planes encrypt under `keyring`. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +fn make_encrypted_engine(id: &str, port: u16, keyring: Keyring) -> SerfEngine { + let cfg = Options::new() + .with_port(port) + .with_close_timeout(Duration::from_secs(10)); + let ep_cfg = EndpointOptions::new(SmolStr::new(id), node_addr(port)); + let transform = + TransformOptions::default().with_encryption(EncryptionOptions::new().with_keyring(keyring)); + let now = Instant::from_origin(Duration::from_secs(86_400)); + SerfEngine::try_new_at(cfg, transform, ep_cfg, SerfOptions::new(), now, test_rng()) + .expect("valid encrypted configuration must construct") +} + +/// A `KeyRequest` carrying `op` and `key` from a synthetic originator, with a +/// future deadline (the mutation path ignores the deadline; `respond_key` uses it). +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +fn key_op(op: KeyRequestOperation, key: Option) -> KeyRequest { + let from = memberlist_proto::Node::new(SmolStr::new("op"), node_addr(6000)); + let deadline = Instant::from_origin(Duration::from_secs(86_400)) + Duration::from_secs(30); + KeyRequest::test_with_deadline(op, 1, from, key, deadline) +} + +/// `apply_key_request` mutates the engine's LIVE wire keyring (the coordinator's, +/// not a driver-held shadow): install adds a secondary leaving the primary intact, +/// use promotes it, remove drops a secondary, removing the current primary is +/// refused with the ring unchanged, and list snapshots the live post-op state. +/// Every op's effect is visible through the `keyring()` accessor — the single +/// source of truth. `handle_key_request` then applies-then-responds, so a valid op +/// lands on the live keyring even when the response cannot be routed. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn key_ops_mutate_the_live_keyring() { + let k1 = secret_key(0x11); + let k2 = secret_key(0x22); + let mut engine = make_encrypted_engine("a", 7946, Keyring::new(k1)); + + // Baseline: primary K1, no secondaries. + assert_eq!(engine.keyring().expect("encrypted").primary_ref(), &k1); + assert!(engine.keyring().unwrap().secondaries().is_empty()); + + // install K2 -> K2 becomes a secondary; the primary is untouched. + let resp = engine.apply_key_request(&key_op(KeyRequestOperation::Install, Some(k2))); + assert!(resp.result, "install must succeed"); + { + let kr = engine.keyring().expect("encrypted"); + assert_eq!(kr.primary_ref(), &k1, "install must not move the primary"); + assert!( + kr.secondaries().contains(&k2), + "install must add K2 as a secondary" + ); + } + + // use K2 -> K2 is promoted to primary. + let resp = engine.apply_key_request(&key_op(KeyRequestOperation::Use, Some(k2))); + assert!(resp.result, "use must succeed"); + assert_eq!( + engine.keyring().unwrap().primary_ref(), + &k2, + "use must promote K2 to primary" + ); + + // remove K1 (now a secondary) -> gone from the ring. + let resp = engine.apply_key_request(&key_op(KeyRequestOperation::Remove, Some(k1))); + assert!(resp.result, "remove of a secondary must succeed"); + { + let kr = engine.keyring().expect("encrypted"); + assert_eq!(kr.primary_ref(), &k2); + assert!( + !kr.secondaries().contains(&k1), + "remove must drop K1 from the ring" + ); + } + + // remove the CURRENT primary (K2) -> refused; the ring is unchanged. + let before_primary = *engine.keyring().unwrap().primary_ref(); + let before_secondaries = engine.keyring().unwrap().secondaries().to_vec(); + let resp = engine.apply_key_request(&key_op(KeyRequestOperation::Remove, Some(k2))); + assert!(!resp.result, "removing the current primary must be refused"); + { + let kr = engine.keyring().expect("encrypted"); + assert_eq!( + *kr.primary_ref(), + before_primary, + "a refused remove must not move the primary" + ); + assert_eq!( + kr.secondaries(), + before_secondaries.as_slice(), + "a refused remove must not change the ring" + ); + } + + // list -> reports the live primary and every installed key; no wire change. + let resp = engine.apply_key_request(&key_op(KeyRequestOperation::List, None)); + assert!(resp.result, "list must succeed"); + assert_eq!(resp.primary_key, Some(k2), "list reports the live primary"); + assert!( + resp.keys.contains(&k2), + "list reports the primary among the keys" + ); + + // handle_key_request composes apply-then-respond: the op lands on the live + // keyring even though the synthetic originator is unroutable (respond is + // best-effort, so its Result is not asserted here). + let k3 = secret_key(0x33); + let now = Instant::from_origin(Duration::from_secs(86_400)); + let _ = engine.handle_key_request(&key_op(KeyRequestOperation::Install, Some(k3)), now); + assert!( + engine.keyring().unwrap().secondaries().contains(&k3), + "handle_key_request must apply the op to the live keyring regardless of respond routing" + ); +} + +/// The wire proof: after a rotation applied through the live-keyring path, the +/// engine's gossip crypto runs under the NEW primary and REJECTS a frame under the +/// removed key. A rotates to K2 and drops K1; a peer B holding K2 decrypts A's +/// post-rotation frame (rotated traffic flows), while a frame captured under K1 no +/// longer decrypts on A (old-key traffic rejected). Driven through the real +/// `encrypt_gossip` / `decrypt_gossip` paths, not keyring-state asserts alone. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn key_rotation_reencrypts_gossip_and_rejects_removed_key() { + let k1 = secret_key(0x11); + let k2 = secret_key(0x22); + // A starts {primary K1, secondary K2}; B holds only K2. + let mut a = make_encrypted_engine("a", 7946, Keyring::with_secondaries(k1, [k2])); + let b = make_encrypted_engine("b", 7947, Keyring::new(k2)); + + let plaintext = b"gossip-probe-payload-0123456789"; + + // A frame A emits under its CURRENT primary (K1), captured before the rotation. + let under_k1 = a + .endpoint + .encrypt_gossip(plaintext) + .expect("encrypt under K1"); + assert_eq!( + a.endpoint + .decrypt_gossip(&under_k1) + .expect("A decrypts its own K1 frame pre-rotation"), + plaintext + ); + + // Rotate A to K2 and drop K1 through the single live-keyring chokepoint. + assert!( + a.apply_key_request(&key_op(KeyRequestOperation::Use, Some(k2))) + .result + ); + assert!( + a.apply_key_request(&key_op(KeyRequestOperation::Remove, Some(k1))) + .result + ); + + // Rotated traffic flows: A now encrypts under K2, and B (holding K2) decrypts it. + let under_k2 = a + .endpoint + .encrypt_gossip(plaintext) + .expect("encrypt under K2"); + assert_eq!( + b.endpoint + .decrypt_gossip(&under_k2) + .expect("a peer holding the new key decrypts A's rotated gossip"), + plaintext, + "post-rotation gossip must decrypt under the promoted key" + ); + + // Old-key traffic rejected: A dropped K1, so a frame under K1 no longer decrypts. + assert!( + a.endpoint.decrypt_gossip(&under_k1).is_err(), + "a frame under the removed key K1 must be rejected by the rotated engine" + ); +} diff --git a/serf-proto/src/event/mod.rs b/serf-proto/src/event/mod.rs index 83e9fca1..6a65fd21 100644 --- a/serf-proto/src/event/mod.rs +++ b/serf-proto/src/event/mod.rs @@ -393,23 +393,24 @@ impl KeyRequest { self.deadline } - /// Construct a `KeyRequest` with an explicit response `deadline`, for the - /// downstream driver tests that exercise deadline-based control-queue bounding - /// (a driver holding these on a non-lossy queue must prune the past-deadline, - /// unanswerable ones). Gated behind the non-default `test-support` feature; NOT - /// a production build path and NOT part of the wire contract. The - /// operation-carrying fields are inert placeholders — only `id`, `from`, `key`, - /// and `deadline` are caller-chosen. + /// Construct a `KeyRequest` with an explicit `op` and response `deadline`, for + /// the downstream driver tests that exercise key-op application and + /// deadline-based control-queue bounding (a driver holding these on a non-lossy + /// queue must prune the past-deadline, unanswerable ones). Gated behind the + /// non-default `test-support` feature; NOT a production build path and NOT part + /// of the wire contract. Only `op`, `id`, `from`, `key`, and `deadline` are + /// caller-chosen; the remaining wire fields are inert placeholders. #[cfg(any(test, feature = "test-support"))] #[cfg_attr(docsrs, doc(cfg(feature = "test-support")))] pub fn test_with_deadline( + op: KeyRequestOperation, id: u32, from: Node, key: Option, deadline: Instant, ) -> Self { Self { - op: KeyRequestOperation::Install, + op, key, id, ltime: LamportTime::ZERO, diff --git a/serf-proto/src/stream_endpoint/mod.rs b/serf-proto/src/stream_endpoint/mod.rs index 0fb3aeda..a986d293 100644 --- a/serf-proto/src/stream_endpoint/mod.rs +++ b/serf-proto/src/stream_endpoint/mod.rs @@ -648,6 +648,36 @@ where self.core.respond_key(&mut self.transport, req, resp, now) } + /// The coordinator's live cross-transport [`memberlist_proto::EncryptionOptions`] + /// — the single source of truth for the gossip and reliable-plane AEAD keyring. + /// + /// A driver applying a key-management op reads this, mutates the keyring, and + /// pushes it back via [`set_encryption_options`](Self::set_encryption_options), + /// so the reported key state and the bytes on the wire cannot diverge. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn encryption_options(&self) -> &memberlist_proto::EncryptionOptions { + self.transport.encryption_options() + } + + /// Replace the coordinator's live encryption options, re-keying the gossip and + /// reliable planes in lockstep. + /// + /// The post-construction counterpart to the construction-time policy: applying a + /// completed key rotation here rotates the actual AEAD both planes encrypt + /// under, rather than leaving a driver-held shadow to drift from the wire. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn set_encryption_options(&mut self, encryption: memberlist_proto::EncryptionOptions) { + self.transport.set_encryption_options(encryption) + } + /// Forwards to [`Endpoint::leave_broadcast_deadline`]. pub const fn leave_broadcast_deadline(&self) -> Option { self.core.leave_broadcast_deadline() From 311cc1b53e5b44b12e80bcf313439826619f9bd1 Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 10 Jul 2026 15:52:52 +1200 Subject: [PATCH 058/128] fix(serf-smoltcp): drop the shadow keyring for the engine's live keyring The driver-local keyring copy reported successful key rotations the wire never saw; route KeyRequest through the engine's live-keyring application and delete the shadow, so responses and the new keyring() accessor reflect the actual AEAD state; the rotation regression installs/promotes/removes across two nodes and asserts both live keyrings rotated and traffic still flows. --- serf-smoltcp/src/serf/mod.rs | 141 ++++----------- serf-smoltcp/tests/key_management.rs | 246 +++++++++++++++++++++++++++ 2 files changed, 277 insertions(+), 110 deletions(-) create mode 100644 serf-smoltcp/tests/key_management.rs diff --git a/serf-smoltcp/src/serf/mod.rs b/serf-smoltcp/src/serf/mod.rs index f4a919dc..d2114c11 100644 --- a/serf-smoltcp/src/serf/mod.rs +++ b/serf-smoltcp/src/serf/mod.rs @@ -7,8 +7,9 @@ //! stack, drives the engine over a [`SmoltcpGossip`] + [`SmoltcpStream`] view of //! the just-ticked sockets, then acts on serf's mandatory driver-actioned events //! IN the poll cycle (a lost id-conflict [`Event::Shutdown`] flips a stop flag; an -//! encryption [`Event::KeyRequest`] is applied to the local keyring and answered -//! via `respond_key`), buffering every drained event for the app's own +//! encryption [`Event::KeyRequest`] is applied to the engine's live wire keyring and +//! answered, both through the engine's `handle_key_request`), buffering every drained +//! event for the app's own //! [`poll_event`](Serf::poll_event). All protocol work lives in the shared engine; //! this driver supplies only the link layer plus the mandatory-event side effects. @@ -40,8 +41,6 @@ use smoltcp::{ #[cfg(encryption)] use serf_embedded::{Keyring, SecretKey}; -#[cfg(encryption)] -use serf_proto::event::{KeyRequest, KeyRequestOperation, KeyResponseArgs}; use crate::{ InitError, InterfaceOptions, JoinError, Options, Resolver, TransformOptions, @@ -169,86 +168,6 @@ fn min_opt(a: Option, b: Option) -> Option { } } -/// Apply one inbound [`KeyRequest`] to the driver's local keyring, producing the -/// [`KeyResponseArgs`] the poll cycle forwards to `respond_key`. -/// -/// Mirrors serf-reactor's `apply_key_request`: the driver owns its key store and -/// answers the four key-management operations from it. A node with no keyring -/// configured reports failure. The op is applied to the driver-local keyring copy; -/// keeping the live coordinator encryption in lockstep is a deferred refinement -/// (the same shape as serf-reactor, whose keyring delegate is likewise separate -/// from the coordinator's construction-time keyring). -#[cfg(encryption)] -fn apply_key_request( - keyring: &mut Option, - req: &KeyRequest, -) -> KeyResponseArgs { - let Some(kr) = keyring.as_mut() else { - return KeyResponseArgs { - result: false, - message: "no keyring configured on this node".into(), - keys: Vec::new(), - primary_key: None, - }; - }; - match (req.op(), req.key()) { - (KeyRequestOperation::Install, Some(key)) => { - kr.insert_secondary(*key); - KeyResponseArgs { - result: true, - message: "".into(), - keys: Vec::new(), - primary_key: None, - } - } - (KeyRequestOperation::Use, Some(key)) => match kr.promote(key.as_bytes()) { - Ok(()) => KeyResponseArgs { - result: true, - message: "".into(), - keys: Vec::new(), - primary_key: None, - }, - Err(_) => KeyResponseArgs { - result: false, - message: "requested primary key is not installed".into(), - keys: Vec::new(), - primary_key: None, - }, - }, - (KeyRequestOperation::Remove, Some(key)) => match kr.remove_secondary(key.as_bytes()) { - Ok(()) => KeyResponseArgs { - result: true, - message: "".into(), - keys: Vec::new(), - primary_key: None, - }, - Err(_) => KeyResponseArgs { - result: false, - message: "key is not a removable secondary".into(), - keys: Vec::new(), - primary_key: None, - }, - }, - (KeyRequestOperation::List, _) => { - let mut keys: Vec = Vec::with_capacity(1 + kr.secondaries().len()); - keys.push(*kr.primary_ref()); - keys.extend(kr.secondaries().iter().copied()); - KeyResponseArgs { - result: true, - message: "".into(), - keys, - primary_key: Some(*kr.primary_ref()), - } - } - (_, None) => KeyResponseArgs { - result: false, - message: "key-management request missing its required key".into(), - keys: Vec::new(), - primary_key: None, - }, - } -} - /// An executor-free serf node that composes serf's super-machine (via /// [`SerfEngine`](serf_embedded::SerfEngine)) with a smoltcp TCP/IP stack. /// @@ -299,11 +218,6 @@ where /// Set once the driver observed a lost id-conflict [`Event::Shutdown`]; the /// caller reads it via [`is_shutdown`](Self::is_shutdown) and stops polling. shutdown: bool, - /// The driver-local keyring answering inbound [`Event::KeyRequest`]s, seeded from - /// the construction `TransformOptions` encryption keyring (or `None` when the node - /// is unencrypted). - #[cfg(encryption)] - keyring: Option, // `D` is passed to construction and each `poll`; `PhantomData` makes the struct // generic over it without holding it. _device: PhantomData, @@ -525,12 +439,6 @@ where where D: Device, { - // Seed the driver-local keyring from the construction encryption policy BEFORE - // `transform` is moved into the engine, so inbound `KeyRequest`s can be answered - // from the same key material the node was built with. - #[cfg(encryption)] - let keyring = transform.encryption.keyring().cloned(); - let embedded_cfg = embedded_options(&cfg); // 1. Validate the medium up front: smoltcp's `Interface::new` asserts the @@ -742,8 +650,6 @@ where app_events_dropped: 0, loopback: VecDeque::new(), shutdown: false, - #[cfg(encryption)] - keyring, _device: PhantomData, _a: PhantomData, }) @@ -1057,6 +963,20 @@ where self.engine.list_keys(now) } + /// The node's LIVE wire keyring — the keyring the gossip and reliable planes + /// actually encrypt under, and the state an inbound [`Event::KeyRequest`] rotates + /// via the engine. `None` when the node is unencrypted. Unlike + /// [`list_keys`](Self::list_keys) (a cluster-wide query), this is a local read of + /// this node's own keyring for UI / diagnostics / tests. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn keyring(&self) -> Option<&Keyring> { + self.engine.keyring() + } + /// Advance both the smoltcp stack and serf's state machine, act on serf's /// mandatory driver-actioned events, and drive to quiescence within the tick. /// Returns the next wakeup deadline: the minimum of the smoltcp stack's next @@ -1070,8 +990,9 @@ where /// a [`SmoltcpGossip`] + [`SmoltcpStream`] view of the just-ticked sockets /// (computing the next deadline and egressing outbound gossip), then the drain /// acts on the driver-actioned events: [`Event::Shutdown`] flips the stop flag; - /// an [`Event::KeyRequest`] is applied to the local keyring and answered via - /// `respond_key`. Every drained event is buffered for the app's own + /// an [`Event::KeyRequest`] is applied to the engine's live wire keyring and + /// answered via the engine's `handle_key_request`. Every drained event is + /// buffered for the app's own /// [`poll_event`](Self::poll_event), so membership / user / query observations /// and the mandatory events alike remain visible AFTER the driver acted. /// @@ -1137,8 +1058,8 @@ where /// effect on each mandatory event, and buffer every event for the app. /// /// Returns whether the drain queued outbound gossip work the current pump's egress - /// did not see — a successful `respond_key` — so [`poll`](Self::poll) knows to - /// re-pump and egress it within the same tick. + /// did not see — a key response the engine's `handle_key_request` just queued — so + /// [`poll`](Self::poll) knows to re-pump and egress it within the same tick. fn drain_engine_events(&mut self, now: Instant) -> bool { // `now` and the queued-outbound signal are consumed only by the encryption // `KeyRequest` arm; a build without an AEAD backend reads neither and queues no @@ -1154,17 +1075,17 @@ where // A lost id-conflict vote means the local node MUST stop; flag it. The event // still reaches the app via `poll_event`. Event::Shutdown => self.shutdown = true, - // An inbound key-management request: apply the op to the local keyring and - // answer the originator. The response is a directed gossip transmit egressed - // on the re-pump [`poll`](Self::poll) runs while `queued` is set. + // An inbound key-management request: apply the op to the engine's LIVE wire + // keyring and answer the originator in one call. The response is a directed + // gossip transmit egressed on the re-pump [`poll`](Self::poll) runs while + // `queued` is set. #[cfg(encryption)] Event::KeyRequest(req) => { - let resp = apply_key_request(&mut self.keyring, req); - // Ignoring Err: `respond_key` fails only when the response cannot be - // routed; the key op has already applied to the driver keyring. - if self.engine.respond_key(req, resp, now).is_ok() { - queued = true; - } + // `Ok` means a key response was queued (re-pump to egress it). Ignoring the + // Err case: `handle_key_request` has already applied the op to the live + // keyring; an Err means only the best-effort response was past-deadline or + // could not be routed, which queues no outbound work. + queued |= self.engine.handle_key_request(req, now).is_ok(); } _ => {} } diff --git a/serf-smoltcp/tests/key_management.rs b/serf-smoltcp/tests/key_management.rs new file mode 100644 index 00000000..4e1069e8 --- /dev/null +++ b/serf-smoltcp/tests/key_management.rs @@ -0,0 +1,246 @@ +//! Key-management end-to-end: two encrypted nodes rotate their keyring across the +//! cluster and BOTH nodes' LIVE wire keyrings follow. +//! +//! The regression this guards: a driver that applies key ops to a private shadow +//! copy reports a completed rotation the wire never sees. Here A drives +//! `install_key` → `use_key` → `remove_key`; each node's poll drain routes the +//! resulting `Event::KeyRequest` into the engine's `handle_key_request`, mutating +//! the coordinator's LIVE keyring. The fail-on-revert assertions read both nodes' +//! `keyring()` and require primary == K2 with K1 absent — which the old shadow model +//! (live keyring frozen at construction K1) cannot satisfy — then a post-rotation +//! user event still crosses the wire, proving both planes now run under K2. + +mod harness; + +use core::net::{IpAddr, Ipv4Addr, SocketAddr}; + +use serf_smoltcp::{ + Bytes, EndpointOptions, Event, Instant, MaybeResolved, Options, Serf, SerfOptions, + SocketAddrResolver, TransformOptions, +}; +use smol_str::SmolStr; + +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +use serf_smoltcp::{EncryptionOptions, Keyring, SecretKey}; + +fn addr(ip: u8, port: u16) -> SocketAddr { + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, ip)), port) +} + +/// Advance the shared clock the way a real event loop sleeps: a delivered frame +/// wakes its receiver at once, otherwise sleep to the soonest returned deadline. +/// Never stalls — a `now`-valued or absent deadline steps a single millisecond. +fn advance(clk: &mut harness::Clock, targets: &[Option], woke: bool) { + if woke { + clk.advance_ms(1); + return; + } + let target = targets.iter().copied().flatten().min(); + match target { + Some(t) if t > clk.now() => clk.advance_to(t), + _ => clk.advance_ms(1), + } +} + +/// A fixed AEAD key filled with `fill`, in whichever backend is compiled. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +fn secret_key(fill: u8) -> SecretKey { + #[cfg(feature = "aes-gcm")] + { + SecretKey::Aes256([fill; 32]) + } + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + { + SecretKey::ChaCha20Poly1305([fill; 32]) + } +} + +/// Poll both nodes to their returned deadlines until A observes a `KeyResponse`, +/// returning its `(num_resp, num_err)`. All other events are drained so neither +/// node's backlog stalls the gossip. The self-addressed response is looped back by +/// the driver, so the originator counts itself among the responders. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +fn drive_to_key_response( + a: &mut Serf, + b: &mut Serf, + da: &mut harness::PairedDevice, + db: &mut harness::PairedDevice, + clk: &mut harness::Clock, + budget: u32, +) -> Option<(usize, usize)> { + for _ in 0..budget { + let na = a.poll(clk.now(), da); + let nb = b.poll(clk.now(), db); + let mut seen = None; + while let Some(ev) = a.poll_event() { + if let Event::KeyResponse(kr) = ev { + seen = Some((kr.num_resp, kr.num_err)); + } + } + while b.poll_event().is_some() {} + if let Some(counts) = seen { + return Some(counts); + } + let woke = da.inbound_pending() || db.inbound_pending(); + advance(clk, &[na, nb], woke); + } + None +} + +/// Two encrypted nodes share primary K1, then A rotates the cluster to K2 via +/// `install_key` → `use_key` → `remove_key`. Each op propagates and every node's +/// drain applies it to the engine's LIVE wire keyring. The install query collects a +/// response from BOTH nodes; the fail-on-revert check reads both nodes' `keyring()` +/// and requires primary == K2 with K1 gone (unsatisfiable under a per-driver shadow +/// keyring); finally a user event still propagates A → B, proving the wire now runs +/// under K2 on both planes. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn key_rotation_across_two_nodes_rotates_both_live_keyrings() { + const BUDGET: u32 = 8000; + + let (mut da, mut db) = harness::link(1500); + let mut clk = harness::Clock::new(); + let now = clk.now(); + + let k1 = secret_key(0x11); + let k2 = secret_key(0x22); + + let transform_a = TransformOptions::default() + .with_encryption(EncryptionOptions::new().with_keyring(Keyring::new(k1))); + let transform_b = TransformOptions::default() + .with_encryption(EncryptionOptions::new().with_keyring(Keyring::new(k1))); + + let mut a: Serf = Serf::new( + Options::new(), + harness::ip_iface(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1))), + transform_a, + EndpointOptions::new(SmolStr::new("a"), addr(1, 7946)), + SerfOptions::new(), + &SocketAddrResolver, + &mut da, + now, + ); + let mut b: Serf = Serf::new( + Options::new(), + harness::ip_iface(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 2))), + transform_b, + EndpointOptions::new(SmolStr::new("b"), addr(2, 7946)), + SerfOptions::new(), + &SocketAddrResolver, + &mut db, + now, + ); + + a.start(now); + b.start(now); + + // Converge on a 2-member view under K1 (a real encrypted push/pull join). + let jid = b + .join( + &SocketAddrResolver, + &[MaybeResolved::Resolved(addr(1, 7946))], + false, + now, + ) + .expect("join from a running node"); + let mut joined = false; + for _ in 0..BUDGET { + let na = a.poll(clk.now(), &mut da); + let nb = b.poll(clk.now(), &mut db); + let _ = b.poll_join(jid); + while a.poll_event().is_some() {} + while b.poll_event().is_some() {} + if a.num_members() == 2 && b.num_members() == 2 { + joined = true; + break; + } + let woke = da.inbound_pending() || db.inbound_pending(); + advance(&mut clk, &[na, nb], woke); + } + assert!(joined, "the encrypted nodes did not converge under K1"); + + // install K2 across the cluster: the query collects a response from BOTH nodes, + // none in error, and each node's live keyring gains K2 as a secondary. + a.install_key(k2, clk.now()).expect("install_key"); + let (num_resp, num_err) = + drive_to_key_response(&mut a, &mut b, &mut da, &mut db, &mut clk, BUDGET) + .expect("the install_key query must close with a KeyResponse"); + assert!( + num_resp >= 2, + "install_key must collect a response from BOTH nodes (num_resp={num_resp})" + ); + assert_eq!(num_err, 0, "install_key must succeed on every node"); + assert!( + a.keyring() + .expect("a encrypted") + .secondaries() + .contains(&k2), + "A's live keyring must gain K2 as a secondary" + ); + assert!( + b.keyring() + .expect("b encrypted") + .secondaries() + .contains(&k2), + "B's live keyring must gain K2 as a secondary" + ); + + // use K2 across the cluster: both nodes promote K2 to primary. + a.use_key(k2, clk.now()).expect("use_key"); + let (_, num_err) = drive_to_key_response(&mut a, &mut b, &mut da, &mut db, &mut clk, BUDGET) + .expect("the use_key query must close with a KeyResponse"); + assert_eq!(num_err, 0, "use_key must succeed on every node"); + assert_eq!(a.keyring().unwrap().primary_ref(), &k2, "A must promote K2"); + assert_eq!(b.keyring().unwrap().primary_ref(), &k2, "B must promote K2"); + + // remove K1 across the cluster: both nodes drop the old key. + a.remove_key(k1, clk.now()).expect("remove_key"); + let (_, num_err) = drive_to_key_response(&mut a, &mut b, &mut da, &mut db, &mut clk, BUDGET) + .expect("the remove_key query must close with a KeyResponse"); + assert_eq!(num_err, 0, "remove_key must succeed on every node"); + + // FAIL-ON-REVERT: BOTH nodes' LIVE keyrings show primary == K2 and K1 absent. On + // the old shadow model the coordinator keyring never rotated, so B's live keyring + // would still be K1-primary here and these assertions would fail. + for (name, node) in [("a", &a), ("b", &b)] { + let kr = node + .keyring() + .unwrap_or_else(|| panic!("{name} must be encrypted")); + assert_eq!( + kr.primary_ref(), + &k2, + "{name}: primary must be the promoted K2" + ); + assert_ne!(kr.primary_ref(), &k1, "{name}: K1 must not be the primary"); + assert!( + !kr.secondaries().contains(&k1), + "{name}: the removed K1 must be absent from the live keyring" + ); + } + + // Post-rotation traffic proof: a user event still crosses the wire, which now runs + // under K2 on both nodes — the reliable and gossip planes rotated with the keyring. + a.user_event("after-rotation", Bytes::from_static(b"payload"), false) + .expect("user_event from a running node"); + let mut b_saw_user = false; + for _ in 0..BUDGET { + let na = a.poll(clk.now(), &mut da); + let nb = b.poll(clk.now(), &mut db); + while let Some(ev) = b.poll_event() { + if matches!(ev, Event::User(_)) { + b_saw_user = true; + } + } + while a.poll_event().is_some() {} + if b_saw_user { + break; + } + let woke = da.inbound_pending() || db.inbound_pending(); + advance(&mut clk, &[na, nb], woke); + } + assert!( + b_saw_user, + "a user event must still propagate A -> B after the rotation (wire under K2)" + ); +} From 4566e84f04f9bb5cdad138820a8010fd0280e548 Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 10 Jul 2026 16:25:20 +1200 Subject: [PATCH 059/128] fix(serf-embedded): variant-exact key operations, cross-cipher collisions refused With both AEAD backends built in, two keys can share bytes across cipher variants, and the byte-keyed keyring ops could promote or remove the wrong cipher's key while reporting success. Verify exact variant-aware membership before every mutation, refuse installing a cross-cipher byte twin, and reject a construction keyring already carrying one, so byte identity is unambiguous and a reported rotation is always the requested key; dual-backend regressions cover the wrong-variant promote/remove/install and the construction preflight. --- serf-embedded/src/engine/mod.rs | 233 ++++++++++++++++++++++++------ serf-embedded/src/engine/tests.rs | 152 +++++++++++++++++++ 2 files changed, 339 insertions(+), 46 deletions(-) diff --git a/serf-embedded/src/engine/mod.rs b/serf-embedded/src/engine/mod.rs index fe63342c..bb4e4027 100644 --- a/serf-embedded/src/engine/mod.rs +++ b/serf-embedded/src/engine/mod.rs @@ -53,7 +53,7 @@ use serf_proto::{ }; #[cfg(encryption)] -use memberlist_proto::Keyring; +use memberlist_proto::{EncryptionError, Keyring}; #[cfg(encryption)] use serf_proto::{ SecretKey, @@ -562,6 +562,24 @@ where // reused preflight so the deterministic checks live in ONE place. validate_runtime_config(&cfg, &transform, ep_cfg.gossip_mtu())?; + // `validate_runtime_config` probes each key's usability but not the ring's + // internal byte-identity. Refuse a seed keyring that already carries a + // cross-cipher byte twin — two keys sharing a raw byte value across different + // cipher variants (e.g. an AES-256 and a ChaCha20-Poly1305 key with identical + // bytes). `SecretKey` equality is variant-inclusive, but the coordinator's + // rotation ops (`promote`/`remove_secondary`) match on bytes alone, so such a + // ring would make every later byte-keyed op ambiguous and let a rotation + // promote or remove the wrong cipher's key. Failing fast here — surfaced through + // the existing encryption `InitError` channel, the closest typed construction + // error — beats a latent ambiguous rotation, and establishes the chokepoint + // invariant that the live keyring is cross-cipher-collision-free at all times. + #[cfg(encryption)] + if let Some(keyring) = transform.encryption.keyring() { + if keyring_carries_cross_cipher_twin(keyring) { + return Err(EncryptionError::KeyMismatch.into()); + } + } + // Capture the advertise-dependent values before `ep_cfg` is moved. let gossip_mtu = ep_cfg.gossip_mtu(); let advertise = *ep_cfg.advertise_addr_ref(); @@ -786,6 +804,34 @@ where } } +/// Whether `keyring` already holds a key whose raw bytes equal `key`'s but whose +/// cipher variant differs — the cross-cipher collision that makes a byte-keyed +/// keyring lookup ambiguous. +/// +/// [`SecretKey`] equality is variant-inclusive (an AES-256 key and a +/// ChaCha20-Poly1305 key with identical 32 bytes are DISTINCT keys), yet +/// [`Keyring::promote`] and [`Keyring::remove_secondary`] match on raw bytes alone. +/// Admitting such a twin would let a byte-keyed op resolve to the wrong cipher's +/// key, so both the install chokepoint and the construction preflight refuse it. +#[cfg(encryption)] +fn keyring_has_cross_cipher_twin(keyring: &Keyring, key: &SecretKey) -> bool { + core::iter::once(keyring.primary_ref()) + .chain(keyring.secondaries()) + .any(|installed| installed.as_bytes() == key.as_bytes() && installed != key) +} + +/// Whether `keyring` already carries a cross-cipher byte twin among its own keys — +/// any two of {primary, secondaries} sharing a raw byte value across different +/// cipher variants. Such a ring makes the coordinator's byte-keyed rotation ops +/// ambiguous, so construction refuses it up front to keep the live keyring +/// cross-cipher-collision-free. +#[cfg(encryption)] +fn keyring_carries_cross_cipher_twin(keyring: &Keyring) -> bool { + core::iter::once(keyring.primary_ref()) + .chain(keyring.secondaries()) + .any(|key| keyring_has_cross_cipher_twin(keyring, key)) +} + // serf-command and read forwarders — reach serf's super-machine only (not the // reliable plane), so they need node identity and BOTH RNGs (the machine driver // surface bounds the gossip `G: Rng` and serf's `SR: Rng + SeedableRng`) but no @@ -1396,11 +1442,16 @@ where /// shadow the wire never sees. It is the ONLY post-construction keyring mutation /// path, so the reported key state and the on-wire AEAD cannot diverge: /// - /// - `install` inserts the key as a secondary (idempotent), - /// - `use` promotes the key to primary, - /// - `remove` drops a secondary — refusing the current primary, + /// - `install` inserts the key as a secondary (idempotent; a cross-cipher byte + /// twin of an already-present key is refused), + /// - `use` promotes the exact (variant + bytes) key to primary, + /// - `remove` drops the exact secondary — refusing the current primary, /// - `list` snapshots the keys and primary from the post-op live state. /// + /// Every keyed op is variant-exact; see [`apply_key_request`](Self::apply_key_request) + /// for the cross-cipher-collision-free invariant that keeps the byte-keyed + /// coordinator ops unambiguous. + /// /// A node with no keyring configured answers `result = false` and makes no wire /// change. A failed op (unknown key, or removing the primary) answers /// `result = false` with a message and leaves the keyring untouched. @@ -1431,6 +1482,21 @@ where /// Read-modify-write the coordinator's live keyring for one [`KeyRequest`], /// returning the answer built from the post-op live state. The single keyring /// mutation chokepoint behind [`handle_key_request`](Self::handle_key_request). + /// + /// Every mutating op is variant-exact. [`memberlist_proto::SecretKey`] equality is + /// variant-inclusive (an AES-256 key and a ChaCha20-Poly1305 key with the same 32 + /// bytes are DISTINCT keys), but the coordinator's [`Keyring::promote`] and + /// [`Keyring::remove_secondary`] match on raw bytes alone. To keep those byte-keyed + /// ops unambiguous this chokepoint upholds one invariant: the live keyring is + /// cross-cipher-collision-free at all times — no two keys share a byte value across + /// different cipher variants. It is established at construction (the preflight in + /// [`try_new_at_with_rng`](Self::try_new_at_with_rng) rejects a seed keyring that + /// already carries a twin) and preserved here (an `install` whose bytes twin an + /// existing key of another variant is refused). Under that invariant, and after the + /// exact (variant + bytes) membership check each `use` / `remove` performs, the + /// byte-keyed promote / remove resolve to exactly the requested key — so a reported + /// rotation is always the key that was asked for, never a same-byte key of a + /// different cipher. #[cfg(encryption)] fn apply_key_request(&mut self, req: &KeyRequest) -> KeyResponseArgs { let mut encryption = self.endpoint.encryption_options().clone(); @@ -1444,49 +1510,124 @@ where let mut keyring = current.clone(); let (resp, mutated) = match (req.op(), req.key()) { (KeyRequestOperation::Install, Some(key)) => { - keyring.insert_secondary(*key); - ( - KeyResponseArgs { - result: true, - ..Default::default() - }, - true, - ) + // Refuse a key whose raw bytes collide with an already-present key of a + // DIFFERENT cipher variant. `SecretKey` equality is variant-inclusive, but + // `promote` / `remove_secondary` match on bytes alone, so admitting a + // cross-cipher byte twin would make every later byte-keyed op ambiguous. A + // same-variant re-install is the idempotent `insert_secondary` no-op and is + // reported as success. + if keyring_has_cross_cipher_twin(&keyring, key) { + ( + KeyResponseArgs { + result: false, + message: "cross-cipher key collision".into(), + ..Default::default() + }, + false, + ) + } else { + keyring.insert_secondary(*key); + ( + KeyResponseArgs { + result: true, + ..Default::default() + }, + true, + ) + } + } + (KeyRequestOperation::Use, Some(key)) => { + // Verify exact (variant + bytes) membership before the byte-keyed promote. + // Promoting the current primary is a trivial success with no wire change (Go + // keymanager parity); a key absent from the live ring is refused with no + // mutation, so a `use` of one cipher's key can never silently promote a + // byte-twin of another. The collision-free invariant then guarantees the + // byte-keyed promote resolves to exactly this key. + if keyring.primary_ref() == key { + ( + KeyResponseArgs { + result: true, + ..Default::default() + }, + false, + ) + } else if keyring.secondaries().contains(key) { + match keyring.promote(key.as_bytes()) { + Ok(()) => ( + KeyResponseArgs { + result: true, + ..Default::default() + }, + true, + ), + // Unreachable given the exact secondary membership just verified plus the + // collision-free invariant; handled fail-closed rather than trusting bytes. + Err(_) => ( + KeyResponseArgs { + result: false, + message: "requested key is not installed".into(), + ..Default::default() + }, + false, + ), + } + } else { + ( + KeyResponseArgs { + result: false, + message: "requested key is not installed".into(), + ..Default::default() + }, + false, + ) + } + } + (KeyRequestOperation::Remove, Some(key)) => { + // Exact (variant + bytes) membership required, mirroring `use`. Removing the + // current primary is refused (operators promote a secondary first); a key + // absent from the live ring is refused with no mutation, so a `remove` of one + // cipher's key can never drop a byte-twin of another. The collision-free + // invariant makes the byte-keyed remove exact. + if keyring.primary_ref() == key { + ( + KeyResponseArgs { + result: false, + message: "cannot remove the primary key; promote a secondary first".into(), + ..Default::default() + }, + false, + ) + } else if keyring.secondaries().contains(key) { + match keyring.remove_secondary(key.as_bytes()) { + Ok(()) => ( + KeyResponseArgs { + result: true, + ..Default::default() + }, + true, + ), + // Unreachable given the exact secondary membership just verified plus the + // collision-free invariant; handled fail-closed. + Err(_) => ( + KeyResponseArgs { + result: false, + message: "requested key is not installed".into(), + ..Default::default() + }, + false, + ), + } + } else { + ( + KeyResponseArgs { + result: false, + message: "requested key is not installed".into(), + ..Default::default() + }, + false, + ) + } } - (KeyRequestOperation::Use, Some(key)) => match keyring.promote(key.as_bytes()) { - Ok(()) => ( - KeyResponseArgs { - result: true, - ..Default::default() - }, - true, - ), - Err(_) => ( - KeyResponseArgs { - result: false, - message: "requested primary key is not installed".into(), - ..Default::default() - }, - false, - ), - }, - (KeyRequestOperation::Remove, Some(key)) => match keyring.remove_secondary(key.as_bytes()) { - Ok(()) => ( - KeyResponseArgs { - result: true, - ..Default::default() - }, - true, - ), - Err(_) => ( - KeyResponseArgs { - result: false, - message: "key is not a removable secondary".into(), - ..Default::default() - }, - false, - ), - }, (KeyRequestOperation::List, _) => { let mut keys = Vec::with_capacity(1 + keyring.secondaries().len()); keys.push(*keyring.primary_ref()); diff --git a/serf-embedded/src/engine/tests.rs b/serf-embedded/src/engine/tests.rs index d38b309f..c68490f7 100644 --- a/serf-embedded/src/engine/tests.rs +++ b/serf-embedded/src/engine/tests.rs @@ -2076,3 +2076,155 @@ fn key_rotation_reencrypts_gossip_and_rejects_removed_key() { "a frame under the removed key K1 must be rejected by the rotated engine" ); } + +// ── Cross-cipher (dual-backend) ambiguity regressions ──────────────────────── +// +// These need BOTH AEAD backends so an AES-256 key and a ChaCha20-Poly1305 key can +// share the same 32 raw bytes yet be DISTINCT keys — the twin the byte-keyed +// coordinator ops (`promote`/`remove_secondary`) cannot tell apart. Gated on both +// features accordingly. + +/// A fixed AES-256-GCM key filled with `fill`. +#[cfg(all(feature = "aes-gcm", feature = "chacha20-poly1305"))] +fn aes256(fill: u8) -> SecretKey { + SecretKey::Aes256([fill; 32]) +} + +/// A fixed ChaCha20-Poly1305 key filled with `fill` — the cross-cipher byte twin of +/// `aes256(fill)` (identical bytes, different variant, so `!=` under `SecretKey`'s +/// variant-inclusive equality). +#[cfg(all(feature = "aes-gcm", feature = "chacha20-poly1305"))] +fn chacha(fill: u8) -> SecretKey { + SecretKey::ChaCha20Poly1305([fill; 32]) +} + +/// Fallible sibling of [`make_encrypted_engine`] that surfaces the construction +/// `InitError` instead of panicking, for the construction-preflight assertion. +#[cfg(all(feature = "aes-gcm", feature = "chacha20-poly1305"))] +fn try_make_encrypted_engine( + id: &str, + port: u16, + keyring: Keyring, +) -> Result, InitError> { + let cfg = Options::new() + .with_port(port) + .with_close_timeout(Duration::from_secs(10)); + let ep_cfg = EndpointOptions::new(SmolStr::new(id), node_addr(port)); + let transform = + TransformOptions::default().with_encryption(EncryptionOptions::new().with_keyring(keyring)); + let now = Instant::from_origin(Duration::from_secs(86_400)); + SerfEngine::try_new_at(cfg, transform, ep_cfg, SerfOptions::new(), now, test_rng()) +} + +/// The chokepoint is variant-exact: a key op naming one cipher can never touch a +/// byte-twin of another cipher, and installing a twin is refused — while exact +/// (variant + bytes) ops still apply. Every wrong-variant assertion fails on the +/// byte-keyed revert (which would promote/remove/install the wrong cipher's key +/// and report success). +#[cfg(all(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn cross_cipher_key_ops_are_variant_exact() { + let k1 = aes256(0x11); // primary + let x_aes = aes256(0x22); // the AES twin, installed as a secondary + let x_chacha = chacha(0x22); // its cross-cipher byte twin (same bytes, ChaCha) + let mut engine = make_encrypted_engine("a", 7946, Keyring::with_secondaries(k1, [x_aes])); + let before = engine.keyring().unwrap().secondaries().to_vec(); + + // use(ChaCha(X)) must NOT promote the byte-twin AES256(X): the exact ChaCha key is + // absent. Refused, ring unchanged. (Revert: promotes AES256(X), reports success.) + let resp = engine.apply_key_request(&key_op(KeyRequestOperation::Use, Some(x_chacha))); + assert!( + !resp.result, + "use of an absent cross-cipher twin must be refused" + ); + assert_eq!( + engine.keyring().unwrap().primary_ref(), + &k1, + "a refused use must not move the primary" + ); + assert_eq!( + engine.keyring().unwrap().secondaries(), + before.as_slice(), + "a refused use must not change the ring" + ); + + // remove(ChaCha(X)) must NOT drop the byte-twin AES256(X). (Revert: removes it.) + let resp = engine.apply_key_request(&key_op(KeyRequestOperation::Remove, Some(x_chacha))); + assert!( + !resp.result, + "remove of an absent cross-cipher twin must be refused" + ); + assert!( + engine.keyring().unwrap().secondaries().contains(&x_aes), + "the AES twin must remain installed after a refused cross-cipher remove" + ); + + // install(ChaCha(X)) while AES256(X) is present is a cross-cipher collision: + // refused, ring unchanged. (Revert: inserts ChaCha(X), producing an ambiguous + // twin ring and reporting success.) + let resp = engine.apply_key_request(&key_op(KeyRequestOperation::Install, Some(x_chacha))); + assert!( + !resp.result, + "installing a cross-cipher byte twin must be refused" + ); + assert!( + !engine.keyring().unwrap().secondaries().contains(&x_chacha), + "a refused cross-cipher install must not add the twin" + ); + assert_eq!( + engine.keyring().unwrap().secondaries(), + before.as_slice(), + "a refused cross-cipher install must not change the ring" + ); + + // Exact-variant ops still apply. Install a genuinely new ChaCha key (no byte twin + // present), promote it by its exact variant, and remove the AES twin by ITS true + // variant — every one succeeds. + let z_chacha = chacha(0x33); + assert!( + engine + .apply_key_request(&key_op(KeyRequestOperation::Install, Some(z_chacha))) + .result, + "installing a non-colliding ChaCha key must succeed" + ); + assert!(engine.keyring().unwrap().secondaries().contains(&z_chacha)); + assert!( + engine + .apply_key_request(&key_op(KeyRequestOperation::Use, Some(z_chacha))) + .result, + "promoting the exact ChaCha key must succeed" + ); + assert_eq!( + engine.keyring().unwrap().primary_ref(), + &z_chacha, + "the exact ChaCha key must become the primary" + ); + assert!( + engine + .apply_key_request(&key_op(KeyRequestOperation::Remove, Some(x_aes))) + .result, + "removing the AES key by its true variant must succeed" + ); + assert!( + !engine.keyring().unwrap().secondaries().contains(&x_aes), + "the AES key must be gone after an exact-variant remove" + ); +} + +/// Construction refuses a seed keyring that already carries a cross-cipher byte +/// twin, so an ambiguous ring can never reach the running chokepoint. Fails on the +/// revert: without the preflight the ring is individually usable and constructs. +#[cfg(all(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn construction_rejects_a_keyring_with_cross_cipher_byte_twins() { + // AES256(X) primary + ChaCha20-Poly1305(X) secondary: identical bytes, different + // ciphers — an ambiguous ring for the byte-keyed rotation ops. + let keyring = Keyring::with_secondaries(aes256(0x11), [chacha(0x11)]); + match try_make_encrypted_engine("a", 7946, keyring) { + Err(InitError::Encryption(_)) => {} + Err(other) => { + panic!("expected InitError::Encryption for a cross-cipher twin keyring, got {other:?}") + } + Ok(_) => panic!("construction must reject a keyring carrying cross-cipher byte twins"), + } +} From 96e7e39b42500b3c9c31707a621327b7aa991ee8 Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 10 Jul 2026 16:28:35 +1200 Subject: [PATCH 060/128] fix(serf-smoltcp): gate the key-management suite on an AEAD backend The rotation tests were feature-gated item-by-item, leaving the file's shared imports and harness helpers dead in a plaintext build and failing clippy --all-targets under the default features; a file-level cfg empties the suite instead. --- serf-smoltcp/tests/key_management.rs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/serf-smoltcp/tests/key_management.rs b/serf-smoltcp/tests/key_management.rs index 4e1069e8..41c47998 100644 --- a/serf-smoltcp/tests/key_management.rs +++ b/serf-smoltcp/tests/key_management.rs @@ -10,19 +10,21 @@ //! (live keyring frozen at construction K1) cannot satisfy — then a post-rotation //! user event still crosses the wire, proving both planes now run under K2. +// The whole suite exercises key rotation, so without an AEAD backend the file +// compiles to nothing — gating item-by-item would leave the shared harness +// helpers dead in a plaintext build. +#![cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + mod harness; use core::net::{IpAddr, Ipv4Addr, SocketAddr}; use serf_smoltcp::{ - Bytes, EndpointOptions, Event, Instant, MaybeResolved, Options, Serf, SerfOptions, - SocketAddrResolver, TransformOptions, + Bytes, EncryptionOptions, EndpointOptions, Event, Instant, Keyring, MaybeResolved, Options, + SecretKey, Serf, SerfOptions, SocketAddrResolver, TransformOptions, }; use smol_str::SmolStr; -#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] -use serf_smoltcp::{EncryptionOptions, Keyring, SecretKey}; - fn addr(ip: u8, port: u16) -> SocketAddr { SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, ip)), port) } From 4077252fb4514d9803033dd469d801612d6ef652 Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 10 Jul 2026 17:24:37 +1200 Subject: [PATCH 061/128] feat(serf-embassy): async no_std serf driver over embassy-net MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit serf-embassy mirrors memberlist-embassy over serf-embedded's SerfEngine — per-slot socket workers, a pump runner, shared state with a bounded app-event buffer, and an async handle; serf's mandatory events are actioned in the runner before lossy buffering (KeyRequest through the live-keyring chokepoint, Shutdown to the lifecycle state); self-addressed gossip is looped back into the engine's ingress since embassy-net does not mirror an OS socket's self-delivery; the async join awaits the engine's core-owned await-result join; 2-node loopback green. --- Cargo.toml | 1 + serf-embassy/Cargo.toml | 89 +++ serf-embassy/README.md | 41 + serf-embassy/build.rs | 17 + serf-embassy/src/config/mod.rs | 114 +++ serf-embassy/src/config/tests.rs | 26 + serf-embassy/src/error/mod.rs | 266 +++++++ serf-embassy/src/gossip_io.rs | 130 ++++ serf-embassy/src/lib.rs | 86 ++ serf-embassy/src/mailbox.rs | 133 ++++ serf-embassy/src/resolver.rs | 60 ++ serf-embassy/src/runner.rs | 234 ++++++ serf-embassy/src/serf/mod.rs | 818 ++++++++++++++++++++ serf-embassy/src/shared/mod.rs | 213 +++++ serf-embassy/src/stream_io/mod.rs | 263 +++++++ serf-embassy/src/stream_io/tests.rs | 235 ++++++ serf-embassy/src/time/mod.rs | 103 +++ serf-embassy/src/time/tests.rs | 65 ++ serf-embassy/src/worker.rs | 453 +++++++++++ serf-embassy/tests/key_rotation.rs | 187 +++++ serf-embassy/tests/loopback.rs | 167 ++++ serf-embassy/tests/query.rs | 105 +++ serf-embassy/tests/support/cluster.rs | 179 +++++ serf-embassy/tests/support/mod.rs | 5 + serf-embassy/tests/support/paired_device.rs | 185 +++++ serf-embassy/tests/user_event.rs | 58 ++ 26 files changed, 4233 insertions(+) create mode 100644 serf-embassy/Cargo.toml create mode 100644 serf-embassy/README.md create mode 100644 serf-embassy/build.rs create mode 100644 serf-embassy/src/config/mod.rs create mode 100644 serf-embassy/src/config/tests.rs create mode 100644 serf-embassy/src/error/mod.rs create mode 100644 serf-embassy/src/gossip_io.rs create mode 100644 serf-embassy/src/lib.rs create mode 100644 serf-embassy/src/mailbox.rs create mode 100644 serf-embassy/src/resolver.rs create mode 100644 serf-embassy/src/runner.rs create mode 100644 serf-embassy/src/serf/mod.rs create mode 100644 serf-embassy/src/shared/mod.rs create mode 100644 serf-embassy/src/stream_io/mod.rs create mode 100644 serf-embassy/src/stream_io/tests.rs create mode 100644 serf-embassy/src/time/mod.rs create mode 100644 serf-embassy/src/time/tests.rs create mode 100644 serf-embassy/src/worker.rs create mode 100644 serf-embassy/tests/key_rotation.rs create mode 100644 serf-embassy/tests/loopback.rs create mode 100644 serf-embassy/tests/query.rs create mode 100644 serf-embassy/tests/support/cluster.rs create mode 100644 serf-embassy/tests/support/mod.rs create mode 100644 serf-embassy/tests/support/paired_device.rs create mode 100644 serf-embassy/tests/user_event.rs diff --git a/Cargo.toml b/Cargo.toml index 5e6b2a75..4482a693 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "serf-compio", "serf-reactor", "serf-smoltcp", + "serf-embassy", ] exclude = ["legacy"] resolver = "3" diff --git a/serf-embassy/Cargo.toml b/serf-embassy/Cargo.toml new file mode 100644 index 00000000..0b3de73d --- /dev/null +++ b/serf-embassy/Cargo.toml @@ -0,0 +1,89 @@ +[package] +name = "serf-embassy" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +homepage.workspace = true +readme = "README.md" +rust-version = "1.96" +description = "Async no_std serf driver over the embassy-net network stack." + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] + +[features] +default = ["std"] +# `std` and `alloc` are INDEPENDENT capability tiers (std does NOT pull alloc): +# under `std` the real `std` crate supplies the heap; under `alloc` the +# no_std+alloc path aliases `alloc`. Each layers the matching feature on every +# dependency that has one. +std = ["serf-embedded/std", "serf-proto/std", "memberlist-proto/std", "bytes/std"] +alloc = ["serf-embedded/alloc", "serf-proto/alloc", "memberlist-proto/alloc"] + +# CIDR peer-admission allow-list. A `CidrPolicy` set via `Options::with_cidr_policy` +# filters inbound gossip by datagram source and inbound reliable connections by +# peer address at the transport boundary, AND inbound alives by the peer's +# self-advertised address at membership admission. no_std+alloc (pulls the heap tier). +cidr = ["alloc", "serf-embedded/cidr", "memberlist-proto/cidr"] + +# Gossip + reliable-plane AEAD encryption backends. Each pass-through enables the +# matching serf-embedded and machine backend so a `TransformOptions` carrying a +# `Keyring` for that cipher can actually encrypt, and serf's key-management queries +# (install/use/remove/list) plus the driver-side `KeyRequest` response become +# available. Both AEAD backends are no_std+alloc. +aes-gcm = ["alloc", "serf-embedded/aes-gcm", "serf-proto/aes-gcm", "memberlist-proto/aes-gcm"] +chacha20-poly1305 = [ + "alloc", + "serf-embedded/chacha20-poly1305", + "serf-proto/chacha20-poly1305", + "memberlist-proto/chacha20-poly1305", +] +# Umbrella that activates both AEAD backends (convenient for tests / CI). +encryption = ["aes-gcm", "chacha20-poly1305"] + +[dependencies] +# The transport-agnostic no_std serf driving core (SerfEngine + the reused +# memberlist-embedded glue it re-exports); depended on alone for the driving +# surface, mirroring how memberlist-embassy depends on memberlist-embedded. +serf-embedded = { path = "../serf-embedded", default-features = false } +# serf's own protocol types named in the `Serf` command signatures +# (`QueryParams` / `QueryEvent` / `Tags` / `Member` / the key-management types). +serf-proto = { path = "../serf-proto", default-features = false, features = [ + "tcp", +] } +# The Sans-I/O machine types named in the construction / command signatures +# (`Instant` / `EndpointOptions` / `Node` / `SmallRng` / `StreamId`). Pinned to the +# same git+branch as serf-embedded so the graph holds one copy. +memberlist-proto = { git = "https://github.com/al8n/memberlist", branch = "main", default-features = false, features = [ + "tcp", +] } +embassy-net = { version = "0.9", default-features = false, features = [ + "udp", + "tcp", + "proto-ipv4", + "proto-ipv6", + "medium-ethernet", +] } +embassy-time = { version = "0.5", default-features = false } +embassy-sync = { version = "0.8", default-features = false } +embassy-futures = { version = "0.1", default-features = false } +bytes = { version = "1", default-features = false } +getrandom = "0.4" +# Named in the `user_event` / `query` forwarder signatures (`impl Into`), +# matching serf-embedded's command surface. +smol_str = { version = "0.3", default-features = false } + +[dev-dependencies] +# A time driver so `embassy_time::Instant::now()` works in the host tests (the +# production build leaves the driver to the embedded app). `std` is test-only; +# `generic-queue-8` is the self-contained timer queue. +embassy-time = { version = "0.5", default-features = false, features = [ + "std", + "generic-queue-8", +] } +# `block_on` to drive the async harness inline in tests. +futures = { version = "0.3", default-features = false, features = ["executor"] } +# A concrete node-id type for the loopback tests. +smol_str = { version = "0.3", default-features = false, features = ["std"] } diff --git a/serf-embassy/README.md b/serf-embassy/README.md new file mode 100644 index 00000000..cdeeb603 --- /dev/null +++ b/serf-embassy/README.md @@ -0,0 +1,41 @@ +# serf-embassy + +An async `no_std` [serf](https://github.com/al8n/serf) driver over the +[embassy-net](https://github.com/embassy-rs/embassy) network stack. + +`serf-embassy` drives [`serf-embedded`](../serf-embedded)'s transport-agnostic +`SerfEngine` on an embassy-net TCP/IP stack. It mirrors +[`memberlist-embassy`](https://github.com/al8n/memberlist)'s architecture — +per-pool-slot async socket workers mediated by a per-slot mailbox, a synchronous +`pump` runner, and a cloneable async handle sharing state through +`embassy-sync` signals — but folds serf's richer machine (queries, user events, +key management) on top of membership. + +The caller owns the embassy-net `Stack` and supplies a gossip `UdpSocket` plus a +pool of reliable-plane `TcpSocket`s; `Serf::new` wires up the engine and hands +back the handle paired with a `Runner` to spawn as a task. + +## Serf-specific behaviors + +- **Self-addressed gossip loopback.** embassy-net (smoltcp underneath) does not + loop a self-addressed UDP datagram back into `recv` like an OS socket. serf + directs a node's response to its OWN locally-originated query / key request at + its own advertise address, so the gossip view diverts such datagrams into a + driver loopback buffer the next pump ingests. +- **Mandatory events actioned before lossy buffering.** The runner's post-pump + drain applies an `Event::Shutdown` (stop) and an `Event::KeyRequest` (through + the engine's live-keyring `handle_key_request` chokepoint) BEFORE the event + copy is buffered for the app's bounded event queue. +- **Core-owned await-result join.** `Serf::join` dispatches the engine's + await-result join and awaits its `poll_join` outcome, mapping + `Ok(ReachedSet)` / `JoinFailed` into the handle's result. + +## Features + +- `std` (default) / `alloc` — independent capability tiers. +- `cidr` — CIDR peer-admission allow-list. +- `aes-gcm` / `chacha20-poly1305` / `encryption` — gossip + reliable-plane AEAD. + +## License + +Licensed under the MPL-2.0 license. diff --git a/serf-embassy/build.rs b/serf-embassy/build.rs new file mode 100644 index 00000000..1af9ef00 --- /dev/null +++ b/serf-embassy/build.rs @@ -0,0 +1,17 @@ +//! Emits the aggregate `encryption` cfg, set when any AEAD backend feature is +//! enabled, so encryption code gates on `#[cfg(encryption)]` instead of repeating +//! the backend list. serf's gossip plane carries no compression / checksum, so +//! only the encryption aggregate is emitted (mirroring `serf-embedded`). + +fn any_feature(names: &[&str]) -> bool { + names + .iter() + .any(|name| std::env::var_os(format!("CARGO_FEATURE_{name}")).is_some()) +} + +fn main() { + println!("cargo::rustc-check-cfg=cfg(encryption)"); + if any_feature(&["AES_GCM", "CHACHA20_POLY1305"]) { + println!("cargo::rustc-cfg=encryption"); + } +} diff --git a/serf-embassy/src/config/mod.rs b/serf-embassy/src/config/mod.rs new file mode 100644 index 00000000..18cc3103 --- /dev/null +++ b/serf-embassy/src/config/mod.rs @@ -0,0 +1,114 @@ +//! Driver-side sizing and ports for [`Serf`](crate::Serf). +//! +//! The embassy-net [`Stack`](embassy_net::Stack), its interface, and its IP +//! configuration are owned by the caller (built via [`embassy_net::new`]), so — +//! unlike the smoltcp driver — this config carries no medium / address / route +//! knobs. It holds only what the driver itself needs: the bound port, the bridge +//! ring capacities (which also bound the per-slot mailbox byte rings), and the +//! engine's graceful-close timeout. + +use core::time::Duration; + +pub use serf_embedded::DEFAULT_CLOSE_TIMEOUT; + +/// Ports and bridge sizing for [`Serf`](crate::Serf). +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct Options { + /// Local port the node binds. The gossip UDP socket and the reliable-plane TCP + /// listener both use it, and it is the port peers reach the node at — the + /// single-port serf model (one advertised `SocketAddr` serves both planes). + pub port: u16, + /// Per-slot inbound (peer → engine) mailbox ring capacity, in bytes. A worker + /// reads at most this many un-handed-off bytes before the engine drains them. + /// Defaults to a TCP socket's worth. + pub tcp_socket_rx_bytes: usize, + /// Per-slot outbound (engine → peer) mailbox ring capacity, in bytes. The + /// engine's `send` accepts up to this many un-written bytes per slot. Defaults + /// to a TCP socket's worth. + pub tcp_socket_tx_bytes: usize, + /// Maximum time a gracefully-closing reliable connection may stay parked before + /// it is force-aborted and returned to the pool (the engine's close bound). + pub close_timeout: Duration, + /// Inactivity timeout applied to each reliable-plane TCP socket. It bounds how + /// long a worker can block in a `connect` / `write` / `flush` / `read` to an + /// unresponsive peer before embassy-net aborts the socket — so a peer that stops + /// ACKing cannot wedge the worker (and, via the reuse gate, withhold its slot) + /// indefinitely. MUST exceed the longest valid reliable exchange AND the + /// `close_timeout`, or a legitimately slow exchange could be aborted early. + /// Defaults to a generous multiple of the close timeout. + pub socket_timeout: Duration, + /// CIDR peer-admission policy. Filters inbound gossip by datagram source and + /// inbound reliable connections by peer address at the transport boundary, AND + /// inbound alives by the peer's self-advertised address at membership + /// admission. `None` (the default) admits every address. Present only with the + /// `cidr` feature; set it via [`with_cidr_policy`](Options::with_cidr_policy). + #[cfg(feature = "cidr")] + #[cfg_attr(docsrs, doc(cfg(feature = "cidr")))] + pub cidr_policy: Option, +} + +impl Default for Options { + fn default() -> Self { + Self { + port: 7946, + tcp_socket_rx_bytes: 4096, + tcp_socket_tx_bytes: 4096, + close_timeout: DEFAULT_CLOSE_TIMEOUT, + socket_timeout: Duration::from_secs(15), + #[cfg(feature = "cidr")] + cidr_policy: None, + } + } +} + +impl Options { + /// Defaults tuned for a small embedded cluster. + pub fn new() -> Self { + Self::default() + } + + /// Override the local port (the gossip UDP socket and the reliable-plane TCP + /// listener both bind it). + pub fn with_port(mut self, p: u16) -> Self { + self.port = p; + self + } + + /// Override the per-slot inbound mailbox ring capacity (bytes). + pub fn with_tcp_socket_rx_bytes(mut self, n: usize) -> Self { + self.tcp_socket_rx_bytes = n; + self + } + + /// Override the per-slot outbound mailbox ring capacity (bytes). + pub fn with_tcp_socket_tx_bytes(mut self, n: usize) -> Self { + self.tcp_socket_tx_bytes = n; + self + } + + /// Override the graceful-close timeout (see [`Options::close_timeout`]). + pub fn with_close_timeout(mut self, d: Duration) -> Self { + self.close_timeout = d; + self + } + + /// Override the per-socket inactivity timeout (see [`Options::socket_timeout`]). + pub fn with_socket_timeout(mut self, d: Duration) -> Self { + self.socket_timeout = d; + self + } + + /// Install a CIDR peer-admission policy (see [`Options::cidr_policy`]). One + /// policy gates the gossip source and reliable peer at the transport boundary + /// AND the advertised address at membership admission. + #[cfg(feature = "cidr")] + #[cfg_attr(docsrs, doc(cfg(feature = "cidr")))] + pub fn with_cidr_policy(mut self, policy: serf_embedded::CidrPolicy) -> Self { + self.cidr_policy = Some(policy); + self + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-embassy/src/config/tests.rs b/serf-embassy/src/config/tests.rs new file mode 100644 index 00000000..314cd3c5 --- /dev/null +++ b/serf-embassy/src/config/tests.rs @@ -0,0 +1,26 @@ +use super::*; + +#[test] +fn defaults_are_sane_and_overridable() { + let c = Options::new(); + assert_eq!(c.port, 7946); + assert!(c.tcp_socket_rx_bytes > 0); + assert!(c.tcp_socket_tx_bytes > 0); + assert!(!c.close_timeout.is_zero()); + assert!( + c.socket_timeout > c.close_timeout, + "socket timeout must exceed the close timeout" + ); + + let c = Options::new() + .with_port(1234) + .with_tcp_socket_rx_bytes(8192) + .with_tcp_socket_tx_bytes(2048) + .with_close_timeout(Duration::from_secs(3)) + .with_socket_timeout(Duration::from_secs(20)); + assert_eq!(c.port, 1234); + assert_eq!(c.tcp_socket_rx_bytes, 8192); + assert_eq!(c.tcp_socket_tx_bytes, 2048); + assert_eq!(c.close_timeout, Duration::from_secs(3)); + assert_eq!(c.socket_timeout, Duration::from_secs(20)); +} diff --git a/serf-embassy/src/error/mod.rs b/serf-embassy/src/error/mod.rs new file mode 100644 index 00000000..0f50e226 --- /dev/null +++ b/serf-embassy/src/error/mod.rs @@ -0,0 +1,266 @@ +//! Errors from constructing a [`Serf`](crate::Serf) and from an await-result +//! [`join`](crate::Serf::join). + +use alloc::boxed::Box; +use core::{fmt, time::Duration}; + +use serf_embedded::{JoinFailed, SerfError}; + +/// Why constructing a [`Serf`](crate::Serf) node failed. +/// +/// Layers a few embassy-driver construction faults (the TCP socket pool size and +/// the bridge ring capacities) on top of the transport-agnostic +/// [`serf_embedded::InitError`] the shared engine surfaces (port, advertise +/// address, gossip MTU, encryption-keyring, machine-endpoint faults). Every +/// variant is a misconfiguration reported in place of a panic. +#[derive(Debug)] +#[non_exhaustive] +pub enum InitError { + /// Fewer than two TCP sockets were supplied. + /// + /// Construction dedicates one pooled socket to the listener and uses the rest + /// for dials/accepts: zero sockets is no reliable plane at all, and one leaves + /// the listener holding the only socket with none free to dial — the node + /// could never dial a seed to join. The functional minimum is a listener plus + /// one dial/accept socket. The supplied count is carried for diagnostics. + TcpPoolTooSmall(usize), + /// A configured bridge ring capacity + /// ([`Options::tcp_socket_rx_bytes`](crate::Options::tcp_socket_rx_bytes) or + /// [`tcp_socket_tx_bytes`](crate::Options::tcp_socket_tx_bytes)) is zero. + /// + /// A zero-byte inbound ring can never buffer a received byte for the engine to + /// drain, and a zero-byte outbound ring can never accept a byte from the + /// engine to write — a silently-dead reliable plane. Both must be non-zero. + ZeroBridgeRing, + /// The per-socket inactivity timeout is out of the valid range. + /// + /// [`Options::socket_timeout`](crate::Options::socket_timeout), as embassy-net installs it + /// into smoltcp (floored to whole microseconds — the embassy tick count handed over via + /// `as_micros`), must be at least one microsecond and strictly greater than BOTH the + /// graceful-close bound ([`close_timeout`](crate::Options::close_timeout)) and the + /// machine's reliable-exchange deadline (`EndpointOptions::stream_timeout`) — otherwise + /// embassy-net could abort a slow-but-valid exchange before the engine's own policy + /// fires — AND no larger than a sane maximum, so it cannot overflow the + /// embassy-time-to-smoltcp duration conversion into a wrapped (effectively past) + /// deadline. Because the bound is enforced on the value rounded DOWN to whole installed + /// microseconds, a coarse or very fine tick rate can reject a timeout that looks valid as + /// a `core::Duration`; the offending values, the maximum, and the platform tick rate are + /// carried for diagnostics. + SocketTimeoutOutOfRange(SocketTimeoutOutOfRange), + /// The shared engine rejected the configuration (see + /// [`serf_embedded::InitError`]): a zero/over-ceiling gossip MTU, a + /// non-routable or port-mismatched advertise address, a zero port or + /// close-timeout, an unusable encryption keyring, or a machine-endpoint init + /// failure (including an entropy draw failure). + Engine(serf_embedded::InitError), + /// The address resolver failed while resolving the advertise address. + /// + /// The resolver's error type is generic, so it is boxed to preserve the + /// `source()` chain; a caller that knows its concrete resolver can downcast. + /// No `Send`/`Sync` bound — the embassy [`AddressResolver`](crate::AddressResolver) + /// is single-threaded by design, so its error need not cross threads. + Resolve(Box), + /// The address resolver succeeded but yielded no address for the advertise + /// address, so the node would have nothing to advertise. + NoAddresses, + /// The platform entropy source ([`getrandom`]) failed while seeding the default + /// gossip / serf RNGs in [`Serf::new`](crate::Serf::new). Use + /// [`Serf::new_with_rng`](crate::Serf::new_with_rng) to supply your own RNGs + /// and avoid the platform entropy draw entirely. + Entropy, +} + +impl InitError { + /// Whether construction failed because the TCP socket pool had fewer than two + /// sockets. + #[inline] + pub const fn is_tcp_pool_too_small(&self) -> bool { + matches!(self, InitError::TcpPoolTooSmall(_)) + } + + /// Whether a configured bridge ring capacity was zero. + #[inline] + pub const fn is_zero_bridge_ring(&self) -> bool { + matches!(self, InitError::ZeroBridgeRing) + } + + /// Whether the per-socket inactivity timeout was out of range. + #[inline] + pub const fn is_socket_timeout_out_of_range(&self) -> bool { + matches!(self, InitError::SocketTimeoutOutOfRange(_)) + } + + /// Whether the shared engine rejected the configuration. + #[inline] + pub const fn is_engine(&self) -> bool { + matches!(self, InitError::Engine(_)) + } + + /// Whether the resolver failed on the advertise address. + #[inline] + pub const fn is_resolve(&self) -> bool { + matches!(self, InitError::Resolve(_)) + } + + /// Whether the resolver yielded no address for the advertise address. + #[inline] + pub const fn is_no_addresses(&self) -> bool { + matches!(self, InitError::NoAddresses) + } + + /// Whether the platform entropy source failed while seeding the RNGs. + #[inline] + pub const fn is_entropy(&self) -> bool { + matches!(self, InitError::Entropy) + } +} + +/// Payload for [`InitError::SocketTimeoutOutOfRange`]: the configured socket timeout, +/// the two engine deadlines it must exceed, the maximum it must not exceed, and the +/// platform tick rate the rounded comparison used (a coarse rate can reject a timeout +/// that looks valid before it is rounded down to whole ticks). +#[derive(Debug, Clone, Copy)] +pub struct SocketTimeoutOutOfRange { + /// The configured per-socket inactivity timeout. + pub socket_timeout: Duration, + /// The graceful-close bound it must exceed (after flooring to installed microseconds). + pub close_timeout: Duration, + /// The machine's reliable-exchange deadline it must exceed (after flooring to installed + /// microseconds). + pub stream_timeout: Duration, + /// The maximum it must not exceed, so its conversion cannot overflow. + pub max: Duration, + /// The platform `embassy-time` tick rate (Hz) that determines the installed-microsecond + /// flooring the comparison used. + pub tick_hz: u64, +} + +impl fmt::Display for InitError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + InitError::TcpPoolTooSmall(n) => write!( + f, + "the TCP socket pool needs at least 2 sockets (a listener plus one \ + dial/accept socket); got {n}" + ), + InitError::ZeroBridgeRing => { + f.write_str("tcp_socket_rx_bytes and tcp_socket_tx_bytes must both be non-zero") + } + InitError::SocketTimeoutOutOfRange(s) => write!( + f, + "socket_timeout ({:?}), as installed into smoltcp (floored to whole microseconds \ + at the {} Hz platform tick rate), must be at least one microsecond and greater \ + than both close_timeout ({:?}) and stream_timeout ({:?}), and no larger than {:?}", + s.socket_timeout, s.tick_hz, s.close_timeout, s.stream_timeout, s.max + ), + InitError::Engine(e) => write!(f, "{e}"), + InitError::Resolve(e) => write!(f, "advertise address resolution failed: {e}"), + InitError::NoAddresses => f.write_str("advertise address resolution returned no addresses"), + InitError::Entropy => f.write_str("entropy source failed while seeding the RNGs"), + } + } +} + +impl From for InitError { + fn from(e: serf_embedded::InitError) -> Self { + InitError::Engine(e) + } +} + +#[cfg(feature = "std")] +#[cfg_attr(docsrs, doc(cfg(feature = "std")))] +impl std::error::Error for InitError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + InitError::Engine(e) => Some(e), + InitError::Resolve(e) => Some(e.as_ref()), + _ => None, + } + } +} + +/// Why an await-result [`join`](crate::Serf::join) did not succeed. +/// +/// A join first resolves each seed through the supplied resolver, then dispatches +/// the intent to the engine and awaits its outcome; any step can fail, and a +/// fully-dispatched join can still reach no seed. +/// +/// `Resolve` is `!Clone` / `!PartialEq` (it boxes the resolver's generic error), +/// so this enum derives neither — match on the variant (or the `is_*` predicates) +/// instead. +#[derive(Debug)] +#[non_exhaustive] +pub enum JoinError { + /// The address resolver failed while resolving a seed. + /// + /// The resolver's error type is generic, so it is boxed to preserve the + /// `source()` chain; a caller that knows its concrete resolver can downcast. + /// No `Send`/`Sync` bound — the embassy [`AddressResolver`](crate::AddressResolver) + /// is single-threaded by design, so its error need not cross threads. + Resolve(Box), + /// The engine rejected the join up front (e.g. the node is not in the running + /// state). + Control(SerfError), + /// A non-empty seed set resolved to no wire address — a discovery failure + /// rather than a successful no-op join. + NoAddresses, + /// Every dispatched push/pull terminated without contacting a seed. The + /// [`JoinFailed`] payload carries the requested-seed count. + Failed(JoinFailed), +} + +impl JoinError { + /// Whether this is a resolver failure. + #[inline] + pub const fn is_resolve(&self) -> bool { + matches!(self, JoinError::Resolve(_)) + } + + /// Whether the engine rejected the join up front. + #[inline] + pub const fn is_control(&self) -> bool { + matches!(self, JoinError::Control(_)) + } + + /// Whether a non-empty seed set resolved to no wire address. + #[inline] + pub const fn is_no_addresses(&self) -> bool { + matches!(self, JoinError::NoAddresses) + } + + /// Whether every dispatched push/pull terminated without contacting a seed. + #[inline] + pub const fn is_failed(&self) -> bool { + matches!(self, JoinError::Failed(_)) + } +} + +impl fmt::Display for JoinError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + JoinError::Resolve(e) => write!(f, "seed address resolution failed: {e}"), + JoinError::Control(e) => write!(f, "join was rejected: {e}"), + JoinError::NoAddresses => f.write_str("no wire address resolved for any seed"), + JoinError::Failed(e) => write!(f, "{e}"), + } + } +} + +impl From for JoinError { + fn from(e: SerfError) -> Self { + JoinError::Control(e) + } +} + +#[cfg(feature = "std")] +#[cfg_attr(docsrs, doc(cfg(feature = "std")))] +impl std::error::Error for JoinError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + JoinError::Resolve(e) => Some(e.as_ref()), + JoinError::Control(e) => Some(e), + JoinError::Failed(e) => Some(e), + JoinError::NoAddresses => None, + } + } +} diff --git a/serf-embassy/src/gossip_io.rs b/serf-embassy/src/gossip_io.rs new file mode 100644 index 00000000..f4017cd0 --- /dev/null +++ b/serf-embassy/src/gossip_io.rs @@ -0,0 +1,130 @@ +//! The [`GossipIo`](serf_embedded::GossipIo) implementation over an embassy-net +//! [`UdpSocket`], with serf's self-delivery loopback. +//! +//! A short-lived view over the gossip socket, rebuilt each engine pump. The +//! engine reads inbound gossip and writes outbound gossip through it without +//! touching embassy-net; the actual stack progress (link RX/TX) the embassy-net +//! `Stack` drives on its own. +//! +//! embassy-net 0.9 exposes only async + `poll_*` UDP methods (no `try_*`), so +//! the non-blocking [`GossipIo`] ops drive `poll_recv_from` / `poll_send_to` +//! with a no-op [`Waker`]: the engine pump is synchronous and re-polls each +//! socket on the next driver tick, so no datagram is lost by not registering a +//! real waker here (the runner registers the real recv waker around the pump). +//! +//! # Self-delivery loopback +//! +//! A real OS UDP socket loops a datagram addressed to its own bound address back +//! into `recv`; embassy-net (smoltcp underneath) does not — a self-addressed frame +//! goes to the wire and is dropped. serf directs a node's response to its OWN +//! locally-originated query / key-request to the originator's advertise address, +//! i.e. THIS node's address, so on embassy-net that response would be lost and a +//! local op would report `num_resp < num_nodes`. This view emulates the OS +//! self-delivery: [`send`] to the node's own `advertise` address queues the +//! datagram on a driver-owned `loopback` buffer instead of the socket, and +//! [`recv`] returns those queued datagrams FIRST (as if received from `advertise`) +//! so a subsequent pump ingests them exactly as an OS loopback would. The bytes +//! are the already-transformed wire form the engine emitted, so feeding them back +//! through the normal gossip ingress decodes them identically. +//! +//! [`send`]: GossipIo::send +//! [`recv`]: GossipIo::recv + +use core::{ + cell::RefCell, + net::{IpAddr, Ipv4Addr, SocketAddr}, + task::{Context, Poll, Waker}, +}; + +use alloc::{collections::VecDeque, vec::Vec}; + +use embassy_net::{IpEndpoint, udp::UdpSocket}; +use serf_embedded::GossipIo; + +/// A [`GossipIo`] view over a single bound gossip [`UdpSocket`] plus the driver's +/// self-delivery loopback buffer. +/// +/// serf binds one advertise address, so a single socket suffices (no v4/v6 split). +/// Built fresh for each engine pump over the already-progressed socket. +pub struct SerfGossip<'a> { + socket: &'a UdpSocket<'a>, + /// Driver-owned self-delivery buffer: datagrams this node addressed to its own + /// `advertise` address, awaiting loopback ingestion on the next pump. A separate + /// field, never inside the socket borrow. + loopback: &'a RefCell>>, + /// The local node's resolved advertise address — the destination a self-addressed + /// gossip datagram carries. + advertise: SocketAddr, +} + +impl<'a> SerfGossip<'a> { + /// Build the gossip view over the bound `socket`, with the driver's `loopback` + /// self-delivery buffer and the node's `advertise` address. + #[inline] + pub fn new( + socket: &'a UdpSocket<'a>, + loopback: &'a RefCell>>, + advertise: SocketAddr, + ) -> Self { + Self { + socket, + loopback, + advertise, + } + } +} + +impl GossipIo for SerfGossip<'_> { + fn recv(&mut self, buf: &mut [u8]) -> Option<(SocketAddr, usize)> { + // Self-delivery first: return any datagrams this node addressed to its own + // advertise address, exactly as an OS UDP socket would loop them back into recv, + // before draining the real rx ring. + loop { + let datagram = self.loopback.borrow_mut().pop_front(); + let Some(datagram) = datagram else { break }; + let n = datagram.len(); + if n <= buf.len() { + buf[..n].copy_from_slice(&datagram); + return Some((self.advertise, n)); + } + // An own datagram larger than the ingress buffer cannot be delivered; drop it + // and continue, mirroring the rx-ring `Truncated` skip below. A self datagram + // is bounded by this node's own gossip MTU + encryption overhead, so it fits + // the arena-sized ingress buffer in practice — this is defence-in-depth. + } + + let mut cx = Context::from_waker(Waker::noop()); + match self.socket.poll_recv_from(buf, &mut cx) { + Poll::Ready(Ok((len, meta))) => Some((meta.endpoint.into(), len)), + // `RecvError::Truncated` (the only `Poll::Ready(Err)`): an oversized + // datagram — larger than this buffer (the configured gossip MTU plus + // encryption overhead) — was already DEQUEUED by embassy-net before the + // length check, so it is consumed and gone. Surface a zero-length marker + // (like the smoltcp driver's `Truncated` skip) so the engine's drain loop + // treats it as nothing to deliver and RE-POLLS for the next datagram, + // instead of stopping early: one oversized datagram cannot stall the + // in-budget datagrams queued behind it. The source address is irrelevant + // for a zero-length frame; `handle_gossip` on an empty slice is a no-op. + Poll::Ready(Err(_)) => Some((SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 0), 0)), + // No datagram queued: the drain loop ends for this pump. + Poll::Pending => None, + } + } + + fn send(&mut self, bytes: &[u8], dest: SocketAddr) { + if dest == self.advertise { + // Self-addressed: embassy-net will not loop this back into its own rx ring + // the way an OS UDP socket does, so deliver it into the driver's loopback + // buffer for the next pump's ingress instead of dropping it on the wire. + self.loopback.borrow_mut().push_back(bytes.to_vec()); + return; + } + let mut cx = Context::from_waker(Waker::noop()); + // Ignoring the result: gossip is best-effort. A full tx ring (`Poll::Pending`) + // or any `SendError` (no route, socket not bound, packet too large) drops this + // datagram and SWIM recovers on the next gossip round — no error is surfaced. + let _ = self + .socket + .poll_send_to(bytes, IpEndpoint::from(dest), &mut cx); + } +} diff --git a/serf-embassy/src/lib.rs b/serf-embassy/src/lib.rs new file mode 100644 index 00000000..05d6640b --- /dev/null +++ b/serf-embassy/src/lib.rs @@ -0,0 +1,86 @@ +#![doc = include_str!("../README.md")] +#![cfg_attr(not(feature = "std"), no_std)] +#![forbid(unsafe_code)] +#![deny(missing_docs)] +// `collapsible_if`: the nested `if cond { if let ... }` form is kept deliberately — +// flattening multi-level guards into one long let-chain reads worse here. +#![allow(clippy::collapsible_if, clippy::type_complexity, unexpected_cfgs)] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![cfg_attr(docsrs, allow(unused_attributes))] + +#[cfg(not(any(feature = "std", feature = "alloc")))] +compile_error!("serf-embassy requires the `std` or `alloc` feature"); + +// Always link the `alloc` crate so path-qualified `alloc::` access (the mailbox +// rings, the `Rc`/`Vec` the runner and handle use) compiles in every feature +// configuration. On a `std` build `alloc` is re-exported by `std`; on no_std it +// is the bare `alloc` crate. +extern crate alloc; + +// Alias `alloc` to the name `std` so genuine-heap `std::` paths compile unchanged +// under no_std+alloc, matching the `serf-embedded` crate this driver builds on +// (its public API surfaces `std::sync::Arc` / `std::vec::Vec`, which resolve to +// `alloc` here). Core-resident items are imported from `core::` directly. +#[cfg(all(not(feature = "std"), feature = "alloc"))] +extern crate alloc as std; + +mod config; +mod error; +mod gossip_io; +mod mailbox; +mod resolver; +mod runner; +mod serf; +mod shared; +mod stream_io; +mod time; +mod worker; + +pub use bytes::Bytes; +pub use config::Options; +pub use error::{InitError, JoinError, SocketTimeoutOutOfRange}; +pub use gossip_io::SerfGossip; +pub use resolver::{AddressResolver, SocketAddrResolver}; +pub use runner::Runner; +pub use serf::Serf; +pub use stream_io::{SerfStream, SlotId}; +pub use time::{EmbassyInstant, now}; + +// The Sans-I/O machine types named in the public construction / command +// signatures, re-exported so a caller need not depend on `memberlist-proto` +// directly. +pub use memberlist_proto::{EndpointOptions, Instant, Node, Rng, SeedableRng, SmallRng}; +// serf's driving-core surface, re-exported from `serf-embedded` so the embassy API +// is self-contained: the event set, the resolver result + admission predicates, the +// transform config, the await-result join types, and serf's config / error types. +pub use serf_embedded::{ + AliveDelegate, DEFAULT_EVENT_BUFFER_CAP, Event, JoinFailed, JoinId, LabelError, + MAX_RESOLVED_ADDRS_PER_SEED, MaybeOwned, MaybeResolved, MergeDelegate, ReachedSet, ResolvedAddrs, + SerfError, SerfOptions, TransformOptions, socket_addr_is_routable, +}; +// serf's own protocol types named in the command signatures. +pub use serf_proto::{ + endpoint::{QueryId, QueryParams}, + event::QueryEvent, + members::{Member, MemberStatus, SerfState}, + typed::Tags, +}; + +// CIDR peer-admission policy, installed via `Options::with_cidr_policy`. +#[cfg(feature = "cidr")] +#[cfg_attr(docsrs, doc(cfg(feature = "cidr")))] +pub use serf_embedded::{AddrParseError, CidrPolicy, IpNet}; +// AEAD keyring types (for a caller assembling a `TransformOptions` encryption +// policy) and the inbound key-management request/response types the driver acts on. +#[cfg(encryption)] +#[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) +)] +pub use serf_embedded::{EncryptionOptions, Keyring, SecretKey}; +#[cfg(encryption)] +#[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) +)] +pub use serf_proto::event::{KeyRequest, KeyRequestOperation, KeyResponseArgs}; diff --git a/serf-embassy/src/mailbox.rs b/serf-embassy/src/mailbox.rs new file mode 100644 index 00000000..4601241a --- /dev/null +++ b/serf-embassy/src/mailbox.rs @@ -0,0 +1,133 @@ +//! The per-slot mailbox bridging the synchronous engine +//! [`StreamIo`](serf_embedded::StreamIo) to an asynchronous embassy-net +//! `TcpSocket`. +//! +//! embassy-net's TCP API is async-only (`connect` / `accept` / `read` / `write` +//! all `.await`), but the engine drives its reliable plane through a synchronous +//! [`StreamIo`] that never awaits. The bridge is a fixed pool of slots: each slot +//! is a persistent async worker future owning one `TcpSocket`, paired with one +//! `RefCell` that both the worker (async) and the engine (sync) touch. +//! +//! The mailbox is the only shared mutable state between the two. The engine's +//! [`pump`](serf_embedded::SerfEngine::pump) is synchronous — every mailbox +//! borrow it takes completes before the pump returns and before any worker runs — +//! so the pump and the N workers are sibling futures in ONE task whose `RefCell` +//! borrows never overlap an `.await`. + +use alloc::collections::VecDeque; +use core::net::SocketAddr; + +/// A directive the engine (via [`SerfStream`](crate::stream_io::SerfStream)) +/// posts to a slot's worker, consumed on the next worker wake. +/// +/// Newtype/unit variants only: `Dial` carries the remote address; the rest are +/// state transitions with no payload. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum Command { + /// No pending directive. A free slot defaults to this; the worker parks until + /// the engine posts a `Listen` / `Dial` (see [`crate::worker`]). + Idle, + /// Begin listening for an inbound connection on the given local port. + Listen(u16), + /// Begin dialing the given remote address. + Dial(SocketAddr), + /// Gracefully close (FIN, drain, then mark the slot closed). + Close, + /// Abort immediately (RST, discard buffered data). + Abort, +} + +/// The shared per-slot bridge state. +/// +/// One `RefCell` exists per pool slot. The engine reads/writes it +/// synchronously inside `pump`; the worker reads/writes it around its socket +/// awaits. The byte rings (`inbound` / `outbound`) are bounded — see +/// [`Mailbox::new`] — so a stalled peer cannot grow them without bound. +pub(crate) struct Mailbox { + /// The directive the worker should act on next. The engine sets it; the worker + /// consumes it (resetting to [`Command::Idle`] once acted on). + pub command: Command, + /// Whether the slot's connection has completed its handshake (worker set on a + /// successful `connect` / `accept`). Mirrors `StreamIo::is_established`. + pub established: bool, + /// Whether the slot is still usable (`false` once the connection is + /// closed/aborted/reset or a dial failed). The engine reaps a slot whose + /// `open` is `false` via `StreamIo::is_open`. + pub open: bool, + /// The remote address of an accepted/connected peer, set by the worker ONLY + /// once established with a known `remote_endpoint`. Mirrors smoltcp's accept + /// gate (`StreamIo::accepted_peer`). + pub accepted_peer: Option, + /// Whether the worker observed the peer's FIN (a `read()` returning `Ok(0)` / + /// the `Finished` receive state). Combined with a drained `inbound`, this is + /// the one-shot EOF the engine reads via `StreamIo::recv_finished`. + pub peer_fin: bool, + /// The worker's mirror of `TcpSocket::send_queue()` — bytes written to the TX + /// ring the peer has not yet ACKed. Feeds `StreamIo::send_queue` so the + /// engine's drain-before-close gate is faithful. + pub sock_send_queue: usize, + /// Bytes the worker `read()` from the socket, awaiting `StreamIo::recv`. + pub inbound: VecDeque, + /// Bytes the engine `send()` enqueued, awaiting the worker's `write()`. + pub outbound: VecDeque, + /// Capacity bound for `inbound` (the configured TCP socket RX byte size). + pub inbound_cap: usize, + /// Capacity bound for `outbound` (the configured TCP socket TX byte size). + pub outbound_cap: usize, + /// Whether the slot's socket is in a clean, freshly-reset Closed state and so + /// is safe to reuse for a new `Listen` / `Dial`. + /// + /// `true` for a fresh slot (a newly-created `TcpSocket` is Closed) and after the + /// worker's `reset_socket` (`abort()` + flush, then [`Mailbox::reset`]); `false` + /// from the instant the worker picks up a `Listen` / `Dial` (the socket is now in + /// use) until that reset completes. The engine reads it via + /// [`SerfStream::reuse_ready`](crate::stream_io::SerfStream) so it never + /// re-`listen`s / re-`connect`s a slot whose teardown the worker has not yet run — + /// which would clobber the pending `Abort`/`Close` and leak the prior connection's + /// `open` / `accepted_peer` / buffers into the reused slot. This is the + /// acknowledged reset→idle transition that makes the async teardown safe. + pub reset_done: bool, +} + +impl Mailbox { + /// Build a fresh, idle mailbox with the given ring capacities. + /// + /// `inbound_cap` / `outbound_cap` come from the driver's + /// [`Options`](crate::Options) (the TCP socket RX/TX byte sizes), so the bridge + /// rings never hold more than a socket buffer's worth of un-handed-off bytes. + pub(crate) fn new(inbound_cap: usize, outbound_cap: usize) -> Self { + Self { + command: Command::Idle, + established: false, + open: false, + accepted_peer: None, + peer_fin: false, + sock_send_queue: 0, + inbound: VecDeque::new(), + outbound: VecDeque::new(), + inbound_cap, + outbound_cap, + // A freshly-created `TcpSocket` is Closed, so the slot is immediately + // reuse-ready for its first `Listen` / `Dial`. + reset_done: true, + } + } + + /// Reset every per-connection field to the fresh/idle state, preserving the + /// configured ring capacities. Called by the worker after a slot's connection + /// breaks (close/abort/reset) so the slot is clean for its next reuse, and at + /// the same point the engine returns the slot to its free-list. + pub(crate) fn reset(&mut self) { + self.command = Command::Idle; + self.established = false; + self.open = false; + self.accepted_peer = None; + self.peer_fin = false; + self.sock_send_queue = 0; + self.inbound.clear(); + self.outbound.clear(); + // The worker calls `reset()` only AFTER it has `abort()`ed + flushed the + // socket back to a clean Closed state, so the slot is now reuse-ready. + self.reset_done = true; + } +} diff --git a/serf-embassy/src/resolver.rs b/serf-embassy/src/resolver.rs new file mode 100644 index 00000000..0196e1f9 --- /dev/null +++ b/serf-embassy/src/resolver.rs @@ -0,0 +1,60 @@ +//! Asynchronous address resolution: the [`AddressResolver`] trait and the trivial +//! [`SocketAddrResolver`]. +//! +//! Resolution happens only at the driver boundary — construction (the local +//! advertise address) and [`join`](crate::Serf::join) (the seeds) — so a resolver +//! is passed to those methods rather than stored on the [`Serf`](crate::Serf) +//! handle. Everything past the boundary is a wire [`SocketAddr`]; the embedded +//! engine only ever sees resolved addresses. + +use core::net::SocketAddr; + +use serf_embedded::ResolvedAddrs; + +/// Asynchronously resolves an unresolved address (e.g. a `host:port` name) into +/// candidate wire [`SocketAddr`]s. +/// +/// Async to match embassy's async runtime, but free of `Send`/`Sync`/`'static` +/// bounds: the embassy driver runs on a single executor and shares its state by +/// [`Rc`](alloc::rc::Rc), so a resolver is never moved across threads. The +/// resolver returns zero or more candidates, since one name may map to several +/// A/AAAA records. +/// +/// The result is a [`ResolvedAddrs`]: a bounded, no-heap collection capped at +/// [`MAX_RESOLVED_ADDRS_PER_SEED`](serf_embedded::MAX_RESOLVED_ADDRS_PER_SEED). +/// The cap is enforced by the type, so a resolver cannot hand back an unbounded +/// result for the driver to allocate and truncate after the fact — there is no +/// caller `limit` to pass or honor. +pub trait AddressResolver { + /// The unresolved address this resolver accepts. + type Address; + + /// The error returned when resolution fails. `'static` so the driver can box + /// it into a typed [`InitError`](crate::InitError) / [`JoinError`](crate::JoinError) + /// while preserving the `source()` chain. + type Error: core::error::Error + 'static; + + /// Resolves `address` into a bounded set of candidate wire addresses. + fn resolve( + &self, + address: &Self::Address, + ) -> impl core::future::Future>; +} + +/// An [`AddressResolver`] for callers that already hold wire [`SocketAddr`]s: it +/// passes each address through unchanged and never fails. +#[derive(Debug, Clone, Copy, Default)] +pub struct SocketAddrResolver; + +impl AddressResolver for SocketAddrResolver { + type Address = SocketAddr; + type Error = core::convert::Infallible; + + async fn resolve(&self, address: &SocketAddr) -> Result { + let mut addrs = ResolvedAddrs::new(); + // Ignoring Err: pushing one element onto a freshly-created, empty bounded + // vec whose capacity is MAX_RESOLVED_ADDRS_PER_SEED (8) cannot overflow. + let _ = addrs.push(*address); + Ok(addrs) + } +} diff --git a/serf-embassy/src/runner.rs b/serf-embassy/src/runner.rs new file mode 100644 index 00000000..cbfe9303 --- /dev/null +++ b/serf-embassy/src/runner.rs @@ -0,0 +1,234 @@ +//! The single-task run loop: drive the engine `pump` plus the N per-slot workers +//! as sibling futures. +//! +//! One embassy task owns the [`Runner`] and calls [`Runner::run`]. Inside, two +//! kinds of future run concurrently under one `join`: +//! +//! - the **pump loop** — re-pump the engine over a fresh +//! [`SerfGossip`](crate::SerfGossip) + [`SerfStream`](crate::SerfStream) view, +//! drain the machine's events (actioning serf's mandatory events, then buffering +//! observations), then sleep on whichever of {UDP recv-ready, a worker/handle +//! pump-wake, the folded deadline timer} fires first; +//! - the **N workers** — each [`run_slot`](crate::worker::run_slot) owns one +//! `TcpSocket` and its `RefCell`, looping internally forever. +//! +//! Only the pump loop re-pumps; the workers loop on their own. Because +//! [`SerfEngine::pump`](serf_embedded::SerfEngine::pump) is synchronous, the +//! pump's borrows of the engine and the mailboxes complete before its `.await`, +//! and each worker only borrows its own mailbox briefly around its socket awaits — +//! so the pump and the workers never hold overlapping `RefCell` borrows even +//! though they are siblings in one task. +//! +//! # Self-delivery quiescence +//! +//! embassy-net (smoltcp underneath) does not loop a self-addressed UDP datagram +//! back into recv like an OS socket, so [`SerfGossip`](crate::SerfGossip) diverts +//! a datagram this node addressed to its OWN advertise address into a driver +//! `loopback` buffer. The pump computes its deadline and egresses BEFORE the drain +//! runs, so a `respond_key` the drain just queued — or a self-addressed datagram +//! the egress just looped back — would not be reflected by that pass. The pump +//! loop therefore RE-PUMPS at the same `now` while a pass queued a key response OR +//! left the loopback non-empty (bounded by [`MAX_SELF_DELIVERY_ITERS`]), so a +//! self response is collected within the wake and a caller sleeping on the engine +//! deadline never strands it. + +use core::{cell::RefCell, net::SocketAddr}; + +use alloc::{collections::VecDeque, rc::Rc, vec::Vec}; + +use embassy_futures::{ + join::{join, join_array}, + select::{select, select3}, +}; +use embassy_net::{tcp::TcpSocket, udp::UdpSocket}; +use embassy_time::Timer; +use memberlist_proto::{Instant, Rng, SeedableRng}; + +use crate::{ + gossip_io::SerfGossip, + mailbox::Mailbox, + shared::Shared, + stream_io::{SerfStream, SlotId, SlotWake}, + time, + worker::run_slot, +}; + +/// The most pump → drain passes one wake makes to reach quiescence. +/// +/// A single wake re-pumps while a pass produced new work the current deadline / +/// egress has not yet reflected: a `respond_key` the drain just queued, or a +/// self-addressed datagram the pump's egress just looped back (see +/// [`SerfGossip`](crate::SerfGossip)). Each self query / key response settles in a +/// few passes; this caps a pathological self-delivery cycle so a single wake +/// cannot spin forever. On hitting the cap with work still pending, the loop folds +/// `now` into its returned deadline so the caller re-polls at once. +const MAX_SELF_DELIVERY_ITERS: usize = 8; + +/// Returns the earlier of two optional deadlines. If only one is `Some`, that +/// deadline wins; if both are `None` the result is `None`. +fn min_opt(a: Option, b: Option) -> Option { + match (a, b) { + (Some(x), Some(y)) => Some(core::cmp::min(x, y)), + (x, y) => x.or(y), + } +} + +/// The owned run-loop state for a node: the shared engine, the gossip UDP socket, +/// the `N` reliable-plane TCP sockets and their per-slot mailboxes + command +/// wakes, and the driver-side free-list. +/// +/// `I` is the node id type; `N` is the TCP socket pool size; `G` is the gossip RNG +/// and `SR` serf's own core RNG (both defaulting to +/// [`SmallRng`](memberlist_proto::SmallRng)). Built by +/// [`Serf::new`](crate::Serf::new), which hands back the paired [`Serf`](crate::Serf) +/// handle. +pub struct Runner< + 'a, + I, + const N: usize, + G = memberlist_proto::SmallRng, + SR = memberlist_proto::SmallRng, +> where + I: Eq + core::hash::Hash, +{ + pub(crate) shared: Rc>, + pub(crate) udp: UdpSocket<'a>, + pub(crate) tcp: [TcpSocket<'a>; N], + pub(crate) mailboxes: [RefCell; N], + pub(crate) cmd_wakes: [SlotWake; N], + /// Per-socket inactivity timeout (already in the embassy-time tick domain) each + /// worker applies to its `TcpSocket` so a blocking connect/write/flush/read to an + /// unresponsive peer cannot wedge it. + pub(crate) socket_timeout: embassy_time::Duration, + pub(crate) free: Vec, +} + +impl Runner<'_, I, N, G, SR> +where + I: memberlist_proto::Id + Clone, + G: Rng, + SR: Rng + SeedableRng, +{ + /// Drive the node forever: pump the engine and run the `N` workers concurrently. + /// + /// Never returns under normal operation; spawn it as an embassy task (or drive + /// it with `select` against an operation in a test). + pub async fn run(self) -> ! { + let Runner { + shared, + udp, + mut tcp, + mailboxes, + cmd_wakes, + socket_timeout, + mut free, + } = self; + + // The driver-owned self-delivery buffer: gossip datagrams this node addressed + // to its own advertise address, replayed into the next pump's ingress. Lives + // for the whole run so a self response queued in one wake is drained in the + // same wake's re-pump loop. + let loopback: RefCell>> = RefCell::new(VecDeque::new()); + + // Build the N worker futures, each owning a distinct `&mut TcpSocket` (via + // `each_mut`, which yields `N` non-aliasing mutable refs) paired with its + // mailbox and command wake by index, plus the shared pump wake. + let mut socket_iter = tcp.each_mut().into_iter(); + let workers = core::array::from_fn::<_, N, _>(|i| { + let sock = socket_iter + .next() + .expect("from_fn yields indices 0..N and the socket array has exactly N elements"); + run_slot( + sock, + &mailboxes[i], + &cmd_wakes[i], + &shared.pump_wake, + socket_timeout, + ) + }); + + // The pump loop and the workers run as siblings under one join. The pump loop + // diverges (loops forever); `join_array` of the diverging workers likewise + // never completes, so `join` never resolves — matching the `-> !` contract. + join( + pump_loop(&shared, &udp, &mailboxes, &cmd_wakes, &mut free, &loopback), + join_array(workers), + ) + .await; + + core::unreachable!("the run loop and workers never complete") + } +} + +/// The engine-pump half of [`Runner::run`]: re-pump to self-delivery quiescence on +/// each wake, actioning serf's mandatory events and folding join completions. +async fn pump_loop( + shared: &Shared, + udp: &UdpSocket<'_>, + mailboxes: &[RefCell], + cmd_wakes: &[SlotWake], + free: &mut Vec, + loopback: &RefCell>>, +) -> ! +where + I: memberlist_proto::Id + Clone, + G: Rng, + SR: Rng + SeedableRng, +{ + let advertise: SocketAddr = shared.advertise; + loop { + let now = time::now(); + + // Pump → drain, re-running at the same `now` until neither a queued key + // response nor a looped-back self-datagram remains, so both are handled within + // this wake. The gossip and stream views are synchronous: their borrows of the + // engine and mailboxes complete inside the block, before any `.await`. + let mut next = None; + let mut settled = false; + for _ in 0..MAX_SELF_DELIVERY_ITERS { + next = { + let mut gossip = SerfGossip::new(udp, loopback, advertise); + let mut stream = SerfStream::new(mailboxes, cmd_wakes, free); + shared + .engine + .borrow_mut() + .pump(now, &mut gossip, &mut stream) + }; + // Action serf's mandatory events (Shutdown → stop, KeyRequest → live-keyring + // chokepoint) and buffer every observation. Returns whether a key response + // was queued the egress above did not see. + let queued = shared.drain_events(now); + if !queued && loopback.borrow().is_empty() { + settled = true; + break; + } + } + + // On an unsettled loop (work still pending at the cap), fold `now` into the + // deadline so the next wake re-pumps at once rather than sleeping past it. + if !settled { + next = min_opt(next, Some(now)); + } + + // Wait for the next thing worth re-pumping for: an inbound gossip datagram, a + // worker/handle pump-wake, or the folded deadline. + match next { + Some(deadline) => { + let raw = time::machine_to_raw(deadline); + // Ignoring the `Either3`: which arm woke is irrelevant — any wake re-runs + // the whole pump, which re-derives all work and the next deadline. + let _ = select3( + udp.wait_recv_ready(), + shared.pump_wake.wait(), + Timer::at(raw), + ) + .await; + } + // No scheduled machine work: wake only on I/O or a pump-wake. + None => { + // Ignoring the `Either`: as above, any wake simply re-runs the pump. + let _ = select(udp.wait_recv_ready(), shared.pump_wake.wait()).await; + } + } + } +} diff --git a/serf-embassy/src/serf/mod.rs b/serf-embassy/src/serf/mod.rs new file mode 100644 index 00000000..cf2df089 --- /dev/null +++ b/serf-embassy/src/serf/mod.rs @@ -0,0 +1,818 @@ +//! The cloneable async [`Serf`] handle and node construction. +//! +//! [`Serf::new`] builds the shared [`SerfEngine`](serf_embedded::SerfEngine) over +//! the caller's embassy-net sockets, seeds the reliable-plane pool with the `N` +//! TCP slots, installs the first listener, and hands back the handle paired with +//! the [`Runner`] the caller drives. The handle is a thin shared reference +//! (`Rc`); clone it freely to issue commands and read membership from +//! multiple places on the single executor. +//! +//! The async ops (`join`) enqueue work on the engine and park on the run loop's +//! signals; the sync commands (`user_event` / `query` / `set_tags` / key +//! management) enqueue and wake the pump; the sync accessors borrow the engine +//! directly. serf's mandatory driver-actioned events (a lost-conflict +//! [`Event::Shutdown`], an inbound [`Event::KeyRequest`]) are handled by the +//! [`Runner`] in its post-pump drain, before the observation is buffered — so the +//! handle only ever OBSERVES them via [`poll_event`](Serf::poll_event). + +use core::{marker::PhantomData, net::SocketAddr}; + +use alloc::{boxed::Box, rc::Rc, vec::Vec}; +use std::sync::Arc; + +use embassy_futures::select::select; +use embassy_net::{tcp::TcpSocket, udp::UdpSocket}; +use embassy_time::Timer; +use memberlist_proto::{EndpointOptions, Instant, Rng, SeedableRng, SmallRng}; +use serf_embedded::{ + Event, JoinId, MaybeResolved, ReachedSet, SerfEngine, SerfError, SerfOptions, TransformOptions, + validate_runtime_config, +}; +use serf_proto::{ + endpoint::{QueryId, QueryParams}, + event::QueryEvent, + members::{Member, SerfState}, + typed::Tags, +}; + +#[cfg(encryption)] +use serf_embedded::{Keyring, SecretKey}; + +use crate::{ + config::Options, + error::{InitError, JoinError, SocketTimeoutOutOfRange}, + mailbox::{Command, Mailbox}, + resolver::AddressResolver, + runner::Runner, + shared::Shared, + stream_io::{SlotId, SlotWake}, + time, +}; + +/// The largest [`Options::socket_timeout`](crate::Options::socket_timeout) +/// [`Serf::new`] accepts. A per-socket inactivity backstop longer than a day is +/// nonsensical for serf (reliable exchanges complete in milliseconds), and +/// rejecting larger values keeps the timeout safely within EVERY downstream duration +/// domain — the `embassy_time` tick count, its `as_micros` conversion (which +/// multiplies before dividing), and smoltcp's `i64` `Instant` arithmetic — at ANY +/// tick rate, so no configurable value can overflow that chain into a wrapped, +/// effectively-past deadline that would abort a TCP slot immediately. +const MAX_SOCKET_TIMEOUT: core::time::Duration = core::time::Duration::from_secs(86_400); + +/// Floor a portable `core::Duration` to whole `embassy_time` ticks at `tick_hz`, +/// exactly as [`embassy_time::Duration::from_ticks`] will store it. +/// +/// Pure and parameterized on `tick_hz` (rather than reading the +/// [`embassy_time::TICK_HZ`] constant directly) so the coarse-rate rounding behavior is +/// unit-testable without rebuilding `embassy-time` at another tick rate. The nanosecond +/// basis keeps the conversion exact at fine tick rates (a microsecond basis would +/// silently drop sub-microsecond resolution); the `u128` saturating multiply and `u64` +/// clamp keep it total for any input, so an out-of-range duration converts to a +/// saturated tick count rather than panicking. +fn duration_to_ticks(d: core::time::Duration, tick_hz: u128) -> u64 { + let ticks = d.as_nanos().saturating_mul(tick_hz) / 1_000_000_000; + u64::try_from(ticks).unwrap_or(u64::MAX) +} + +/// The whole-microsecond timeout embassy-net actually installs into smoltcp for an +/// already-floored embassy tick count at `tick_hz`. +/// +/// embassy-net hands smoltcp `embassy_time::Duration::as_micros()` — a SECOND floor on +/// top of the tick flooring: at a tick rate finer than 1 MHz the tick count carries +/// sub-microsecond resolution this floor discards. Reproduced here (saturating `u128`) so +/// validation reasons about the value smoltcp receives, microseconds, not the +/// intermediate tick count. +fn installed_micros(ticks: u64, tick_hz: u128) -> u128 { + u128::from(ticks).saturating_mul(1_000_000) / tick_hz +} + +/// Validate `socket_timeout` against the engine deadlines IN THE INSTALLED MICROSECOND +/// DOMAIN at `tick_hz`, returning the embassy tick count the worker installs when it is +/// in range. +/// +/// The value that actually gates the TCP socket is `socket_timeout` floored twice — to +/// whole embassy ticks (`from_ticks`) and then to whole microseconds (embassy-net's +/// `as_micros`, see [`installed_micros`]). The ordering invariant must hold on that +/// microsecond value, not on the portable input or the intermediate tick count: a coarse +/// tick rate can floor a portable-valid value below a deadline, and a tick rate finer than +/// 1 MHz can clear a tick comparison yet still install the same (or zero) microsecond +/// value. The deadlines are floored to microseconds too — the engine enforces them at full +/// resolution, but an installed whole-microsecond timeout exceeds the real deadline exactly +/// when it exceeds the deadline's microsecond floor. `socket_us > close_us` with +/// non-negative `close_us` also forces at least one installed microsecond, so an accepted +/// timeout is never the zero value smoltcp treats as an immediate abort. The upper bound is +/// checked first, in the portable domain, so the conversions cannot overflow (see +/// [`MAX_SOCKET_TIMEOUT`]). +fn checked_socket_timeout( + socket: core::time::Duration, + close: core::time::Duration, + stream: core::time::Duration, + tick_hz: u128, +) -> Option { + if socket > MAX_SOCKET_TIMEOUT { + return None; + } + let socket_ticks = duration_to_ticks(socket, tick_hz); + let socket_us = installed_micros(socket_ticks, tick_hz); + (socket_us > close.as_micros() && socket_us > stream.as_micros()).then_some(socket_ticks) +} + +/// Assemble the [`serf_embedded::Options`] the engine reads from the driver's +/// [`crate::Options`]. +/// +/// The driver's `crate::Options` carries link-layer sizing (the bridge ring +/// capacities, the per-socket timeout) that stays on the driver, while +/// `serf_embedded::Options` carries only the port and close timeout (plus the CIDR +/// policy) the engine reads directly. Built once, up front, so the same value +/// drives both the construction preflight +/// ([`serf_embedded::validate_runtime_config`]) and the engine itself. +fn embedded_options(cfg: &Options) -> serf_embedded::Options { + let opts = serf_embedded::Options::new() + .with_port(cfg.port) + .with_close_timeout(cfg.close_timeout); + #[cfg(feature = "cidr")] + let opts = match cfg.cidr_policy.clone() { + Some(policy) => opts.with_cidr_policy(policy), + None => opts, + }; + opts +} + +/// Cancel an in-flight await-result join if its future is dropped before the join +/// resolves (a `select` timeout, or the caller abandoning the await). +/// +/// [`SerfEngine::join`](serf_embedded::SerfEngine::join) mints a [`JoinId`] whose +/// entry the engine reaps only once the caller polls or cancels it. A join future +/// dropped mid-await therefore leaves that entry lingering; this guard cancels the +/// join on drop so no abandoned join accumulates. On a resolved join the awaiting +/// method [`disarm`](Self::disarm)s the guard first, so a completed join is never +/// double-cancelled. Drop runs at a suspension point where no engine borrow is +/// live (borrows never span an `.await`), so the `borrow_mut` here cannot alias. +struct JoinGuard<'a, I, G, SR> +where + I: memberlist_proto::Id + Clone, + G: Rng, + SR: Rng + SeedableRng, +{ + shared: &'a Shared, + id: Option, +} + +impl JoinGuard<'_, I, G, SR> +where + I: memberlist_proto::Id + Clone, + G: Rng, + SR: Rng + SeedableRng, +{ + /// Disarm the guard so a resolved join is not cancelled on drop. + fn disarm(&mut self) { + self.id = None; + } +} + +impl Drop for JoinGuard<'_, I, G, SR> +where + I: memberlist_proto::Id + Clone, + G: Rng, + SR: Rng + SeedableRng, +{ + fn drop(&mut self) { + if let Some(id) = self.id.take() { + self.shared.engine.borrow_mut().cancel_join(id); + } + } +} + +/// A cloneable handle to an embassy-net serf node. +/// +/// Holds a shared reference to the node's +/// [`SerfEngine`](serf_embedded::SerfEngine) and the run loop's coordination +/// signals. `join` enqueues work on the engine and awaits the run loop; the sync +/// commands enqueue and wake the pump; the sync accessors borrow the engine +/// directly. Every method takes `&self`, so the handle is shared across the +/// executor's tasks. +/// +/// `I` is the node identifier type (e.g. `smol_str::SmolStr`). `A` is the +/// resolver's unresolved address type — the advertise address is resolved to a +/// wire [`SocketAddr`] at construction and the seeds at [`join`](Self::join), so +/// the engine only ever sees `SocketAddr`. `G` is the memberlist gossip RNG and +/// `SR` is serf's own core RNG (both defaulting to [`SmallRng`]); the two are +/// seeded independently so fresh nodes never share a query-id schedule. +pub struct Serf +where + I: memberlist_proto::Id, +{ + shared: Rc>, + // Ties the handle to the resolver's unresolved address type. `fn(A)` keeps the + // marker contravariant in `A` and free of drop/auto-trait obligations. + _a: PhantomData, +} + +impl Clone for Serf +where + I: memberlist_proto::Id, +{ + fn clone(&self) -> Self { + Self { + shared: self.shared.clone(), + _a: PhantomData, + } + } +} + +impl Serf +where + I: memberlist_proto::Id + Clone, +{ + /// Construct a node over the caller's embassy-net sockets, returning the handle + /// and the [`Runner`] to drive. + /// + /// The caller owns the embassy-net [`Stack`](embassy_net::Stack) and supplies a + /// gossip [`UdpSocket`] and the reliable-plane pool of `N` [`TcpSocket`]s. `new` + /// binds the UDP socket to `cfg.port`, wires up the transport-agnostic + /// [`SerfEngine`](serf_embedded::SerfEngine), seeds the reliable-plane pool with + /// the `N` slots, dedicates one to the listener, and arms serf's schedulers. No + /// I/O occurs here. Drive the returned [`Runner`] with [`Runner::run`] and the + /// embassy-net stack `Runner` separately. + /// + /// Seeds BOTH the gossip RNG and serf's core RNG from independent draws of the + /// platform [`getrandom`] backend; use [`new_with_rng`](Self::new_with_rng) to + /// inject your own. + /// + /// # Errors + /// + /// As [`new_with_rng`](Self::new_with_rng), plus [`InitError::Entropy`] if the + /// platform entropy backend fails while seeding the default RNGs. + #[allow(clippy::too_many_arguments)] + pub async fn new<'a, Res, const N: usize>( + cfg: Options, + transform: TransformOptions, + ep_cfg: EndpointOptions, + serf_opts: SerfOptions, + resolver: &Res, + udp_socket: UdpSocket<'a>, + tcp_sockets: [TcpSocket<'a>; N], + now: Instant, + ) -> Result<(Self, Runner<'a, I, N, SmallRng, SmallRng>), InitError> + where + Res: AddressResolver
, + { + // Draw two independent 64-bit seeds from the platform entropy backend — one + // for the gossip RNG, one for serf's core RNG — so a fresh node never shares a + // `(ltime, id)` query-id schedule with a peer. + let mut b = [0u8; 16]; + getrandom::fill(&mut b).map_err(|_| InitError::Entropy)?; + let word = |i: usize| { + u64::from_le_bytes([ + b[i], + b[i + 1], + b[i + 2], + b[i + 3], + b[i + 4], + b[i + 5], + b[i + 6], + b[i + 7], + ]) + }; + Self::new_with_rng( + cfg, + transform, + ep_cfg, + serf_opts, + resolver, + udp_socket, + tcp_sockets, + now, + SmallRng::seed_from_u64(word(0)), + SmallRng::seed_from_u64(word(8)), + ) + .await + } +} + +impl Serf +where + I: memberlist_proto::Id + Clone, + SR: SeedableRng, +{ + /// Like [`new`](Self::new) but with caller-supplied gossip + serf RNGs, returning + /// the handle and the [`Runner`] to drive. + /// + /// Identical to [`new`](Self::new) except the RNGs: the caller owns seeding both + /// `gossip_rng` (memberlist peer selection / timing jitter) and `serf_rng` + /// (serf's query ids / relay selection), so no platform entropy is drawn here. + /// + /// # Errors + /// + /// - [`InitError::TcpPoolTooSmall`] — `N < 2` (a listener plus one dial/accept + /// socket is the functional minimum). + /// - [`InitError::ZeroBridgeRing`] — a zero bridge ring capacity. + /// - [`InitError::SocketTimeoutOutOfRange`] — the socket timeout, as installed + /// into smoltcp, is not strictly greater than both `close_timeout` and the + /// machine's `stream_timeout`, or is above the safe maximum. + /// - [`InitError::Resolve`] / [`InitError::NoAddresses`] — advertise resolution. + /// - [`InitError::Engine`] — the shared engine rejected the configuration. + /// + /// # Panics + /// + /// Panics if binding the supplied `udp_socket` to `cfg.port` fails — which, with + /// a non-zero port and a fresh socket, embassy-net does not do. + #[allow(clippy::too_many_arguments)] + pub async fn new_with_rng<'a, Res, const N: usize>( + cfg: Options, + transform: TransformOptions, + ep_cfg: EndpointOptions, + serf_opts: SerfOptions, + resolver: &Res, + mut udp_socket: UdpSocket<'a>, + tcp_sockets: [TcpSocket<'a>; N], + now: Instant, + gossip_rng: G, + serf_rng: SR, + ) -> Result<(Self, Runner<'a, I, N, G, SR>), InitError> + where + Res: AddressResolver
, + // Arming the schedulers (`SerfEngine::start`, below) draws from both RNGs. + G: Rng, + SR: Rng, + { + // Deterministic local-config guards first, before resolving or binding. + if N < 2 { + return Err(InitError::TcpPoolTooSmall(N)); + } + if cfg.tcp_socket_rx_bytes == 0 || cfg.tcp_socket_tx_bytes == 0 { + return Err(InitError::ZeroBridgeRing); + } + // The per-socket inactivity timeout must fire strictly AFTER the engine's own + // deadlines, enforced on the whole-microsecond value embassy-net installs into + // smoltcp (see `checked_socket_timeout`). + let stream_timeout = ep_cfg.stream_timeout(); + let socket_ticks = checked_socket_timeout( + cfg.socket_timeout, + cfg.close_timeout, + stream_timeout, + embassy_time::TICK_HZ as u128, + ) + .ok_or(InitError::SocketTimeoutOutOfRange( + SocketTimeoutOutOfRange { + socket_timeout: cfg.socket_timeout, + close_timeout: cfg.close_timeout, + stream_timeout, + max: MAX_SOCKET_TIMEOUT, + tick_hz: embassy_time::TICK_HZ, + }, + ))?; + let socket_timeout = embassy_time::Duration::from_ticks(socket_ticks); + + // Engine advertise-independent preflight before resolving / binding, so a zero + // port or over-ceiling MTU fails deterministically. + let embedded_cfg = embedded_options(&cfg); + validate_runtime_config(&embedded_cfg, &transform, ep_cfg.gossip_mtu()) + .map_err(InitError::from)?; + + // Resolve the advertise address into a single wire `SocketAddr`, then re-type + // `ep_cfg` so the rest of construction — and the engine — only sees the resolved + // address. + let resolved = resolver + .resolve(ep_cfg.advertise_addr_ref()) + .await + .map_err(|e| InitError::Resolve(Box::new(e)))? + .into_iter() + .next() + .ok_or(InitError::NoAddresses)?; + let ep_cfg = ep_cfg.map_advertise(|_| resolved); + let advertise = *ep_cfg.advertise_addr_ref(); + + // Bind the gossip socket. The preflight rejected port 0, so with a fresh socket + // this cannot fail; a misuse (already-bound socket) is a programming error. + udp_socket + .bind(cfg.port) + .expect("binding the gossip UDP socket to the configured port failed"); + + // Build the engine, seeding BOTH RNGs. `try_new_at_with_rng` maps a + // machine/keyring/advertise failure to a typed `InitError`. + let mut engine: SerfEngine = SerfEngine::try_new_at_with_rng( + embedded_cfg, + transform, + ep_cfg, + serf_opts, + now, + gossip_rng, + serf_rng, + ) + .map_err(InitError::from)?; + + // Seed the reliable-plane pool with every slot id, then dedicate one to the + // listener. The engine owns this pool (it reaches it directly, not through the + // `StreamIo` view), exactly like the smoltcp driver. + for i in 0..N { + engine.plane_mut().pool.push(SlotId(i)); + } + + // Per-slot mailboxes + command wakes; the ring capacities come from the driver + // config so a slot's bridge never holds more than a socket buffer's worth of + // un-handed-off bytes. + let mailboxes: [_; N] = core::array::from_fn(|_| { + core::cell::RefCell::new(Mailbox::new( + cfg.tcp_socket_rx_bytes, + cfg.tcp_socket_tx_bytes, + )) + }); + let cmd_wakes: [SlotWake; N] = core::array::from_fn(|_| SlotWake::new()); + + // Dedicate one pooled slot to the listener and post its worker a `Listen` + // directive so it begins accepting on the bound port at startup. + if let Some(listener) = engine.plane_mut().pool.take() { + mailboxes[listener.0].borrow_mut().command = Command::Listen(cfg.port); + engine.set_listener(listener); + } + + // Arm serf's periodic probe / gossip / push-pull schedulers so they are live + // from the first pump. + engine.start(now); + + let shared = Rc::new(Shared::new(engine, advertise)); + let runner = Runner { + shared: shared.clone(), + udp: udp_socket, + tcp: tcp_sockets, + mailboxes, + cmd_wakes, + socket_timeout, + // The engine owns and drives its own pool, so the driver-side free-list + // starts empty (and stays unused — see `SerfStream`'s pool methods). + free: Vec::new(), + }; + + Ok(( + Self { + shared, + _a: PhantomData, + }, + runner, + )) + } +} + +// Pure driver / reliable-plane reads — needing neither RNG. +impl Serf +where + I: memberlist_proto::Id + Clone, +{ + /// The local node's advertised `SocketAddr`. + #[inline] + pub fn advertise_address(&self) -> SocketAddr { + self.shared.advertise + } + + /// Whether the run loop has observed a lost id-conflict [`Event::Shutdown`] and + /// the node should stop. + #[inline] + pub fn is_shutdown(&self) -> bool { + self.shared.is_shutdown() + } + + /// Drain one application-visible serf event the run loop buffered, mandatory + /// driver-actioned events first (the run loop has ALREADY acted on them), then + /// passive observations. `None` when the queue is empty. + #[inline] + pub fn poll_event(&self) -> Option> { + self.shared.pop_app_event() + } + + /// Number of inbound reliable connections accepted since construction. + #[doc(hidden)] + #[inline] + pub fn accepted_inbound_count(&self) -> u64 { + self.shared.engine.borrow().accepted_inbound_count() + } + + /// Number of pooled TCP slots currently free. + #[doc(hidden)] + #[inline] + pub fn pool_free_count(&self) -> usize { + self.shared.engine.borrow().pool_free_count() + } + + /// Number of TCP slots currently parked mid-close. + #[doc(hidden)] + #[inline] + pub fn closing_count(&self) -> usize { + self.shared.engine.borrow().closing_count() + } + + /// Number of reliable exchanges currently half-closed. + #[doc(hidden)] + #[inline] + pub fn half_closed_count(&self) -> usize { + self.shared.engine.borrow().half_closed_count() + } + + /// Whether a passive-open listener slot is currently installed. + #[doc(hidden)] + #[inline] + pub fn listener_present(&self) -> bool { + self.shared.engine.borrow().listener_present() + } + + /// Number of reliable exchanges still in `PendingDial`. + #[doc(hidden)] + #[inline] + pub fn pending_dial_count(&self) -> usize { + self.shared.engine.borrow().pending_dial_count() + } + + /// Number of await-result joins currently tracked. + #[doc(hidden)] + #[inline] + pub fn pending_join_count(&self) -> usize { + self.shared.engine.borrow().pending_join_count() + } +} + +// serf reads + command surface + the async join — reach serf's super-machine, so +// they carry the gossip `G: Rng` and serf's `SR: Rng + SeedableRng` bounds. The +// engine's connection handle is the concrete `SlotId` (Copy + Eq + Hash), so the +// pump's `C` bound is satisfied without an extra parameter. +impl Serf +where + I: memberlist_proto::Id + Clone, + G: Rng, + SR: Rng + SeedableRng, +{ + /// serf's current lifecycle state. + #[inline] + pub fn state(&self) -> SerfState { + self.shared.engine.borrow().state() + } + + /// Number of serf members currently tracked (including the local node). + #[inline] + pub fn num_members(&self) -> usize { + self.shared.engine.borrow().num_members() + } + + /// A snapshot of every serf member currently tracked (alive, leaving, left, or + /// failed within the reap window). + #[inline] + pub fn members(&self) -> Vec>> { + self.shared.engine.borrow().members_snapshot() + } + + /// The local node's id. + #[inline] + pub fn local_id(&self) -> I { + self.shared.engine.borrow().local_id().clone() + } + + /// The local node's serf member Lamport clock. + #[inline] + pub fn member_time(&self) -> u64 { + self.shared.engine.borrow().member_time() + } + + /// The local node's serf event Lamport clock. + #[inline] + pub fn event_time(&self) -> u64 { + self.shared.engine.borrow().event_time() + } + + /// The local node's serf query Lamport clock. + #[inline] + pub fn query_time(&self) -> u64 { + self.shared.engine.borrow().query_time() + } + + /// The number of app events shed because a consumer did not keep up: the engine's + /// own passive-observation drops plus this driver's + /// [`poll_event`](Self::poll_event) backlog drops. + #[inline] + pub fn events_dropped(&self) -> u64 { + self.shared.events_dropped() + } + + /// Announce the local node's join intent and await the result: resolve each seed, + /// dispatch an await-result join on the engine, and await its outcome. + /// + /// Each seed is resolved through `resolver` (a [`MaybeResolved::Resolved`] address + /// is used verbatim, a [`MaybeResolved::Unresolved`] one is expanded into the wire + /// addresses the resolver yields). The run loop initiates a push/pull to each + /// routable seed and folds every completion into the join; this call resolves once + /// every dispatched push/pull has terminated: `Ok(ReachedSet)` with the reached + /// addresses if any seed was contacted, else [`JoinError::Failed`]. If the future + /// is dropped before it resolves (e.g. a `select` timeout), the in-flight join is + /// cancelled so nothing leaks. + /// + /// When `ignore_old` is set, each seed's push/pull suppresses replay of the peer's + /// pre-join user events. + /// + /// # Errors + /// + /// [`JoinError::Control`] when the engine rejects the join (e.g. the node is not + /// running), [`JoinError::Resolve`] on a resolver failure, [`JoinError::NoAddresses`] + /// when a non-empty seed set resolves to no address, or [`JoinError::Failed`] when + /// every dispatched push/pull terminated without contacting a seed. + pub async fn join( + &self, + resolver: &Res, + seeds: &[MaybeResolved], + ignore_old: bool, + ) -> Result + where + Res: AddressResolver
, + { + let now = time::now(); + let mut resolved = Vec::with_capacity(seeds.len()); + for seed in seeds { + match seed { + MaybeResolved::Resolved(s) => resolved.push(*s), + MaybeResolved::Unresolved(a) => resolved.extend( + resolver + .resolve(a) + .await + .map_err(|e| JoinError::Resolve(Box::new(e)))?, + ), + } + } + if !seeds.is_empty() && resolved.is_empty() { + return Err(JoinError::NoAddresses); + } + + let handle = self + .shared + .engine + .borrow_mut() + .join(&resolved, ignore_old, now) + .map_err(JoinError::Control)?; + self.shared.wake_pump(); + + // Cancel the in-flight join if this future is dropped before it resolves. + let mut guard = JoinGuard { + shared: &self.shared, + id: Some(handle), + }; + + // Park until the pump folds the terminal completion into the join. The run loop + // pulses `join_wake` after every drain; race it against a short timer so a wake + // the single-consumer signal delivered to another concurrent joiner only costs + // an interval, never a hang. + loop { + let outcome = self.shared.engine.borrow_mut().poll_join(handle); + if let Some(outcome) = outcome { + // Resolved: do not cancel on drop. + guard.disarm(); + return outcome.map_err(JoinError::Failed); + } + // Ignoring the `Either`: whichever of the join wake or the timer fired, the + // loop simply re-checks `poll_join`. + let _ = select( + self.shared.join_wake.wait(), + Timer::after(embassy_time::Duration::from_millis(20)), + ) + .await; + } + } + + /// Begin leaving the cluster. Gossips the departure and ultimately emits + /// [`Event::LeftCluster`] via [`poll_event`](Self::poll_event). + /// + /// Returns [`SerfError`] if the node is not in a running state (already left or a + /// refused leave). + pub fn leave(&self) -> Result<(), SerfError> { + let now = time::now(); + let r = self.shared.engine.borrow_mut().leave(now); + self.shared.wake_pump(); + r + } + + /// Force a named node out of the cluster (an operator-driven removal). + pub fn force_leave(&self, id: I, prune: bool) -> Result<(), SerfError> { + let now = time::now(); + let r = self.shared.engine.borrow_mut().force_leave(id, prune, now); + self.shared.wake_pump(); + r + } + + /// Broadcast an application user event to the cluster. `coalesce` requests that + /// identical events be coalesced by name. Peers observe it as [`Event::User`]. + pub fn user_event( + &self, + name: impl Into, + payload: bytes::Bytes, + coalesce: bool, + ) -> Result<(), SerfError> { + let r = self + .shared + .engine + .borrow_mut() + .user_event(name, payload, coalesce); + self.shared.wake_pump(); + r + } + + /// Issue a cluster-wide query, returning its [`QueryId`]. Responders observe it as + /// [`Event::Query`] and answer via [`respond`](Self::respond). + pub fn query( + &self, + name: impl Into, + payload: bytes::Bytes, + params: QueryParams, + ) -> Result { + let now = time::now(); + let r = self + .shared + .engine + .borrow_mut() + .query(name, payload, params, now); + self.shared.wake_pump(); + r + } + + /// Answer a received query. `token` is the [`QueryEvent`] delivered via + /// [`Event::Query`]. + pub fn respond( + &self, + token: &QueryEvent, + payload: bytes::Bytes, + ) -> Result<(), SerfError> { + let now = time::now(); + let r = self.shared.engine.borrow_mut().respond(token, payload, now); + self.shared.wake_pump(); + r + } + + /// Replace the local node's tags, re-advertising them and refreshing the local + /// member in the membership store. + pub fn set_tags(&self, tags: Tags) -> Result<(), SerfError> { + let r = self.shared.engine.borrow_mut().set_tags(tags); + self.shared.wake_pump(); + r + } + + /// Issue a cluster-wide `install_key` query to add `key` to every node's keyring. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn install_key(&self, key: SecretKey) -> Result { + let now = time::now(); + let r = self.shared.engine.borrow_mut().install_key(key, now); + self.shared.wake_pump(); + r + } + + /// Issue a cluster-wide `use_key` query to promote `key` to primary. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn use_key(&self, key: SecretKey) -> Result { + let now = time::now(); + let r = self.shared.engine.borrow_mut().use_key(key, now); + self.shared.wake_pump(); + r + } + + /// Issue a cluster-wide `remove_key` query to remove `key` from all nodes. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn remove_key(&self, key: SecretKey) -> Result { + let now = time::now(); + let r = self.shared.engine.borrow_mut().remove_key(key, now); + self.shared.wake_pump(); + r + } + + /// Issue a cluster-wide `list_keys` query to enumerate installed keys. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn list_keys(&self) -> Result { + let now = time::now(); + let r = self.shared.engine.borrow_mut().list_keys(now); + self.shared.wake_pump(); + r + } + + /// A clone of the node's LIVE wire keyring — the keyring the gossip and reliable + /// planes actually encrypt under, and the state an inbound [`Event::KeyRequest`] + /// rotates via the engine. `None` when the node is unencrypted. Unlike + /// [`list_keys`](Self::list_keys) (a cluster-wide query), this is a local read of + /// this node's own keyring for UI / diagnostics / tests. Returned by value because + /// the engine lives behind interior mutability. + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn keyring(&self) -> Option { + self.shared.engine.borrow().keyring().cloned() + } +} diff --git a/serf-embassy/src/shared/mod.rs b/serf-embassy/src/shared/mod.rs new file mode 100644 index 00000000..3bade455 --- /dev/null +++ b/serf-embassy/src/shared/mod.rs @@ -0,0 +1,213 @@ +//! State shared between the [`Serf`](crate::Serf) handle and the +//! [`Runner`](crate::Runner) run loop. +//! +//! Single-executor (`!Send`) cooperative sharing: the engine lives behind a +//! [`RefCell`] and the two sides coordinate through `embassy-sync` +//! [`Signal`]s. The handle borrows the engine to enqueue work and parks on a +//! signal; the Runner (the only `pump` caller) drains the machine's events each +//! loop — actioning serf's mandatory driver-actioned events (a lost-conflict +//! [`Event::Shutdown`] stops the node, an [`Event::KeyRequest`] rotates the live +//! wire keyring) BEFORE buffering the observation — and pulses the parked waiters. +//! +//! Because [`SerfEngine::pump`](serf_embedded::SerfEngine::pump) is synchronous, +//! every `RefCell` borrow either side takes completes before the next `.await`, +//! so no borrow ever spans a suspension point. + +use core::{ + cell::{Cell, RefCell}, + hash::Hash, + net::SocketAddr, +}; + +use alloc::collections::VecDeque; + +use embassy_sync::{blocking_mutex::raw::NoopRawMutex, signal::Signal}; +use memberlist_proto::{Instant, Rng, SeedableRng, SmallRng}; +use serf_embedded::{DEFAULT_EVENT_BUFFER_CAP, Event, SerfEngine}; + +use crate::stream_io::SlotId; + +/// The state both the handle and the run loop reach. +/// +/// Shared via [`Rc`](alloc::rc::Rc) (single-core cooperative). `I` is the node id +/// type; `G` is the memberlist gossip RNG and `SR` is serf's own core RNG (both +/// defaulting to [`SmallRng`]). +pub(crate) struct Shared +where + // Mandated by `SerfEngine`'s membership store keyed by `I`. Every impl bounds + // `I: Id`, which implies these; the field type needs them here. + I: Eq + Hash, +{ + /// The transport-agnostic serf driving core (serf's super-machine, the + /// reliable-plane state + pool, the gossip codec, the join/await-result queues), + /// behind interior mutability. + pub(crate) engine: RefCell>, + /// The pump loop's single wake. Producers: the handle (when it enqueues a + /// command / join / leave) AND every worker (when it advances its mailbox). + /// Sole consumer: the pump loop. A single-consumer, many-producer [`Signal`] is + /// sound because the pump drains EVERY mailbox each tick, so one pulse re-pumps + /// all pending work regardless of which producer fired it. + pub(crate) pump_wake: Signal, + /// Pulsed by the Runner after each drain (the pump folds every push/pull + /// completion into its await-result join), so a parked [`join`](crate::Serf::join) + /// re-checks `poll_join`. A parked `join` also races a short timer, so a missed + /// pulse (this `Signal` wakes only one of several concurrent joiners) costs at + /// most that interval, never a hang. + pub(crate) join_wake: Signal, + /// Application-facing events the Runner drained from the machine, buffered for + /// the handle's [`Serf::poll_event`](crate::Serf::poll_event). + /// + /// The Runner is the sole `poll_event` caller on the engine (it drains events + /// each pump to action the mandatory ones); draining is destructive, so every + /// event — including the mandatory ones the driver has ALREADY acted on — is + /// re-buffered here for the app. Bounded at [`DEFAULT_EVENT_BUFFER_CAP`] with + /// drop-oldest so a never-polling app cannot grow it without bound. + pub(crate) app_events: RefCell>>, + /// Count of app events shed from `app_events` because the app never drained + /// [`poll_event`](crate::Serf::poll_event) fast enough and the backlog hit the + /// cap. Summed with the engine's own load-shed counter by + /// [`Serf::events_dropped`](crate::Serf::events_dropped). + pub(crate) app_events_dropped: Cell, + /// Set once the Runner observed a lost id-conflict [`Event::Shutdown`]; the + /// handle reads it via [`Serf::is_shutdown`](crate::Serf::is_shutdown). + pub(crate) shutdown: Cell, + /// The local node's resolved advertise address, captured at construction (the + /// engine does not surface it). Read by the handle's `advertise_address` and by + /// the runner's gossip view (the self-delivery loopback destination). + pub(crate) advertise: SocketAddr, +} + +impl Shared +where + I: memberlist_proto::Id, +{ + /// Wrap a constructed engine as shared state with empty signals/buffers. + pub(crate) fn new(engine: SerfEngine, advertise: SocketAddr) -> Self { + Self { + engine: RefCell::new(engine), + pump_wake: Signal::new(), + join_wake: Signal::new(), + app_events: RefCell::new(VecDeque::new()), + app_events_dropped: Cell::new(0), + shutdown: Cell::new(false), + advertise, + } + } + + /// Pop one buffered application event for the handle's `poll_event`. + #[inline] + pub(crate) fn pop_app_event(&self) -> Option> { + self.app_events.borrow_mut().pop_front() + } + + /// Wake the pump loop (a handle op enqueued work). + #[inline] + pub(crate) fn wake_pump(&self) { + self.pump_wake.signal(()); + } + + /// Whether the Runner has observed a lost id-conflict [`Event::Shutdown`]. + #[inline] + pub(crate) fn is_shutdown(&self) -> bool { + self.shutdown.get() + } + + /// Buffer one event for [`poll_event`](crate::Serf::poll_event), bounding the + /// backlog at [`DEFAULT_EVENT_BUFFER_CAP`] with drop-oldest so a never-draining + /// app cannot grow memory without bound. + fn push_app_event(&self, ev: Event) { + let mut q = self.app_events.borrow_mut(); + if q.len() >= DEFAULT_EVENT_BUFFER_CAP { + q.pop_front(); + self + .app_events_dropped + .set(self.app_events_dropped.get() + 1); + } + q.push_back(ev); + } +} + +impl Shared +where + I: memberlist_proto::Id + Clone, + G: Rng, + SR: Rng + SeedableRng, +{ + /// Drain the engine's event queue (mandatory-first), take the driver-owned side + /// effect on each mandatory event, buffer every event for the app, and pulse + /// `join_wake` so parked joins re-check `poll_join`. + /// + /// Called by the Runner once per pump, AFTER `pump` (so it sees this tick's + /// freshly-emitted events). The mandatory ACTIONS run BEFORE the lossy buffering: + /// an [`Event::Shutdown`] sets the shutdown state (the node stops), and an + /// [`Event::KeyRequest`] is applied to the engine's LIVE wire keyring and + /// answered in one call through `handle_key_request` — the live-keyring + /// chokepoint — never a driver-local shadow. A dropped observation is fine; a + /// dropped action is not, so the actions are taken here regardless of the app + /// ever polling. + /// + /// Returns whether the drain queued outbound gossip work the current pump's + /// egress did not see — a key response `handle_key_request` just queued — so the + /// Runner knows to re-pump and egress it (and loop back a self-addressed one) + /// within the same tick. + pub(crate) fn drain_events(&self, now: Instant) -> bool { + // `now` and the queued-outbound signal are consumed only by the encryption + // `KeyRequest` arm; a build without an AEAD backend reads neither and queues no + // key response, so its drain never re-pumps on this account. + #[cfg(not(encryption))] + let _ = now; + #[cfg(encryption)] + let mut queued = false; + #[cfg(not(encryption))] + let queued = false; + + loop { + let ev = self.engine.borrow_mut().poll_event(); + let Some(ev) = ev else { break }; + match &ev { + // A lost id-conflict vote means the local node MUST stop; flag it. The + // event still reaches the app via `poll_event`. + Event::Shutdown => self.shutdown.set(true), + // An inbound key-management request: apply the op to the engine's LIVE wire + // keyring and answer the originator in one call. The response is a directed + // gossip transmit egressed on the re-pump the runner performs while `queued` + // is set. + #[cfg(encryption)] + Event::KeyRequest(req) => { + // `Ok` means a key response was queued (re-pump to egress it). Ignoring + // the Err case: `handle_key_request` has already applied the op to the + // live keyring; an Err means only the best-effort response was + // past-deadline or could not be routed, which queues no outbound work. + queued |= self + .engine + .borrow_mut() + .handle_key_request(req, now) + .is_ok(); + } + _ => {} + } + self.push_app_event(ev); + } + + // Every drain re-checks parked joins: the pump folds each completion into its + // await-result join, so a resolved outcome is now visible to `poll_join`. + self.join_wake.signal(()); + queued + } + + /// The number of app events shed because a consumer did not keep up: the + /// engine's own passive-observation drops PLUS this driver's `poll_event` + /// backlog drops. + /// + /// A single pump can shed observations INSIDE the engine (its bounded queue) + /// before the driver's queue — freshly drained each pump — ever fills, so this + /// sums BOTH counters; reporting only the driver's would under-count real loss. + #[inline] + pub(crate) fn events_dropped(&self) -> u64 { + self + .engine + .borrow() + .events_dropped() + .saturating_add(self.app_events_dropped.get()) + } +} diff --git a/serf-embassy/src/stream_io/mod.rs b/serf-embassy/src/stream_io/mod.rs new file mode 100644 index 00000000..9d0381d7 --- /dev/null +++ b/serf-embassy/src/stream_io/mod.rs @@ -0,0 +1,263 @@ +//! The [`StreamIo`](serf_embedded::StreamIo) implementation over the per-slot +//! mailboxes, keyed by [`SlotId`]. +//! +//! A short-lived view the [`Runner`](crate::Runner) rebuilds each engine pump +//! over the slot mailboxes, the free-list, and the per-slot worker wake signals. +//! The engine drives the reliable plane entirely through this view; it NEVER +//! touches a `TcpSocket` — that is the worker's job (see [`crate::worker`]). Each +//! method takes a brief `borrow`/`borrow_mut` of one mailbox and, for the +//! command-posting methods, fires that slot's wake so its worker promptly acts on +//! the new directive. +//! +//! Because the engine's [`pump`](serf_embedded::SerfEngine::pump) is +//! synchronous, every borrow these methods take completes before `pump` returns — +//! and before any worker future runs — so the view's borrows never overlap a +//! worker's. + +use core::{cell::RefCell, net::SocketAddr}; + +use embassy_sync::{blocking_mutex::raw::NoopRawMutex, signal::Signal}; +use serf_embedded::{StreamIo, StreamIoError}; + +use crate::mailbox::{Command, Mailbox}; +use alloc::vec::Vec; + +/// An opaque pool-slot identifier — the engine's reliable-plane connection +/// handle (`StreamIo::Conn`) for this driver. A plain index into the slot arrays. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct SlotId(pub(crate) usize); + +impl SlotId { + /// The raw slot index. + #[inline] + pub const fn index(self) -> usize { + self.0 + } +} + +/// The per-slot wake signal type: a one-shot `()` signal on the single-executor +/// (`!Send`) [`NoopRawMutex`], fired by the engine view to nudge a slot's worker. +pub(crate) type SlotWake = Signal; + +/// A [`StreamIo`] view over the per-slot mailboxes plus the pool free-list and +/// the worker wake signals. +/// +/// Rebuilt by the [`Runner`](crate::Runner) each pump. Holds shared references to +/// the per-slot `RefCell`es and wake [`Signal`]s, and a mutable +/// reference to the driver-owned free-list of [`SlotId`]s. +pub struct SerfStream<'a> { + /// One mailbox per pool slot, indexed by [`SlotId`]. + pub(crate) slots: &'a [RefCell], + /// One command wake per pool slot, indexed by [`SlotId`]. Firing slot `c`'s + /// wake nudges exactly that slot's worker to act on the freshly-posted command. + pub(crate) cmd_wakes: &'a [SlotWake], + /// The driver-owned free-list of currently-unassigned slots. + pub(crate) free: &'a mut Vec, +} + +impl<'a> SerfStream<'a> { + /// Build the view over the slot mailboxes, command wakes, and free-list. + #[inline] + pub(crate) fn new( + slots: &'a [RefCell], + cmd_wakes: &'a [SlotWake], + free: &'a mut Vec, + ) -> Self { + Self { + slots, + cmd_wakes, + free, + } + } + + /// Post a command to slot `c`'s mailbox and wake its worker. + #[inline] + fn post(&self, c: SlotId, cmd: Command) { + self.slots[c.0].borrow_mut().command = cmd; + self.cmd_wakes[c.0].signal(()); + } +} + +impl StreamIo for SerfStream<'_> { + type Conn = SlotId; + + // ── Pool (driver-owned free-list) ─────────────────────────────────────────── + // + // Unlike the smoltcp view (whose engine owns the `SocketHandle` free-list and + // never calls these), this driver backs its slots from its OWN `free` Vec, so + // these three are implemented faithfully. The engine's `ReliablePlane::pool` + // is still seeded with every `SlotId` at construction; `take_free` / `give` + // here are the driver-side mirror the engine uses through the view when it + // routes pool churn through `StreamIo` (the embedded engine seeds and reaches + // its own pool, so in practice these are exercised by the engine's pool calls + // that go through the trait — kept correct so a slot is never double-issued). + + fn take_free(&mut self) -> Option { + self.free.pop() + } + + fn give(&mut self, c: Self::Conn) { + self.free.push(c); + } + + fn free_count(&self) -> usize { + self.free.len() + } + + fn reuse_ready(&self, c: Self::Conn) -> bool { + // embassy-net teardown is ASYNCHRONOUS: `abort`/`close` post a command to the + // slot's worker, which resets the `TcpSocket` on a later wake — so a slot the + // engine just `give`s back to the pool is NOT yet clean. The worker clears + // `reset_done` the instant it begins a `Listen`/`Dial` and re-sets it only after + // `reset_socket` (`abort()` + flush + mailbox clear), so this gate is `true` + // exactly when the slot's socket is back in a clean Closed state. The engine + // consults it before reusing any pooled slot, so a freed-but-still-tearing-down + // slot is skipped this tick and retried once its worker has finished the reset — + // never re-`listen`ed/`connect`ed over a pending teardown (which would leak the + // prior connection's `open`/`accepted_peer`/buffers into the reused slot). + self.slots[c.0].borrow().reset_done + } + + // ── Listener / accept ─────────────────────────────────────────────────────── + + fn listen(&mut self, c: Self::Conn, port: u16) -> Result<(), StreamIoError> { + // embassy-net binds its listen endpoint to the port; port 0 is rejected by + // `accept` (`InvalidPort`). The engine only ever passes its non-zero bound + // port, so this never fires in practice; surface a non-fatal error rather + // than asserting. + if port == 0 { + return Err(StreamIoError::Unaddressable); + } + self.post(c, Command::Listen(port)); + Ok(()) + } + + fn accepted_peer(&self, c: Self::Conn) -> Option { + // The worker sets `accepted_peer` to `Some` ONLY once the slot is established + // with a known remote endpoint (the embassy-net accept gate), matching + // smoltcp's `may_send()`-gated `remote_endpoint()`. + self.slots[c.0].borrow().accepted_peer + } + + // ── Dial ──────────────────────────────────────────────────────────────────── + + fn connect( + &mut self, + c: Self::Conn, + remote: SocketAddr, + _local_port: u16, + ) -> Result<(), StreamIoError> { + // `_local_port` is unused: embassy-net's `TcpSocket::connect` binds its own + // ephemeral local port from the stack (`get_local_port`), so the engine's + // requested local port has no effect here. (smoltcp threaded it through; the + // embassy-net stack owns ephemeral-port selection.) + self.post(c, Command::Dial(remote)); + Ok(()) + } + + // ── Lifecycle predicates ────────────────────────────────────────────────────── + + fn may_send(&self, c: Self::Conn) -> bool { + // Established AND the outbound ring has room for at least one more byte. The + // engine gates `send` on this; reporting capacity here keeps it from + // attempting a push that the bounded ring would partially reject. + let mb = self.slots[c.0].borrow(); + mb.established && mb.outbound.len() < mb.outbound_cap + } + + fn may_recv(&self, c: Self::Conn) -> bool { + // Mirrors the smoltcp view's `may_recv()`: readable while established and not + // (peer-FIN'd with the inbound ring drained), OR whenever buffered inbound + // bytes remain (so a half-closed peer's already-delivered bytes still read). + let mb = self.slots[c.0].borrow(); + (mb.established && !(mb.peer_fin && mb.inbound.is_empty())) || !mb.inbound.is_empty() + } + + fn is_open(&self, c: Self::Conn) -> bool { + self.slots[c.0].borrow().open + } + + fn is_established(&self, c: Self::Conn) -> bool { + self.slots[c.0].borrow().established + } + + fn recv_finished(&self, c: Self::Conn) -> bool { + // The one-shot EOF: the worker saw the peer's FIN AND the engine has drained + // every byte the peer sent before it (`inbound` empty). False while the ring + // still holds bytes, while handshaking (`peer_fin` unset), and for an + // established-but-momentarily-empty ring — so no spurious EOF reaches the + // machine and the FIN is delivered exactly once, after the data. + let mb = self.slots[c.0].borrow(); + mb.peer_fin && mb.inbound.is_empty() + } + + // ── Byte I/O ────────────────────────────────────────────────────────────────── + + fn recv(&mut self, c: Self::Conn, buf: &mut [u8]) -> Option { + // Drain up to `buf.len()` bytes from the inbound ring. `None` when empty — + // "no readable bytes this tick", NOT end-of-stream (that is `recv_finished`). + let (drained, was_full) = { + let mut mb = self.slots[c.0].borrow_mut(); + if mb.inbound.is_empty() { + return None; + } + let was_full = mb.inbound.len() >= mb.inbound_cap; + let mut n = 0; + while n < buf.len() { + match mb.inbound.pop_front() { + Some(b) => { + buf[n] = b; + n += 1; + } + None => break, + } + } + (n, was_full) + }; + // Wake the worker if this drain relieved a full inbound ring, so a worker + // parked on backpressure resumes reading from the socket. No-op in the common + // case where the ring had room. + if was_full { + self.cmd_wakes[c.0].signal(()); + } + Some(drained) + } + + fn send(&mut self, c: Self::Conn, bytes: &[u8]) -> usize { + // Push into the outbound ring up to its capacity; return how many bytes were + // accepted (a partial accept on a near-full ring, like a TCP tx ring). Wake + // the worker so it writes the freshly-queued bytes without waiting for an + // unrelated event. + let mut mb = self.slots[c.0].borrow_mut(); + let room = mb.outbound_cap.saturating_sub(mb.outbound.len()); + let take = room.min(bytes.len()); + mb.outbound.extend(bytes[..take].iter().copied()); + drop(mb); + if take > 0 { + self.cmd_wakes[c.0].signal(()); + } + take + } + + fn send_queue(&self, c: Self::Conn) -> usize { + // Un-written bytes (still in the outbound ring) PLUS written-but-unACKed + // bytes (the worker's mirror of `TcpSocket::send_queue()`). Reaches 0 only + // when both rings have fully drained to the peer, so the engine's + // drain-before-close gate never FINs ahead of an in-flight reply. + let mb = self.slots[c.0].borrow(); + mb.outbound.len() + mb.sock_send_queue + } + + // ── Close ───────────────────────────────────────────────────────────────────── + + fn close(&mut self, c: Self::Conn) { + self.post(c, Command::Close); + } + + fn abort(&mut self, c: Self::Conn) { + self.post(c, Command::Abort); + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-embassy/src/stream_io/tests.rs b/serf-embassy/src/stream_io/tests.rs new file mode 100644 index 00000000..8efd9cf0 --- /dev/null +++ b/serf-embassy/src/stream_io/tests.rs @@ -0,0 +1,235 @@ +use super::{Command, Mailbox, SerfStream, SlotId, SlotWake}; +use alloc::vec::Vec; +use core::{cell::RefCell, net::SocketAddr}; +use serf_embedded::{StreamIo, StreamIoError}; + +/// Build `n` fresh mailboxes (each `cap` bytes per ring) and `n` wakes for a +/// view. Returns them so the test frame owns them for the view's lifetime. +fn slots(n: usize, cap: usize) -> (Vec>, Vec) { + let mb = (0..n) + .map(|_| RefCell::new(Mailbox::new(cap, cap))) + .collect(); + let wakes = (0..n).map(|_| SlotWake::new()).collect(); + (mb, wakes) +} + +fn sa(last: u8) -> SocketAddr { + SocketAddr::from(([169, 254, 0, last], 7946)) +} + +#[test] +fn slot_id_exposes_its_raw_index() { + assert_eq!(SlotId(3).index(), 3); +} + +/// The driver-owned free-list is a faithful LIFO: `give` then `take_free` +/// returns the same slot, and `free_count` tracks the length. +#[test] +fn free_list_take_give_and_count() { + let (mb, wakes) = slots(2, 64); + let mut free = Vec::new(); + let mut view = SerfStream::new(&mb, &wakes, &mut free); + + assert_eq!(view.free_count(), 0); + assert!(view.take_free().is_none(), "an empty free-list yields None"); + + view.give(SlotId(0)); + view.give(SlotId(1)); + assert_eq!(view.free_count(), 2); + assert_eq!( + view.take_free(), + Some(SlotId(1)), + "LIFO: last given comes first" + ); + assert_eq!(view.take_free(), Some(SlotId(0))); + assert_eq!(view.free_count(), 0); +} + +/// `reuse_ready` reads the slot's `reset_done`: true for a fresh slot, false +/// once the worker has begun using it (cleared `reset_done`). +#[test] +fn reuse_ready_tracks_reset_done() { + let (mb, wakes) = slots(1, 64); + let mut free = Vec::new(); + let view = SerfStream::new(&mb, &wakes, &mut free); + + assert!(view.reuse_ready(SlotId(0)), "a fresh slot is reuse-ready"); + mb[0].borrow_mut().reset_done = false; + assert!( + !view.reuse_ready(SlotId(0)), + "a slot whose worker is mid-use (reset_done cleared) is not reuse-ready" + ); +} + +/// `listen` posts a `Listen` command and wakes the worker for a valid port, and +/// rejects port 0 with `Unaddressable` without posting. +#[test] +fn listen_posts_for_a_valid_port_and_rejects_port_zero() { + let (mb, wakes) = slots(1, 64); + let mut free = Vec::new(); + let mut view = SerfStream::new(&mb, &wakes, &mut free); + + assert!(matches!( + view.listen(SlotId(0), 0), + Err(StreamIoError::Unaddressable) + )); + assert_eq!( + mb[0].borrow().command, + Command::Idle, + "a rejected listen does not post a command" + ); + + view + .listen(SlotId(0), 7946) + .expect("a non-zero port is accepted"); + assert_eq!(mb[0].borrow().command, Command::Listen(7946)); + assert!( + wakes[0].signaled(), + "the worker is woken for the new directive" + ); +} + +/// `connect` posts a `Dial` to the slot regardless of the engine's requested +/// local port (embassy-net owns ephemeral-port selection). +#[test] +fn connect_posts_a_dial() { + let (mb, wakes) = slots(1, 64); + let mut free = Vec::new(); + let mut view = SerfStream::new(&mb, &wakes, &mut free); + + view.connect(SlotId(0), sa(2), 1234).expect("connect posts"); + assert_eq!(mb[0].borrow().command, Command::Dial(sa(2))); + assert!(wakes[0].signaled()); +} + +/// The established-state predicates read straight from the mailbox: an +/// established, open slot with a known peer reports each accordingly. +#[test] +fn lifecycle_predicates_reflect_the_mailbox() { + let (mb, wakes) = slots(1, 64); + let mut free = Vec::new(); + let view = SerfStream::new(&mb, &wakes, &mut free); + + // A fresh slot is neither open nor established and has no peer. + assert!(!view.is_open(SlotId(0))); + assert!(!view.is_established(SlotId(0))); + assert!(view.accepted_peer(SlotId(0)).is_none()); + + { + let mut m = mb[0].borrow_mut(); + m.established = true; + m.open = true; + m.accepted_peer = Some(sa(2)); + } + assert!(view.is_open(SlotId(0))); + assert!(view.is_established(SlotId(0))); + assert_eq!(view.accepted_peer(SlotId(0)), Some(sa(2))); + // Established with room ⇒ may_send; established and not (peer-FIN'd + drained) + // ⇒ may_recv. + assert!(view.may_send(SlotId(0))); + assert!(view.may_recv(SlotId(0))); +} + +/// `send` fills the outbound ring up to capacity (partial accept on a full +/// ring) and `may_send` reports false once the ring is full. +#[test] +fn send_respects_outbound_capacity() { + let (mb, wakes) = slots(1, 4); + let mut free = Vec::new(); + let mut view = SerfStream::new(&mb, &wakes, &mut free); + mb[0].borrow_mut().established = true; + + assert_eq!(view.send(SlotId(0), b"abcdef"), 4, "only 4 of 6 bytes fit"); + assert_eq!(mb[0].borrow().outbound.len(), 4); + assert!( + !view.may_send(SlotId(0)), + "a full outbound ring cannot accept more" + ); + assert_eq!(view.send(SlotId(0), b"z"), 0, "a full ring accepts nothing"); +} + +/// `recv` drains buffered inbound up to the caller's buffer length and returns +/// `None` (not end-of-stream) when the ring is empty. +#[test] +fn recv_drains_inbound_and_returns_none_when_empty() { + let (mb, wakes) = slots(1, 64); + let mut free = Vec::new(); + let mut view = SerfStream::new(&mb, &wakes, &mut free); + + assert!( + view.recv(SlotId(0), &mut [0u8; 4]).is_none(), + "empty ⇒ None" + ); + + mb[0].borrow_mut().inbound.extend(b"hello".iter().copied()); + let mut buf = [0u8; 3]; + assert_eq!(view.recv(SlotId(0), &mut buf), Some(3)); + assert_eq!(&buf, b"hel"); + let mut rest = [0u8; 8]; + assert_eq!(view.recv(SlotId(0), &mut rest), Some(2)); + assert_eq!(&rest[..2], b"lo"); + assert!(view.recv(SlotId(0), &mut rest).is_none()); +} + +/// `send_queue` sums the un-written outbound bytes and the worker's mirror of +/// the socket's un-ACKed send queue, reaching zero only when both drain. +#[test] +fn send_queue_sums_unwritten_and_unacked() { + let (mb, wakes) = slots(1, 64); + let mut free = Vec::new(); + let view = SerfStream::new(&mb, &wakes, &mut free); + + assert_eq!(view.send_queue(SlotId(0)), 0); + { + let mut m = mb[0].borrow_mut(); + m.outbound.extend(b"ab".iter().copied()); + m.sock_send_queue = 3; + } + assert_eq!(view.send_queue(SlotId(0)), 5, "2 unwritten + 3 un-ACKed"); +} + +/// `recv_finished` is the one-shot EOF: true only with a peer FIN AND a drained +/// inbound ring, so the FIN is delivered after the data, exactly once. +#[test] +fn recv_finished_is_the_drained_peer_fin() { + let (mb, wakes) = slots(1, 64); + let mut free = Vec::new(); + let view = SerfStream::new(&mb, &wakes, &mut free); + + assert!(!view.recv_finished(SlotId(0)), "no FIN yet ⇒ not finished"); + + { + let mut m = mb[0].borrow_mut(); + m.peer_fin = true; + m.inbound.extend(b"tail".iter().copied()); + } + assert!( + !view.recv_finished(SlotId(0)), + "a FIN with buffered bytes still pending is not yet finished" + ); + // `may_recv` stays true while buffered bytes remain even after the FIN. + assert!(view.may_recv(SlotId(0))); + + mb[0].borrow_mut().inbound.clear(); + assert!( + view.recv_finished(SlotId(0)), + "FIN + drained inbound ⇒ end-of-stream" + ); +} + +/// `close` and `abort` post their respective commands and wake the worker. +#[test] +fn close_and_abort_post_commands() { + let (mb, wakes) = slots(1, 64); + let mut free = Vec::new(); + let mut view = SerfStream::new(&mb, &wakes, &mut free); + + view.close(SlotId(0)); + assert_eq!(mb[0].borrow().command, Command::Close); + assert!(wakes[0].signaled()); + + wakes[0].reset(); + view.abort(SlotId(0)); + assert_eq!(mb[0].borrow().command, Command::Abort); + assert!(wakes[0].signaled()); +} diff --git a/serf-embassy/src/time/mod.rs b/serf-embassy/src/time/mod.rs new file mode 100644 index 00000000..999f6940 --- /dev/null +++ b/serf-embassy/src/time/mod.rs @@ -0,0 +1,103 @@ +//! Bridge the `embassy-time` clock to the machine's portable +//! [`Instant`](memberlist_proto::Instant). +//! +//! The SWIM machine carries time as a [`Duration`](core::time::Duration) since an +//! opaque origin the driver chooses (`memberlist_proto::Instant`); only relative +//! arithmetic is meaningful. embassy-time's own [`Instant`](embassy_time::Instant) +//! counts monotonically from device boot (tick 0 = [`Instant::MIN`]), so anchoring +//! the machine origin at boot makes the mapping a plain "uptime as a `Duration`": +//! monotonic, starting at ~0, and far below `Duration::MAX`, so machine-side +//! deadline arithmetic (`now + interval`) never saturates in practice. +//! +//! Microsecond granularity is preserved (`as_micros`) rather than the smoltcp +//! driver's millisecond rounding, since embassy-time can resolve finer than a +//! millisecond depending on the configured tick rate. + +use core::time::Duration; + +use embassy_time::Instant as RawInstant; +use memberlist_proto::Instant as MachineInstant; + +/// An [`embassy_time::Instant`] wrapper carrying the driver's runtime clock +/// reading. +/// +/// The machine consumes `memberlist_proto::Instant` (a portable `Duration`-based +/// newtype), so the driver maps each raw reading through [`EmbassyInstant::machine`] +/// (or the free [`now`] helper) at the boundary. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct EmbassyInstant(pub RawInstant); + +impl From for EmbassyInstant { + #[inline(always)] + fn from(raw: RawInstant) -> Self { + Self(raw) + } +} + +impl From for RawInstant { + #[inline(always)] + fn from(wrapped: EmbassyInstant) -> Self { + wrapped.0 + } +} + +impl EmbassyInstant { + /// Map this embassy-time reading to the machine [`Instant`](memberlist_proto::Instant), + /// anchoring the machine origin at device boot ([`embassy_time::Instant::MIN`]). + /// + /// The result is the uptime since boot as a [`Duration`], so it is monotonic in + /// lockstep with `embassy_time::Instant` and starts at ~0. + #[inline] + pub fn machine(self) -> MachineInstant { + // `duration_since(MIN)` is the full uptime; embassy-time's `Duration` is in + // ticks, so convert through microseconds for the machine's `core::Duration`. + let since_boot = self.0.duration_since(RawInstant::MIN); + MachineInstant::from_origin(Duration::from_micros(since_boot.as_micros())) + } +} + +/// The current machine [`Instant`](memberlist_proto::Instant) from the embassy-time +/// clock, anchored at device boot. +/// +/// Reads [`embassy_time::Instant::now`] and maps it through +/// [`EmbassyInstant::machine`]. This is the `now` the driver hands the engine each +/// pump. +#[inline] +pub fn now() -> MachineInstant { + EmbassyInstant(RawInstant::now()).machine() +} + +/// Map a machine [`Instant`](memberlist_proto::Instant) deadline the engine +/// returns back to an [`embassy_time::Instant`] for [`Timer::at`](embassy_time::Timer). +/// +/// The inverse of [`EmbassyInstant::machine`]: the machine instant is the uptime +/// since boot as a [`Duration`], so adding it to [`embassy_time::Instant::MIN`] +/// (tick 0 = boot) reconstructs the raw reading. embassy-time's `Duration` counts +/// ticks at the configured rate; `from_micros` truncates the machine instant's +/// microseconds to the nearest tick at or below it. A monotonic timer cannot fire +/// before its deadline, so this is acceptable only because the driver re-pumps on +/// every wake and the engine returns a fresh, still-future deadline if the timer +/// fired a tick early — the loop converges rather than spinning, since each +/// re-pump that finds no work due yet returns the same deadline and the next tick +/// crosses it. +/// +/// Saturates at [`embassy_time::Instant::MAX`] for a deadline whose tick count +/// overflows embassy-time's `u64` tick domain (e.g. a machine `Instant` built +/// from a saturated/extreme timeout), so an "effectively never" deadline maps to +/// a far-future timer rather than panicking. +#[inline] +pub fn machine_to_raw(deadline: MachineInstant) -> RawInstant { + let micros = deadline.since_origin().as_micros(); + // embassy-time `Duration::from_micros` takes a `u64`; clamp an out-of-range + // microsecond count to the max tick instant rather than truncating the high + // bits (which would alias an extreme deadline to a near one). + match u64::try_from(micros) { + Ok(us) => RawInstant::MIN + .checked_add(embassy_time::Duration::from_micros(us)) + .unwrap_or(RawInstant::MAX), + Err(_) => RawInstant::MAX, + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-embassy/src/time/tests.rs b/serf-embassy/src/time/tests.rs new file mode 100644 index 00000000..7eaa98a3 --- /dev/null +++ b/serf-embassy/src/time/tests.rs @@ -0,0 +1,65 @@ +use core::time::Duration; + +use embassy_time::Instant as RawInstant; +use memberlist_proto::Instant as MachineInstant; + +use super::{EmbassyInstant, machine_to_raw, now}; + +/// Two embassy-time instants 5ms apart map to two machine instants whose +/// `duration_since` is 5ms — the bridge preserves elapsed time. +#[test] +fn five_ms_gap_is_preserved_through_the_machine_instant() { + let t0 = RawInstant::now(); + let t1 = t0 + embassy_time::Duration::from_millis(5); + + let m0 = EmbassyInstant(t0).machine(); + let m1 = EmbassyInstant(t1).machine(); + + // Exact at any tick rate that resolves a millisecond (the std test driver + // ticks at 1 MHz, so 5ms = 5000 ticks is represented exactly). + assert_eq!(m1.duration_since(m0), Duration::from_millis(5)); + // The mapping is monotonic: the later instant is strictly greater. + assert!(m1 > m0); + // Out-of-order input saturates to zero rather than panicking. + assert_eq!(m0.duration_since(m1), Duration::ZERO); +} + +/// The two `From` conversions are mutual inverses: wrapping a raw reading and +/// unwrapping it yields the same `embassy_time::Instant`. +#[test] +fn raw_wrapper_conversions_round_trip() { + let raw = RawInstant::now(); + let wrapped: EmbassyInstant = raw.into(); + let back: RawInstant = wrapped.into(); + assert_eq!(raw, back); +} + +/// `machine_to_raw` is the inverse of `EmbassyInstant::machine`: a raw reading +/// mapped to the machine domain and back reconstructs the original tick (exact +/// at the std driver's 1 MHz, where every microsecond is a whole tick). +#[test] +fn machine_to_raw_inverts_machine_at_microsecond_resolution() { + let raw = RawInstant::MIN + embassy_time::Duration::from_micros(123_456); + let machine = EmbassyInstant(raw).machine(); + assert_eq!(machine_to_raw(machine), raw); +} + +/// A machine deadline whose microsecond count overflows embassy-time's `u64` +/// tick domain saturates to `Instant::MAX` rather than wrapping to a near +/// (effectively-past) instant. +#[test] +fn machine_to_raw_saturates_on_overflow_instead_of_wrapping() { + // `u128::MAX` microseconds cannot fit a `u64`, so the conversion takes the + // saturating branch. + let extreme = MachineInstant::from_origin(Duration::new(u64::MAX, 0)); + assert_eq!(machine_to_raw(extreme), RawInstant::MAX); +} + +/// `now()` reads the live clock and advances monotonically: a later call is +/// never earlier than an earlier one. +#[test] +fn now_is_monotonic() { + let a = now(); + let b = now(); + assert!(b >= a, "now() must be monotonic non-decreasing"); +} diff --git a/serf-embassy/src/worker.rs b/serf-embassy/src/worker.rs new file mode 100644 index 00000000..24f5ddaf --- /dev/null +++ b/serf-embassy/src/worker.rs @@ -0,0 +1,453 @@ +//! The per-slot async worker future. +//! +//! Each pool slot has one [`run_slot`] future owning one embassy-net +//! [`TcpSocket`] and sharing a `RefCell` with the engine. The worker is +//! the ONLY code that touches the socket; it translates the engine's mailbox +//! [`Command`]s into socket operations and mirrors the socket's inbound bytes, +//! send-queue depth, and peer-FIN/reset state back into the mailbox for the +//! engine's synchronous [`StreamIo`](serf_embedded::StreamIo) view to read. +//! +//! # Borrow discipline +//! +//! The worker NEVER holds a `Mailbox` borrow across an `.await`. Every step +//! copies what it needs out of the mailbox (dropping the borrow), awaits the +//! socket op, then re-borrows to record the result. Combined with the engine's +//! synchronous pump (whose borrows complete before any worker runs), this keeps +//! the pump and the N sibling workers free of overlapping `RefCell` borrows. + +use core::cell::RefCell; + +use embassy_futures::select::{Either, select}; +use embassy_net::tcp::TcpSocket; + +use crate::{ + mailbox::{Command, Mailbox}, + stream_io::SlotWake, +}; + +/// Scratch buffer for one `read()` per readable wake. A modest size keeps the +/// worker's stack frame small; the engine's TCP socket RX ring (not this buffer) +/// bounds how much the peer can have in flight, and a larger payload simply takes +/// more read wakes to drain. +const READ_CHUNK: usize = 1024; + +/// Drive one pool slot's socket forever, mediated by its mailbox. +/// +/// Loops over the slot's lifecycle: wait for a [`Command::Listen`] / +/// [`Command::Dial`], complete the handshake, run the established read/write loop +/// until close/abort/peer-reset, reset the socket for reuse, and repeat. +/// +/// An [`Command::Idle`] slot PARKS on its command wake rather than listening. +/// This diverges deliberately from a "every idle slot is a candidate acceptor" +/// design: the shared engine ([`serf_embedded::SerfEngine`]) tracks exactly ONE +/// listener slot and only inspects that slot's `accepted_peer`. If a non-listener +/// idle slot also listened on the same port, the stack could hand an inbound SYN +/// to it instead of the engine's listener slot, and the engine would never map +/// that connection — wedging the inbound exchange and stranding the slot. So a +/// worker listens ONLY when the engine commands it via `stream.listen` (the +/// construction-time listener slot and each replenished listener), keeping exactly +/// one acceptor live at a time. The construction-time listener is seeded by +/// posting `Command::Listen(port)` to that slot's mailbox before the workers run +/// (see [`Serf::new`](crate::Serf::new)). +/// +/// `cmd_wake` is the slot's command signal (engine → worker): the engine-view's +/// command posters pulse it so the worker promptly acts on a new directive. +/// `pump_wake` is the shared pump wake (worker → pump): the worker pulses it after +/// every mailbox change the pump must observe (inbound bytes ready, outbound +/// drained, a handshake settled, a FIN/reset), so the synchronous pump re-runs and +/// hands those bytes/state transitions to the machine. +pub(crate) async fn run_slot( + socket: &mut TcpSocket<'_>, + mb: &RefCell, + cmd_wake: &SlotWake, + pump_wake: &SlotWake, + socket_timeout: embassy_time::Duration, +) -> ! { + // Bound EVERY blocking socket await (connect / write / flush / read) so a peer + // that stops responding cannot wedge this worker: embassy-net aborts the socket + // after `socket_timeout` of inactivity, turning the pending op into an error the + // worker's failure arms convert to a teardown + `reset_socket` (so `reset_done` + // becomes true and the slot is reusable again). Without this, a flush awaiting an + // ACK from a black-holed peer never returns, the slot never resets, and the + // round-1 reuse gate would withhold it forever. Set once; it persists across the + // reuse loop below (the socket object is reused, not recreated). The timeout was + // converted into the embassy-time tick domain and validated against the engine + // deadlines at construction (see `Serf::new`), so it installs directly. + socket.set_timeout(Some(socket_timeout)); + loop { + // 1. Read the pending command. An idle slot parks on its wake (see the + // single-listener rationale above) until the engine posts Listen/Dial. + let command = { + let m = mb.borrow(); + m.command + }; + + match command { + Command::Listen(port) => { + // The slot is now in use (its socket leaves the clean Closed state), so it + // is no longer reuse-ready until the next `reset_socket`. Clearing this + // BEFORE the first await is what closes the abort/reuse race: once the + // engine has handed this slot a `Listen`, `reuse_ready` reports `false`, so + // a later same-pump `give` + re-take cannot re-`listen`/`connect` it (and + // clobber a pending teardown) until the worker has actually reset it. + mb.borrow_mut().reset_done = false; + // `accept(port)` puts the socket in Listen and resolves once the + // three-way handshake completes (the socket leaves Listen/SynSent/ + // SynReceived). Race it against the command wake so a command change (e.g. + // the engine re-purposing this slot for a dial) can pre-empt a pending + // accept. Consume the wake that delivered THIS command first: it is latched + // (the command was read from the mailbox, not the signal), so without this + // the race fires `Either::Second` immediately on the already-consumed signal. + cmd_wake.reset(); + match select(socket.accept(port), cmd_wake.wait()).await { + Either::First(Ok(())) => { + let peer = socket.remote_endpoint().map(Into::into); + { + let mut m = mb.borrow_mut(); + // Accept gate: surface the peer ONLY with a known remote endpoint, + // matching smoltcp's `may_send()`-gated `accepted_peer`. + m.accepted_peer = peer; + m.established = peer.is_some(); + m.open = peer.is_some(); + m.command = Command::Idle; + m.sock_send_queue = socket.send_queue(); + } + // The pump must observe the new accept so `check_listener` maps it. + pump_wake.signal(()); + if peer.is_some() { + run_established(socket, mb, cmd_wake, pump_wake).await; + } + reset_socket(socket, mb).await; + pump_wake.signal(()); + } + Either::First(Err(_)) => { + // Listen failed (e.g. the socket was not in a clean state). Reset and + // retry on the next loop. + reset_socket(socket, mb).await; + } + Either::Second(()) => { + // A new command arrived mid-accept; abandon the listen and re-read it. + // `abort()` returns the socket to Closed so the next command's + // accept/connect starts cleanly. + socket.abort(); + // Ignoring Err: a best-effort RST flush on a socket with no peer is a + // no-op; the reset below re-clears the mailbox regardless. + let _ = socket.flush().await; + } + } + } + Command::Dial(remote) => { + // The slot is now in use; it is no longer reuse-ready until `reset_socket` + // runs (see the `Listen` arm). Clear it before the connect await so the + // engine cannot reuse this slot mid-teardown. + mb.borrow_mut().reset_done = false; + // `connect(remote)` spans the handshake; on success the slot is established + // and open. Race it against the command wake so a `Close`/`Abort` (the + // engine retiring this dial at the exchange deadline) pre-empts a connect to + // a black-holed peer; the socket inactivity timeout bounds the connect + // itself as a backstop. On failure the engine reaps the slot via + // `is_open() == false`. Consume the wake that delivered THIS `Dial` before + // racing: it is latched (the command was read from the mailbox, not the + // signal), so without this the race fires `Either::Second` immediately on + // the already-consumed signal and abandons every dial. + cmd_wake.reset(); + match select(socket.connect(remote), cmd_wake.wait()).await { + Either::First(Ok(())) => { + let peer = socket.remote_endpoint().map(Into::into); + { + let mut m = mb.borrow_mut(); + m.accepted_peer = peer; + m.established = true; + m.open = true; + m.command = Command::Idle; + m.sock_send_queue = socket.send_queue(); + } + // The pump must observe the established slot to promote its exchange. + pump_wake.signal(()); + run_established(socket, mb, cmd_wake, pump_wake).await; + reset_socket(socket, mb).await; + pump_wake.signal(()); + } + Either::First(Err(_)) => { + // Dial failed (refused, unreachable, or the socket timed out): mark the + // slot closed so the engine reaps it, then reset the socket for reuse. + { + let mut m = mb.borrow_mut(); + m.established = false; + m.open = false; + m.command = Command::Idle; + } + // The pump must observe `open == false` to reap the failed dial. + pump_wake.signal(()); + reset_socket(socket, mb).await; + } + Either::Second(()) => { + // A command arrived mid-connect (the engine posted `Abort`, retiring this + // dial at the exchange deadline): abandon the dial, reset the socket so + // the next command starts cleanly, and wake the pump so it observes the + // slot is reuse-ready (`reset_done`) and can dispatch any deferred dial — + // without this the dial-churn would stall (freed slots never re-observed). + reset_socket(socket, mb).await; + pump_wake.signal(()); + } + } + } + // A Close/Abort posted to an un-established slot has nothing to tear down; + // clear it and park again. + Command::Close | Command::Abort => { + reset_socket(socket, mb).await; + } + // Idle: park until the engine posts a command. Re-read after the wake. + Command::Idle => { + cmd_wake.wait().await; + } + } + } +} + +/// The established read/write loop: shuttle bytes between the socket and the +/// mailbox until the peer closes/resets, or the engine commands a close/abort. +/// +/// Returns once the connection is finished (so the caller resets the socket for +/// reuse). Mirrors `sock_send_queue` after every write and on every read wake so +/// the engine's `send_queue` drain-before-close gate stays accurate. +async fn run_established( + socket: &mut TcpSocket<'_>, + mb: &RefCell, + cmd_wake: &SlotWake, + pump_wake: &SlotWake, +) { + // Half-close state machine. A push/pull exchange is a full-duplex request/reply + // over ONE connection that BOTH sides half-close: the requester writes its + // request, FINs, then reads the reply + the peer's FIN; the responder reads the + // request + the peer's FIN, then writes its reply and FINs. So: + // + // - `we_finned` — the engine commanded `Close`, we sent our write-half FIN. + // We keep READING afterward (the peer's reply + FIN still arrive). + // - `peer_finned` — we read the peer's FIN (`read` returned `Ok(0)`). We keep + // WRITING afterward (the responder still owes its reply). + // + // The connection is fully done — slot reaped — once BOTH halves are shut and the + // engine has drained every inbound byte: `we_finned && peer_finned`, the + // outbound ring is empty, and the inbound ring is empty. `abort` / a transport + // error tears down immediately. + let mut we_finned = false; + let mut peer_finned = false; + + loop { + // Snapshot the command and whether there is outbound work, releasing the + // borrow before any await. + let (command, has_outbound) = { + let m = mb.borrow(); + (m.command, !m.outbound.is_empty()) + }; + + match command { + Command::Close => { + // Graceful half-close: FIN our write half (the last queued segment carries + // the FIN). The engine issues Close only after its own outbound bytes have + // drained (its `send_queue` gate), so nothing is left to write. CRUCIAL: do + // NOT return — the read half stays open so the peer's reply + FIN still + // pump inbound. Consume the command and fall through to the I/O select. + if !we_finned { + socket.close(); + we_finned = true; + { + let mut m = mb.borrow_mut(); + m.command = Command::Idle; + m.sock_send_queue = socket.send_queue(); + } + pump_wake.signal(()); + } + } + Command::Abort => { + socket.abort(); + // Ignoring Err: abort flush waits for the RST to be sent; failure means + // it is already gone. The slot is reset next regardless. + let _ = socket.flush().await; + { + let mut m = mb.borrow_mut(); + m.open = false; + m.sock_send_queue = 0; + } + pump_wake.signal(()); + return; + } + // Idle / Listen / Dial while established: no transition to act on; the + // command was already consumed at handshake time. Fall through to the I/O + // select. + Command::Idle | Command::Listen(_) | Command::Dial(_) => {} + } + + // Full-close completion: both halves FIN'd and every byte handed off. Flush so + // OUR FIN is actually sent and ACKed by the peer BEFORE the caller resets the + // socket — `reset_socket`'s `abort()` would otherwise RST a still-pending FIN, + // and the peer (mid graceful close) could see a reset instead of our clean FIN + // and fail to deliver its inbound message. `flush` resolves once the FIN is + // ACKed (the peer has received it), so the graceful handshake completes first. + if we_finned && peer_finned { + let drained = { + let m = mb.borrow(); + m.inbound.is_empty() && m.outbound.is_empty() + }; + if drained { + // Ignoring Err: a reset during the final flush just means the peer already + // tore down; our FIN is moot then and the slot is reset regardless. + let _ = socket.flush().await; + { + let mut m = mb.borrow_mut(); + m.open = false; + m.sock_send_queue = 0; + } + pump_wake.signal(()); + return; + } + } + + // Write-before-read, serialized. A push/pull exchange is request→reply (never + // interleaved on one side), so the worker can fully drain its outbound, then + // wait to read — which avoids racing `read` (needs `&mut socket`) against a + // write in one `select`. CRUCIAL for correctness: reads use `socket.read()` + // directly, NOT `wait_read_ready`. embassy-net's `poll_read_ready` is ready + // only when `can_recv()` is true, which is FALSE for a pure peer-FIN (EOF with + // no data) — so `wait_read_ready` never fires on a graceful close and the EOF + // would be missed. `read()` correctly resolves `Ok(0)` on the FIN and parks on + // the recv waker otherwise (which smoltcp wakes on the FIN). + + // 1. Drain outbound (only while our write half is open). Write the whole ring, + // then flush so the bytes are ACKed and `sock_send_queue` reaches 0 — the + // engine's `send_queue` drain-before-close gate depends on this (an ACK + // alone does not wake a readiness future, so without the flush the gate + // would never see the queue drain and the FIN would be deferred forever). + if has_outbound && !we_finned { + let chunk: alloc::vec::Vec = { + let m = mb.borrow(); + m.outbound.iter().copied().collect() + }; + match socket.write(&chunk).await { + Ok(written) => { + { + let mut m = mb.borrow_mut(); + for _ in 0..written { + m.outbound.pop_front(); + } + m.sock_send_queue = socket.send_queue(); + } + // Ignoring Err: a reset mid-flush means the connection is gone; the read + // below observes it and tears the slot down. Re-sync the send queue from + // the socket regardless. + let _ = socket.flush().await; + { + let mut m = mb.borrow_mut(); + m.sock_send_queue = socket.send_queue(); + } + pump_wake.signal(()); + } + Err(_) => { + { + let mut m = mb.borrow_mut(); + m.open = false; + m.sock_send_queue = 0; + } + pump_wake.signal(()); + return; + } + } + // Loop to re-read the command (the engine may now Close) and re-drain. + continue; + } + + // 2. No outbound to write. If the peer has FIN'd there is nothing left to read + // either; park on the command wake (the engine drives the remaining + // teardown: it processes the EOF, finishes the exchange, and commands + // Close, which wakes us here). + if peer_finned { + cmd_wake.wait().await; + continue; + } + + // 3. Read the next inbound chunk, bounded by the ring's remaining capacity so a + // slow-to-drain engine cannot grow the ring past `inbound_cap`. Race the + // read against the command wake so a Close/Abort (or freshly-queued + // outbound) pre-empts a blocked read. + let room = { + let m = mb.borrow(); + m.inbound_cap.saturating_sub(m.inbound.len()) + }; + if room == 0 { + // Ring full: park until the engine's `recv` drains it (it pulses this slot's + // command wake on a drain) or a close/abort arrives. + cmd_wake.wait().await; + continue; + } + let cap = room.min(READ_CHUNK); + let mut tmp = [0u8; READ_CHUNK]; + match select(socket.read(&mut tmp[..cap]), cmd_wake.wait()).await { + // `Ok(0)` is the peer's FIN (embassy-net maps `RecvError::Finished` to + // `Ok(0)`): record the one-shot EOF and stop reading. Do NOT tear down — the + // responder still owes its reply (its engine queues it AFTER seeing this + // EOF), and the engine still needs to drain any buffered inbound and observe + // the EOF via `recv_finished`. `peer_finned` routes subsequent loops to the + // write/park paths above. + Either::First(Ok(0)) => { + peer_finned = true; + { + let mut m = mb.borrow_mut(); + m.peer_fin = true; + m.sock_send_queue = socket.send_queue(); + } + pump_wake.signal(()); + } + Either::First(Ok(n)) => { + { + let mut m = mb.borrow_mut(); + m.inbound.extend(tmp[..n].iter().copied()); + m.sock_send_queue = socket.send_queue(); + } + // The pump must observe the new inbound bytes to feed the machine. + pump_wake.signal(()); + } + Either::First(Err(_)) => { + // A reset / invalid-state read: the connection FAILED (a peer RST, not a + // graceful close). Mark the slot closed (`open = false`) but DO NOT set + // `peer_fin`: a reset is NOT an orderly end-of-stream. Latching `peer_fin` + // here would make `SerfStream::recv_finished` (`peer_fin && + // inbound.is_empty()`) report a clean EOF for a reset once the buffered + // inbound drained, and the machine maps a `UserMessage` transport EOF to a + // SUCCESSFUL completion — so a mid-send RST would falsely resolve an + // exchange as Ok. Surfacing only `open = false` keeps the reset a failure: + // the exchange's bridge elapses at its `stream_timeout` deadline and the + // machine emits `StreamAction::Abort`, exactly as the smoltcp driver treats + // a RST-`Closed` socket (its `recv_finished` likewise excludes the reset + // state). Any already-buffered inbound still drains via `recv` before then; + // it simply is not followed by a spurious EOF anchor. + { + let mut m = mb.borrow_mut(); + m.open = false; + m.sock_send_queue = 0; + } + pump_wake.signal(()); + return; + } + // Command wake fired (Close/Abort or new outbound): loop to handle it. The + // in-flight `read` future is dropped; smoltcp keeps the received bytes + // buffered, so the next read still sees them — no data is lost. + Either::Second(()) => {} + } + } +} + +/// Return the socket to a clean Closed state and reset the mailbox for the slot's +/// next reuse. +/// +/// `abort()` is idempotent on an already-closed socket and guarantees the socket +/// leaves any half-open/closing state so the next `accept`/`connect` starts +/// fresh; the subsequent `flush` lets a pending RST go out. The mailbox reset +/// clears every per-connection field (preserving ring capacities). +async fn reset_socket(socket: &mut TcpSocket<'_>, mb: &RefCell) { + socket.abort(); + // Ignoring Err: flushing the RST on an already-dead socket is a no-op; the + // socket is returned to Closed either way. + let _ = socket.flush().await; + mb.borrow_mut().reset(); +} diff --git a/serf-embassy/tests/key_rotation.rs b/serf-embassy/tests/key_rotation.rs new file mode 100644 index 00000000..d3e49b65 --- /dev/null +++ b/serf-embassy/tests/key_rotation.rs @@ -0,0 +1,187 @@ +//! Key-management end-to-end: two encrypted nodes rotate their keyring across the +//! cluster and BOTH nodes' LIVE wire keyrings follow. +//! +//! The regression this guards: a driver that applies key ops to a private shadow +//! copy reports a completed rotation the wire never sees. Here A drives +//! `install_key` -> `use_key` -> `remove_key`; each node's runner drain routes the +//! resulting `Event::KeyRequest` into the engine's `handle_key_request`, mutating +//! the coordinator's LIVE keyring. The fail-on-revert assertions read both nodes' +//! `keyring()` and require primary == K2 with K1 absent, then a post-rotation user +//! event still crosses the wire, proving both planes now run under K2. + +// The whole suite exercises key rotation, so without an AEAD backend the file +// compiles to nothing — gating item-by-item would leave the shared harness +// helpers dead in a plaintext build. +#![cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#![allow(clippy::collapsible_if)] + +mod support; + +use core::net::SocketAddr; + +use embassy_net::StackResources; +use embassy_time::{Duration, Timer}; +use futures::executor::block_on; +use serf_embassy::{ + Bytes, EncryptionOptions, Event, Keyring, SecretKey, Serf, TransformOptions, now, +}; +use smol_str::SmolStr; + +use support::cluster::{ + NodeBufs, POOL, build_node, build_sockets, build_stack, devices, drive, join_and_converge, +}; + +/// A fixed AEAD key filled with `fill`, in whichever backend is compiled. +fn secret_key(fill: u8) -> SecretKey { + #[cfg(feature = "aes-gcm")] + { + SecretKey::Aes256([fill; 32]) + } + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + { + SecretKey::ChaCha20Poly1305([fill; 32]) + } +} + +/// Poll A until it observes a `KeyResponse`, returning its `(num_resp, num_err)`. +/// B's events are drained too so neither node's backlog stalls the gossip. The +/// self-addressed response is looped back by the driver, so the originator counts +/// itself among the responders. +async fn drive_to_key_response( + a: &Serf, + b: &Serf, +) -> (usize, usize) { + loop { + let mut seen = None; + while let Some(ev) = a.poll_event() { + if let Event::KeyResponse(kr) = ev { + seen = Some((kr.num_resp, kr.num_err)); + } + } + while b.poll_event().is_some() {} + if let Some(counts) = seen { + return counts; + } + Timer::after(Duration::from_millis(5)).await; + } +} + +/// Two encrypted nodes share primary K1, then A rotates the cluster to K2 via +/// `install_key` -> `use_key` -> `remove_key`. Each op propagates and every node's +/// drain applies it to the LIVE wire keyring. The fail-on-revert check reads both +/// nodes' `keyring()` and requires primary == K2 with K1 gone; finally a user event +/// still propagates A -> B, proving the wire now runs under K2 on both planes. +#[test] +fn key_rotation_across_two_nodes_rotates_both_live_keyrings() { + let k1 = secret_key(0x11); + let k2 = secret_key(0x22); + + let transform_a = TransformOptions::default() + .with_encryption(EncryptionOptions::new().with_keyring(Keyring::new(k1))); + let transform_b = TransformOptions::default() + .with_encryption(EncryptionOptions::new().with_keyring(Keyring::new(k1))); + + let (dev_a, dev_b) = devices(); + let mut res_a = StackResources::<{ POOL + 2 }>::new(); + let mut res_b = StackResources::<{ POOL + 2 }>::new(); + let (stack_a, mut net_a) = build_stack(dev_a, &mut res_a, 1, 0x1111_2222); + let (stack_b, mut net_b) = build_stack(dev_b, &mut res_b, 2, 0x3333_4444); + + let mut bufs_a = NodeBufs::new(); + let mut bufs_b = NodeBufs::new(); + let (udp_a, tcp_a) = build_sockets(stack_a, &mut bufs_a); + let (udp_b, tcp_b) = build_sockets(stack_b, &mut bufs_b); + + let now = now(); + let (ml_a, run_a) = build_node(udp_a, tcp_a, "a", 1, now, transform_a); + let (ml_b, run_b) = build_node(udp_b, tcp_b, "b", 2, now, transform_b); + + block_on(async { + let op = async { + // Converge on a 2-member view under K1 (a real encrypted push/pull join). + join_and_converge(&ml_a, &ml_b).await; + + // install K2 across the cluster: the query collects a response from BOTH + // nodes, none in error, and each node's live keyring gains K2 as a secondary. + ml_a.install_key(k2).expect("install_key"); + let (num_resp, num_err) = drive_to_key_response(&ml_a, &ml_b).await; + assert!( + num_resp >= 2, + "install_key must collect a response from BOTH nodes (num_resp={num_resp})" + ); + assert_eq!(num_err, 0, "install_key must succeed on every node"); + assert!( + ml_a + .keyring() + .expect("a encrypted") + .secondaries() + .contains(&k2), + "A's live keyring must gain K2 as a secondary" + ); + assert!( + ml_b + .keyring() + .expect("b encrypted") + .secondaries() + .contains(&k2), + "B's live keyring must gain K2 as a secondary" + ); + + // use K2 across the cluster: both nodes promote K2 to primary. + ml_a.use_key(k2).expect("use_key"); + let (_, num_err) = drive_to_key_response(&ml_a, &ml_b).await; + assert_eq!(num_err, 0, "use_key must succeed on every node"); + assert_eq!( + ml_a.keyring().expect("a encrypted").primary_ref(), + &k2, + "A must promote K2" + ); + assert_eq!( + ml_b.keyring().expect("b encrypted").primary_ref(), + &k2, + "B must promote K2" + ); + + // remove K1 across the cluster: both nodes drop the old key. + ml_a.remove_key(k1).expect("remove_key"); + let (_, num_err) = drive_to_key_response(&ml_a, &ml_b).await; + assert_eq!(num_err, 0, "remove_key must succeed on every node"); + + // FAIL-ON-REVERT: BOTH nodes' LIVE keyrings show primary == K2 and K1 absent. + for (name, kr) in [ + ("a", ml_a.keyring().expect("a encrypted")), + ("b", ml_b.keyring().expect("b encrypted")), + ] { + assert_eq!( + kr.primary_ref(), + &k2, + "{name}: primary must be the promoted K2" + ); + assert!( + !kr.secondaries().contains(&k1), + "{name}: the removed K1 must be absent from the live keyring" + ); + } + + // Post-rotation traffic proof: a user event still crosses the wire, which now + // runs under K2 on both nodes. + ml_a + .user_event("after-rotation", Bytes::from_static(b"payload"), false) + .expect("user_event from a running node"); + loop { + if let Some(ev) = ml_b.poll_event() { + if matches!(ev, Event::User(_)) { + return true; + } + } else { + Timer::after(Duration::from_millis(5)).await; + } + } + }; + let ok = drive(op, run_a, run_b, &mut net_a, &mut net_b).await; + assert!( + ok, + "a user event must still propagate A -> B after the rotation" + ); + }); +} diff --git a/serf-embassy/tests/loopback.rs b/serf-embassy/tests/loopback.rs new file mode 100644 index 00000000..12ab41e0 --- /dev/null +++ b/serf-embassy/tests/loopback.rs @@ -0,0 +1,167 @@ +//! Two-node loopback over real embassy-net stacks wired by a channel-backed +//! paired [`Driver`](embassy_net::driver::Driver). +//! +//! Each test stands up two embassy-net stacks (static IPs `169.254.1.1/.2`), a +//! `Serf` + `Runner` on each, and drives every future concurrently under one +//! [`block_on`](futures::executor::block_on): the two serf run loops, the two +//! embassy-net stack run loops, and the operation under test, raced against a +//! wall-clock timeout so a regression fails fast instead of hanging. +//! +//! This is the behavioral gate for the embassy driver: an async join converging +//! over TCP push/pull, and a graceful leave — over real sockets, not mocks. + +// nested `if let X = ev { if cond }` kept for readability, as in the crate roots. +#![allow(clippy::collapsible_if)] + +mod support; + +use std::time::Instant as StdInstant; + +use embassy_futures::select::{Either, select}; +use embassy_net::{IpEndpoint, StackResources}; +use embassy_time::{Duration, Timer}; +use futures::executor::block_on; +use serf_embassy::{Event, TransformOptions, now}; + +use support::cluster::{ + NodeBufs, POOL, TEST_TIMEOUT, addr, build_node, build_sockets, build_stack, devices, drive, + join_and_converge, +}; + +/// Substrate check: a raw UDP datagram crosses the two paired-device stacks. +/// Isolates the device/stack/waker plumbing from the serf protocol so a +/// convergence failure can be attributed correctly. +#[test] +fn raw_udp_crosses_the_paired_link() { + let (dev_a, dev_b) = devices(); + let mut res_a = StackResources::<{ POOL + 2 }>::new(); + let mut res_b = StackResources::<{ POOL + 2 }>::new(); + let (stack_a, mut net_a) = build_stack(dev_a, &mut res_a, 1, 0x1111_2222); + let (stack_b, mut net_b) = build_stack(dev_b, &mut res_b, 2, 0x3333_4444); + + let mut bufs_a = NodeBufs::new(); + let mut bufs_b = NodeBufs::new(); + let (mut udp_a, _tcp_a) = build_sockets(stack_a, &mut bufs_a); + let (mut udp_b, _tcp_b) = build_sockets(stack_b, &mut bufs_b); + + block_on(async { + udp_a.bind(7000).expect("bind a"); + udp_b.bind(7000).expect("bind b"); + let op = async { + let dst = IpEndpoint::from(addr(2, 7000)); + let mut buf = [0u8; 32]; + loop { + // Ignoring Err: best-effort retry — resend until B's socket is up and the + // recv below observes the datagram, so a dropped early send is harmless. + let _ = udp_a.send_to(b"ping", dst).await; + match select( + udp_b.recv_from(&mut buf), + Timer::after(Duration::from_millis(50)), + ) + .await + { + Either::First(Ok((n, _))) => return buf[..n].to_vec(), + _ => continue, + } + } + }; + let nets = select(net_a.run(), net_b.run()); + let got = match select(op, select(nets, Timer::after(TEST_TIMEOUT))).await { + Either::First(v) => v, + Either::Second(_) => panic!("raw UDP did not cross the link within {TEST_TIMEOUT:?}"), + }; + assert_eq!(got, b"ping"); + }); +} + +/// Two nodes; B joins A as a seed; both converge on a 2-member view; then B leaves +/// gracefully and observes `Event::LeftCluster`. +#[test] +fn two_node_join_converges_and_leaves() { + let (dev_a, dev_b) = devices(); + let mut res_a = StackResources::<{ POOL + 2 }>::new(); + let mut res_b = StackResources::<{ POOL + 2 }>::new(); + let (stack_a, mut net_a) = build_stack(dev_a, &mut res_a, 1, 0x1111_2222); + let (stack_b, mut net_b) = build_stack(dev_b, &mut res_b, 2, 0x3333_4444); + + let mut bufs_a = NodeBufs::new(); + let mut bufs_b = NodeBufs::new(); + let (udp_a, tcp_a) = build_sockets(stack_a, &mut bufs_a); + let (udp_b, tcp_b) = build_sockets(stack_b, &mut bufs_b); + + let now = now(); + let (ml_a, run_a) = build_node(udp_a, tcp_a, "a", 1, now, TransformOptions::default()); + let (ml_b, run_b) = build_node(udp_b, tcp_b, "b", 2, now, TransformOptions::default()); + + let start = StdInstant::now(); + block_on(async { + let op = async { + // B joins A; the await resolves once the push/pull reaches the seed. + let reached = ml_b + .join( + &serf_embassy::SocketAddrResolver, + &[serf_embassy::MaybeResolved::Resolved(addr(1, 7946))], + false, + ) + .await + .expect("join from a running node"); + assert!( + reached.iter().any(|a| *a == addr(1, 7946)), + "the join must report A as reached: {reached:?}" + ); + // Wait for BOTH to converge (A learns B a tick after the exchange). + loop { + if ml_a.num_members() == 2 && ml_b.num_members() == 2 { + break; + } + Timer::after(Duration::from_millis(10)).await; + } + // B leaves gracefully and must observe LeftCluster. + ml_b.leave().expect("leave from a running node"); + loop { + if let Some(ev) = ml_b.poll_event() { + if matches!(ev, Event::LeftCluster) { + return true; + } + } else { + Timer::after(Duration::from_millis(5)).await; + } + } + }; + let left = drive(op, run_a, run_b, &mut net_a, &mut net_b).await; + assert!(left, "B did not observe LeftCluster after leaving"); + }); + let elapsed = start.elapsed(); + + assert_eq!(ml_a.num_members(), 2, "A did not converge to 2 members"); + println!("two_node_join_converges_and_leaves: converged in {elapsed:?}"); +} + +/// The `join_and_converge` helper is exercised implicitly by the other suites; +/// this asserts it standalone so a convergence regression is attributed here. +#[test] +fn join_and_converge_helper_reaches_two_members() { + let (dev_a, dev_b) = devices(); + let mut res_a = StackResources::<{ POOL + 2 }>::new(); + let mut res_b = StackResources::<{ POOL + 2 }>::new(); + let (stack_a, mut net_a) = build_stack(dev_a, &mut res_a, 1, 0x1111_2222); + let (stack_b, mut net_b) = build_stack(dev_b, &mut res_b, 2, 0x3333_4444); + + let mut bufs_a = NodeBufs::new(); + let mut bufs_b = NodeBufs::new(); + let (udp_a, tcp_a) = build_sockets(stack_a, &mut bufs_a); + let (udp_b, tcp_b) = build_sockets(stack_b, &mut bufs_b); + + let now = now(); + let (ml_a, run_a) = build_node(udp_a, tcp_a, "a", 1, now, TransformOptions::default()); + let (ml_b, run_b) = build_node(udp_b, tcp_b, "b", 2, now, TransformOptions::default()); + + block_on(async { + let op = async { + join_and_converge(&ml_a, &ml_b).await; + (ml_a.num_members(), ml_b.num_members()) + }; + let (a_n, b_n) = drive(op, run_a, run_b, &mut net_a, &mut net_b).await; + assert_eq!((a_n, b_n), (2, 2)); + }); +} diff --git a/serf-embassy/tests/query.rs b/serf-embassy/tests/query.rs new file mode 100644 index 00000000..625711b3 --- /dev/null +++ b/serf-embassy/tests/query.rs @@ -0,0 +1,105 @@ +//! Query-plane end-to-end over the embassy-net gossip self-delivery loopback. +//! +//! serf directs a node's response to its OWN locally-originated query to the +//! originator's advertise address = THIS node's address. A real OS UDP socket +//! loops such a self-addressed datagram back into recv; embassy-net (smoltcp +//! underneath) does not, so the driver emulates the OS self-delivery. This test +//! exercises that path: the query round-trip proves the originator observes its +//! OWN (local, self-addressed) response alongside the remote peer's. Without the +//! driver's self-delivery loopback the originator never observes its own response, +//! so `a_resp` would hold only `"b"` and the test fails. + +#![allow(clippy::collapsible_if)] + +mod support; + +use core::{net::SocketAddr, time::Duration as CoreDuration}; + +use embassy_net::StackResources; +use embassy_time::{Duration, Timer}; +use futures::executor::block_on; +use serf_embassy::{Bytes, Event, QueryParams, Serf, TransformOptions, now}; +use smol_str::SmolStr; + +use support::cluster::{ + NodeBufs, POOL, build_node, build_sockets, build_stack, devices, drive, join_and_converge, +}; + +/// Drain a node's buffered events: answer every observed `Event::Query` (directing +/// the reply to the query's originator), and record every `Event::QueryResponse`'s +/// responder id. +fn service_queries(node: &Serf, responders: &mut Vec) { + let mut pending = Vec::new(); + while let Some(ev) = node.poll_event() { + match ev { + Event::Query(qe) => pending.push(qe), + Event::QueryResponse(qr) => responders.push(qr.from().id_ref().clone()), + _ => {} + } + } + for qe in pending { + // Ignoring Err: a duplicate / past-deadline respond is a no-op the test tolerates. + let _ = node.respond(&qe, Bytes::from_static(b"pong")); + } +} + +/// Two converged nodes; A issues a query; the originator must collect responses +/// from BOTH the remote peer AND itself. The local node self-processes its own +/// query and directs its reply to its own advertise address — a self-addressed +/// gossip datagram embassy-net does not loop back on its own. +#[test] +fn query_collects_local_and_remote_responses() { + let (dev_a, dev_b) = devices(); + let mut res_a = StackResources::<{ POOL + 2 }>::new(); + let mut res_b = StackResources::<{ POOL + 2 }>::new(); + let (stack_a, mut net_a) = build_stack(dev_a, &mut res_a, 1, 0x1111_2222); + let (stack_b, mut net_b) = build_stack(dev_b, &mut res_b, 2, 0x3333_4444); + + let mut bufs_a = NodeBufs::new(); + let mut bufs_b = NodeBufs::new(); + let (udp_a, tcp_a) = build_sockets(stack_a, &mut bufs_a); + let (udp_b, tcp_b) = build_sockets(stack_b, &mut bufs_b); + + let now = now(); + let (ml_a, run_a) = build_node(udp_a, tcp_a, "a", 1, now, TransformOptions::default()); + let (ml_b, run_b) = build_node(udp_b, tcp_b, "b", 2, now, TransformOptions::default()); + + block_on(async { + let op = async { + join_and_converge(&ml_a, &ml_b).await; + + ml_a + .query( + "ping", + Bytes::from_static(b"q"), + QueryParams { + timeout: CoreDuration::from_secs(4), + ..Default::default() + }, + ) + .expect("query from a running node"); + + // A must collect responses from BOTH itself ("a", via the loopback) and B + // ("b", over the wire). Both nodes answer every Query they observe. + let mut a_resp: Vec = Vec::new(); + let mut b_resp: Vec = Vec::new(); + loop { + service_queries(&ml_a, &mut a_resp); + service_queries(&ml_b, &mut b_resp); + if a_resp.iter().any(|id| id == "a") && a_resp.iter().any(|id| id == "b") { + return a_resp; + } + Timer::after(Duration::from_millis(5)).await; + } + }; + let a_resp = drive(op, run_a, run_b, &mut net_a, &mut net_b).await; + assert!( + a_resp.iter().any(|id| id == "a"), + "originator did not observe its OWN self-addressed, looped-back response: {a_resp:?}" + ); + assert!( + a_resp.iter().any(|id| id == "b"), + "originator did not observe the remote peer's response: {a_resp:?}" + ); + }); +} diff --git a/serf-embassy/tests/support/cluster.rs b/serf-embassy/tests/support/cluster.rs new file mode 100644 index 00000000..849b3b85 --- /dev/null +++ b/serf-embassy/tests/support/cluster.rs @@ -0,0 +1,179 @@ +//! Shared two-node cluster scaffolding over the paired embassy-net driver: the +//! per-node socket buffers, stack + socket construction, node construction, and +//! the `drive` harness that runs both serf loops, both stack loops, and the +//! operation under test against a wall-clock timeout. + +#![allow(dead_code)] + +use core::net::{IpAddr, Ipv4Addr, SocketAddr}; + +use embassy_futures::select::{Either, select}; +use embassy_net::{ + Config as NetConfig, Ipv4Cidr, Runner as NetRunner, Stack, StackResources, StaticConfigV4, + tcp::TcpSocket, + udp::{PacketMetadata, UdpSocket}, +}; +use embassy_time::{Duration, Timer}; +use futures::executor::block_on; +use memberlist_proto::{Instant, SeedableRng, SmallRng}; +use serf_embassy::{ + EndpointOptions, MaybeResolved, Options, Runner, Serf, SerfOptions, SocketAddrResolver, + TransformOptions, +}; +use smol_str::SmolStr; + +use super::paired_device::{PairedDevice, pair}; + +/// TCP socket pool size per node (a listener plus dial/accept sockets). +pub const POOL: usize = 4; +/// Per-TCP-socket rx/tx buffer bytes. +pub const TCP_BUF: usize = 4096; +/// Wall-clock cap on each test so a wedged plane fails fast. +/// Wall-clock cap on each test so a wedged plane fails fast. Generous because a +/// serf key-management query runs to its own multi-second deadline, and the key +/// rotation suite issues three of them back to back. +pub const TEST_TIMEOUT: Duration = Duration::from_secs(15); + +/// A node's `169.254.1.:` wire address. +pub fn addr(last: u8, port: u16) -> SocketAddr { + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(169, 254, 1, last)), port) +} + +/// All the owned buffers one node's sockets borrow. Declared in the test frame so +/// the sockets (and the `Serf`/`Runner` that hold them) can borrow them for the +/// whole `block_on`. +pub struct NodeBufs { + udp_rx_meta: [PacketMetadata; 16], + udp_rx: [u8; 16 * 1024], + udp_tx_meta: [PacketMetadata; 16], + udp_tx: [u8; 16 * 1024], + tcp_rx: [[u8; TCP_BUF]; POOL], + tcp_tx: [[u8; TCP_BUF]; POOL], +} + +impl Default for NodeBufs { + fn default() -> Self { + Self::new() + } +} + +impl NodeBufs { + pub fn new() -> Self { + Self { + udp_rx_meta: [PacketMetadata::EMPTY; 16], + udp_rx: [0u8; 16 * 1024], + udp_tx_meta: [PacketMetadata::EMPTY; 16], + udp_tx: [0u8; 16 * 1024], + tcp_rx: [[0u8; TCP_BUF]; POOL], + tcp_tx: [[0u8; TCP_BUF]; POOL], + } + } +} + +/// Build one node's `UdpSocket` + `[TcpSocket; POOL]` over its `Stack` and bufs. +pub fn build_sockets<'a>( + stack: Stack<'a>, + bufs: &'a mut NodeBufs, +) -> (UdpSocket<'a>, [TcpSocket<'a>; POOL]) { + let udp = UdpSocket::new( + stack, + &mut bufs.udp_rx_meta, + &mut bufs.udp_rx, + &mut bufs.udp_tx_meta, + &mut bufs.udp_tx, + ); + let mut rx_iter = bufs.tcp_rx.iter_mut(); + let mut tx_iter = bufs.tcp_tx.iter_mut(); + let tcp = core::array::from_fn::<_, POOL, _>(|_| { + let rx = rx_iter.next().expect("POOL rx buffers"); + let tx = tx_iter.next().expect("POOL tx buffers"); + TcpSocket::new(stack, rx, tx) + }); + (udp, tcp) +} + +/// Build a static-IPv4 embassy-net stack over a paired device. +pub fn build_stack<'a>( + device: PairedDevice, + resources: &'a mut StackResources<{ POOL + 2 }>, + last: u8, + seed: u64, +) -> (Stack<'a>, NetRunner<'a, PairedDevice>) { + let config = NetConfig::ipv4_static(StaticConfigV4 { + address: Ipv4Cidr::new(Ipv4Addr::new(169, 254, 1, last), 16), + gateway: None, + dns_servers: Default::default(), + }); + embassy_net::new(device, config, resources, seed) +} + +/// Build one serf node over its sockets, seeding the gossip and serf RNGs +/// distinctly (from `last` so two nodes never share a schedule). +pub fn build_node<'a>( + udp: UdpSocket<'a>, + tcp: [TcpSocket<'a>; POOL], + id: &str, + last: u8, + now_: Instant, + transform: TransformOptions, +) -> (Serf, Runner<'a, SmolStr, POOL>) { + block_on(Serf::new_with_rng::<_, POOL>( + Options::new(), + transform, + EndpointOptions::new(SmolStr::new(id), addr(last, 7946)), + SerfOptions::new(), + &SocketAddrResolver, + udp, + tcp, + now_, + SmallRng::seed_from_u64(u64::from(last)), + SmallRng::seed_from_u64(u64::from(last) + 100), + )) + .expect("build node") +} + +/// Convenience: create the two paired stacks and return their runners plus a +/// closure-free tuple of the pieces a test drives. Kept as separate calls in each +/// test so the borrows (buffers / resources) live in the test frame. +pub type PairedStacks<'a> = (Stack<'a>, NetRunner<'a, PairedDevice>); + +/// B joins A as a seed and both converge on a 2-member view. Panics via the outer +/// timeout if convergence stalls. +pub async fn join_and_converge(a: &Serf, b: &Serf) { + b.join( + &SocketAddrResolver, + &[MaybeResolved::Resolved(addr(1, 7946))], + false, + ) + .await + .expect("join from a running node"); + loop { + if a.num_members() == 2 && b.num_members() == 2 { + break; + } + Timer::after(Duration::from_millis(10)).await; + } +} + +/// Drive `op` to completion against both serf run loops, both embassy-net stack +/// run loops, and the test timeout. Returns the op's value, or panics on timeout. +pub async fn drive( + op: impl core::future::Future, + ml_a: Runner<'_, SmolStr, POOL>, + ml_b: Runner<'_, SmolStr, POOL>, + net_a: &mut NetRunner<'_, PairedDevice>, + net_b: &mut NetRunner<'_, PairedDevice>, +) -> T { + let nets = select(net_a.run(), net_b.run()); + let mls = select(ml_a.run(), ml_b.run()); + let infra = select(nets, mls); + match select(op, select(infra, Timer::after(TEST_TIMEOUT))).await { + Either::First(v) => v, + Either::Second(_) => panic!("test timed out after {TEST_TIMEOUT:?}"), + } +} + +/// Build the two cross-wired paired devices for a test. +pub fn devices() -> (PairedDevice, PairedDevice) { + pair() +} diff --git a/serf-embassy/tests/support/mod.rs b/serf-embassy/tests/support/mod.rs new file mode 100644 index 00000000..5fd255fa --- /dev/null +++ b/serf-embassy/tests/support/mod.rs @@ -0,0 +1,5 @@ +//! Shared test support: the channel-backed paired embassy-net driver and the +//! two-node cluster scaffolding. + +pub mod cluster; +pub mod paired_device; diff --git a/serf-embassy/tests/support/paired_device.rs b/serf-embassy/tests/support/paired_device.rs new file mode 100644 index 00000000..e4714197 --- /dev/null +++ b/serf-embassy/tests/support/paired_device.rs @@ -0,0 +1,185 @@ +//! A channel-backed paired [`embassy_net_driver::Driver`] for host loopback tests. +//! +//! Two `PairedDevice`s are cross-wired by [`pair`]: each device's transmit pushes +//! a frame into a shared channel that is the OTHER device's receive channel, and +//! wakes the other device's stack so its `embassy_net::Runner` re-polls and drains +//! the frame. This moves real ethernet frames between two embassy-net stacks on a +//! single `block_on` thread — the substrate the loopback tests run a two-node serf +//! cluster over. + +#![allow(dead_code)] + +use std::{ + cell::{Cell, RefCell}, + collections::VecDeque, + rc::Rc, + task::{Context, Waker}, +}; + +use embassy_net::driver::{Capabilities, Driver, HardwareAddress, LinkState, RxToken, TxToken}; + +/// The link MTU advertised by the paired devices (a standard ethernet frame). +const MTU: usize = 1514; + +/// A shared frame FIFO between the two devices. +type Wire = Rc>>>; +/// A shared slot holding a stack's most recent receive waker. +type WakerSlot = Rc>>; + +/// One end of a cross-wired virtual ethernet link. +/// +/// Reads frames from `rx` (frames the peer transmitted) and writes frames to `tx` +/// (delivered to the peer's `rx`). `peer_waker` is the peer stack's receive waker, +/// woken on every transmit so the peer promptly drains the frame. +pub struct PairedDevice { + rx: Wire, + tx: Wire, + /// This device's own receive waker, registered by `receive`/`link_state` and + /// woken by the peer's transmit. + my_waker: WakerSlot, + /// The peer device's receive waker, woken by this device's transmit. + peer_waker: WakerSlot, + /// When `false`, this device's transmits are DROPPED (not delivered to the peer, + /// no peer wake) — simulating a peer/link that has gone silent (stops ACKing). A + /// test closes this via [`PairedDevice::tx_gate`] to exercise a worker's socket + /// inactivity timeout. + tx_open: Rc>, + mac: [u8; 6], +} + +/// Build the two ends of one virtual link, cross-wiring their channels and wakers. +/// +/// Frames device `A` transmits arrive at device `B`'s receive queue (and wake +/// `B`'s stack), and vice versa. +pub fn pair() -> (PairedDevice, PairedDevice) { + let a2b: Wire = Rc::new(RefCell::new(VecDeque::new())); + let b2a: Wire = Rc::new(RefCell::new(VecDeque::new())); + let waker_a: WakerSlot = Rc::new(RefCell::new(None)); + let waker_b: WakerSlot = Rc::new(RefCell::new(None)); + ( + PairedDevice { + rx: b2a.clone(), + tx: a2b.clone(), + my_waker: waker_a.clone(), + peer_waker: waker_b.clone(), + tx_open: Rc::new(Cell::new(true)), + mac: [0x02, 0, 0, 0, 0, 1], + }, + PairedDevice { + rx: a2b, + tx: b2a, + my_waker: waker_b, + peer_waker: waker_a, + tx_open: Rc::new(Cell::new(true)), + mac: [0x02, 0, 0, 0, 0, 2], + }, + ) +} + +impl PairedDevice { + /// Register the current task's waker as this device's receive waker, so the + /// peer's transmit can wake the stack. + fn register(&self, cx: &mut Context<'_>) { + *self.my_waker.borrow_mut() = Some(cx.waker().clone()); + } + + /// A handle to this device's transmit gate. Setting it to `false` drops every + /// subsequent frame this device transmits, so the peer stops receiving (and so + /// stops ACKing) — letting a test exercise a worker's socket inactivity timeout. + pub fn tx_gate(&self) -> Rc> { + self.tx_open.clone() + } +} + +/// A receive token carrying one delivered frame. +pub struct PairedRx(Vec); + +/// A transmit token that, on consume, writes the frame to the peer's receive +/// queue and wakes the peer's stack. +pub struct PairedTx { + tx: Wire, + peer_waker: WakerSlot, + tx_open: Rc>, +} + +impl RxToken for PairedRx { + fn consume(self, f: F) -> R + where + F: FnOnce(&mut [u8]) -> R, + { + let mut buf = self.0; + f(&mut buf) + } +} + +impl TxToken for PairedTx { + fn consume(self, len: usize, f: F) -> R + where + F: FnOnce(&mut [u8]) -> R, + { + let mut buf = std::vec![0u8; len]; + let r = f(&mut buf); + // A closed tx gate DROPS the frame, simulating a peer/link gone silent: it never + // reaches the peer and never wakes it, so the peer stops ACKing. + if self.tx_open.get() { + self.tx.borrow_mut().push_back(buf); + // Wake the peer's stack so its `embassy_net::Runner` re-polls and drains this + // frame; without this the peer would not process inbound traffic until some + // unrelated event polled it. + if let Some(w) = self.peer_waker.borrow_mut().take() { + w.wake(); + } + } + r + } +} + +impl Driver for PairedDevice { + type RxToken<'a> + = PairedRx + where + Self: 'a; + type TxToken<'a> + = PairedTx + where + Self: 'a; + + fn receive(&mut self, cx: &mut Context<'_>) -> Option<(PairedRx, PairedTx)> { + // Always (re)register the waker so a frame arriving after this poll wakes us. + self.register(cx); + let frame = self.rx.borrow_mut().pop_front()?; + Some(( + PairedRx(frame), + PairedTx { + tx: self.tx.clone(), + peer_waker: self.peer_waker.clone(), + tx_open: self.tx_open.clone(), + }, + )) + } + + fn transmit(&mut self, _cx: &mut Context<'_>) -> Option { + Some(PairedTx { + tx: self.tx.clone(), + peer_waker: self.peer_waker.clone(), + tx_open: self.tx_open.clone(), + }) + } + + fn link_state(&mut self, cx: &mut Context<'_>) -> LinkState { + // Keep the waker fresh on the link-state poll path too (the stack polls this), + // so a frame delivered between data polls still wakes the stack. + self.register(cx); + LinkState::Up + } + + fn capabilities(&self) -> Capabilities { + let mut caps = Capabilities::default(); + caps.max_transmission_unit = MTU; + caps + } + + fn hardware_address(&self) -> HardwareAddress { + HardwareAddress::Ethernet(self.mac) + } +} diff --git a/serf-embassy/tests/user_event.rs b/serf-embassy/tests/user_event.rs new file mode 100644 index 00000000..35bddfc0 --- /dev/null +++ b/serf-embassy/tests/user_event.rs @@ -0,0 +1,58 @@ +//! User-event propagation end-to-end: after two nodes converge, a user event +//! broadcast by A crosses the gossip plane and B observes it as `Event::User`. + +#![allow(clippy::collapsible_if)] + +mod support; + +use embassy_time::{Duration, Timer}; +use futures::executor::block_on; +use serf_embassy::{Bytes, Event, TransformOptions, now}; + +use embassy_net::StackResources; +use support::cluster::{ + NodeBufs, POOL, build_node, build_sockets, build_stack, devices, drive, join_and_converge, +}; + +/// After convergence, A broadcasts a user event; B must observe it via +/// `poll_event` as `Event::User`. +#[test] +fn user_event_propagates_a_to_b() { + let (dev_a, dev_b) = devices(); + let mut res_a = StackResources::<{ POOL + 2 }>::new(); + let mut res_b = StackResources::<{ POOL + 2 }>::new(); + let (stack_a, mut net_a) = build_stack(dev_a, &mut res_a, 1, 0x1111_2222); + let (stack_b, mut net_b) = build_stack(dev_b, &mut res_b, 2, 0x3333_4444); + + let mut bufs_a = NodeBufs::new(); + let mut bufs_b = NodeBufs::new(); + let (udp_a, tcp_a) = build_sockets(stack_a, &mut bufs_a); + let (udp_b, tcp_b) = build_sockets(stack_b, &mut bufs_b); + + let now = now(); + let (ml_a, run_a) = build_node(udp_a, tcp_a, "a", 1, now, TransformOptions::default()); + let (ml_b, run_b) = build_node(udp_b, tcp_b, "b", 2, now, TransformOptions::default()); + + block_on(async { + let op = async { + join_and_converge(&ml_a, &ml_b).await; + + ml_a + .user_event("greet", Bytes::from_static(b"hello"), false) + .expect("user_event from a running node"); + + // B observes the user event on its gossip plane. + loop { + if let Some(ev) = ml_b.poll_event() { + if matches!(ev, Event::User(_)) { + return true; + } + } else { + Timer::after(Duration::from_millis(5)).await; + } + } + }; + let saw = drive(op, run_a, run_b, &mut net_a, &mut net_b).await; + assert!(saw, "B did not observe the user event"); + }); +} From 54da7ea21b5989ea393d9857177537619a6115b3 Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 10 Jul 2026 18:17:35 +1200 Subject: [PATCH 062/128] fix(serf-embassy): make the conflict-loss shutdown terminal A lost id-conflict Shutdown only set a flag while the runner kept pumping and the handle kept accepting commands, so the losing node stayed active under the duplicate identity. The drain now poisons the shared state (pending joins and later commands fail fast), the pump loop does a final drain then exits so Runner::run completes and the slot workers and sockets wind down, and the buffered Shutdown stays observable via poll_event; a conflict-loss e2e proves the loser stops while the winner runs. --- serf-embassy/src/error/mod.rs | 92 ++++++++++- serf-embassy/src/lib.rs | 2 +- serf-embassy/src/runner.rs | 55 +++++-- serf-embassy/src/serf/mod.rs | 137 +++++++++++++--- serf-embassy/src/shared/mod.rs | 37 ++++- serf-embassy/tests/conflict_shutdown.rs | 201 ++++++++++++++++++++++++ 6 files changed, 481 insertions(+), 43 deletions(-) create mode 100644 serf-embassy/tests/conflict_shutdown.rs diff --git a/serf-embassy/src/error/mod.rs b/serf-embassy/src/error/mod.rs index 0f50e226..1b9166a4 100644 --- a/serf-embassy/src/error/mod.rs +++ b/serf-embassy/src/error/mod.rs @@ -207,6 +207,12 @@ pub enum JoinError { /// Every dispatched push/pull terminated without contacting a seed. The /// [`JoinFailed`] payload carries the requested-seed count. Failed(JoinFailed), + /// The run loop stopped after a lost id-conflict `Event::Shutdown` before this + /// join could resolve — the node is no longer active. A join in flight when the + /// conflict-loss lands resolves here rather than hanging, and a join attempted + /// after shutdown fails fast with it. Mirrors serf-reactor's `SerfError::Shutdown` + /// for the join reply. + Shutdown, } impl JoinError { @@ -233,6 +239,13 @@ impl JoinError { pub const fn is_failed(&self) -> bool { matches!(self, JoinError::Failed(_)) } + + /// Whether the run loop stopped after a lost id-conflict shutdown before the + /// join resolved. + #[inline] + pub const fn is_shutdown(&self) -> bool { + matches!(self, JoinError::Shutdown) + } } impl fmt::Display for JoinError { @@ -242,6 +255,9 @@ impl fmt::Display for JoinError { JoinError::Control(e) => write!(f, "join was rejected: {e}"), JoinError::NoAddresses => f.write_str("no wire address resolved for any seed"), JoinError::Failed(e) => write!(f, "{e}"), + JoinError::Shutdown => { + f.write_str("the node shut down after losing an id-conflict vote before the join resolved") + } } } } @@ -260,7 +276,81 @@ impl std::error::Error for JoinError { JoinError::Resolve(e) => Some(e.as_ref()), JoinError::Control(e) => Some(e), JoinError::Failed(e) => Some(e), - JoinError::NoAddresses => None, + JoinError::NoAddresses | JoinError::Shutdown => None, + } + } +} + +/// Why a [`Serf`](crate::Serf) command (`user_event`, `query`, `leave`, +/// `force_leave`, `respond`, `set_tags`, key management) could not run. +/// +/// Either the engine rejected the command ([`Serf`](Self::Serf)), or the run loop +/// has already stopped after a lost id-conflict `Event::Shutdown` +/// ([`Shutdown`](Self::Shutdown)) — after which the node is no longer active and +/// the handle rejects every command up front, before touching the (now +/// winding-down) engine, so a losing node cannot keep mutating serf state under the +/// duplicate identity. +#[derive(Debug)] +#[non_exhaustive] +pub enum OpError { + /// The run loop stopped after a lost id-conflict `Event::Shutdown`; the node is + /// no longer active and rejects further commands. Mirrors serf-reactor's + /// `SerfError::Shutdown`. + Shutdown, + /// The engine rejected the command (e.g. an oversized user event, a + /// past-deadline `respond`, or a leave/join from an invalid lifecycle state). + Serf(SerfError), +} + +impl OpError { + /// Whether the node has shut down after a lost id-conflict vote and no longer + /// accepts commands. + #[inline] + pub const fn is_shutdown(&self) -> bool { + matches!(self, OpError::Shutdown) + } + + /// Whether the engine rejected the command. + #[inline] + pub const fn is_serf(&self) -> bool { + matches!(self, OpError::Serf(_)) + } + + /// The underlying [`SerfError`] when the engine rejected the command, else + /// `None` (a shutdown refusal carries no engine error). + #[inline] + pub const fn as_serf(&self) -> Option<&SerfError> { + match self { + OpError::Serf(e) => Some(e), + OpError::Shutdown => None, + } + } +} + +impl fmt::Display for OpError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + OpError::Shutdown => f.write_str( + "the node has shut down after losing an id-conflict vote; it no longer accepts commands", + ), + OpError::Serf(e) => write!(f, "{e}"), + } + } +} + +impl From for OpError { + fn from(e: SerfError) -> Self { + OpError::Serf(e) + } +} + +#[cfg(feature = "std")] +#[cfg_attr(docsrs, doc(cfg(feature = "std")))] +impl std::error::Error for OpError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + OpError::Serf(e) => Some(e), + OpError::Shutdown => None, } } } diff --git a/serf-embassy/src/lib.rs b/serf-embassy/src/lib.rs index 05d6640b..eecc1b62 100644 --- a/serf-embassy/src/lib.rs +++ b/serf-embassy/src/lib.rs @@ -38,7 +38,7 @@ mod worker; pub use bytes::Bytes; pub use config::Options; -pub use error::{InitError, JoinError, SocketTimeoutOutOfRange}; +pub use error::{InitError, JoinError, OpError, SocketTimeoutOutOfRange}; pub use gossip_io::SerfGossip; pub use resolver::{AddressResolver, SocketAddrResolver}; pub use runner::Runner; diff --git a/serf-embassy/src/runner.rs b/serf-embassy/src/runner.rs index cbfe9303..de421db8 100644 --- a/serf-embassy/src/runner.rs +++ b/serf-embassy/src/runner.rs @@ -2,7 +2,7 @@ //! as sibling futures. //! //! One embassy task owns the [`Runner`] and calls [`Runner::run`]. Inside, two -//! kinds of future run concurrently under one `join`: +//! kinds of future race under one `select`: //! //! - the **pump loop** — re-pump the engine over a fresh //! [`SerfGossip`](crate::SerfGossip) + [`SerfStream`](crate::SerfStream) view, @@ -12,6 +12,11 @@ //! - the **N workers** — each [`run_slot`](crate::worker::run_slot) owns one //! `TcpSocket` and its `RefCell`, looping internally forever. //! +//! Both diverge under normal operation, so the `select` never resolves and the task +//! runs forever. A lost id-conflict [`Event::Shutdown`](serf_embedded::Event) is the +//! one terminal exit: the pump loop observes the poisoned shared state and returns, +//! collapsing the `select` — the worker futures drop and every socket winds down. +//! //! Only the pump loop re-pumps; the workers loop on their own. Because //! [`SerfEngine::pump`](serf_embedded::SerfEngine::pump) is synchronous, the //! pump's borrows of the engine and the mailboxes complete before its `.await`, @@ -37,7 +42,7 @@ use core::{cell::RefCell, net::SocketAddr}; use alloc::{collections::VecDeque, rc::Rc, vec::Vec}; use embassy_futures::{ - join::{join, join_array}, + join::join_array, select::{select, select3}, }; use embassy_net::{tcp::TcpSocket, udp::UdpSocket}; @@ -109,11 +114,16 @@ where G: Rng, SR: Rng + SeedableRng, { - /// Drive the node forever: pump the engine and run the `N` workers concurrently. + /// Drive the node: pump the engine and run the `N` workers concurrently. /// - /// Never returns under normal operation; spawn it as an embassy task (or drive - /// it with `select` against an operation in a test). - pub async fn run(self) -> ! { + /// Runs forever under normal operation — spawn it as an embassy task (or drive + /// it with `select` against an operation in a test). It returns ONLY after a lost + /// id-conflict [`Event::Shutdown`](serf_embedded::Event::Shutdown): the pump loop + /// observes the poisoned shared state, performs its final drain, and returns, + /// which resolves the `select` below and drops the worker futures — so this frame + /// unwinds and the owned sockets (the TCP pool and the gossip UDP socket) close, + /// taking the losing node off the wire. + pub async fn run(self) { let Runner { shared, udp, @@ -147,16 +157,23 @@ where ) }); - // The pump loop and the workers run as siblings under one join. The pump loop - // diverges (loops forever); `join_array` of the diverging workers likewise - // never completes, so `join` never resolves — matching the `-> !` contract. - join( + // The pump loop and the workers race under one `select`. Under normal operation + // the pump loop diverges (loops forever) and `join_array` of the diverging + // workers likewise never completes, so the `select` never resolves and `run` + // never returns. On a lost id-conflict shutdown the pump loop RETURNS after its + // final drain; the `select` then completes and DROPS the worker futures, + // releasing their `&mut TcpSocket` borrows. `run` then returns and the owned + // sockets close as this frame unwinds — the structural teardown that stops the + // losing node without any per-worker abort. + // + // Ignoring the `Either`: only the pump-loop arm can ever resolve (the workers + // diverge), and its `()` output carries nothing — reaching here means shutdown, + // and `run` simply returns. + let _ = select( pump_loop(&shared, &udp, &mailboxes, &cmd_wakes, &mut free, &loopback), join_array(workers), ) .await; - - core::unreachable!("the run loop and workers never complete") } } @@ -169,8 +186,7 @@ async fn pump_loop( cmd_wakes: &[SlotWake], free: &mut Vec, loopback: &RefCell>>, -) -> ! -where +) where I: memberlist_proto::Id + Clone, G: Rng, SR: Rng + SeedableRng, @@ -210,6 +226,17 @@ where next = min_opt(next, Some(now)); } + // A lost id-conflict `Event::Shutdown` observed by the drain above poisoned the + // shared state. The pump+drain that just ran was the FINAL one — every remaining + // event is buffered for the app's `poll_event` — so stop pumping now instead of + // sleeping. Returning ends this future, which resolves `Runner::run`'s `select` + // and collapses the workers so the sockets wind down (an abrupt stop: a lost + // node does not gossip a leave for the id it just lost, which would only + // confuse the winner). + if shared.is_shutdown() { + return; + } + // Wait for the next thing worth re-pumping for: an inbound gossip datagram, a // worker/handle pump-wake, or the folded deadline. match next { diff --git a/serf-embassy/src/serf/mod.rs b/serf-embassy/src/serf/mod.rs index cf2df089..e8d3dbe3 100644 --- a/serf-embassy/src/serf/mod.rs +++ b/serf-embassy/src/serf/mod.rs @@ -25,7 +25,7 @@ use embassy_net::{tcp::TcpSocket, udp::UdpSocket}; use embassy_time::Timer; use memberlist_proto::{EndpointOptions, Instant, Rng, SeedableRng, SmallRng}; use serf_embedded::{ - Event, JoinId, MaybeResolved, ReachedSet, SerfEngine, SerfError, SerfOptions, TransformOptions, + Event, JoinId, MaybeResolved, ReachedSet, SerfEngine, SerfOptions, TransformOptions, validate_runtime_config, }; use serf_proto::{ @@ -40,7 +40,7 @@ use serf_embedded::{Keyring, SecretKey}; use crate::{ config::Options, - error::{InitError, JoinError, SocketTimeoutOutOfRange}, + error::{InitError, JoinError, OpError, SocketTimeoutOutOfRange}, mailbox::{Command, Mailbox}, resolver::AddressResolver, runner::Runner, @@ -472,6 +472,23 @@ where self.shared.is_shutdown() } + /// Stand in for a lost id-conflict vote in driver tests: buffer the terminal + /// [`Event::Shutdown`] for [`poll_event`](Self::poll_event) and poison the shared + /// state, exactly as the run loop's post-pump drain does when the engine emits + /// `Event::Shutdown` from its conflict-resolution tally. + /// + /// The engine emits `Event::Shutdown` only from that vote tally, which the paired + /// two-node test harness cannot drive to a deterministic majority, so this exposes + /// the same terminal input to exercise the driver's shutdown enforcement (pump + /// stop, socket teardown, command rejection, join poisoning) end-to-end. Present + /// only in the host (`std`) test build. + #[cfg(any(test, feature = "std"))] + #[doc(hidden)] + #[inline] + pub fn simulate_conflict_shutdown(&self) { + self.shared.simulate_conflict_shutdown(); + } + /// Drain one application-visible serf event the run loop buffered, mandatory /// driver-actioned events first (the run loop has ALREADY acted on them), then /// passive observations. `None` when the queue is empty. @@ -621,6 +638,13 @@ where where Res: AddressResolver
, { + // Fail fast if the node already lost an id-conflict vote: a join under a + // duplicate identity is meaningless, and the stopped run loop would never + // dispatch its push/pulls. + if self.shared.is_shutdown() { + return Err(JoinError::Shutdown); + } + let now = time::now(); let mut resolved = Vec::with_capacity(seeds.len()); for seed in seeds { @@ -663,6 +687,13 @@ where guard.disarm(); return outcome.map_err(JoinError::Failed); } + // The run loop stopped mid-join after a lost id-conflict shutdown: resolve + // with the shutdown error rather than spin the backstop forever. The guard is + // left armed so its drop cancels the now-orphaned engine-side join entry + // exactly once (no borrow is live here, so the drop's `borrow_mut` is safe). + if self.shared.is_shutdown() { + return Err(JoinError::Shutdown); + } // Ignoring the `Either`: whichever of the join wake or the timer fired, the // loop simply re-checks `poll_join`. let _ = select( @@ -676,48 +707,78 @@ where /// Begin leaving the cluster. Gossips the departure and ultimately emits /// [`Event::LeftCluster`] via [`poll_event`](Self::poll_event). /// - /// Returns [`SerfError`] if the node is not in a running state (already left or a - /// refused leave). - pub fn leave(&self) -> Result<(), SerfError> { + /// # Errors + /// + /// [`OpError::Shutdown`] if the node already lost an id-conflict vote (the run + /// loop has stopped), or [`OpError::Serf`] if the engine rejects the leave (not in + /// a running state — already left or a refused leave). + pub fn leave(&self) -> Result<(), OpError> { + if self.shared.is_shutdown() { + return Err(OpError::Shutdown); + } let now = time::now(); let r = self.shared.engine.borrow_mut().leave(now); self.shared.wake_pump(); - r + r.map_err(OpError::from) } /// Force a named node out of the cluster (an operator-driven removal). - pub fn force_leave(&self, id: I, prune: bool) -> Result<(), SerfError> { + /// + /// # Errors + /// + /// [`OpError::Shutdown`] after a lost id-conflict vote, else the engine's + /// rejection as [`OpError::Serf`]. + pub fn force_leave(&self, id: I, prune: bool) -> Result<(), OpError> { + if self.shared.is_shutdown() { + return Err(OpError::Shutdown); + } let now = time::now(); let r = self.shared.engine.borrow_mut().force_leave(id, prune, now); self.shared.wake_pump(); - r + r.map_err(OpError::from) } /// Broadcast an application user event to the cluster. `coalesce` requests that /// identical events be coalesced by name. Peers observe it as [`Event::User`]. + /// + /// # Errors + /// + /// [`OpError::Shutdown`] after a lost id-conflict vote, else the engine's + /// rejection as [`OpError::Serf`] (e.g. an oversized event). pub fn user_event( &self, name: impl Into, payload: bytes::Bytes, coalesce: bool, - ) -> Result<(), SerfError> { + ) -> Result<(), OpError> { + if self.shared.is_shutdown() { + return Err(OpError::Shutdown); + } let r = self .shared .engine .borrow_mut() .user_event(name, payload, coalesce); self.shared.wake_pump(); - r + r.map_err(OpError::from) } /// Issue a cluster-wide query, returning its [`QueryId`]. Responders observe it as /// [`Event::Query`] and answer via [`respond`](Self::respond). + /// + /// # Errors + /// + /// [`OpError::Shutdown`] after a lost id-conflict vote, else the engine's + /// rejection as [`OpError::Serf`]. pub fn query( &self, name: impl Into, payload: bytes::Bytes, params: QueryParams, - ) -> Result { + ) -> Result { + if self.shared.is_shutdown() { + return Err(OpError::Shutdown); + } let now = time::now(); let r = self .shared @@ -725,28 +786,44 @@ where .borrow_mut() .query(name, payload, params, now); self.shared.wake_pump(); - r + r.map_err(OpError::from) } /// Answer a received query. `token` is the [`QueryEvent`] delivered via /// [`Event::Query`]. + /// + /// # Errors + /// + /// [`OpError::Shutdown`] after a lost id-conflict vote, else the engine's + /// rejection as [`OpError::Serf`] (e.g. a duplicate or past-deadline respond). pub fn respond( &self, token: &QueryEvent, payload: bytes::Bytes, - ) -> Result<(), SerfError> { + ) -> Result<(), OpError> { + if self.shared.is_shutdown() { + return Err(OpError::Shutdown); + } let now = time::now(); let r = self.shared.engine.borrow_mut().respond(token, payload, now); self.shared.wake_pump(); - r + r.map_err(OpError::from) } /// Replace the local node's tags, re-advertising them and refreshing the local /// member in the membership store. - pub fn set_tags(&self, tags: Tags) -> Result<(), SerfError> { + /// + /// # Errors + /// + /// [`OpError::Shutdown`] after a lost id-conflict vote, else the engine's + /// rejection as [`OpError::Serf`]. + pub fn set_tags(&self, tags: Tags) -> Result<(), OpError> { + if self.shared.is_shutdown() { + return Err(OpError::Shutdown); + } let r = self.shared.engine.borrow_mut().set_tags(tags); self.shared.wake_pump(); - r + r.map_err(OpError::from) } /// Issue a cluster-wide `install_key` query to add `key` to every node's keyring. @@ -755,11 +832,14 @@ where docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] - pub fn install_key(&self, key: SecretKey) -> Result { + pub fn install_key(&self, key: SecretKey) -> Result { + if self.shared.is_shutdown() { + return Err(OpError::Shutdown); + } let now = time::now(); let r = self.shared.engine.borrow_mut().install_key(key, now); self.shared.wake_pump(); - r + r.map_err(OpError::from) } /// Issue a cluster-wide `use_key` query to promote `key` to primary. @@ -768,11 +848,14 @@ where docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] - pub fn use_key(&self, key: SecretKey) -> Result { + pub fn use_key(&self, key: SecretKey) -> Result { + if self.shared.is_shutdown() { + return Err(OpError::Shutdown); + } let now = time::now(); let r = self.shared.engine.borrow_mut().use_key(key, now); self.shared.wake_pump(); - r + r.map_err(OpError::from) } /// Issue a cluster-wide `remove_key` query to remove `key` from all nodes. @@ -781,11 +864,14 @@ where docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] - pub fn remove_key(&self, key: SecretKey) -> Result { + pub fn remove_key(&self, key: SecretKey) -> Result { + if self.shared.is_shutdown() { + return Err(OpError::Shutdown); + } let now = time::now(); let r = self.shared.engine.borrow_mut().remove_key(key, now); self.shared.wake_pump(); - r + r.map_err(OpError::from) } /// Issue a cluster-wide `list_keys` query to enumerate installed keys. @@ -794,11 +880,14 @@ where docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] - pub fn list_keys(&self) -> Result { + pub fn list_keys(&self) -> Result { + if self.shared.is_shutdown() { + return Err(OpError::Shutdown); + } let now = time::now(); let r = self.shared.engine.borrow_mut().list_keys(now); self.shared.wake_pump(); - r + r.map_err(OpError::from) } /// A clone of the node's LIVE wire keyring — the keyring the gossip and reliable diff --git a/serf-embassy/src/shared/mod.rs b/serf-embassy/src/shared/mod.rs index 3bade455..00322ed9 100644 --- a/serf-embassy/src/shared/mod.rs +++ b/serf-embassy/src/shared/mod.rs @@ -112,6 +112,19 @@ where self.shutdown.get() } + /// Poison the shared state on a lost id-conflict [`Event::Shutdown`]: latch the + /// one-way shutdown flag, then wake the parked joins (so a join in flight + /// resolves with the shutdown error rather than hang) and the pump loop (so it + /// observes the latch after its final drain and stops, collapsing the workers so + /// every socket winds down). Idempotent — a repeated shutdown re-signals but the + /// latch never clears. + #[inline] + pub(crate) fn begin_shutdown(&self) { + self.shutdown.set(true); + self.join_wake.signal(()); + self.pump_wake.signal(()); + } + /// Buffer one event for [`poll_event`](crate::Serf::poll_event), bounding the /// backlog at [`DEFAULT_EVENT_BUFFER_CAP`] with drop-oldest so a never-draining /// app cannot grow memory without bound. @@ -125,6 +138,22 @@ where } q.push_back(ev); } + + /// Drive the exact effect the post-pump drain takes when it observes a lost + /// id-conflict [`Event::Shutdown`]: buffer the terminal event for the app's + /// [`poll_event`](crate::Serf::poll_event), then poison the shared state + /// ([`begin_shutdown`](Self::begin_shutdown)). + /// + /// The engine emits `Event::Shutdown` only from its conflict-resolution vote + /// tally (`close_conflict_query`), which the paired two-node harness cannot drive + /// to a deterministic majority; this stands in for that vote loss so the driver's + /// terminal-shutdown enforcement (pump stop, socket teardown, command rejection, + /// join poisoning) is testable end-to-end without a fabricated cluster. + #[cfg(any(test, feature = "std"))] + pub(crate) fn simulate_conflict_shutdown(&self) { + self.push_app_event(Event::Shutdown); + self.begin_shutdown(); + } } impl Shared @@ -165,9 +194,11 @@ where let ev = self.engine.borrow_mut().poll_event(); let Some(ev) = ev else { break }; match &ev { - // A lost id-conflict vote means the local node MUST stop; flag it. The - // event still reaches the app via `poll_event`. - Event::Shutdown => self.shutdown.set(true), + // A lost id-conflict vote means the local node MUST stop: poison the shared + // state so the pump loop halts after this drain and any parked join resolves + // with the shutdown error. The event still reaches the app via `poll_event` + // (buffered below). + Event::Shutdown => self.begin_shutdown(), // An inbound key-management request: apply the op to the engine's LIVE wire // keyring and answer the originator in one call. The response is a directed // gossip transmit egressed on the re-pump the runner performs while `queued` diff --git a/serf-embassy/tests/conflict_shutdown.rs b/serf-embassy/tests/conflict_shutdown.rs new file mode 100644 index 00000000..216cfccc --- /dev/null +++ b/serf-embassy/tests/conflict_shutdown.rs @@ -0,0 +1,201 @@ +//! The conflict-loss shutdown is TERMINAL: a node that loses an id-conflict vote +//! stops pumping, closes its sockets, and rejects further commands — so two nodes +//! can never stay active under one identity. +//! +//! serf emits `Event::Shutdown` only from its conflict-resolution vote tally, which +//! the paired two-node harness cannot drive to a deterministic majority, so these +//! tests inject the same terminal signal via the driver's `simulate_conflict_shutdown` +//! seam (buffer the `Event::Shutdown` + poison the shared state, exactly as the run +//! loop's post-pump drain does on a real vote loss) and assert the ENFORCEMENT the +//! driver adds on top of that signal. + +#![allow(clippy::collapsible_if)] + +mod support; + +use core::cell::Cell; + +use embassy_futures::select::{Either, select}; +use embassy_net::StackResources; +use embassy_time::{Duration, Timer}; +use futures::executor::block_on; +use serf_embassy::{ + Bytes, Event, JoinError, MaybeResolved, OpError, ReachedSet, SocketAddrResolver, + TransformOptions, now, +}; + +use support::cluster::{ + NodeBufs, POOL, TEST_TIMEOUT, addr, build_node, build_sockets, build_stack, devices, + join_and_converge, +}; + +/// Two converged nodes; A loses an id-conflict vote. A's shutdown must be terminal — +/// its `Runner::run` future completes (the pump stopped and the sockets wound down), +/// a subsequent command fails fast with [`OpError::Shutdown`], yet the buffered +/// terminal `Event::Shutdown` is still drainable via `poll_event` — while the winner +/// B keeps running and keeps accepting commands. +/// +/// Reverting the enforcement to flag-only (the pump keeps looping, the handle keeps +/// accepting commands) makes A's runner never complete and the post-shutdown command +/// succeed, so both assertions below fail. +#[test] +fn conflict_loss_stops_the_loser_and_spares_the_winner() { + let (dev_a, dev_b) = devices(); + let mut res_a = StackResources::<{ POOL + 2 }>::new(); + let mut res_b = StackResources::<{ POOL + 2 }>::new(); + let (stack_a, mut net_a) = build_stack(dev_a, &mut res_a, 1, 0x1111_2222); + let (stack_b, mut net_b) = build_stack(dev_b, &mut res_b, 2, 0x3333_4444); + + let mut bufs_a = NodeBufs::new(); + let mut bufs_b = NodeBufs::new(); + let (udp_a, tcp_a) = build_sockets(stack_a, &mut bufs_a); + let (udp_b, tcp_b) = build_sockets(stack_b, &mut bufs_b); + + let now = now(); + let (ml_a, run_a) = build_node(udp_a, tcp_a, "a", 1, now, TransformOptions::default()); + let (ml_b, run_b) = build_node(udp_b, tcp_b, "b", 2, now, TransformOptions::default()); + + block_on(async { + // Latched by A's runner future the instant it completes, so the orchestration + // can observe the terminal stop (and then park so the composed infra keeps B and + // both stacks alive while the assertions run). + let a_stopped = Cell::new(false); + let run_a_then_park = async { + run_a.run().await; + a_stopped.set(true); + core::future::pending::<()>().await; + }; + + let orchestrate = async { + // Converge: B joins A, both reach a 2-member view. + join_and_converge(&ml_a, &ml_b).await; + + // A loses the id-conflict vote: the drain observes the terminal Event::Shutdown. + ml_a.simulate_conflict_shutdown(); + + // The loser's flag flips AND its runner future completes (the pump stopped and + // `Runner::run`'s select collapsed the workers, closing the sockets). + loop { + if ml_a.is_shutdown() && a_stopped.get() { + break; + } + Timer::after(Duration::from_millis(2)).await; + } + + // The loser rejects further commands fast — it cannot keep acting under the + // duplicate identity. + let post = ml_a.user_event("post-shutdown", Bytes::from_static(b"x"), false); + assert!( + matches!(post, Err(OpError::Shutdown)), + "a command after shutdown must fail with OpError::Shutdown, got {post:?}" + ); + assert!( + ml_a.leave().is_err(), + "leave after shutdown must also be rejected" + ); + + // The app can still drain the buffered terminal Event::Shutdown. + let mut saw_shutdown = false; + while let Some(ev) = ml_a.poll_event() { + if matches!(ev, Event::Shutdown) { + saw_shutdown = true; + break; + } + } + assert!( + saw_shutdown, + "the buffered Event::Shutdown must remain observable via poll_event" + ); + + // The winner keeps running: not shut down, and still accepting commands. + assert!(!ml_b.is_shutdown(), "the winner must not be shut down"); + ml_b + .user_event("winner-still-live", Bytes::from_static(b"y"), false) + .expect("the winner keeps accepting commands"); + true + }; + + // Drive A's (stopping) runner, B's (forever) runner, both stacks, and the + // orchestration against the wall-clock timeout. + let infra = select( + select(run_a_then_park, run_b.run()), + select(net_a.run(), net_b.run()), + ); + match select(orchestrate, select(infra, Timer::after(TEST_TIMEOUT))).await { + Either::First(ok) => assert!(ok), + Either::Second(_) => panic!("conflict-shutdown test timed out after {TEST_TIMEOUT:?}"), + } + }); +} + +/// A join in flight when the conflict-loss lands must resolve with +/// [`JoinError::Shutdown`] rather than spin its backstop forever, and the +/// join's drop-cancel of the orphaned engine-side entry must be clean (no panic). +#[test] +fn join_in_flight_resolves_on_shutdown() { + // A single node is enough: it joins an unreachable seed so the push/pull stays in + // flight, then loses the vote before the dial can resolve. + let (dev_a, _dev_b) = devices(); + let mut res_a = StackResources::<{ POOL + 2 }>::new(); + let (stack_a, mut net_a) = build_stack(dev_a, &mut res_a, 1, 0x1111_2222); + + let mut bufs_a = NodeBufs::new(); + let (udp_a, tcp_a) = build_sockets(stack_a, &mut bufs_a); + + let now = now(); + let (ml_a, run_a) = build_node(udp_a, tcp_a, "a", 1, now, TransformOptions::default()); + + block_on(async { + let run_a_then_park = async { + run_a.run().await; + core::future::pending::<()>().await; + }; + + let orchestrate = async { + // Start a join to an address no node answers, so it parks in flight. The seed + // slice must outlive the held `join_fut`, so bind it. + let seeds = [MaybeResolved::Resolved(addr(9, 7946))]; + let join_fut = ml_a.join(&SocketAddrResolver, &seeds, false); + // Once the join is registered and in flight, lose the vote. + let trigger = async { + loop { + if ml_a.pending_join_count() > 0 { + break; + } + Timer::after(Duration::from_millis(2)).await; + } + ml_a.simulate_conflict_shutdown(); + // The join resolves; this branch just keeps the trigger alive meanwhile. + core::future::pending::>().await + }; + let result = match select(join_fut, trigger).await { + Either::First(r) => r, + Either::Second(unreached) => unreached, + }; + assert!( + matches!(result, Err(JoinError::Shutdown)), + "a join in flight when shutdown lands must resolve with JoinError::Shutdown, got {result:?}" + ); + // A join attempted AFTER shutdown also fails fast (and the guard drop above was + // clean — a panic there would have aborted the test). + let again = ml_a + .join( + &SocketAddrResolver, + &[MaybeResolved::Resolved(addr(9, 7946))], + false, + ) + .await; + assert!( + matches!(again, Err(JoinError::Shutdown)), + "a join after shutdown must fail fast with JoinError::Shutdown, got {again:?}" + ); + true + }; + + let infra = select(run_a_then_park, net_a.run()); + match select(orchestrate, select(infra, Timer::after(TEST_TIMEOUT))).await { + Either::First(ok) => assert!(ok), + Either::Second(_) => panic!("join-vs-shutdown test timed out after {TEST_TIMEOUT:?}"), + } + }); +} From 910dcf5ed934a9b72dc248c5e96884dbe0596d46 Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 10 Jul 2026 18:41:51 +1200 Subject: [PATCH 063/128] feat(serf-embassy): public shutdown command, drop the test seam The `#[doc(hidden)] simulate_conflict_shutdown` terminal-stop hook was gated `any(test, feature = "std")`, so it shipped in every std production build. Promote it to the real public `Serf::shutdown()` the reactor already exposes (abrupt local stop, idempotent, no leave gossip, no synthesized event), and delete the seam. The conflict drain arm is now covered by a private `shared` unit test, and the e2e drives the public API. --- serf-embassy/src/serf/mod.rs | 30 +++--- serf-embassy/src/shared/mod.rs | 128 ++++++++++++------------ serf-embassy/src/shared/tests.rs | 75 ++++++++++++++ serf-embassy/tests/conflict_shutdown.rs | 72 +++++-------- 4 files changed, 184 insertions(+), 121 deletions(-) create mode 100644 serf-embassy/src/shared/tests.rs diff --git a/serf-embassy/src/serf/mod.rs b/serf-embassy/src/serf/mod.rs index e8d3dbe3..8a9043aa 100644 --- a/serf-embassy/src/serf/mod.rs +++ b/serf-embassy/src/serf/mod.rs @@ -472,21 +472,25 @@ where self.shared.is_shutdown() } - /// Stand in for a lost id-conflict vote in driver tests: buffer the terminal - /// [`Event::Shutdown`] for [`poll_event`](Self::poll_event) and poison the shared - /// state, exactly as the run loop's post-pump drain does when the engine emits - /// `Event::Shutdown` from its conflict-resolution tally. + /// Abruptly stop the local node. /// - /// The engine emits `Event::Shutdown` only from that vote tally, which the paired - /// two-node test harness cannot drive to a deterministic majority, so this exposes - /// the same terminal input to exercise the driver's shutdown enforcement (pump - /// stop, socket teardown, command rejection, join poisoning) end-to-end. Present - /// only in the host (`std`) test build. - #[cfg(any(test, feature = "std"))] - #[doc(hidden)] + /// Latches the terminal shutdown state and wakes the run loop: the + /// [`Runner`](crate::Runner) returns after its next drain, collapsing its workers + /// so the gossip and reliable-plane sockets wind down. Pending [`join`](Self::join)s + /// and every subsequent command fail fast with the shutdown error + /// ([`JoinError::Shutdown`](crate::JoinError::Shutdown) / + /// [`OpError::Shutdown`](crate::OpError::Shutdown)); events already buffered stay + /// drainable via [`poll_event`](Self::poll_event). + /// + /// This does NOT gossip a leave — call [`leave`](Self::leave) for a graceful + /// departure that notifies peers. The node initiated the stop, so no + /// [`Event::Shutdown`](serf_embedded::Event::Shutdown) is synthesized for it; a + /// lost id-conflict vote drives this same terminal path but, being unsolicited, + /// DOES surface that event through [`poll_event`](Self::poll_event). Idempotent: a + /// second call re-signals the run loop but the shutdown latch never clears. #[inline] - pub fn simulate_conflict_shutdown(&self) { - self.shared.simulate_conflict_shutdown(); + pub fn shutdown(&self) { + self.shared.begin_shutdown(); } /// Drain one application-visible serf event the run loop buffered, mandatory diff --git a/serf-embassy/src/shared/mod.rs b/serf-embassy/src/shared/mod.rs index 00322ed9..485eedf5 100644 --- a/serf-embassy/src/shared/mod.rs +++ b/serf-embassy/src/shared/mod.rs @@ -138,22 +138,6 @@ where } q.push_back(ev); } - - /// Drive the exact effect the post-pump drain takes when it observes a lost - /// id-conflict [`Event::Shutdown`]: buffer the terminal event for the app's - /// [`poll_event`](crate::Serf::poll_event), then poison the shared state - /// ([`begin_shutdown`](Self::begin_shutdown)). - /// - /// The engine emits `Event::Shutdown` only from its conflict-resolution vote - /// tally (`close_conflict_query`), which the paired two-node harness cannot drive - /// to a deterministic majority; this stands in for that vote loss so the driver's - /// terminal-shutdown enforcement (pump stop, socket teardown, command rejection, - /// join poisoning) is testable end-to-end without a fabricated cluster. - #[cfg(any(test, feature = "std"))] - pub(crate) fn simulate_conflict_shutdown(&self) { - self.push_app_event(Event::Shutdown); - self.begin_shutdown(); - } } impl Shared @@ -162,27 +146,50 @@ where G: Rng, SR: Rng + SeedableRng, { - /// Drain the engine's event queue (mandatory-first), take the driver-owned side - /// effect on each mandatory event, buffer every event for the app, and pulse - /// `join_wake` so parked joins re-check `poll_join`. + /// Drain the engine's event queue (mandatory-first), routing each event through + /// [`route_drained_event`](Self::route_drained_event) — the driver-owned side + /// effect, then the app buffering — and pulse `join_wake` so parked joins + /// re-check `poll_join`. /// /// Called by the Runner once per pump, AFTER `pump` (so it sees this tick's - /// freshly-emitted events). The mandatory ACTIONS run BEFORE the lossy buffering: - /// an [`Event::Shutdown`] sets the shutdown state (the node stops), and an - /// [`Event::KeyRequest`] is applied to the engine's LIVE wire keyring and - /// answered in one call through `handle_key_request` — the live-keyring - /// chokepoint — never a driver-local shadow. A dropped observation is fine; a - /// dropped action is not, so the actions are taken here regardless of the app - /// ever polling. - /// - /// Returns whether the drain queued outbound gossip work the current pump's - /// egress did not see — a key response `handle_key_request` just queued — so the - /// Runner knows to re-pump and egress it (and loop back a self-addressed one) - /// within the same tick. + /// freshly-emitted events). Returns whether the drain queued outbound gossip work + /// the current pump's egress did not see — a key response `handle_key_request` + /// just queued — so the Runner knows to re-pump and egress it (and loop back a + /// self-addressed one) within the same tick. pub(crate) fn drain_events(&self, now: Instant) -> bool { + let mut queued = false; + loop { + let ev = self.engine.borrow_mut().poll_event(); + let Some(ev) = ev else { break }; + queued |= self.route_drained_event(ev, now); + } + + // Every drain re-checks parked joins: the pump folds each completion into its + // await-result join, so a resolved outcome is now visible to `poll_join`. + self.join_wake.signal(()); + queued + } + + /// Route one drained machine event: take serf's mandatory driver-owned side + /// effect on it, then buffer it for the app's + /// [`poll_event`](crate::Serf::poll_event). Returns whether the side effect + /// queued outbound gossip work (a key response) the current pump's egress did not + /// see. + /// + /// The mandatory ACTION runs BEFORE the lossy buffering. A lost id-conflict + /// [`Event::Shutdown`] poisons the shared state + /// ([`begin_shutdown`](Self::begin_shutdown)) — the node stops, the pump loop + /// halts after this drain, and any parked join resolves with the shutdown error; + /// this is the same terminal path the handle's + /// [`Serf::shutdown`](crate::Serf::shutdown) reaches directly. An + /// [`Event::KeyRequest`] is applied to the engine's LIVE wire keyring and answered + /// in one call through `handle_key_request` — the live-keyring chokepoint, never a + /// driver-local shadow. A dropped observation is fine; a dropped action is not, so + /// the action is taken regardless of the app ever polling. + fn route_drained_event(&self, ev: Event, now: Instant) -> bool { // `now` and the queued-outbound signal are consumed only by the encryption // `KeyRequest` arm; a build without an AEAD backend reads neither and queues no - // key response, so its drain never re-pumps on this account. + // key response. #[cfg(not(encryption))] let _ = now; #[cfg(encryption)] @@ -190,39 +197,31 @@ where #[cfg(not(encryption))] let queued = false; - loop { - let ev = self.engine.borrow_mut().poll_event(); - let Some(ev) = ev else { break }; - match &ev { - // A lost id-conflict vote means the local node MUST stop: poison the shared - // state so the pump loop halts after this drain and any parked join resolves - // with the shutdown error. The event still reaches the app via `poll_event` - // (buffered below). - Event::Shutdown => self.begin_shutdown(), - // An inbound key-management request: apply the op to the engine's LIVE wire - // keyring and answer the originator in one call. The response is a directed - // gossip transmit egressed on the re-pump the runner performs while `queued` - // is set. - #[cfg(encryption)] - Event::KeyRequest(req) => { - // `Ok` means a key response was queued (re-pump to egress it). Ignoring - // the Err case: `handle_key_request` has already applied the op to the - // live keyring; an Err means only the best-effort response was - // past-deadline or could not be routed, which queues no outbound work. - queued |= self - .engine - .borrow_mut() - .handle_key_request(req, now) - .is_ok(); - } - _ => {} + match &ev { + // A lost id-conflict vote means the local node MUST stop: poison the shared + // state so the pump loop halts after this drain and any parked join resolves + // with the shutdown error. The event still reaches the app via `poll_event` + // (buffered below). + Event::Shutdown => self.begin_shutdown(), + // An inbound key-management request: apply the op to the engine's LIVE wire + // keyring and answer the originator in one call. The response is a directed + // gossip transmit egressed on the re-pump the runner performs while `queued` + // is set. + #[cfg(encryption)] + Event::KeyRequest(req) => { + // `Ok` means a key response was queued (re-pump to egress it). Ignoring + // the Err case: `handle_key_request` has already applied the op to the + // live keyring; an Err means only the best-effort response was + // past-deadline or could not be routed, which queues no outbound work. + queued |= self + .engine + .borrow_mut() + .handle_key_request(req, now) + .is_ok(); } - self.push_app_event(ev); + _ => {} } - - // Every drain re-checks parked joins: the pump folds each completion into its - // await-result join, so a resolved outcome is now visible to `poll_join`. - self.join_wake.signal(()); + self.push_app_event(ev); queued } @@ -242,3 +241,6 @@ where .saturating_add(self.app_events_dropped.get()) } } + +#[cfg(test)] +mod tests; diff --git a/serf-embassy/src/shared/tests.rs b/serf-embassy/src/shared/tests.rs new file mode 100644 index 00000000..fcb99da2 --- /dev/null +++ b/serf-embassy/src/shared/tests.rs @@ -0,0 +1,75 @@ +use super::*; + +use core::{ + net::{IpAddr, Ipv4Addr}, + time::Duration, +}; + +use memberlist_proto::EndpointOptions; +use serf_embedded::{Options, SerfOptions, TransformOptions}; +use smol_str::SmolStr; + +/// A fixed instant well past the origin, for the deterministic engine construction +/// and the drain call. +fn at() -> Instant { + Instant::from_origin(Duration::from_secs(86_400)) +} + +/// A minimal single-node [`Shared`] over a real engine bound to a routable advertise +/// address, wrapped with the fresh (unsignaled) signals and empty buffers. +fn shared() -> Shared { + let advertise = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)), 7946); + let engine = SerfEngine::::try_new_at( + Options::new() + .with_port(7946) + .with_close_timeout(Duration::from_secs(10)), + TransformOptions::default(), + EndpointOptions::new(SmolStr::new("test"), advertise), + SerfOptions::new(), + at(), + SmallRng::seed_from_u64(42), + ) + .expect("a routable single-node configuration constructs"); + Shared::new(engine, advertise) +} + +/// The production conflict pathway: a drained `Event::Shutdown` — the terminal event +/// serf emits when the local node loses an id-conflict vote — routes through +/// `route_drained_event` to `begin_shutdown` (latching the one-way shutdown state and +/// pulsing both wakes so the pump halts and a parked join resolves with the shutdown +/// error), AND is buffered for the app's `poll_event`. +/// +/// This is the drain arm the conflict e2e no longer exercises: the public +/// `shutdown()` reaches `begin_shutdown` directly, without an engine event, so it +/// buffers no `Event::Shutdown`. Only a real vote loss surfaces that event, and this +/// covers that route. +#[test] +fn drained_shutdown_event_poisons_and_buffers() { + let shared = shared(); + + // A fresh node is live, its wakes unsignaled, its app buffer empty — so the + // transition below is caused by the drained event alone. + assert!(!shared.is_shutdown()); + assert!(!shared.join_wake.signaled()); + assert!(!shared.pump_wake.signaled()); + assert!(shared.pop_app_event().is_none()); + + // Route the terminal event exactly as the pump's drain does for each drained event. + let queued = shared.route_drained_event(Event::Shutdown, at()); + + assert!(!queued, "a Shutdown event queues no outbound gossip work"); + // `begin_shutdown` ran: the one-way latch is set and both waiters are poisoned — + // the pump wake so the loop stops after this drain, the join wake so a parked join + // re-checks and resolves with the shutdown error. + assert!(shared.is_shutdown(), "the shutdown latch must be set"); + assert!(shared.join_wake.signaled(), "parked joins must be woken"); + assert!( + shared.pump_wake.signaled(), + "the pump loop must be woken to observe the latch and stop" + ); + // The terminal event still reaches the app. + assert!( + matches!(shared.pop_app_event(), Some(Event::Shutdown)), + "the terminal Event::Shutdown must be buffered for poll_event" + ); +} diff --git a/serf-embassy/tests/conflict_shutdown.rs b/serf-embassy/tests/conflict_shutdown.rs index 216cfccc..62674f46 100644 --- a/serf-embassy/tests/conflict_shutdown.rs +++ b/serf-embassy/tests/conflict_shutdown.rs @@ -1,15 +1,13 @@ -//! The conflict-loss shutdown is TERMINAL: a node that loses an id-conflict vote -//! stops pumping, closes its sockets, and rejects further commands — so two nodes -//! can never stay active under one identity. +//! Shutdown is TERMINAL: a stopped node — whether it called `Serf::shutdown` or lost +//! an id-conflict vote — stops pumping, closes its sockets, and rejects further +//! commands, so two nodes can never stay active under one identity. //! -//! serf emits `Event::Shutdown` only from its conflict-resolution vote tally, which -//! the paired two-node harness cannot drive to a deterministic majority, so these -//! tests inject the same terminal signal via the driver's `simulate_conflict_shutdown` -//! seam (buffer the `Event::Shutdown` + poison the shared state, exactly as the run -//! loop's post-pump drain does on a real vote loss) and assert the ENFORCEMENT the -//! driver adds on top of that signal. - -#![allow(clippy::collapsible_if)] +//! Both triggers reach the same terminal path (`begin_shutdown`). These end-to-end +//! tests drive the public `Serf::shutdown` and assert that ENFORCEMENT: the runner +//! future completes, the sockets wind down, in-flight and subsequent commands fail +//! with the shutdown error, and a peer is unaffected. The extra `Event::Shutdown` a +//! real vote loss surfaces on top (the engine emits it and the drain buffers it for +//! the app) is covered by the driver's own `shared` unit test. mod support; @@ -20,8 +18,7 @@ use embassy_net::StackResources; use embassy_time::{Duration, Timer}; use futures::executor::block_on; use serf_embassy::{ - Bytes, Event, JoinError, MaybeResolved, OpError, ReachedSet, SocketAddrResolver, - TransformOptions, now, + Bytes, JoinError, MaybeResolved, OpError, ReachedSet, SocketAddrResolver, TransformOptions, now, }; use support::cluster::{ @@ -29,17 +26,16 @@ use support::cluster::{ join_and_converge, }; -/// Two converged nodes; A loses an id-conflict vote. A's shutdown must be terminal — -/// its `Runner::run` future completes (the pump stopped and the sockets wound down), -/// a subsequent command fails fast with [`OpError::Shutdown`], yet the buffered -/// terminal `Event::Shutdown` is still drainable via `poll_event` — while the winner -/// B keeps running and keeps accepting commands. +/// Two converged nodes; A calls the public `shutdown()`. A's stop must be terminal — +/// its `Runner::run` future completes (the pump stopped and the sockets wound down) +/// and every subsequent command fails fast with [`OpError::Shutdown`] — while the +/// peer B keeps running and keeps accepting commands. /// /// Reverting the enforcement to flag-only (the pump keeps looping, the handle keeps /// accepting commands) makes A's runner never complete and the post-shutdown command /// succeed, so both assertions below fail. #[test] -fn conflict_loss_stops_the_loser_and_spares_the_winner() { +fn shutdown_stops_the_node_and_spares_the_peer() { let (dev_a, dev_b) = devices(); let mut res_a = StackResources::<{ POOL + 2 }>::new(); let mut res_b = StackResources::<{ POOL + 2 }>::new(); @@ -70,10 +66,10 @@ fn conflict_loss_stops_the_loser_and_spares_the_winner() { // Converge: B joins A, both reach a 2-member view. join_and_converge(&ml_a, &ml_b).await; - // A loses the id-conflict vote: the drain observes the terminal Event::Shutdown. - ml_a.simulate_conflict_shutdown(); + // A stops abruptly via the public shutdown command. + ml_a.shutdown(); - // The loser's flag flips AND its runner future completes (the pump stopped and + // A's flag flips AND its runner future completes (the pump stopped and // `Runner::run`'s select collapsed the workers, closing the sockets). loop { if ml_a.is_shutdown() && a_stopped.get() { @@ -82,8 +78,7 @@ fn conflict_loss_stops_the_loser_and_spares_the_winner() { Timer::after(Duration::from_millis(2)).await; } - // The loser rejects further commands fast — it cannot keep acting under the - // duplicate identity. + // A rejects further commands fast — a stopped node cannot keep acting. let post = ml_a.user_event("post-shutdown", Bytes::from_static(b"x"), false); assert!( matches!(post, Err(OpError::Shutdown)), @@ -94,24 +89,11 @@ fn conflict_loss_stops_the_loser_and_spares_the_winner() { "leave after shutdown must also be rejected" ); - // The app can still drain the buffered terminal Event::Shutdown. - let mut saw_shutdown = false; - while let Some(ev) = ml_a.poll_event() { - if matches!(ev, Event::Shutdown) { - saw_shutdown = true; - break; - } - } - assert!( - saw_shutdown, - "the buffered Event::Shutdown must remain observable via poll_event" - ); - - // The winner keeps running: not shut down, and still accepting commands. - assert!(!ml_b.is_shutdown(), "the winner must not be shut down"); + // The peer keeps running: not shut down, and still accepting commands. + assert!(!ml_b.is_shutdown(), "the peer must not be shut down"); ml_b - .user_event("winner-still-live", Bytes::from_static(b"y"), false) - .expect("the winner keeps accepting commands"); + .user_event("peer-still-live", Bytes::from_static(b"y"), false) + .expect("the peer keeps accepting commands"); true }; @@ -128,13 +110,13 @@ fn conflict_loss_stops_the_loser_and_spares_the_winner() { }); } -/// A join in flight when the conflict-loss lands must resolve with +/// A join in flight when a `shutdown()` lands must resolve with /// [`JoinError::Shutdown`] rather than spin its backstop forever, and the /// join's drop-cancel of the orphaned engine-side entry must be clean (no panic). #[test] fn join_in_flight_resolves_on_shutdown() { // A single node is enough: it joins an unreachable seed so the push/pull stays in - // flight, then loses the vote before the dial can resolve. + // flight, then shuts down before the dial can resolve. let (dev_a, _dev_b) = devices(); let mut res_a = StackResources::<{ POOL + 2 }>::new(); let (stack_a, mut net_a) = build_stack(dev_a, &mut res_a, 1, 0x1111_2222); @@ -156,7 +138,7 @@ fn join_in_flight_resolves_on_shutdown() { // slice must outlive the held `join_fut`, so bind it. let seeds = [MaybeResolved::Resolved(addr(9, 7946))]; let join_fut = ml_a.join(&SocketAddrResolver, &seeds, false); - // Once the join is registered and in flight, lose the vote. + // Once the join is registered and in flight, shut the node down. let trigger = async { loop { if ml_a.pending_join_count() > 0 { @@ -164,7 +146,7 @@ fn join_in_flight_resolves_on_shutdown() { } Timer::after(Duration::from_millis(2)).await; } - ml_a.simulate_conflict_shutdown(); + ml_a.shutdown(); // The join resolves; this branch just keeps the trigger alive meanwhile. core::future::pending::>().await }; From 8b9f29b6f9c2ffcd4b6de00880ce6923917f2ca4 Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 10 Jul 2026 19:31:03 +1200 Subject: [PATCH 064/128] fix(serf-embassy): close the post-latch pump and parked-resolution shutdown windows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runner checked the shutdown latch only after an egress-capable pump, so a wake-up or quiescence re-pump could still transmit after the node stopped — the latch is now checked before every pump and after every drain, with one final no-pump drain and queued outbound dropped by the abrupt-stop contract. A join parked in seed resolution now races the latch and re-checks it after every resolver await and before engine dispatch, so it resolves with the shutdown error instead of hanging or touching the stopped engine; peer-observed and parked-resolver regressions prove both windows closed. --- serf-embassy/src/runner.rs | 49 ++-- serf-embassy/src/serf/mod.rs | 70 ++++-- serf-embassy/tests/conflict_shutdown.rs | 286 +++++++++++++++++++++++- 3 files changed, 372 insertions(+), 33 deletions(-) diff --git a/serf-embassy/src/runner.rs b/serf-embassy/src/runner.rs index de421db8..5c41dd2e 100644 --- a/serf-embassy/src/runner.rs +++ b/serf-embassy/src/runner.rs @@ -117,12 +117,15 @@ where /// Drive the node: pump the engine and run the `N` workers concurrently. /// /// Runs forever under normal operation — spawn it as an embassy task (or drive - /// it with `select` against an operation in a test). It returns ONLY after a lost - /// id-conflict [`Event::Shutdown`](serf_embedded::Event::Shutdown): the pump loop - /// observes the poisoned shared state, performs its final drain, and returns, - /// which resolves the `select` below and drops the worker futures — so this frame - /// unwinds and the owned sockets (the TCP pool and the gossip UDP socket) close, - /// taking the losing node off the wire. + /// it with `select` against an operation in a test). It returns ONLY after the + /// shared shutdown latch is set — a lost id-conflict + /// [`Event::Shutdown`](serf_embedded::Event::Shutdown) or a public + /// [`Serf::shutdown`](crate::Serf::shutdown): the pump loop observes the latch + /// before its next egress-capable pump, performs one no-pump final drain, and + /// returns, which resolves the `select` below and drops the worker futures — so + /// this frame unwinds and the owned sockets (the TCP pool and the gossip UDP + /// socket) close, taking the stopped node off the wire without flushing queued + /// traffic. pub async fn run(self) { let Runner { shared, @@ -193,6 +196,18 @@ async fn pump_loop( { let advertise: SocketAddr = shared.advertise; loop { + // Check the terminal latch BEFORE any egress-capable pump. A public `shutdown()` + // — or a lost id-conflict vote latched on a prior tick — that woke this loop must + // not get one more pump: an abrupt stop does not flush in-flight traffic. Run one + // no-pump final drain so this node's engine-buffered events still reach + // `app_events` for the app, then stop. `drain_events` neither pumps nor egresses, + // so it cannot re-arm work — any queued outbound (key responses, loopback + // self-datagrams, undisseminated gossip) is dropped by the abrupt-stop contract. + if shared.is_shutdown() { + shared.drain_events(time::now()); + return; + } + let now = time::now(); // Pump → drain, re-running at the same `now` until neither a queued key @@ -214,6 +229,17 @@ async fn pump_loop( // chokepoint) and buffer every observation. Returns whether a key response // was queued the egress above did not see. let queued = shared.drain_events(now); + // Check the latch immediately after the drain, BEFORE the re-pump decision. The + // drain may have observed a lost id-conflict `Event::Shutdown` and latched the + // terminal state; that drain was then the FINAL one — it buffered this tick's + // events into `app_events` — so stop now rather than re-pump. Any queued outbound + // (a key response, a looped-back self-datagram) is dropped by the abrupt-stop + // contract: a lost node does not gossip on the id it just lost, which would only + // confuse the winner. Returning ends this future, which resolves `Runner::run`'s + // `select` and collapses the workers so the sockets wind down. + if shared.is_shutdown() { + return; + } if !queued && loopback.borrow().is_empty() { settled = true; break; @@ -226,17 +252,6 @@ async fn pump_loop( next = min_opt(next, Some(now)); } - // A lost id-conflict `Event::Shutdown` observed by the drain above poisoned the - // shared state. The pump+drain that just ran was the FINAL one — every remaining - // event is buffered for the app's `poll_event` — so stop pumping now instead of - // sleeping. Returning ends this future, which resolves `Runner::run`'s `select` - // and collapses the workers so the sockets wind down (an abrupt stop: a lost - // node does not gossip a leave for the id it just lost, which would only - // confuse the winner). - if shared.is_shutdown() { - return; - } - // Wait for the next thing worth re-pumping for: an inbound gossip datagram, a // worker/handle pump-wake, or the folded deadline. match next { diff --git a/serf-embassy/src/serf/mod.rs b/serf-embassy/src/serf/mod.rs index 8a9043aa..fb0525a1 100644 --- a/serf-embassy/src/serf/mod.rs +++ b/serf-embassy/src/serf/mod.rs @@ -20,7 +20,7 @@ use core::{marker::PhantomData, net::SocketAddr}; use alloc::{boxed::Box, rc::Rc, vec::Vec}; use std::sync::Arc; -use embassy_futures::select::select; +use embassy_futures::select::{Either, select}; use embassy_net::{tcp::TcpSocket, udp::UdpSocket}; use embassy_time::Timer; use memberlist_proto::{EndpointOptions, Instant, Rng, SeedableRng, SmallRng}; @@ -183,6 +183,21 @@ where } } +/// Resolve once the shared shutdown latch flips, polled on the same 20ms cadence the +/// join wait loop uses. Raced against each unresolved-seed lookup in +/// [`Serf::join`](Serf::join) so a resolver parked in a never-completing future cannot +/// leave the join pending past an abrupt stop. `join_wake` is a single-consumer +/// [`Signal`](embassy_sync::signal::Signal), so this polls the latch rather than +/// hanging a second consumer off it. +async fn shutdown_backstop(shared: &Shared) +where + I: memberlist_proto::Id, +{ + while !shared.is_shutdown() { + Timer::after(embassy_time::Duration::from_millis(20)).await; + } +} + /// A cloneable handle to an embassy-net serf node. /// /// Holds a shared reference to the node's @@ -475,13 +490,20 @@ where /// Abruptly stop the local node. /// /// Latches the terminal shutdown state and wakes the run loop: the - /// [`Runner`](crate::Runner) returns after its next drain, collapsing its workers - /// so the gossip and reliable-plane sockets wind down. Pending [`join`](Self::join)s - /// and every subsequent command fail fast with the shutdown error + /// [`Runner`](crate::Runner) observes the latch BEFORE its next egress-capable pump, + /// runs one no-pump final drain, and returns, collapsing its workers so the gossip + /// and reliable-plane sockets wind down. Pending [`join`](Self::join)s and every + /// subsequent command fail fast with the shutdown error /// ([`JoinError::Shutdown`](crate::JoinError::Shutdown) / /// [`OpError::Shutdown`](crate::OpError::Shutdown)); events already buffered stay /// drainable via [`poll_event`](Self::poll_event). /// + /// An abrupt stop does NOT flush in-flight traffic: any outbound the latch + /// pre-empts — an undisseminated gossip broadcast, a queued key response, a + /// looped-back self-datagram — is dropped rather than transmitted, so a stopped node + /// (a manual stop or a conflict-losing duplicate) never emits on the wire after the + /// terminal state is observable. + /// /// This does NOT gossip a leave — call [`leave`](Self::leave) for a graceful /// departure that notifies peers. The node initiated the stop, so no /// [`Event::Shutdown`](serf_embedded::Event::Shutdown) is synthesized for it; a @@ -631,8 +653,11 @@ where /// /// [`JoinError::Control`] when the engine rejects the join (e.g. the node is not /// running), [`JoinError::Resolve`] on a resolver failure, [`JoinError::NoAddresses`] - /// when a non-empty seed set resolves to no address, or [`JoinError::Failed`] when - /// every dispatched push/pull terminated without contacting a seed. + /// when a non-empty seed set resolves to no address, [`JoinError::Failed`] when + /// every dispatched push/pull terminated without contacting a seed, or + /// [`JoinError::Shutdown`] if the node lost an id-conflict vote or was stopped — + /// before dispatch, while a seed was still resolving, or while the join was in + /// flight — so a stopped node never dispatches or hangs. pub async fn join( &self, resolver: &Res, @@ -642,8 +667,8 @@ where where Res: AddressResolver
, { - // Fail fast if the node already lost an id-conflict vote: a join under a - // duplicate identity is meaningless, and the stopped run loop would never + // Fail fast if the node already lost an id-conflict vote or was stopped: a join + // under a duplicate identity is meaningless, and the stopped run loop would never // dispatch its push/pulls. if self.shared.is_shutdown() { return Err(JoinError::Shutdown); @@ -654,18 +679,35 @@ where for seed in seeds { match seed { MaybeResolved::Resolved(s) => resolved.push(*s), - MaybeResolved::Unresolved(a) => resolved.extend( - resolver - .resolve(a) - .await - .map_err(|e| JoinError::Resolve(Box::new(e)))?, - ), + MaybeResolved::Unresolved(a) => { + // Race each unresolved-seed lookup against the shutdown latch: a resolver + // that never completes must not leave the join pending past an abrupt stop, + // and one that resolves only after the stop must not reach the stopped + // engine. The backstop resolves only when the latch flips. + let result = match select(resolver.resolve(a), shutdown_backstop(&self.shared)).await { + Either::First(r) => r, + Either::Second(()) => return Err(JoinError::Shutdown), + }; + // Re-check after the await so a latch that flipped just as the resolver won + // the race still stops the join here: a post-shutdown resolver error or empty + // result resolves as `Shutdown`, never `Resolve` / `NoAddresses`. + if self.shared.is_shutdown() { + return Err(JoinError::Shutdown); + } + resolved.extend(result.map_err(|e| JoinError::Resolve(Box::new(e)))?); + } } } if !seeds.is_empty() && resolved.is_empty() { return Err(JoinError::NoAddresses); } + // A shutdown latched after the final resolver await — or during an all-`Resolved` + // seed set that raced no resolver — must not dispatch onto the stopped engine. + if self.shared.is_shutdown() { + return Err(JoinError::Shutdown); + } + let handle = self .shared .engine diff --git a/serf-embassy/tests/conflict_shutdown.rs b/serf-embassy/tests/conflict_shutdown.rs index 62674f46..95c25888 100644 --- a/serf-embassy/tests/conflict_shutdown.rs +++ b/serf-embassy/tests/conflict_shutdown.rs @@ -8,17 +8,23 @@ //! with the shutdown error, and a peer is unaffected. The extra `Event::Shutdown` a //! real vote loss surfaces on top (the engine emits it and the drain buffers it for //! the app) is covered by the driver's own `shared` unit test. +//! +//! Two terminality RACE windows are also pinned here: the runner must check the latch +//! BEFORE its next egress-capable pump (so a wake-up never pumps out queued work), and +//! a join parked in seed resolution must race the latch (so it resolves with the +//! shutdown error instead of hanging or dispatching onto the stopped engine). mod support; -use core::cell::Cell; +use core::{cell::Cell, convert::Infallible, net::SocketAddr}; use embassy_futures::select::{Either, select}; use embassy_net::StackResources; use embassy_time::{Duration, Timer}; use futures::executor::block_on; use serf_embassy::{ - Bytes, JoinError, MaybeResolved, OpError, ReachedSet, SocketAddrResolver, TransformOptions, now, + AddressResolver, Bytes, JoinError, MaybeResolved, OpError, ReachedSet, ResolvedAddrs, + SocketAddrResolver, TransformOptions, now, }; use support::cluster::{ @@ -26,6 +32,48 @@ use support::cluster::{ join_and_converge, }; +/// A resolver whose lookup never completes, modelling a stuck DNS query: it flags that +/// it was entered (so the test can latch shutdown once the join is parked in +/// resolution) and then parks forever. +struct NeverResolver<'a> { + entered: &'a Cell, +} + +impl AddressResolver for NeverResolver<'_> { + type Address = SocketAddr; + type Error = Infallible; + + async fn resolve(&self, _address: &SocketAddr) -> Result { + self.entered.set(true); + core::future::pending::>().await + } +} + +/// A resolver that parks until `release` is set, then resolves SUCCESSFULLY. The test +/// latches shutdown before releasing it, so the successful resolution lands AFTER the +/// stop — the late-success case that must not reach the engine. +struct LateResolver<'a> { + entered: &'a Cell, + release: &'a Cell, +} + +impl AddressResolver for LateResolver<'_> { + type Address = SocketAddr; + type Error = Infallible; + + async fn resolve(&self, address: &SocketAddr) -> Result { + self.entered.set(true); + while !self.release.get() { + Timer::after(Duration::from_millis(2)).await; + } + let mut out = ResolvedAddrs::new(); + // Ignoring Err: one push onto a fresh bounded vec (cap MAX_RESOLVED_ADDRS_PER_SEED) + // cannot overflow. + let _ = out.push(*address); + Ok(out) + } +} + /// Two converged nodes; A calls the public `shutdown()`. A's stop must be terminal — /// its `Runner::run` future completes (the pump stopped and the sockets wound down) /// and every subsequent command fails fast with [`OpError::Shutdown`] — while the @@ -181,3 +229,237 @@ fn join_in_flight_resolves_on_shutdown() { } }); } + +/// Reliable work queued the instant before a `shutdown()` must NOT be pumped out after +/// the latch: the runner checks the latch BEFORE its next egress-capable pump, so the +/// wake-up never dispatches the queued dial. Observed on A itself — the paired harness +/// drops A's sockets the instant its runner returns, so a peer can never observe the +/// (absent) egress; the reliable-pool state is the deterministic driver-side proof. +/// +/// A single `select` atomically dispatches a join to a routable seed (queuing it for +/// the pump's dial dispatch) and latches shutdown with no runner pump in between: +/// `select` polls the join future first (its first poll runs `engine.join`, queuing the +/// seed, then parks), then the second arm latches shutdown before the runner is polled. +/// +/// Reverting the pre-pump latch check → the wake-up pumps once and dispatches the +/// queued dial → a reliable slot is consumed (or a dial goes pending) → an assertion +/// below flips. +#[test] +fn shutdown_before_wake_drops_queued_reliable_work() { + let (dev_a, _dev_b) = devices(); + let mut res_a = StackResources::<{ POOL + 2 }>::new(); + let (stack_a, mut net_a) = build_stack(dev_a, &mut res_a, 1, 0x1111_2222); + + let mut bufs_a = NodeBufs::new(); + let (udp_a, tcp_a) = build_sockets(stack_a, &mut bufs_a); + + let now = now(); + let (ml_a, run_a) = build_node(udp_a, tcp_a, "a", 1, now, TransformOptions::default()); + + // A lone node's reliable pool is the full pool minus the dedicated listener slot, + // with nothing dialing — the baseline a post-latch pump would disturb. + let free_before = ml_a.pool_free_count(); + assert_eq!( + free_before, + POOL - 1, + "baseline: full pool minus the listener slot" + ); + assert_eq!(ml_a.pending_dial_count(), 0, "baseline: nothing dialing"); + + block_on(async { + let run_a_then_park = async { + run_a.run().await; + core::future::pending::<()>().await; + }; + + let orchestrate = async { + let seeds = [MaybeResolved::Resolved(addr(9, 7946))]; + let join_fut = ml_a.join(&SocketAddrResolver, &seeds, false); + let stop = async { + ml_a.shutdown(); + core::future::pending::>().await + }; + let result = match select(join_fut, stop).await { + Either::First(r) => r, + Either::Second(unreached) => unreached, + }; + assert!( + matches!(result, Err(JoinError::Shutdown)), + "the in-flight join must resolve with JoinError::Shutdown, got {result:?}" + ); + // The wake never pumped: the queued seed was never dispatched, so no reliable + // slot was consumed and no dial is pending. A revert pumps once on wake and + // dispatches the dial, flipping one of these. + assert_eq!( + ml_a.pool_free_count(), + free_before, + "a post-latch wake pumped and consumed a reliable slot for the queued dial" + ); + assert_eq!( + ml_a.pending_dial_count(), + 0, + "a post-latch wake pumped and queued a dial for the queued seed" + ); + true + }; + + let infra = select(run_a_then_park, net_a.run()); + match select(orchestrate, select(infra, Timer::after(TEST_TIMEOUT))).await { + Either::First(ok) => assert!(ok), + Either::Second(_) => panic!("post-latch-pump test timed out after {TEST_TIMEOUT:?}"), + } + }); +} + +/// A join parked in seed resolution when `shutdown()` lands must resolve with +/// [`JoinError::Shutdown`] within the backstop cadence, not hang: each unresolved-seed +/// lookup now races the shutdown latch. The resolver here never completes (a stuck DNS +/// query), so only the race can end the join. +/// +/// Reverting the resolution race → `resolve().await` parks forever → the join never +/// resolves → the test times out. +#[test] +fn join_parked_in_resolution_resolves_on_shutdown() { + let (dev_a, _dev_b) = devices(); + let mut res_a = StackResources::<{ POOL + 2 }>::new(); + let (stack_a, mut net_a) = build_stack(dev_a, &mut res_a, 1, 0x1111_2222); + + let mut bufs_a = NodeBufs::new(); + let (udp_a, tcp_a) = build_sockets(stack_a, &mut bufs_a); + + let now = now(); + let (ml_a, run_a) = build_node(udp_a, tcp_a, "a", 1, now, TransformOptions::default()); + + block_on(async { + let run_a_then_park = async { + run_a.run().await; + core::future::pending::<()>().await; + }; + + let entered = Cell::new(false); + let resolver = NeverResolver { entered: &entered }; + + let orchestrate = async { + // The seed slice must outlive the held join future, so bind it. + let seeds = [MaybeResolved::Unresolved(addr(9, 7946))]; + let join_fut = ml_a.join(&resolver, &seeds, false); + let trigger = async { + // Latch shutdown once the join is parked inside the resolver. + loop { + if entered.get() { + break; + } + Timer::after(Duration::from_millis(2)).await; + } + ml_a.shutdown(); + core::future::pending::>().await + }; + let result = match select(join_fut, trigger).await { + Either::First(r) => r, + Either::Second(unreached) => unreached, + }; + assert!( + matches!(result, Err(JoinError::Shutdown)), + "a join parked in resolution must resolve with JoinError::Shutdown, got {result:?}" + ); + // Resolution never completed, so no engine-side join was ever dispatched. + assert_eq!( + ml_a.pending_join_count(), + 0, + "a resolution-parked join dispatched nothing" + ); + true + }; + + let infra = select(run_a_then_park, net_a.run()); + match select(orchestrate, select(infra, Timer::after(TEST_TIMEOUT))).await { + Either::First(ok) => assert!(ok), + Either::Second(_) => panic!("resolution-parked-join test timed out after {TEST_TIMEOUT:?}"), + } + }); +} + +/// A seed resolution that completes SUCCESSFULLY only after shutdown is latched must +/// not touch the stopped engine: the post-await latch re-check (and the pre-dispatch +/// check) return [`JoinError::Shutdown`] before `engine.join`, so no join intent is +/// announced. `engine.join`'s `broadcast_join` bumps the member Lamport clock and +/// cancellation does not roll it back, so an unchanged `member_time` proves the engine +/// was never reached. +/// +/// Reverting the post-await re-check → the late `Ok` resolution reaches `engine.join` +/// → the member clock is bumped → the `member_time` assertion fails. +#[test] +fn late_resolution_does_not_touch_the_stopped_engine() { + let (dev_a, _dev_b) = devices(); + let mut res_a = StackResources::<{ POOL + 2 }>::new(); + let (stack_a, mut net_a) = build_stack(dev_a, &mut res_a, 1, 0x1111_2222); + + let mut bufs_a = NodeBufs::new(); + let (udp_a, tcp_a) = build_sockets(stack_a, &mut bufs_a); + + let now = now(); + let (ml_a, run_a) = build_node(udp_a, tcp_a, "a", 1, now, TransformOptions::default()); + + // The member clock a spurious `engine.join` would witness-bump. A lone idle node + // never advances it on its own, so any increase is attributable to a dispatch. + let member_time_before = ml_a.member_time(); + + block_on(async { + let run_a_then_park = async { + run_a.run().await; + core::future::pending::<()>().await; + }; + + let entered = Cell::new(false); + let release = Cell::new(false); + let resolver = LateResolver { + entered: &entered, + release: &release, + }; + + let orchestrate = async { + let seeds = [MaybeResolved::Unresolved(addr(9, 7946))]; + let join_fut = ml_a.join(&resolver, &seeds, false); + let trigger = async { + // Once the resolver is parked, latch shutdown, THEN release the resolver so its + // success lands after the stop. + loop { + if entered.get() { + break; + } + Timer::after(Duration::from_millis(2)).await; + } + ml_a.shutdown(); + release.set(true); + core::future::pending::>().await + }; + let result = match select(join_fut, trigger).await { + Either::First(r) => r, + Either::Second(unreached) => unreached, + }; + assert!( + matches!(result, Err(JoinError::Shutdown)), + "a late-resolving seed after shutdown must yield JoinError::Shutdown, got {result:?}" + ); + // Nothing was dispatched: no join intent announced (member clock untouched) and + // no engine-side join entry left behind. + assert_eq!( + ml_a.member_time(), + member_time_before, + "the stopped engine was reached: engine.join announced a join intent" + ); + assert_eq!( + ml_a.pending_join_count(), + 0, + "a refused late join dispatched nothing" + ); + true + }; + + let infra = select(run_a_then_park, net_a.run()); + match select(orchestrate, select(infra, Timer::after(TEST_TIMEOUT))).await { + Either::First(ok) => assert!(ok), + Either::Second(_) => panic!("late-resolution test timed out after {TEST_TIMEOUT:?}"), + } + }); +} From 2fd88d33f333a804f93f3026d8fd55face88828c Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 10 Jul 2026 20:07:56 +1200 Subject: [PATCH 065/128] feat(serf): bare-metal qemu execution proof + embedded CI tests/serf-embassy-qemu boots two serf-embassy nodes on an emulated Cortex-M4 (mps2-an386, semihosting exit code): they join, converge, and propagate a user event over the in-memory paired link with a deterministic fixed-seed rng. The crate is workspace-excluded with its own lockfile and qemu runner config. embedded.yml is activated from its placeholder with the bare-metal check, host-test, and qemu jobs adjusted to serf's crates and aes-gcm,cidr feature set. --- .../{embedded.yml.bk => embedded.yml} | 13 +- Cargo.toml | 6 +- tests/serf-embassy-qemu/.cargo/config.toml | 29 + tests/serf-embassy-qemu/Cargo.lock | 1141 +++++++++++++++++ tests/serf-embassy-qemu/Cargo.toml | 79 ++ tests/serf-embassy-qemu/README.md | 48 + tests/serf-embassy-qemu/memory.x | 15 + tests/serf-embassy-qemu/src/heap.rs | 44 + tests/serf-embassy-qemu/src/main.rs | 52 + tests/serf-embassy-qemu/src/node.rs | 260 ++++ tests/serf-embassy-qemu/src/paired_device.rs | 166 +++ tests/serf-embassy-qemu/src/rng.rs | 66 + tests/serf-embassy-qemu/src/systick_driver.rs | 145 +++ 13 files changed, 2052 insertions(+), 12 deletions(-) rename .github/workflows/{embedded.yml.bk => embedded.yml} (86%) create mode 100644 tests/serf-embassy-qemu/.cargo/config.toml create mode 100644 tests/serf-embassy-qemu/Cargo.lock create mode 100644 tests/serf-embassy-qemu/Cargo.toml create mode 100644 tests/serf-embassy-qemu/README.md create mode 100644 tests/serf-embassy-qemu/memory.x create mode 100644 tests/serf-embassy-qemu/src/heap.rs create mode 100644 tests/serf-embassy-qemu/src/main.rs create mode 100644 tests/serf-embassy-qemu/src/node.rs create mode 100644 tests/serf-embassy-qemu/src/paired_device.rs create mode 100644 tests/serf-embassy-qemu/src/rng.rs create mode 100644 tests/serf-embassy-qemu/src/systick_driver.rs diff --git a/.github/workflows/embedded.yml.bk b/.github/workflows/embedded.yml similarity index 86% rename from .github/workflows/embedded.yml.bk rename to .github/workflows/embedded.yml index 36fe6960..81e5946e 100644 --- a/.github/workflows/embedded.yml.bk +++ b/.github/workflows/embedded.yml @@ -1,12 +1,3 @@ -# INACTIVE PLACEHOLDER — GitHub Actions does NOT run `.bk` files. -# -# This is a ready-to-activate template for FUTURE serf no_std / embedded driver -# support. serf has no embedded/smoltcp/embassy crates yet; the crate names below -# (serf-embedded / serf-smoltcp / serf-embassy) are anticipated equivalents of -# memberlist's embedded stack. When those crates exist, rename this file -# `embedded.yml.bk` -> `embedded.yml` and adjust the crate names / feature flags -# to match what the crates actually expose. - name: embedded on: @@ -83,9 +74,9 @@ jobs: - name: Format run: cargo fmt -p serf-smoltcp -p serf-embassy --check - name: Clippy - run: cargo clippy -p serf-smoltcp -p serf-embassy --all-targets --features aes-gcm,coordinates --no-deps -- -D warnings + run: cargo clippy -p serf-smoltcp -p serf-embassy --all-targets --features aes-gcm,cidr --no-deps -- -D warnings - name: Test - run: cargo test -p serf-smoltcp -p serf-embassy --features aes-gcm,coordinates + run: cargo test -p serf-smoltcp -p serf-embassy --features aes-gcm,cidr # Execution proof: build the `serf-embassy-qemu` firmware and run it under QEMU. # The crate's `.cargo/config.toml` supplies the target, the linker script, the diff --git a/Cargo.toml b/Cargo.toml index 4482a693..7300091b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -8,7 +8,11 @@ members = [ "serf-smoltcp", "serf-embassy", ] -exclude = ["legacy"] +# `serf-embassy-qemu` is the bare-metal qemu execution proof for the embassy +# driver. It pins a foreign default target (a Cortex-M machine) and a cortex-m-rt +# runtime, so it cannot participate in the host `--workspace` build; it is built +# and run on its own via `cd tests/serf-embassy-qemu && cargo run`. +exclude = ["legacy", "tests/serf-embassy-qemu"] resolver = "3" [workspace.package] diff --git a/tests/serf-embassy-qemu/.cargo/config.toml b/tests/serf-embassy-qemu/.cargo/config.toml new file mode 100644 index 00000000..7cebac78 --- /dev/null +++ b/tests/serf-embassy-qemu/.cargo/config.toml @@ -0,0 +1,29 @@ +# Build + run config for the bare-metal qemu execution proof. +# +# The default target is `thumbv7em-none-eabihf` (Cortex-M4 with hardware float), +# matching the `mps2-an386` machine and the target the embassy driver's +# compile-check already covers. `cargo run` here builds the ELF and launches it +# under qemu with semihosting on, so the binary's `semihosting::process::exit` +# code becomes qemu's process exit code — that is the pass/fail signal. + +[build] +target = "thumbv7em-none-eabihf" + +[target.thumbv7em-none-eabihf] +# `-kernel ` loads the image into the AN386's SSRAM1-as-flash region and +# boots from its reset vector; `-semihosting-config target=native` routes ARM +# semihosting calls (stdout writes and `SYS_EXIT`) to the host so the program can +# print over semihosting and set the qemu process exit code. +runner = "qemu-system-arm -cpu cortex-m4 -machine mps2-an386 -nographic -semihosting-config enable=on,target=native -kernel" +rustflags = [ + # Link against the cortex-m-rt linker script, which pulls in `memory.x`. + "-C", "link-arg=--nmagic", + "-C", "link-arg=-Tlink.x", + # No on-board entropy source on this emulated core, so the gossip RNG seed is + # drawn from a fixed-seed custom `getrandom` backend provided in this binary + # (deterministic — this is a reproducible test, not a security context). + "--cfg", "getrandom_backend=\"custom\"", +] + +[env] +# `defmt`/`semihosting` need no extra env; declared block kept minimal. diff --git a/tests/serf-embassy-qemu/Cargo.lock b/tests/serf-embassy-qemu/Cargo.lock new file mode 100644 index 00000000..0479f81c --- /dev/null +++ b/tests/serf-embassy-qemu/Cargo.lock @@ -0,0 +1,1141 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "bare-metal" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5deb64efa5bd81e31fcd1938615a6d98c82eafcbcd787162b6f63b91d6bac5b3" +dependencies = [ + "rustc_version 0.2.3", +] + +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + +[[package]] +name = "bitfield" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46afbd2983a5d5a7bd740ccb198caf5b82f45c40c09c0eed36052d91cb92e719" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4388bee8683e3d04af747c73422af53102d2bd24d9eadb6cbc100baef4b43f8" + +[[package]] +name = "buffa" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33f29a40702df4b86ccd84211bfde8cee0bce6d0811450ade4a86a7d0958a23a" +dependencies = [ + "bytes", + "foldhash 0.1.5", + "hashbrown 0.15.5", + "once_cell", + "rustversion", + "serde", + "serde_json", + "smoothutf8", + "thiserror", +] + +[[package]] +name = "buffa-build" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef33217cabddfad99c0a54d42ddb7ca5b73fa492f7b16ddb621132ef51107556" +dependencies = [ + "buffa", + "buffa-codegen", + "tempfile", +] + +[[package]] +name = "buffa-codegen" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6681b562b18ea719622d0d12684e88ecbdca600d517557dc7bcef7704631e28" +dependencies = [ + "buffa", + "buffa-descriptor", + "prettyplease", + "proc-macro2", + "quote", + "syn 2.0.118", + "thiserror", +] + +[[package]] +name = "buffa-descriptor" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57ed423c4ecec86d1500879ce42e7e5f6def01bc632985ac756c1fd723fa21fc" +dependencies = [ + "buffa", + "rustversion", + "serde", + "serde_json", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "cheap-clone" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23310282f1a049e4d2ec0e993cb18c0258fb318ad33db710b1e961f775a7c240" +dependencies = [ + "bytes", + "paste", + "seq-macro", + "smol_str", + "triomphe", +] + +[[package]] +name = "const-default" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b396d1f76d455557e1218ec8066ae14bba60b4b36ecd55577ba979f5db7ecaa" + +[[package]] +name = "convert_case" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "633458d4ef8c78b72454de2d54fd6ab2e60f9e02be22f3c6104cdc8a4e0fceb9" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "cortex-m" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ec610d8f49840a5b376c69663b6369e71f4b34484b9b2eb29fb918d92516cb9" +dependencies = [ + "bare-metal", + "bitfield", + "critical-section", + "embedded-hal 0.2.7", + "volatile-register", +] + +[[package]] +name = "cortex-m-rt" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "801d4dec46b34c299ccf6b036717ae0fce602faa4f4fe816d9013b9a7c9f5ba6" +dependencies = [ + "cortex-m-rt-macros", +] + +[[package]] +name = "cortex-m-rt-macros" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e37549a379a9e0e6e576fd208ee60394ccb8be963889eebba3ffe0980364f472" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.118", +] + +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "derive_more" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d751e9e49156b02b44f9c1815bcb94b984cdcc4396ecc32521c739452808b134" +dependencies = [ + "derive_more-impl", +] + +[[package]] +name = "derive_more-impl" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "799a97264921d8623a957f6c3b9011f3b5492f557bbb7a5a19b7fa6d06ba8dcb" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version 0.4.1", + "syn 2.0.118", + "unicode-xid", +] + +[[package]] +name = "document-features" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4b8a88685455ed29a21542a33abd9cb6510b6b129abadabdcef0f4c55bc8f61" +dependencies = [ + "litrs", +] + +[[package]] +name = "either" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91622ff5e7162018101f2fea40d6ebf4a78bbe5a49736a2020649edf9693679e" + +[[package]] +name = "embassy-executor" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06070468370195e0e86f241c8e5004356d696590a678d47d6676795b2e439c6b" +dependencies = [ + "cortex-m", + "critical-section", + "document-features", + "embassy-executor-macros", + "embassy-executor-timer-queue", +] + +[[package]] +name = "embassy-executor-macros" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfdddc3a04226828316bf31393b6903ee162238576b1584ee2669af215d55472" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "embassy-executor-timer-queue" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fc328bf943af66b80b98755db9106bf7e7471b0cf47dc8559cd9a6be504cc9c" + +[[package]] +name = "embassy-futures" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc2d050bdc5c21e0862a89256ed8029ae6c290a93aecefc73084b3002cdebb01" + +[[package]] +name = "embassy-net" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "347bc855bdbdf50ed9c5a1d80e8204badb0ba149b8732dde38e1e9708ed9d313" +dependencies = [ + "document-features", + "embassy-net-driver", + "embassy-sync", + "embassy-time", + "embedded-io-async", + "embedded-nal-async", + "heapless", + "managed", + "smoltcp", +] + +[[package]] +name = "embassy-net-driver" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524eb3c489760508f71360112bca70f6e53173e6fe48fc5f0efd0f5ab217751d" + +[[package]] +name = "embassy-sync" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bbd85cf5a5ae56bdf26f618364af642d1d0a4e245cdd75cd9aabda382f65a81" +dependencies = [ + "cfg-if", + "critical-section", + "embedded-io-async", + "futures-core", + "futures-sink", + "heapless", +] + +[[package]] +name = "embassy-time" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "592b0c143ec626e821d4d90da51a2bd91d559d6c442b7c74a47d368c9e23d97a" +dependencies = [ + "cfg-if", + "critical-section", + "document-features", + "embassy-time-driver", + "embassy-time-queue-utils", + "embedded-hal 0.2.7", + "embedded-hal 1.0.0", + "embedded-hal-async", + "futures-core", +] + +[[package]] +name = "embassy-time-driver" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ee71af1b3a0deaa53eaf2d39252f83504c853646e472400b763060389b9fcc9" +dependencies = [ + "document-features", +] + +[[package]] +name = "embassy-time-queue-utils" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "168297bf80aaf114b3c9ad589bf38b01b3009b9af7f97cd18086c5bbf96f5693" +dependencies = [ + "embassy-executor-timer-queue", + "heapless", +] + +[[package]] +name = "embedded-alloc" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f2de9133f68db0d4627ad69db767726c99ff8585272716708227008d3f1bddd" +dependencies = [ + "const-default", + "critical-section", + "linked_list_allocator", + "rlsf", +] + +[[package]] +name = "embedded-hal" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35949884794ad573cf46071e41c9b60efb0cb311e3ca01f7af807af1debc66ff" +dependencies = [ + "nb 0.1.3", + "void", +] + +[[package]] +name = "embedded-hal" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "361a90feb7004eca4019fb28352a9465666b24f840f5c3cddf0ff13920590b89" + +[[package]] +name = "embedded-hal-async" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c4c685bbef7fe13c3c6dd4da26841ed3980ef33e841cddfa15ce8a8fb3f1884" +dependencies = [ + "embedded-hal 1.0.0", +] + +[[package]] +name = "embedded-io" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9eb1aa714776b75c7e67e1da744b81a129b3ff919c8712b5e1b32252c1f07cc7" + +[[package]] +name = "embedded-io-async" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2564b9f813c544241430e147d8bc454815ef9ac998878d30cc3055449f7fd4c0" +dependencies = [ + "embedded-io", +] + +[[package]] +name = "embedded-nal" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56a28be191a992f28f178ec338a0bf02f63d7803244add736d026a471e6ed77" +dependencies = [ + "nb 1.1.0", +] + +[[package]] +name = "embedded-nal-async" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb5a1bd585135d302f8f6d7de329310938093da6271b37a6c94b8798795c0c6d" +dependencies = [ + "embedded-io-async", + "embedded-nal", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + +[[package]] +name = "foldhash" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "hash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47d60b12902ba28e2730cd37e95b8c9223af2808df9e902d4df49588d1470606" +dependencies = [ + "byteorder", +] + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +dependencies = [ + "foldhash 0.2.0", +] + +[[package]] +name = "heapless" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25ba4bd83f9415b58b4ed8dc5714c76e626a105be4646c02630ad730ad3b5aa4" +dependencies = [ + "hash32", + "stable_deref_trait", +] + +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + +[[package]] +name = "indexmap" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +dependencies = [ + "equivalent", + "hashbrown 0.17.1", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.186" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "linked_list_allocator" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b23ac50abb8261cb38c6e2a7192d3302e0836dac1628f6a93b82b4fad185897" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "litrs" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11d3d7f243d5c5a8b9bb5d6dd2b1602c0cb0b9db1621bafc7ed66e35ff9fe092" + +[[package]] +name = "managed" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca88d725a0a943b096803bd34e73a4437208b6077654cc4ecb2947a5f91618d" + +[[package]] +name = "memberlist-embedded" +version = "0.1.0" +source = "git+https://github.com/al8n/memberlist?branch=main#2c6d2a15b554675bde472f07130817f984a3d4f5" +dependencies = [ + "bytes", + "getrandom", + "hashbrown 0.17.1", + "heapless", + "memberlist-proto", + "smallvec-wrapper", +] + +[[package]] +name = "memberlist-proto" +version = "0.4.0" +source = "git+https://github.com/al8n/memberlist?branch=main#2c6d2a15b554675bde472f07130817f984a3d4f5" +dependencies = [ + "buffa", + "buffa-build", + "bytes", + "cheap-clone", + "derive_more", + "getrandom", + "hashbrown 0.17.1", + "indexmap", + "libm", + "paste", + "rand", + "rustc-hash", + "smallvec", + "smallvec-wrapper", + "smol_str", + "thiserror", + "triomphe", + "varing", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "nb" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "801d31da0513b6ec5214e9bf433a77966320625a37860f910be265be6e18d06f" +dependencies = [ + "nb 1.1.0", +] + +[[package]] +name = "nb" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d5439c4ad607c3c23abf66de8c8bf57ba8adcd1f129e699851a6e43935d339d" + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.118", +] + +[[package]] +name = "proc-macro2" +version = "1.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fd00f0bb2e90d81d1044c2b32617f68fcb9fa3bb7640c23e9c748e53fb30934" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbc457d0c7a0759a614551b11a6409e5951f6c7537be1f1b7682b9ae9230368" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7f5fa3a058cd35567ef9bfa5e75732bee0f9e4c55fa90477bef2dfcdbc4be80" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" + +[[package]] +name = "rlsf" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1646a59a9734b8b7a0ac51689388a60fe1625d4b956348e9de07591a1478457a" +dependencies = [ + "cfg-if", + "const-default", + "libc", + "rustversion", + "svgbobdoc", +] + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver 0.9.0", +] + +[[package]] +name = "rustc_version" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92" +dependencies = [ + "semver 1.0.28", +] + +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags 2.13.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" + +[[package]] +name = "semihosting" +version = "0.1.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8e4abf97879f4e80db69a9fba7bd64998e9bdad25f58ef045a778e191172fd4" + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "serde_json" +version = "1.0.150" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "serf-embassy" +version = "0.5.2" +dependencies = [ + "bytes", + "embassy-futures", + "embassy-net", + "embassy-sync", + "embassy-time", + "getrandom", + "memberlist-proto", + "serf-embedded", + "serf-proto", + "smol_str", +] + +[[package]] +name = "serf-embassy-qemu" +version = "0.0.0" +dependencies = [ + "bytes", + "cortex-m", + "cortex-m-rt", + "critical-section", + "embassy-executor", + "embassy-futures", + "embassy-net", + "embassy-sync", + "embassy-time", + "embassy-time-driver", + "embassy-time-queue-utils", + "embedded-alloc", + "getrandom", + "semihosting", + "serf-embassy", + "smol_str", + "static_cell", +] + +[[package]] +name = "serf-embedded" +version = "0.5.2" +dependencies = [ + "bytes", + "getrandom", + "hashbrown 0.17.1", + "heapless", + "memberlist-embedded", + "memberlist-proto", + "serf-proto", + "smallvec-wrapper", + "smol_str", +] + +[[package]] +name = "serf-proto" +version = "0.5.2" +dependencies = [ + "bitflags 2.13.0", + "buffa", + "buffa-build", + "bytes", + "derive_more", + "hashbrown 0.17.1", + "libm", + "memberlist-proto", + "rand", + "rustc-hash", + "smol_str", + "thiserror", +] + +[[package]] +name = "simdutf8" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e" + +[[package]] +name = "smallvec" +version = "1.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" + +[[package]] +name = "smallvec-wrapper" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e7ac95ddfb355a82e23e3bdce91b646b847307bb481b1f7bc7f44942461d571" +dependencies = [ + "either", + "paste", + "smallvec", +] + +[[package]] +name = "smol_str" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aaa7368fcf4852a4c2dd92df0cace6a71f2091ca0a23391ce7f3a31833f1523" + +[[package]] +name = "smoltcp" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f73d40463bba65efc9adc6370b56df76d563cc46e2482bba58351b4afb7535e" +dependencies = [ + "bitflags 1.3.2", + "byteorder", + "cfg-if", + "heapless", + "managed", +] + +[[package]] +name = "smoothutf8" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36358427d32ecdb1624616deed99eccfef0a167fe5bf40ddb51efe6980bc1ec8" +dependencies = [ + "simdutf8", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "static_cell" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0530892bb4fa575ee0da4b86f86c667132a94b74bb72160f58ee5a4afec74c23" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "svgbobdoc" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2c04b93fc15d79b39c63218f15e3fdffaa4c227830686e3b7c5f41244eb3e50" +dependencies = [ + "base64", + "proc-macro2", + "quote", + "syn 1.0.109", + "unicode-width", +] + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.118" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9ae57f904213ebb649ce6895b8a66c66f0203b9319718f69a5612a065b1422" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "triomphe" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b40688ea6389c8171614b25491f71d4a27946e0c7ce2da1c6de27e25abf1a0ae" + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode-xid" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" + +[[package]] +name = "varing" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15fc8771f370eb5378b51aac8ce0ef92418ee540e13cfae5fec34347bcfb923b" +dependencies = [ + "paste", + "seq-macro", + "thiserror", +] + +[[package]] +name = "vcell" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77439c1b53d2303b20d9459b1ade71a83c716e3f9c34f3228c00e6f185d6c002" + +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + +[[package]] +name = "volatile-register" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de437e2a6208b014ab52972a27e59b33fa2920d3e00fe05026167a1c509d19cc" +dependencies = [ + "vcell", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "zmij" +version = "1.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" diff --git a/tests/serf-embassy-qemu/Cargo.toml b/tests/serf-embassy-qemu/Cargo.toml new file mode 100644 index 00000000..7a7a41f8 --- /dev/null +++ b/tests/serf-embassy-qemu/Cargo.toml @@ -0,0 +1,79 @@ +[package] +name = "serf-embassy-qemu" +version = "0.0.0" +edition = "2024" +license = "MPL-2.0" +publish = false +description = "Bare-metal QEMU execution proof: two serf-embassy nodes joining on an emulated Cortex-M." + +[dependencies] +# serf-embassy driver, built no_std (+ alloc) like a real firmware would. +serf-embassy = { path = "../../serf-embassy", default-features = false, features = [ + "alloc", +] } +# Re-exported through the driver, but named directly here for socket buffers and +# the in-memory paired device. The version tracks serf-embassy's own pin. +embassy-net = { version = "0.9", default-features = false, features = [ + "udp", + "tcp", + "proto-ipv4", + "proto-ipv6", + "medium-ethernet", +] } +embassy-time = { version = "0.5", default-features = false, features = [ + # 1 MHz timebase: `now()` is reported in microseconds. The SysTick handler + # supplies the actual wake granularity (coarser); the queue handles waker + # storage. + "tick-hz-1_000_000", + # 8-slot software timer queue. With this feature embassy-time does NOT provide + # the `schedule_wake` symbol itself, so this crate's SysTick driver supplies + # the full `embassy_time_driver::Driver` (now + schedule_wake) backed by it. + "generic-queue-8", +] } +embassy-time-driver = { version = "0.2", default-features = false } +# The 8-slot software timer queue the SysTick driver stores wakers in. embassy-net +# pulls it transitively (via embassy-time's `generic-queue-8`), but the driver +# names `Queue` directly, so it is a direct dependency here with the matching +# capacity feature. +embassy-time-queue-utils = { version = "0.3", default-features = false, features = [ + "generic-queue-8", +] } +embassy-executor = { version = "0.9", default-features = false, features = [ + "arch-cortex-m", + "executor-thread", +] } +embassy-sync = { version = "0.8", default-features = false } +embassy-futures = { version = "0.1", default-features = false } + +# Cortex-M runtime: reset handler, vector table, the `#[entry]` macro, and the +# `link.x` linker script that consumes `memory.x`. +cortex-m = { version = "0.7", features = ["critical-section-single-core"] } +cortex-m-rt = "0.7" +# Critical sections on a single-core M-profile core; the SysTick driver guards +# its queue with `critical_section::Mutex`. +critical-section = "1" +# A global allocator over a static heap, for the driver's `alloc` use. +embedded-alloc = "0.6" +# ARM semihosting: stdout writes and `process::exit` (the qemu exit code), plus +# a `panic_handler` that prints the message over semihosting and exits non-zero. +semihosting = { version = "0.1", features = ["stdio", "panic-handler"] } +# `&'static mut` homes for the executor and the per-node buffer bundles without a +# hand-rolled `static mut` for each. +static_cell = "2" + +# A fixed-seed `getrandom` custom backend so the gossip RNG has entropy on a +# core with no hardware RNG. Deterministic on purpose (reproducible test). +getrandom = "0.4" + +# Node-id type for the two nodes. +smol_str = { version = "0.3", default-features = false } +bytes = { version = "1", default-features = false } + +[profile.dev] +# `opt-level = 1` keeps the debug build small enough and fast enough to converge +# within the qemu deadline while preserving panic messages. +opt-level = 1 + +[profile.release] +opt-level = "s" +lto = true diff --git a/tests/serf-embassy-qemu/README.md b/tests/serf-embassy-qemu/README.md new file mode 100644 index 00000000..dae6923a --- /dev/null +++ b/tests/serf-embassy-qemu/README.md @@ -0,0 +1,48 @@ +
+

serf-embassy-qemu

+
+
+ +Bare-metal QEMU execution proof for the **serf** embassy driver. + +[github][Github-url] + +
+ +## Introduction + +`serf-embassy-qemu` boots [`serf-embassy`] on an emulated Cortex-M (the `mps2-an386` +machine) under QEMU and joins two nodes over embassy-net, then propagates a user event +between them — an end-to-end proof that the `no_std` async serf driver runs on real bare +metal, with its own `cortex-m-rt` runtime, a SysTick embassy-time driver, and a no_std +entropy backend. + +Two serf nodes converge on a two-member view and then one broadcasts a user event the +other observes as `Event::User` — the minimal serf-above-memberlist signal, exercising the +whole stack (SWIM membership plus serf's gossip plane) on the emulated core. Success is +reported via the semihosting process exit code (0 = pass). + +It is the CI execution gate for the embassy driver and doubles as a complete, runnable +wiring example. Because it pins a foreign default target, it is **excluded** from the host +workspace and built on its own: + +```sh +cd tests/serf-embassy-qemu && cargo run +``` + +This is an internal proof binary (`publish = false`) for the [serf] workspace. + +## License + +`serf-embassy-qemu` is under the terms of the MPL-2.0 license. + +See [LICENSE] for details. + +Copyright (c) 2025 Al Liu. + +Copyright (c) 2013 HashiCorp, Inc. + +[serf]: https://github.com/al8n/serf +[`serf-embassy`]: https://crates.io/crates/serf-embassy +[LICENSE]: https://github.com/al8n/serf/blob/main/LICENSE +[Github-url]: https://github.com/al8n/serf/ diff --git a/tests/serf-embassy-qemu/memory.x b/tests/serf-embassy-qemu/memory.x new file mode 100644 index 00000000..4af50e28 --- /dev/null +++ b/tests/serf-embassy-qemu/memory.x @@ -0,0 +1,15 @@ +/* Memory layout for the QEMU `mps2-an386` machine (ARM MPS2 with the AN386 + * FPGA image, a Cortex-M4). + * + * The board exposes two 4 MB ZBT SSRAM blocks: SSRAM1 at 0x0000_0000 is used in + * lieu of flash and is where QEMU's `-kernel` loader places the image, and + * SSRAM2&3 at 0x2000_0000 is the main RAM. Either block is far larger than a + * real microcontroller's, which is exactly why this machine is chosen for the + * two-node proof: two embassy-net stacks, two engines, and the heap fit + * comfortably. (Addresses per QEMU's hw/arm/mps2.c for AN386.) + */ +MEMORY +{ + FLASH : ORIGIN = 0x00000000, LENGTH = 4M + RAM : ORIGIN = 0x20000000, LENGTH = 4M +} diff --git a/tests/serf-embassy-qemu/src/heap.rs b/tests/serf-embassy-qemu/src/heap.rs new file mode 100644 index 00000000..9ccecd28 --- /dev/null +++ b/tests/serf-embassy-qemu/src/heap.rs @@ -0,0 +1,44 @@ +//! The global allocator over a static heap. +//! +//! The serf driver is `no_std + alloc`, so the firmware must register a global +//! allocator. [`embedded_alloc::LlffHeap`] manages a fixed static byte region; +//! [`init`] hands it that region once at startup. + +use core::mem::MaybeUninit; + +use embedded_alloc::LlffHeap as Heap; + +/// Backing heap size in bytes. +/// +/// Sized for two embassy-net stacks plus two serf engines (the serf super-machine +/// over memberlist: gossip codec scratch, the reliable-plane connection state, +/// membership maps, and serf's event/query bookkeeping) running concurrently on +/// the AN386's 4 MB RAM. Generous because the machine has the +/// room and convergence allocates transiently; the static socket/stack buffers +/// live outside this heap (in `.bss`). +pub const HEAP_SIZE: usize = 512 * 1024; + +#[global_allocator] +static HEAP: Heap = Heap::empty(); + +/// The heap's backing storage, in `.bss`. +static mut HEAP_MEM: [MaybeUninit; HEAP_SIZE] = [MaybeUninit::uninit(); HEAP_SIZE]; + +/// Initialize the global allocator. Call once, before any allocation. +/// +/// # Safety +/// +/// Must be called exactly once and before the first heap allocation. The +/// `addr_of_mut!` read of the `static mut` produces a single pointer that is +/// handed to the allocator, which thereafter owns the region; no other code +/// touches `HEAP_MEM`. +pub fn init() { + // SAFETY: called once at the top of `main` before anything allocates, and + // `HEAP_MEM` is referenced nowhere else, so this is the sole live reference to + // the static. `HEAP.init` records the base/size and the allocator owns it from + // here on. + unsafe { + let ptr = core::ptr::addr_of_mut!(HEAP_MEM) as usize; + HEAP.init(ptr, HEAP_SIZE); + } +} diff --git a/tests/serf-embassy-qemu/src/main.rs b/tests/serf-embassy-qemu/src/main.rs new file mode 100644 index 00000000..a131b523 --- /dev/null +++ b/tests/serf-embassy-qemu/src/main.rs @@ -0,0 +1,52 @@ +//! Bare-metal QEMU execution proof for the `serf-embassy` driver. +//! +//! Boots a Cortex-M4 under QEMU's `mps2-an386` machine, stands up the embassy +//! thread-mode executor on a SysTick time driver, and runs two serf nodes over an +//! in-memory paired embassy-net link until they converge on a two-member view AND +//! a user event broadcast by one node is observed by the other — then exits 0 via +//! semihosting. A deadline guard exits non-zero if either step stalls, so the qemu +//! process exit code is the pass/fail signal. +//! +//! This is the runtime counterpart to the host loopback / user-event tests in +//! `serf-embassy/tests/`: same two-node join and user-event crossing, but on a +//! real emulated core with no `std`. + +#![no_std] +#![no_main] + +extern crate alloc; + +mod heap; +mod node; +mod paired_device; +mod rng; +mod systick_driver; + +use cortex_m_rt::entry; +use embassy_executor::Executor; +use static_cell::StaticCell; + +/// The thread-mode executor, owned for the program's lifetime. +static EXECUTOR: StaticCell = StaticCell::new(); + +#[entry] +fn main() -> ! { + // Heap first: the driver is `alloc`-based, so nothing may allocate before this. + heap::init(); + + // SysTick time driver: take the core peripherals, hand SysTick to the driver so + // `embassy_time` has a clock and the executor wakes each tick. + let core = cortex_m::Peripherals::take().expect("core peripherals available once at boot"); + systick_driver::init(core.SYST); + + // Start the thread-mode executor; `run` never returns. The spawned task drives + // the whole proof and calls `semihosting::process::exit`, which (with qemu + // semihosting on) terminates the emulator with that code. + let executor = EXECUTOR.init(Executor::new()); + executor.run(|spawner| { + // The entry task receives the `Spawner` so it can launch the four run loops + // (two serf Runners + two embassy-net Runners) once it has built them. + // `must_spawn`: this single-slot task is spawned exactly once. + spawner.must_spawn(node::main_task(spawner)); + }); +} diff --git a/tests/serf-embassy-qemu/src/node.rs b/tests/serf-embassy-qemu/src/node.rs new file mode 100644 index 00000000..b62ac263 --- /dev/null +++ b/tests/serf-embassy-qemu/src/node.rs @@ -0,0 +1,260 @@ +//! The proof task: stand up two serf nodes over an in-memory paired embassy-net +//! link, converge them, and cross a user event on the emulated core. +//! +//! This is the bare-metal twin of `serf-embassy/tests/loopback.rs` + +//! `user_event.rs`: two embassy-net stacks (static IPs `169.254.1.1` and `.2`) +//! cross-wired by a [`paired_device`], a [`Serf`] + [`Runner`] on each, node B +//! joining node A as a seed, a wait for both to see two members, and then a user +//! event broadcast by A that B must observe as [`Event::User`]. Where the host +//! tests race every future under one `block_on`, here the four run loops (two serf +//! [`Runner`]s + two `embassy_net::Runner`s) are spawned as embassy tasks and the +//! main task drives the join, convergence, and the user-event crossing. +//! +//! All borrowed state (stack resources, socket buffers) is promoted to `'static` +//! via [`StaticCell`] so the run loops can be `'static` embassy tasks. On the +//! user-event crossing the emulator exits 0; a deadline guard exits 1. + +use core::net::{IpAddr, Ipv4Addr, SocketAddr}; + +use embassy_executor::Spawner; +use embassy_net::{ + Config as NetConfig, Ipv4Cidr, Runner as NetRunner, Stack, StackResources, StaticConfigV4, + tcp::TcpSocket, + udp::{PacketMetadata, UdpSocket}, +}; +use embassy_time::{Duration, Timer}; +use semihosting::println; +use serf_embassy::{ + Bytes, EndpointOptions, Event, MaybeResolved, Options, Runner, SeedableRng, Serf, SerfOptions, + SmallRng, SocketAddrResolver, TransformOptions, now, +}; +use smol_str::SmolStr; +use static_cell::StaticCell; + +use crate::paired_device::{PairedDevice, pair}; + +/// TCP socket pool size per node (a listener plus dial/accept sockets), matching +/// the host loopback test. +const POOL: usize = 4; +/// Per-TCP-socket rx/tx buffer bytes. +const TCP_BUF: usize = 4096; +/// Per-node UDP datagram buffer bytes (gossip datagrams are small; this holds a +/// handful in flight). +const UDP_BUF: usize = 8 * 1024; +/// Per-node UDP packet-metadata slots. +const UDP_META: usize = 16; +/// `StackResources` socket budget: the TCP pool plus headroom for UDP + DHCP-less +/// control sockets, mirroring the host test's `POOL + 2`. +const SOCKS: usize = POOL + 2; +/// Convergence + user-event deadline in virtual time. The SysTick driver advances +/// ~1 ms of virtual time per tick, so this is ~20_000 SysTicks — ample for a +/// two-node join plus a gossiped user event yet bounded so a regression fails fast. +const DEADLINE: Duration = Duration::from_secs(20); +/// Poll cadence while waiting for convergence and the user event. +const POLL: Duration = Duration::from_millis(10); + +/// Gossip/TCP port both nodes bind (the single-port memberlist model serf runs on). +const PORT: u16 = 7946; + +/// Build a `SocketAddr` in the link-local `169.254.1.0/24` test subnet. +fn addr(last: u8, port: u16) -> SocketAddr { + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(169, 254, 1, last)), port) +} + +/// All the owned buffers one node's sockets borrow, promoted to `'static` so the +/// sockets (and the `Serf`/`Runner` holding them) are `'static`. +struct NodeBufs { + udp_rx_meta: [PacketMetadata; UDP_META], + udp_rx: [u8; UDP_BUF], + udp_tx_meta: [PacketMetadata; UDP_META], + udp_tx: [u8; UDP_BUF], + tcp_rx: [[u8; TCP_BUF]; POOL], + tcp_tx: [[u8; TCP_BUF]; POOL], +} + +impl NodeBufs { + const fn new() -> Self { + Self { + udp_rx_meta: [PacketMetadata::EMPTY; UDP_META], + udp_rx: [0u8; UDP_BUF], + udp_tx_meta: [PacketMetadata::EMPTY; UDP_META], + udp_tx: [0u8; UDP_BUF], + tcp_rx: [[0u8; TCP_BUF]; POOL], + tcp_tx: [[0u8; TCP_BUF]; POOL], + } + } +} + +/// Build one node's `UdpSocket` + `[TcpSocket; POOL]` over its `'static` stack and +/// `'static` bufs. +fn build_sockets( + stack: Stack<'static>, + bufs: &'static mut NodeBufs, +) -> (UdpSocket<'static>, [TcpSocket<'static>; POOL]) { + let udp = UdpSocket::new( + stack, + &mut bufs.udp_rx_meta, + &mut bufs.udp_rx, + &mut bufs.udp_tx_meta, + &mut bufs.udp_tx, + ); + // Pair each rx buffer with its tx buffer in lockstep, yielding `POOL` sockets. + let mut rx_iter = bufs.tcp_rx.iter_mut(); + let mut tx_iter = bufs.tcp_tx.iter_mut(); + let tcp = core::array::from_fn::<_, POOL, _>(|_| { + let rx = rx_iter.next().expect("POOL rx buffers"); + let tx = tx_iter.next().expect("POOL tx buffers"); + TcpSocket::new(stack, rx, tx) + }); + (udp, tcp) +} + +/// Build a static-IPv4 embassy-net stack over a paired device. +fn build_stack( + device: PairedDevice, + resources: &'static mut StackResources, + last: u8, + seed: u64, +) -> (Stack<'static>, NetRunner<'static, PairedDevice>) { + let config = NetConfig::ipv4_static(StaticConfigV4 { + address: Ipv4Cidr::new(Ipv4Addr::new(169, 254, 1, last), 24), + gateway: None, + dns_servers: Default::default(), + }); + embassy_net::new(device, config, resources, seed) +} + +/// Drive one embassy-net stack's run loop. Two instances (one per stack) move +/// frames across the paired devices. +#[embassy_executor::task(pool_size = 2)] +async fn net_task(mut runner: NetRunner<'static, PairedDevice>) -> ! { + runner.run().await +} + +/// Drive one serf node's run loop (engine pump + the `POOL` reliable-plane +/// workers). Two instances, one per node. `Runner::run` returns only on a terminal +/// shutdown, which the proof never triggers, so each stays live for the run. +#[embassy_executor::task(pool_size = 2)] +async fn serf_task(runner: Runner<'static, SmolStr, POOL>) { + runner.run().await; +} + +/// The single spawned entry task: build both nodes, spawn the four run loops, +/// drive the join, wait for convergence, then cross a user event and exit the +/// emulator with the pass/fail code. +#[embassy_executor::task] +pub async fn main_task(spawner: Spawner) -> ! { + // Promote every borrowed resource to `'static`. Distinct cells per node so the + // two stacks and two socket sets do not alias. + static RES_A: StaticCell> = StaticCell::new(); + static RES_B: StaticCell> = StaticCell::new(); + static BUFS_A: StaticCell = StaticCell::new(); + static BUFS_B: StaticCell = StaticCell::new(); + + let (dev_a, dev_b) = pair(); + let (stack_a, net_a) = build_stack(dev_a, RES_A.init(StackResources::new()), 1, 0x1111_2222); + let (stack_b, net_b) = build_stack(dev_b, RES_B.init(StackResources::new()), 2, 0x3333_4444); + + let (udp_a, tcp_a) = build_sockets(stack_a, BUFS_A.init(NodeBufs::new())); + let (udp_b, tcp_b) = build_sockets(stack_b, BUFS_B.init(NodeBufs::new())); + + let clock = now(); + // Each node seeds its gossip RNG and serf's own core RNG independently so no two + // nodes share a `(ltime, id)` query-id schedule; the fixed seeds keep the run + // reproducible. + let (ml_a, run_a) = Serf::new_with_rng::<_, POOL>( + Options::new(), + TransformOptions::default(), + EndpointOptions::new(SmolStr::new("a"), addr(1, PORT)), + SerfOptions::new(), + &SocketAddrResolver, + udp_a, + tcp_a, + clock, + SmallRng::seed_from_u64(1), + SmallRng::seed_from_u64(101), + ) + .await + .expect("build node a"); + let (ml_b, run_b) = Serf::new_with_rng::<_, POOL>( + Options::new(), + TransformOptions::default(), + EndpointOptions::new(SmolStr::new("b"), addr(2, PORT)), + SerfOptions::new(), + &SocketAddrResolver, + udp_b, + tcp_b, + clock, + SmallRng::seed_from_u64(2), + SmallRng::seed_from_u64(102), + ) + .await + .expect("build node b"); + + // Spawn the four run loops. `must_spawn`: each pool slot is used exactly once, + // so the spawn cannot fail. + spawner.must_spawn(net_task(net_a)); + spawner.must_spawn(net_task(net_b)); + spawner.must_spawn(serf_task(run_a)); + spawner.must_spawn(serf_task(run_b)); + + // B joins A as a seed; the convergence wait below bounds success by the deadline. + // Ignoring Err: a failed seed join surfaces as the convergence loop timing out, + // which the proof already treats as the failure signal. `ignore_old = false` + // keeps the default replay of A's pre-join user events (there are none here). + let _ = ml_b + .join( + &SocketAddrResolver, + &[MaybeResolved::Resolved(addr(1, PORT))], + false, + ) + .await; + println!("join issued; waiting for convergence"); + + let mut waited = Duration::from_secs(0); + + // Phase 1: both views reach two members (A learns B from the push/pull a tick + // after the exchange). + loop { + let a = ml_a.num_members(); + let b = ml_b.num_members(); + if a == 2 && b == 2 { + println!("converged: A and B both see 2 members"); + break; + } + if waited >= DEADLINE { + println!("timed out before convergence: a={} b={}", a, b); + semihosting::process::exit(1); + } + Timer::after(POLL).await; + waited += POLL; + } + + // Phase 2: the serf-above-memberlist proof — A broadcasts a user event and B must + // observe it as `Event::User`, which can only happen if serf's gossip plane truly + // disseminated it across the emulated link, not just SWIM membership. + ml_a + .user_event("greet", Bytes::from_static(b"hello"), false) + .expect("broadcast a user event from a running node"); + println!("user event broadcast; waiting for B to observe it"); + + loop { + match ml_b.poll_event() { + Some(Event::User(_)) => { + println!("user event crossed: B observed Event::User; proof complete"); + semihosting::process::exit(0); + } + // Drain and ignore any other buffered events (e.g. the membership batch from + // the join) and keep polling for the user event. + Some(_) => continue, + None => { + if waited >= DEADLINE { + println!("timed out before B observed the user event"); + semihosting::process::exit(1); + } + Timer::after(POLL).await; + waited += POLL; + } + } + } +} diff --git a/tests/serf-embassy-qemu/src/paired_device.rs b/tests/serf-embassy-qemu/src/paired_device.rs new file mode 100644 index 00000000..f4a7fafa --- /dev/null +++ b/tests/serf-embassy-qemu/src/paired_device.rs @@ -0,0 +1,166 @@ +//! A `no_std` channel-backed paired [`embassy_net::driver::Driver`]. +//! +//! This is the bare-metal twin of `serf-embassy`'s host-test +//! `support::paired_device`: two `PairedDevice`s are cross-wired by [`pair`] so +//! each device's transmit pushes a frame into the OTHER device's receive queue +//! and wakes the other stack, moving real ethernet frames between two embassy-net +//! stacks on a single-threaded executor. It is the in-memory link the two serf +//! nodes converge over, standing in for an MII/RMII peripheral. +//! +//! Everything runs on one thread-mode executor with interrupts the only +//! preemption, and the embassy-net stack only touches the device from task +//! context, so the shared queues are plain `Rc>` (no locking) exactly +//! as in the host port. + +use alloc::{collections::VecDeque, rc::Rc, vec, vec::Vec}; +use core::{ + cell::RefCell, + task::{Context, Waker}, +}; + +use embassy_net::driver::{Capabilities, Driver, HardwareAddress, LinkState, RxToken, TxToken}; + +/// The link MTU advertised by the paired devices (a standard ethernet frame). +const MTU: usize = 1514; + +/// A shared frame FIFO between the two devices. +type Wire = Rc>>>; +/// A shared slot holding a stack's most recent receive waker. +type WakerSlot = Rc>>; + +/// One end of a cross-wired virtual ethernet link. +/// +/// Reads frames from `rx` (frames the peer transmitted) and writes frames to `tx` +/// (delivered to the peer's `rx`). `peer_waker` is the peer stack's receive waker, +/// woken on every transmit so the peer promptly drains the frame. +pub struct PairedDevice { + rx: Wire, + tx: Wire, + /// This device's own receive waker, registered by `receive`/`link_state` and + /// woken by the peer's transmit. + my_waker: WakerSlot, + /// The peer device's receive waker, woken by this device's transmit. + peer_waker: WakerSlot, + mac: [u8; 6], +} + +/// Build the two ends of one virtual link, cross-wiring their channels and wakers. +/// +/// Frames device `A` transmits arrive at device `B`'s receive queue (and wake +/// `B`'s stack), and vice versa. +pub fn pair() -> (PairedDevice, PairedDevice) { + let a2b: Wire = Rc::new(RefCell::new(VecDeque::new())); + let b2a: Wire = Rc::new(RefCell::new(VecDeque::new())); + let waker_a: WakerSlot = Rc::new(RefCell::new(None)); + let waker_b: WakerSlot = Rc::new(RefCell::new(None)); + ( + PairedDevice { + rx: b2a.clone(), + tx: a2b.clone(), + my_waker: waker_a.clone(), + peer_waker: waker_b.clone(), + mac: [0x02, 0, 0, 0, 0, 1], + }, + PairedDevice { + rx: a2b, + tx: b2a, + my_waker: waker_b, + peer_waker: waker_a, + mac: [0x02, 0, 0, 0, 0, 2], + }, + ) +} + +impl PairedDevice { + /// Register the current task's waker as this device's receive waker, so the + /// peer's transmit can wake the stack. + fn register(&self, cx: &mut Context<'_>) { + *self.my_waker.borrow_mut() = Some(cx.waker().clone()); + } +} + +/// A receive token carrying one delivered frame. +pub struct PairedRx(Vec); + +/// A transmit token that, on consume, writes the frame to the peer's receive +/// queue and wakes the peer's stack. +pub struct PairedTx { + tx: Wire, + peer_waker: WakerSlot, +} + +impl RxToken for PairedRx { + fn consume(self, f: F) -> R + where + F: FnOnce(&mut [u8]) -> R, + { + let mut buf = self.0; + f(&mut buf) + } +} + +impl TxToken for PairedTx { + fn consume(self, len: usize, f: F) -> R + where + F: FnOnce(&mut [u8]) -> R, + { + let mut buf = vec![0u8; len]; + let r = f(&mut buf); + self.tx.borrow_mut().push_back(buf); + // Wake the peer's stack so its `embassy_net::Runner` re-polls and drains this + // frame; without this the peer would not process inbound traffic until some + // unrelated event polled it. + if let Some(w) = self.peer_waker.borrow_mut().take() { + w.wake(); + } + r + } +} + +impl Driver for PairedDevice { + type RxToken<'a> + = PairedRx + where + Self: 'a; + type TxToken<'a> + = PairedTx + where + Self: 'a; + + fn receive(&mut self, cx: &mut Context<'_>) -> Option<(PairedRx, PairedTx)> { + // Always (re)register the waker so a frame arriving after this poll wakes us. + self.register(cx); + let frame = self.rx.borrow_mut().pop_front()?; + Some(( + PairedRx(frame), + PairedTx { + tx: self.tx.clone(), + peer_waker: self.peer_waker.clone(), + }, + )) + } + + fn transmit(&mut self, _cx: &mut Context<'_>) -> Option { + Some(PairedTx { + tx: self.tx.clone(), + peer_waker: self.peer_waker.clone(), + }) + } + + fn link_state(&mut self, cx: &mut Context<'_>) -> LinkState { + // Keep the waker fresh on the link-state poll path too (the stack polls this), + // so a frame delivered between data polls still wakes the stack. + self.register(cx); + LinkState::Up + } + + fn capabilities(&self) -> Capabilities { + let mut caps = Capabilities::default(); + caps.max_transmission_unit = MTU; + caps + } + + fn hardware_address(&self) -> HardwareAddress { + HardwareAddress::Ethernet(self.mac) + } +} diff --git a/tests/serf-embassy-qemu/src/rng.rs b/tests/serf-embassy-qemu/src/rng.rs new file mode 100644 index 00000000..958ddfc2 --- /dev/null +++ b/tests/serf-embassy-qemu/src/rng.rs @@ -0,0 +1,66 @@ +//! A fixed-seed `getrandom` custom backend. +//! +//! The emulated Cortex-M has no hardware RNG, so the gossip layer's default +//! entropy draw (via [`getrandom`]) needs a backend symbol. This crate builds +//! with `--cfg getrandom_backend="custom"`, which makes `getrandom` call the +//! externally defined `__getrandom_v03_custom`; this module provides it. +//! +//! The stream is a deterministic SplitMix64 seeded from a fixed constant. That is +//! intentional: this is a reproducible convergence test, not a security context, +//! so a fixed seed makes every run identical. (Each node additionally receives its +//! own gossip and serf RNGs as seeded `SmallRng`s passed to `Serf::new_with_rng`, +//! so this backend only ever services whatever incidental entropy the protocol +//! draws beyond those.) +//! +//! The state is a `Cell` behind a `critical_section::Mutex` rather than an +//! atomic, because ARMv7-M has no 64-bit atomic and this runs on a single core. + +use core::cell::Cell; + +use critical_section::Mutex; + +/// SplitMix64 state, advanced on every fill. A fixed start makes the run +/// deterministic. +static STATE: Mutex> = Mutex::new(Cell::new(0x0123_4567_89AB_CDEF)); + +/// One SplitMix64 step under the critical section: mix and return the next value. +fn next_u64() -> u64 { + critical_section::with(|cs| { + let cell = STATE.borrow(cs); + let z = cell.get().wrapping_add(0x9E37_79B9_7F4A_7C15); + cell.set(z); + let z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + let z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + }) +} + +/// The `getrandom` custom backend entry point. +/// +/// `getrandom` (built with `getrandom_backend="custom"`) links its `fill_inner` +/// to this symbol. Fills `len` bytes at `dest` from the SplitMix64 stream and +/// reports success. +/// +/// # Safety +/// +/// `dest` must point to at least `len` writable bytes (guaranteed by `getrandom`, +/// which derives both from a `&mut [MaybeUninit]`). This writes exactly `len` +/// bytes and reads nothing from `dest`. +#[unsafe(no_mangle)] +unsafe extern "Rust" fn __getrandom_v03_custom( + dest: *mut u8, + len: usize, +) -> Result<(), getrandom::Error> { + let mut written = 0usize; + while written < len { + let chunk = next_u64().to_le_bytes(); + let take = core::cmp::min(8, len - written); + // SAFETY: `dest + written` stays within the `len`-byte region the caller + // guarantees, and `take <= 8` so this never reads past `chunk`. + unsafe { + core::ptr::copy_nonoverlapping(chunk.as_ptr(), dest.add(written), take); + } + written += take; + } + Ok(()) +} diff --git a/tests/serf-embassy-qemu/src/systick_driver.rs b/tests/serf-embassy-qemu/src/systick_driver.rs new file mode 100644 index 00000000..2599369f --- /dev/null +++ b/tests/serf-embassy-qemu/src/systick_driver.rs @@ -0,0 +1,145 @@ +//! A SysTick-backed [`embassy_time_driver::Driver`] for the emulated Cortex-M. +//! +//! embassy-time 0.5 with the `generic-queue-8` feature does *not* ship a +//! `schedule_wake` implementation on a bare-metal target — it only provides the +//! 8-slot software timer queue ([`embassy_time_queue_utils::Queue`]). This module +//! supplies the missing half: a full [`Driver`] (both [`Driver::now`] and +//! [`Driver::schedule_wake`]) driven by the core's SysTick exception. +//! +//! # How time advances +//! +//! [`init`] programs SysTick to fire periodically, and the handler maintains a +//! monotonic 64-bit tick counter that [`Driver::now`] reads. Because the gossip +//! protocol only ever measures *relative* intervals, the absolute rate of this +//! virtual clock against wall-clock time is irrelevant; all that matters is that +//! it is monotonic and that it advances whenever the executor is parked. SysTick +//! gives exactly that: it is an exception, so it resumes the thread-mode executor +//! out of its `WFE` sleep on every tick. +//! +//! # How timers fire +//! +//! [`Driver::schedule_wake`] stores the waker in the queue. There is no separate +//! hardware compare/alarm to program — instead the SysTick handler, after bumping +//! the counter, calls [`Queue::next_expiration`], which dequeues every timer whose +//! deadline has passed and wakes its waker. Waking an embassy task runs the +//! executor's pender (which issues `SEV`), so the parked executor wakes and +//! re-polls. The result is that timers are serviced on the first SysTick at or +//! after their deadline — a wake granularity of one SysTick period. +//! +//! # Why a 64-bit counter lives behind the critical section +//! +//! ARMv7-M has no 64-bit atomic instructions, so the monotonic counter cannot be +//! an `AtomicU64`. It instead lives in the same `critical_section::Mutex` as the +//! queue, as a `Cell`; reads and the handler's increment are single-core +//! critical sections (a PRIMASK toggle), which is cheap and race-free here. + +use core::cell::{Cell, RefCell}; + +use cortex_m::peripheral::{SYST, syst::SystClkSource}; +use critical_section::Mutex; +use embassy_time_driver::Driver; +use embassy_time_queue_utils::Queue; + +/// SysTick reload value (counts down from this to zero, firing on the underflow). +/// +/// The AN386 model clocks SysTick from the (emulated) processor clock, so a +/// reload of `N` produces an interrupt every `N + 1` core cycles of *virtual* +/// time. A modest reload keeps timer-wake latency low; the exact wall-clock rate +/// does not matter (see the module docs), only that ticks accrue while the +/// executor sleeps. +const SYSTICK_RELOAD: u32 = 6_000; + +/// How many [`embassy_time_driver::TICK_HZ`] ticks one SysTick period represents. +/// +/// The timebase is fixed at 1 MHz (`tick-hz-1_000_000`), so `now()` is reported +/// in microseconds. Each SysTick advances that virtual microsecond clock by this +/// many ticks; it sets how fast virtual time runs and therefore how many SysTick +/// interrupts a given protocol interval costs. 1000 µs per SysTick keeps the +/// emulated run short (a 10 s protocol deadline is ~10_000 SysTicks) while still +/// leaving sub-deadline wake granularity. +const TICKS_PER_SYSTICK: u64 = 1_000; + +/// The SysTick-backed time driver: the monotonic tick counter plus the software +/// timer queue, both guarded by one critical-section mutex. The SysTick handler +/// owns advancing the counter and draining the queue; [`Driver::now`] reads the +/// counter and [`Driver::schedule_wake`] enqueues. +struct SystickDriver { + inner: Mutex, +} + +/// The driver's mutable state behind the critical-section mutex. +struct Inner { + /// Monotonic virtual time in `TICK_HZ` ticks, advanced by the SysTick handler. + now: Cell, + /// The 8-slot timer queue holding scheduled wakers. + queue: RefCell, +} + +embassy_time_driver::time_driver_impl!(static DRIVER: SystickDriver = SystickDriver { + inner: Mutex::new(Inner { + now: Cell::new(0), + queue: RefCell::new(Queue::new()), + }), +}); + +impl Driver for SystickDriver { + fn now(&self) -> u64 { + critical_section::with(|cs| self.inner.borrow(cs).now.get()) + } + + fn schedule_wake(&self, at: u64, waker: &core::task::Waker) { + critical_section::with(|cs| { + let inner = self.inner.borrow(cs); + // Enqueue (or advance) this waker's deadline. There is no hardware alarm to + // (re)arm: the free-running SysTick handler drains the queue every tick, so + // a newly scheduled timer is picked up on the next SysTick at or after `at`. + // `next_expiration` is still called when the queue changed so a timer whose + // deadline is already in the past fires immediately rather than waiting a + // tick. + let mut queue = inner.queue.borrow_mut(); + if queue.schedule_wake(at, waker) { + queue.next_expiration(inner.now.get()); + } + }); + } +} + +/// Start the SysTick time driver: configure and enable the SysTick exception. +/// +/// Call once, early in `main`, before the executor starts. Consumes the [`SYST`] +/// peripheral so nothing else can reprogram it. +pub fn init(mut syst: SYST) { + // Clock SysTick from the processor clock and fire on underflow. + syst.set_clock_source(SystClkSource::Core); + syst.set_reload(SYSTICK_RELOAD); + syst.clear_current(); + syst.enable_interrupt(); + syst.enable_counter(); + // Keep the peripheral handle from dropping the counter's enable on scope exit: + // dropping `SYST` does not stop the timer, but moving it out of `init` would + // let a later `take()` of the peripherals hand it out again. Forgetting it pins + // ownership here for the program's life. + core::mem::forget(syst); +} + +/// The SysTick exception handler: advance the virtual clock and dispatch any +/// timers that have now expired. +/// +/// `cortex-m-rt` routes the SysTick exception here by the reserved name +/// `SysTick`. It runs in interrupt context; the state access takes the same +/// `critical_section` the driver uses, and `next_expiration` waking a task runs +/// the executor pender (`SEV`), resuming the parked executor. +#[cortex_m_rt::exception] +fn SysTick() { + critical_section::with(|cs| { + let inner = DRIVER.inner.borrow(cs); + let now = inner.now.get().wrapping_add(TICKS_PER_SYSTICK); + inner.now.set(now); + // Wake every timer whose deadline is at or before the new `now`; the returned + // next deadline is unused because the handler re-checks every tick rather than + // programming a one-shot alarm. + // Ignoring the next-expiration return: SysTick free-runs and re-drains every + // tick, so there is no alarm register to reprogram from it. + let _ = inner.queue.borrow_mut().next_expiration(now); + }); +} From 2465b3755bdaf7404c9e184a14803650003496f2 Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 10 Jul 2026 20:19:44 +1200 Subject: [PATCH 066/128] fix(serf-embassy-qemu): bound the whole scenario with a phase-labeled watchdog The per-phase deadline loops only started bounding once join() had returned, so a join that never resolved hung the emulator with no exit code; a watchdog task armed before the join now exits 1 naming the stalled phase after the deadline plus margin, and the qemu CI job gains a timeout-minutes backstop for a hang that stops SysTick itself. --- .github/workflows/embedded.yml | 3 +++ tests/serf-embassy-qemu/src/node.rs | 36 ++++++++++++++++++++++++++++- 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/.github/workflows/embedded.yml b/.github/workflows/embedded.yml index 81e5946e..f475b036 100644 --- a/.github/workflows/embedded.yml +++ b/.github/workflows/embedded.yml @@ -90,6 +90,9 @@ jobs: # from the workspace, so it is built only by this `cargo run`. qemu: runs-on: ubuntu-latest + # Backstop for a firmware hang the in-image watchdog cannot catch (e.g. a dead + # SysTick stops the watchdog timer itself): bound the job, don't hang the runner. + timeout-minutes: 15 steps: - uses: actions/checkout@v7 - uses: dtolnay/rust-toolchain@master diff --git a/tests/serf-embassy-qemu/src/node.rs b/tests/serf-embassy-qemu/src/node.rs index b62ac263..840e2eb5 100644 --- a/tests/serf-embassy-qemu/src/node.rs +++ b/tests/serf-embassy-qemu/src/node.rs @@ -14,7 +14,10 @@ //! via [`StaticCell`] so the run loops can be `'static` embassy tasks. On the //! user-event crossing the emulator exits 0; a deadline guard exits 1. -use core::net::{IpAddr, Ipv4Addr, SocketAddr}; +use core::{ + net::{IpAddr, Ipv4Addr, SocketAddr}, + sync::atomic::{AtomicU8, Ordering}, +}; use embassy_executor::Spawner; use embassy_net::{ @@ -53,6 +56,32 @@ const DEADLINE: Duration = Duration::from_secs(20); /// Poll cadence while waiting for convergence and the user event. const POLL: Duration = Duration::from_millis(10); +/// Whole-scenario watchdog. The per-phase `DEADLINE` loops only start bounding +/// once `join(..)` has returned, so a join (or any pre-loop await) that never +/// resolves would otherwise hang the emulator with no exit code. Armed before +/// the join, this exits 1 with the stalled phase after `DEADLINE` plus margin — +/// long enough that a live run's own phase messages always fire first. +const WATCHDOG: Duration = Duration::from_secs(30); + +/// The scenario phase the watchdog reports on a stall: 0 = joining, +/// 1 = converging, 2 = awaiting the user event. +static PHASE: AtomicU8 = AtomicU8::new(0); + +/// Phase names indexed by [`PHASE`]. +const PHASE_NAMES: [&str; 3] = ["joining", "converging", "awaiting the user event"]; + +#[embassy_executor::task] +async fn watchdog() { + Timer::after(WATCHDOG).await; + let phase = PHASE.load(Ordering::Relaxed) as usize; + println!( + "watchdog: scenario stalled while {} ({}s elapsed)", + PHASE_NAMES.get(phase).unwrap_or(&"in an unknown phase"), + WATCHDOG.as_secs(), + ); + semihosting::process::exit(1); +} + /// Gossip/TCP port both nodes bind (the single-port memberlist model serf runs on). const PORT: u16 = 7946; @@ -197,6 +226,9 @@ pub async fn main_task(spawner: Spawner) -> ! { spawner.must_spawn(net_task(net_b)); spawner.must_spawn(serf_task(run_a)); spawner.must_spawn(serf_task(run_b)); + // Armed BEFORE the join so the whole scenario — including a join that never + // resolves — is bounded by a non-zero exit rather than an emulator hang. + spawner.must_spawn(watchdog()); // B joins A as a seed; the convergence wait below bounds success by the deadline. // Ignoring Err: a failed seed join surfaces as the convergence loop timing out, @@ -209,6 +241,7 @@ pub async fn main_task(spawner: Spawner) -> ! { false, ) .await; + PHASE.store(1, Ordering::Relaxed); println!("join issued; waiting for convergence"); let mut waited = Duration::from_secs(0); @@ -233,6 +266,7 @@ pub async fn main_task(spawner: Spawner) -> ! { // Phase 2: the serf-above-memberlist proof — A broadcasts a user event and B must // observe it as `Event::User`, which can only happen if serf's gossip plane truly // disseminated it across the emulated link, not just SWIM membership. + PHASE.store(2, Ordering::Relaxed); ml_a .user_event("greet", Bytes::from_static(b"hello"), false) .expect("broadcast a user event from a running node"); From 4a81fcb4dcda04adab91e81a6228ccec6a031731 Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 10 Jul 2026 20:55:06 +1200 Subject: [PATCH 067/128] fix(serf-proto): tcp-gate the drain-now test seam, pin alloc on the minimal CI combo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two pre-existing red proto matrix entries: the drain-now test adapter is called only from the tcp-gated unit suite, so a quic-only clippy flagged it dead — gate it like its callers; and the no-default entry predates the crate's std/alloc feature tiers, whose absence now fails with the deliberate compile_error — the minimal combo pins alloc instead. --- .github/workflows/ci-core.yml | 9 ++++++--- serf-proto/src/endpoint/mod.rs | 6 +++++- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml index 396be658..c70a39e1 100644 --- a/.github/workflows/ci-core.yml +++ b/.github/workflows/ci-core.yml @@ -79,8 +79,11 @@ jobs: run: cargo test -p serf-proto --features "$FEATURES" # serf-proto is pure-CPU, so exercise the interesting feature permutations on a - # single OS (default, the coordinate + AEAD combos, all-features, no-default, - # coordinates+aes+tcp, and quic-rustls-ring alone). + # single OS (default, the coordinate + AEAD combos, all-features, the minimal + # no_std alloc tier, coordinates+aes+tcp, and quic-rustls-ring alone). A bare + # `--no-default-features` build is impossible by design — the crate requires + # the `std` or `alloc` tier and fails with a deliberate compile_error — so the + # minimal combo pins `alloc`. proto-features: name: proto - ${{ matrix.name }} runs-on: ubuntu-latest @@ -95,7 +98,7 @@ jobs: - { name: "coords-aes-gcm", flags: "--features coordinates,aes-gcm" } - { name: "coords-chacha", flags: "--features coordinates,chacha20-poly1305" } - { name: "all-features", flags: "--all-features" } - - { name: "no-default", flags: "--no-default-features" } + - { name: "no-default", flags: "--no-default-features --features alloc" } - { name: "coords-aes-gcm-tcp", flags: "--features coordinates,aes-gcm,tcp" } - { name: "quic-rustls-ring", flags: "--no-default-features --features quic-rustls-ring" } steps: diff --git a/serf-proto/src/endpoint/mod.rs b/serf-proto/src/endpoint/mod.rs index d5fbc92b..c6b735f0 100644 --- a/serf-proto/src/endpoint/mod.rs +++ b/serf-proto/src/endpoint/mod.rs @@ -4315,7 +4315,11 @@ where /// handler directly (e.g. `test_handle_query`) use this to advance the /// endpoint's current-time reference between calls without going through /// `handle_timeout`. - #[cfg(test)] + /// + /// Gated like the unit suite that calls it (`mod tests` is `tcp`-gated because + /// it drives through `StreamEndpoint`), so a `quic`-only build carries no + /// uncalled seam. + #[cfg(all(test, feature = "tcp"))] pub(crate) fn test_set_drain_now(&mut self, now: Instant) { self.drain_now = now; } From 93645586d8d62b48656d20143e675421f8ea2bfd Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 10 Jul 2026 21:06:39 +1200 Subject: [PATCH 068/128] chore(codecov): absorb async-coverage noise with a project threshold The driver suites cover timing-dependent async paths, so two runs of an identical tree differ by a few hit lines; with target auto and no threshold a zero-line PR fails on a 0.02% stochastic dip. A 0.5% threshold keeps the gate while absorbing the noise. --- .codecov.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.codecov.yml b/.codecov.yml index cda5191a..d30a8506 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -13,6 +13,10 @@ coverage: project: # Overall project status default: target: auto + # The driver suites exercise timing-dependent async paths, so repeated + # runs of an identical tree differ by a few hit lines; absorb that + # measurement noise instead of failing a zero-line diff on a 0.02% dip. + threshold: 0.5% if_not_found: success only_pulls: false patch: # Status for the patch in pull requests From f4acc66982eb48c48d60cc8048c7fab5ed87d50b Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 10 Jul 2026 22:36:36 +1200 Subject: [PATCH 069/128] feat(serf-proto): expose the live encryption options on the QUIC endpoint Mirrors the stream endpoint's forwarders so a QUIC driver can read and rotate the live wire keyring at runtime. --- serf-proto/src/quic_endpoint/mod.rs | 36 +++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/serf-proto/src/quic_endpoint/mod.rs b/serf-proto/src/quic_endpoint/mod.rs index 9acc8cb4..2afdbcd3 100644 --- a/serf-proto/src/quic_endpoint/mod.rs +++ b/serf-proto/src/quic_endpoint/mod.rs @@ -401,6 +401,42 @@ where self.transport.decrypt_gossip(datagram) } + /// The coordinator's live cross-transport [`memberlist_proto::EncryptionOptions`] + /// — the single source of truth for the AEAD keyring. On QUIC the keyring + /// governs the gossip datagram plane only; the reliable path always skips + /// (quinn already encrypts the stream). + /// + /// A driver applying a key-management op reads this, mutates the keyring, and + /// pushes it back via [`set_encryption_options`](Self::set_encryption_options), + /// so the reported key state and the bytes on the wire cannot diverge. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn encryption_options(&self) -> &memberlist_proto::EncryptionOptions { + self.transport.encryption_options() + } + + /// Replace the coordinator's live encryption options, re-keying the gossip + /// plane. + /// + /// The post-construction counterpart to the construction-time policy: applying + /// a completed key rotation here rotates the actual AEAD the gossip datagrams + /// encrypt under, rather than leaving a driver-held shadow to drift from the + /// wire. The QUIC reliable bridges force-disable encryption regardless (quinn + /// encrypts the stream), so the propagation is a no-op there; the coordinator + /// also drops its buffered inbound gossip so a datagram queued under the old + /// policy is never decrypted under the new one. + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn set_encryption_options(&mut self, encryption: memberlist_proto::EncryptionOptions) { + self.transport.set_encryption_options(encryption) + } + /// The coordinator's maximum reliable-stream frame size — the driver uses it /// to bound the observation byte-backstop budget. pub fn max_stream_frame_size(&self) -> usize { From 978c22dd3fb3cf49e0a10ac10d26b1bcaa322075 Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 10 Jul 2026 22:36:37 +1200 Subject: [PATCH 070/128] fix(serf-reactor,serf-compio): rotate the live wire keyring, demote the delegate to an observer Key ops previously applied to a caller delegate that authored the responses while the endpoint kept encrypting with the construction-time keys, so a rotation reported complete with the old key still live. The drivers now read-modify-write the endpoint's live keyring through a shared variant-exact helper (exact membership, primary-removal and cross-cipher-collision refusals, responses from live state) and the delegate observes successful rotations for persistence. --- serf-compio/Cargo.toml | 8 +- serf-compio/src/delegate/mod.rs | 50 +++--- serf-compio/src/delegate/void.rs | 51 +------ serf-compio/src/driver/quic/mod.rs | 59 ++++--- serf-compio/src/driver/stream/mod.rs | 59 ++++--- serf-compio/src/quic/mod.rs | 3 + serf-compio/src/tcp/mod.rs | 3 + serf-compio/src/tls/mod.rs | 3 + serf-compio/src/transport/mod.rs | 28 ++++ serf-driver/Cargo.toml | 10 +- serf-driver/build.rs | 16 ++ serf-driver/src/keyring/mod.rs | 212 ++++++++++++++++++++++++++ serf-driver/src/keyring/tests.rs | 190 +++++++++++++++++++++++ serf-driver/src/lib.rs | 11 ++ serf-reactor/Cargo.toml | 8 +- serf-reactor/src/delegate/mod.rs | 53 +++---- serf-reactor/src/delegate/void.rs | 51 +------ serf-reactor/src/driver/quic/mod.rs | 58 ++++--- serf-reactor/src/driver/stream/mod.rs | 57 ++++--- serf-reactor/src/quic/mod.rs | 3 + serf-reactor/src/tcp/mod.rs | 3 + serf-reactor/src/tls/mod.rs | 3 + serf-reactor/src/transport/mod.rs | 28 ++++ 23 files changed, 728 insertions(+), 239 deletions(-) create mode 100644 serf-driver/build.rs create mode 100644 serf-driver/src/keyring/mod.rs create mode 100644 serf-driver/src/keyring/tests.rs diff --git a/serf-compio/Cargo.toml b/serf-compio/Cargo.toml index 555005ef..8b0c0692 100644 --- a/serf-compio/Cargo.toml +++ b/serf-compio/Cargo.toml @@ -20,8 +20,12 @@ quic-rustls-ring = ["quic", "serf-proto/quic-rustls-ring", "serf-driver/quic-rus coordinates = ["serf-proto/coordinates", "serf-driver/coordinates"] # Encryption backends — forward to serf-proto (key-management messages) and to # memberlist-proto (the coordinator's gossip-encryption keyring + encrypt/decrypt). -aes-gcm = ["serf-proto/aes-gcm", "memberlist-proto/aes-gcm"] -chacha20-poly1305 = ["serf-proto/chacha20-poly1305", "memberlist-proto/chacha20-poly1305"] +aes-gcm = ["serf-proto/aes-gcm", "serf-driver/aes-gcm", "memberlist-proto/aes-gcm"] +chacha20-poly1305 = [ + "serf-proto/chacha20-poly1305", + "serf-driver/chacha20-poly1305", + "memberlist-proto/chacha20-poly1305", +] # Regex-backed tag-filter matching. tag-regex = ["serf-proto/tag-regex", "serf-driver/tag-regex"] # Emit `tracing` spans around the public driver operations. diff --git a/serf-compio/src/delegate/mod.rs b/serf-compio/src/delegate/mod.rs index 603ca9b2..a6a8001b 100644 --- a/serf-compio/src/delegate/mod.rs +++ b/serf-compio/src/delegate/mod.rs @@ -25,9 +25,7 @@ use std::sync::Arc; use serf_proto::{event::QueryEvent, members::Member, typed::UserEventMessage}; #[cfg(encryption)] -use memberlist_proto::SecretKey; -#[cfg(encryption)] -use serf_proto::KeyResponseArgs; +use memberlist_proto::Keyring; /// Async observation hooks for serf membership events. /// @@ -138,17 +136,22 @@ pub trait Delegate: type Address; } -/// Synchronous delegate the driver implements to apply key-management operations -/// to its local keyring. +/// Observer the driver notifies after it rotates the LIVE wire keyring, so an +/// application can persist the new key material. /// -/// The driver's event loop calls the matching method when it receives a -/// [`serf_proto::event::Event::KeyRequest`] event, then forwards the result to -/// `StreamEndpoint::respond_key` / `QuicEndpoint::respond_key` so the response -/// is routed back to the originating node. +/// The wire keyring lives in the endpoint (the coordinator's `EncryptionOptions`), +/// and the driver is its single source of truth: on an inbound +/// [`Event::KeyRequest`](serf_proto::event::Event::KeyRequest) it read-modify-writes +/// that live keyring directly — install adds a secondary, use promotes the primary, +/// remove drops a secondary, every op variant-exact — and answers the originator +/// from the post-op live state via `respond_key`. This delegate does NOT author +/// those responses; it only OBSERVES a successful rotation, receiving the new live +/// [`Keyring`] so the application can persist it. A `list` and every refused or +/// no-op request do not fire it. /// -/// All methods are **synchronous and non-blocking**: key storage is the -/// driver's concern. If a key store requires async I/O, shadow the result -/// through pre-computed state the synchronous method can read. +/// [`keyring_updated`](Self::keyring_updated) is **synchronous and non-blocking**: +/// it runs on the driver pump. If persistence needs async I/O, hand the ring off to +/// a channel the observer owns and drain it elsewhere. /// /// Requires the `aes-gcm` or `chacha20-poly1305` feature. #[cfg(encryption)] @@ -157,21 +160,14 @@ pub trait Delegate: doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] pub trait KeyringDelegate: 'static { - /// Install a new key into the keyring. Returns the response args passed to - /// `respond_key`. - fn install(&self, key: SecretKey) -> KeyResponseArgs; - - /// Promote a key to the primary encryption key. Returns the response args - /// passed to `respond_key`. - fn use_key(&self, key: SecretKey) -> KeyResponseArgs; - - /// Remove a key from the keyring. Returns the response args passed to - /// `respond_key`. - fn remove(&self, key: SecretKey) -> KeyResponseArgs; - - /// List all installed keys and the current primary. Returns the response args - /// (with `keys` populated) passed to `respond_key`. - fn list(&self) -> KeyResponseArgs; + /// Called after a key-management request successfully rotated the live wire + /// keyring, with the new ring the gossip and reliable planes now encrypt under. + /// Not called for a `list` or any refused or no-op request. The default is a + /// no-op — the rotation is applied to the wire regardless; overriding this only + /// adds out-of-band persistence. + fn keyring_updated(&self, keyring: &Keyring) { + let _ = keyring; // Unused: default no-op; override to persist the rotation. + } } /// Async veto hook invoked by the driver on the join path before accepting diff --git a/serf-compio/src/delegate/void.rs b/serf-compio/src/delegate/void.rs index 8faa6510..230d08ce 100644 --- a/serf-compio/src/delegate/void.rs +++ b/serf-compio/src/delegate/void.rs @@ -9,8 +9,6 @@ use super::{Delegate, MemberDelegate, MergeDelegate, QueryDelegate, UserEventDel #[cfg(encryption)] use super::KeyringDelegate; -#[cfg(encryption)] -use serf_proto::KeyResponseArgs; /// Zero-cost default observation delegate. Every hook is a no-op. /// @@ -101,14 +99,14 @@ where type Error = core::convert::Infallible; } -/// A keyring delegate that manages no key material. +/// A keyring delegate that persists nothing. /// -/// The default for nodes that do not participate in key management: every -/// operation reports `result = false` with an explanatory message and an empty -/// key list, so a cluster-wide key query records this node as not key-managed -/// rather than silently timing out. A node that DOES manage encryption keys -/// supplies its own [`KeyringDelegate`](super::KeyringDelegate) bound to the -/// keyring it shares with the transport's encryption configuration. +/// The default for nodes that do not need to observe key rotations. The driver +/// still applies every inbound key-management op to the live wire keyring and +/// answers from that live state; this delegate simply does not persist the +/// result. A node that wants to persist rotated key material supplies its own +/// [`KeyringDelegate`](super::KeyringDelegate), overriding +/// [`keyring_updated`](super::KeyringDelegate::keyring_updated). /// /// Requires the `aes-gcm` or `chacha20-poly1305` feature. #[cfg(encryption)] @@ -119,37 +117,4 @@ where pub struct VoidKeyringDelegate; #[cfg(encryption)] -impl VoidKeyringDelegate { - /// The response returned by every operation: not key-managed. - fn unmanaged() -> KeyResponseArgs { - KeyResponseArgs { - result: false, - message: "node has no keyring delegate configured".into(), - keys: Vec::new(), - primary_key: None, - } - } -} - -#[cfg(encryption)] -impl KeyringDelegate for VoidKeyringDelegate { - #[inline] - fn install(&self, _key: memberlist_proto::SecretKey) -> KeyResponseArgs { - Self::unmanaged() - } - - #[inline] - fn use_key(&self, _key: memberlist_proto::SecretKey) -> KeyResponseArgs { - Self::unmanaged() - } - - #[inline] - fn remove(&self, _key: memberlist_proto::SecretKey) -> KeyResponseArgs { - Self::unmanaged() - } - - #[inline] - fn list(&self) -> KeyResponseArgs { - Self::unmanaged() - } -} +impl KeyringDelegate for VoidKeyringDelegate {} diff --git a/serf-compio/src/driver/quic/mod.rs b/serf-compio/src/driver/quic/mod.rs index 2ee4c741..c703a781 100644 --- a/serf-compio/src/driver/quic/mod.rs +++ b/serf-compio/src/driver/quic/mod.rs @@ -62,7 +62,7 @@ use crate::{ snapshot::{SerfSnapshot, SnapshotCell}, }; #[cfg(encryption)] -use serf_proto::{KeyRequestOperation, KeyResponseArgs, event::KeyRequest}; +use serf_proto::{KeyResponseArgs, event::KeyRequest}; /// Driver-side state for one outstanding await-result join call. /// @@ -1235,9 +1235,10 @@ where // `respond_key`) ahead of the observation hand-off below. #[cfg(encryption)] if let Event::KeyRequest(req) = &ev { - let resp = apply_key_request(keyring, req); + let resp = apply_key_request_live(endpoint, keyring, req); // Ignoring Err: `respond_key` fails only when the response cannot be routed - // (originator gone / relay dropped); the key op has already applied locally. + // (originator gone / relay dropped); the key op has already applied to the + // live wire keyring. let _ = endpoint.respond_key(req, resp, Instant::now()); } @@ -1324,29 +1325,43 @@ where terminal } -/// Apply one inbound [`KeyRequest`] to the driver's keyring delegate, producing -/// the [`KeyResponseArgs`] the pump forwards to `respond_key`. +/// Read-modify-write `endpoint`'s LIVE wire keyring for one inbound [`KeyRequest`], +/// returning the [`KeyResponseArgs`] built from the post-op live state. /// -/// `Install` / `Use` / `Remove` carry a key (the machine enforces op-shape, so a -/// missing key is reported as a failed response rather than panicking); `List` -/// carries no key and enumerates the keyring. +/// The endpoint-facing wrapper over [`serf_driver::apply_key_request`]: it reads +/// the coordinator's live `encryption_options`, applies the op variant-exactly +/// against the live ring, and on a real mutation publishes the rotated ring back +/// via `set_encryption_options` — re-keying the gossip datagram plane (the QUIC +/// reliable path always skips, quinn encrypts the stream) — then notifies the +/// keyring observer for persistence. A node with no keyring configured answers +/// `result = false` and makes no wire change; a read-only `list` or a refused op +/// leaves the wire untouched. #[cfg(encryption)] -fn apply_key_request( - keyring: &dyn KeyringDelegate, - req: &KeyRequest, -) -> KeyResponseArgs { - match (req.op(), req.key()) { - (KeyRequestOperation::Install, Some(key)) => keyring.install(*key), - (KeyRequestOperation::Use, Some(key)) => keyring.use_key(*key), - (KeyRequestOperation::Remove, Some(key)) => keyring.remove(*key), - (KeyRequestOperation::List, _) => keyring.list(), - (_, None) => KeyResponseArgs { +fn apply_key_request_live( + endpoint: &mut QuicEndpoint, + delegate: &dyn KeyringDelegate, + req: &KeyRequest, +) -> KeyResponseArgs +where + I: memberlist_proto::Id + Clone, + G: Rng, + R: Rng + SeedableRng, +{ + let mut encryption = endpoint.encryption_options().clone(); + let Some(current) = encryption.keyring() else { + return KeyResponseArgs { result: false, - message: "key-management request missing its required key".into(), - keys: Vec::new(), - primary_key: None, - }, + message: "no keyring configured on this node".into(), + ..Default::default() + }; + }; + let (resp, rotated) = serf_driver::apply_key_request(current, req.op(), req.key()).into_parts(); + if let Some(new_ring) = rotated { + encryption.set_keyring(new_ring.clone()); + endpoint.set_encryption_options(encryption); + delegate.keyring_updated(&new_ring); } + resp } /// Per-driver observation task: dispatch each event's [`Delegate`] hook, then fan diff --git a/serf-compio/src/driver/stream/mod.rs b/serf-compio/src/driver/stream/mod.rs index 94592bee..0e6cc275 100644 --- a/serf-compio/src/driver/stream/mod.rs +++ b/serf-compio/src/driver/stream/mod.rs @@ -70,7 +70,7 @@ use crate::{ snapshot::{SerfSnapshot, SnapshotCell}, }; #[cfg(encryption)] -use serf_proto::{KeyRequestOperation, KeyResponseArgs, event::KeyRequest}; +use serf_proto::{KeyResponseArgs, event::KeyRequest}; /// Driver-side state for one outstanding await-result join call. /// @@ -1562,29 +1562,43 @@ where progress } -/// Apply one inbound [`KeyRequest`] to the driver's keyring delegate, producing -/// the [`KeyResponseArgs`] the pump forwards to `respond_key`. +/// Read-modify-write `endpoint`'s LIVE wire keyring for one inbound [`KeyRequest`], +/// returning the [`KeyResponseArgs`] built from the post-op live state. /// -/// `Install` / `Use` / `Remove` carry a key (the machine enforces op-shape, so a -/// missing key is reported as a failed response rather than panicking); `List` -/// carries no key and enumerates the keyring. +/// The endpoint-facing wrapper over [`serf_driver::apply_key_request`]: it reads +/// the coordinator's live `encryption_options`, applies the op variant-exactly +/// against the live ring, and on a real mutation publishes the rotated ring back +/// via `set_encryption_options` — so the gossip and reliable planes re-key in +/// lockstep — then notifies the keyring observer for persistence. A node with no +/// keyring configured answers `result = false` and makes no wire change; a +/// read-only `list` or a refused op leaves the wire untouched. #[cfg(encryption)] -fn apply_key_request( - keyring: &dyn KeyringDelegate, - req: &KeyRequest, -) -> KeyResponseArgs { - match (req.op(), req.key()) { - (KeyRequestOperation::Install, Some(key)) => keyring.install(*key), - (KeyRequestOperation::Use, Some(key)) => keyring.use_key(*key), - (KeyRequestOperation::Remove, Some(key)) => keyring.remove(*key), - (KeyRequestOperation::List, _) => keyring.list(), - (_, None) => KeyResponseArgs { +fn apply_key_request_live( + endpoint: &mut StreamEndpoint, + delegate: &dyn KeyringDelegate, + req: &KeyRequest, +) -> KeyResponseArgs +where + I: memberlist_proto::Id + Clone, + RT: StreamTransport, + G: rand::Rng, + R: rand::Rng + SeedableRng, +{ + let mut encryption = endpoint.encryption_options().clone(); + let Some(current) = encryption.keyring() else { + return KeyResponseArgs { result: false, - message: "key-management request missing its required key".into(), - keys: Vec::new(), - primary_key: None, - }, + message: "no keyring configured on this node".into(), + ..Default::default() + }; + }; + let (resp, rotated) = serf_driver::apply_key_request(current, req.op(), req.key()).into_parts(); + if let Some(new_ring) = rotated { + encryption.set_keyring(new_ring.clone()); + endpoint.set_encryption_options(encryption); + delegate.keyring_updated(&new_ring); } + resp } /// Drain every queued serf [`Event`]: synchronous protocol accounting (leave @@ -1655,9 +1669,10 @@ where // `respond_key`) ahead of the observation hand-off below. #[cfg(encryption)] if let Event::KeyRequest(req) = &ev { - let resp = apply_key_request(keyring, req); + let resp = apply_key_request_live(endpoint, keyring, req); // Ignoring Err: `respond_key` fails only when the response cannot be routed - // (originator gone / relay dropped); the key op has already applied locally. + // (originator gone / relay dropped); the key op has already applied to the + // live wire keyring. let _ = endpoint.respond_key(req, resp, Instant::now()); } diff --git a/serf-compio/src/quic/mod.rs b/serf-compio/src/quic/mod.rs index af185472..8587631c 100644 --- a/serf-compio/src/quic/mod.rs +++ b/serf-compio/src/quic/mod.rs @@ -204,6 +204,9 @@ where RES: Resolver
, AR: AdvertiseAddrResolver, { + // Refuse a seed keyring that carries a cross-cipher byte twin, before binding. + #[cfg(encryption)] + crate::transport::reject_cross_cipher_keyring(&options.encryption)?; let local_id = options.local_id.ok_or_else(|| { SerfError::Io(std::io::Error::new( ErrorKind::InvalidInput, diff --git a/serf-compio/src/tcp/mod.rs b/serf-compio/src/tcp/mod.rs index 1e339774..d274e46c 100644 --- a/serf-compio/src/tcp/mod.rs +++ b/serf-compio/src/tcp/mod.rs @@ -194,6 +194,9 @@ where // (e.g. a zero `bridge_recv_buf_len` makes every bridge read return a // false EOF) BEFORE binding any socket. options.stream.validate()?; + // Refuse a seed keyring that carries a cross-cipher byte twin, before binding. + #[cfg(encryption)] + crate::transport::reject_cross_cipher_keyring(&options.encryption)?; let local_id = options.local_id.ok_or_else(|| { SerfError::Io(std::io::Error::new( diff --git a/serf-compio/src/tls/mod.rs b/serf-compio/src/tls/mod.rs index ac428acf..3a7effa6 100644 --- a/serf-compio/src/tls/mod.rs +++ b/serf-compio/src/tls/mod.rs @@ -268,6 +268,9 @@ where // (e.g. a zero `bridge_recv_buf_len` makes every bridge read return a // false EOF) BEFORE binding any socket. options.stream.validate()?; + // Refuse a seed keyring that carries a cross-cipher byte twin, before binding. + #[cfg(encryption)] + crate::transport::reject_cross_cipher_keyring(&options.encryption)?; let local_id = options.local_id.ok_or_else(|| { SerfError::Io(std::io::Error::new( diff --git a/serf-compio/src/transport/mod.rs b/serf-compio/src/transport/mod.rs index 221ccdbf..6e492327 100644 --- a/serf-compio/src/transport/mod.rs +++ b/serf-compio/src/transport/mod.rs @@ -174,6 +174,34 @@ pub(crate) fn post_bind_setup( crate::os_seeded_std_rng() } +/// Reject a construction-time encryption keyring that already carries a +/// cross-cipher byte twin — two keys sharing a raw byte value across different +/// cipher variants. +/// +/// The coordinator's byte-keyed rotation ops (`promote` / `remove_secondary`) +/// match on bytes alone, so such a ring would make every later key op ambiguous +/// and let a rotation promote or remove the wrong cipher's key. Each transport's +/// `Transport::new` calls this on its stored [`EncryptionOptions`] before binding +/// a socket, establishing the invariant — upheld thereafter by the drivers' live +/// key-op chokepoint — that the live keyring is cross-cipher-collision-free from +/// construction on. +/// +/// [`EncryptionOptions`]: memberlist_proto::EncryptionOptions +#[cfg(encryption)] +pub(crate) fn reject_cross_cipher_keyring( + encryption: &memberlist_proto::EncryptionOptions, +) -> Result<(), crate::SerfError> { + if let Some(keyring) = encryption.keyring() + && serf_driver::keyring_carries_cross_cipher_twin(keyring) + { + return Err(crate::SerfError::Io(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "encryption keyring carries a cross-cipher key collision", + ))); + } + Ok(()) +} + /// The QUIC variant of [`post_bind_setup`]: read the bound socket's address back /// (an ephemeral `:0` resolves to a concrete port here), then run the shared /// advertise validation + RNG draw, returning both. diff --git a/serf-driver/Cargo.toml b/serf-driver/Cargo.toml index d22e6c1f..7adc06d7 100644 --- a/serf-driver/Cargo.toml +++ b/serf-driver/Cargo.toml @@ -15,13 +15,21 @@ quic = ["serf-proto/quic"] quic-rustls-ring = ["quic", "serf-proto/quic-rustls-ring"] coordinates = ["serf-proto/coordinates"] tag-regex = ["serf-proto/tag-regex"] +# Encryption backends — forward to serf-proto (the key-management projection +# types) and memberlist-proto (the live `Keyring` the shared apply logic +# read-modify-writes). Aggregated to the `encryption` cfg by `build.rs`. +aes-gcm = ["serf-proto/aes-gcm", "memberlist-proto/aes-gcm"] +chacha20-poly1305 = ["serf-proto/chacha20-poly1305", "memberlist-proto/chacha20-poly1305"] [dependencies] serf-proto = { path = "../serf-proto", default-features = false } +# Pinned to the same git+branch as serf-proto to guarantee a single copy in the +# resolver; the shared key-management apply logic rotates memberlist's live +# `Keyring` and matches on `SecretKey` under the encryption cfg gate. +memberlist-proto = { git = "https://github.com/al8n/memberlist", branch = "main", default-features = false, features = ["std"] } [dev-dependencies] bytes = { workspace = true } -memberlist-proto = { git = "https://github.com/al8n/memberlist", branch = "main", features = ["std"] } [package.metadata.docs.rs] all-features = true diff --git a/serf-driver/build.rs b/serf-driver/build.rs new file mode 100644 index 00000000..1d5d54a4 --- /dev/null +++ b/serf-driver/build.rs @@ -0,0 +1,16 @@ +//! Emits the aggregate `encryption` cfg when any AEAD cipher backend feature is enabled, +//! so serf-driver code gates on `#[cfg(encryption)]` instead of repeating the full +//! backend list. + +fn any_feature(names: &[&str]) -> bool { + names + .iter() + .any(|name| std::env::var_os(format!("CARGO_FEATURE_{name}")).is_some()) +} + +fn main() { + println!("cargo::rustc-check-cfg=cfg(encryption)"); + if any_feature(&["AES_GCM", "CHACHA20_POLY1305"]) { + println!("cargo::rustc-cfg=encryption"); + } +} diff --git a/serf-driver/src/keyring/mod.rs b/serf-driver/src/keyring/mod.rs new file mode 100644 index 00000000..94e9cbb7 --- /dev/null +++ b/serf-driver/src/keyring/mod.rs @@ -0,0 +1,212 @@ +//! Pure, transport-agnostic key-management apply logic shared by the serf async +//! driver crates. +//! +//! An inbound key-management request must be applied to the driver's LIVE wire +//! keyring — the coordinator's [`EncryptionOptions`] keyring the gossip plane +//! (and, on the stream transports, the reliable plane) encrypts under — so a +//! completed rotation actually re-keys the wire instead of updating a caller-held +//! shadow the wire never sees. This module holds the runtime- and +//! endpoint-independent core of that operation: given the current [`Keyring`] and +//! a request, it computes the [`KeyResponseArgs`] answer plus the rotated ring to +//! publish. Each driver wraps it with the endpoint read-modify-write — read +//! [`encryption_options`], publish via [`set_encryption_options`], notify the +//! persistence observer — which stays per-driver because the endpoint handles +//! differ. +//! +//! Every keyed op is variant-exact. [`SecretKey`] equality is variant-inclusive +//! (an AES-256 key and a ChaCha20-Poly1305 key with the same 32 bytes are +//! DISTINCT), but [`Keyring::promote`] and [`Keyring::remove_secondary`] match on +//! raw bytes alone. To keep those byte-keyed ops unambiguous the drivers uphold +//! one invariant: the live keyring is cross-cipher-collision-free at all times — +//! no two keys share a byte value across different cipher variants. It is +//! established at construction (each driver's build preflight rejects a seed ring +//! carrying a twin via [`keyring_carries_cross_cipher_twin`]) and preserved here +//! (an `install` whose bytes twin an existing key of another variant is refused). +//! Under that invariant, after the exact (variant + bytes) membership check each +//! `use` / `remove` performs, the byte-keyed promote / remove resolve to exactly +//! the requested key. +//! +//! [`EncryptionOptions`]: memberlist_proto::EncryptionOptions +//! [`encryption_options`]: serf_proto::StreamEndpoint::encryption_options +//! [`set_encryption_options`]: serf_proto::StreamEndpoint::set_encryption_options + +#[cfg(test)] +mod tests; + +use memberlist_proto::{Keyring, SecretKey}; +use serf_proto::{KeyRequestOperation, KeyResponseArgs}; + +/// The outcome of applying one key-management [`KeyRequest`] to a live keyring. +/// +/// Carries the [`KeyResponseArgs`] to return to the originator, plus the rotated +/// ring to publish to the wire — `Some` only when the op actually mutated the +/// ring. A read-only `list`, a trivial promote of the current primary, and every +/// refusal leave it `None`, so the wire is never needlessly re-keyed. +pub struct KeyApplyOutcome { + response: KeyResponseArgs, + rotated: Option, +} + +impl KeyApplyOutcome { + /// The response to hand back to the originator via `respond_key`. + #[inline] + pub fn response(&self) -> &KeyResponseArgs { + &self.response + } + + /// The rotated ring to publish to the live wire keyring, or `None` for a + /// read-only or refused op that left the ring unchanged. + #[inline] + pub fn rotated(&self) -> Option<&Keyring> { + self.rotated.as_ref() + } + + /// Split into the response and the optional rotated ring. + #[inline] + pub fn into_parts(self) -> (KeyResponseArgs, Option) { + (self.response, self.rotated) + } +} + +/// Whether `keyring` already holds a key whose raw bytes equal `key`'s but whose +/// cipher variant differs — the cross-cipher collision that makes a byte-keyed +/// keyring lookup ambiguous. +/// +/// [`SecretKey`] equality is variant-inclusive, yet [`Keyring::promote`] and +/// [`Keyring::remove_secondary`] match on raw bytes alone, so admitting such a +/// twin would let a byte-keyed op resolve to the wrong cipher's key. Both the +/// install path and the construction preflight refuse it. +pub fn keyring_has_cross_cipher_twin(keyring: &Keyring, key: &SecretKey) -> bool { + core::iter::once(keyring.primary_ref()) + .chain(keyring.secondaries()) + .any(|installed| installed.as_bytes() == key.as_bytes() && installed != key) +} + +/// Whether `keyring` already carries a cross-cipher byte twin among its own keys — +/// any two of {primary, secondaries} sharing a raw byte value across different +/// cipher variants. Such a ring makes the coordinator's byte-keyed rotation ops +/// ambiguous, so a driver's construction preflight refuses it up front. +pub fn keyring_carries_cross_cipher_twin(keyring: &Keyring) -> bool { + core::iter::once(keyring.primary_ref()) + .chain(keyring.secondaries()) + .any(|key| keyring_has_cross_cipher_twin(keyring, key)) +} + +/// Apply one inbound key-management request — its [`KeyRequestOperation`] and +/// optional key — to `current` (a snapshot of the live wire keyring), returning +/// the answer built from the post-op state and the rotated ring to publish when +/// the op mutated it. +/// +/// Takes the op and key rather than the wire [`KeyRequest`](serf_proto::event::KeyRequest) +/// so it is transport- and test-agnostic; each driver forwards `req.op()` / +/// `req.key()`. +/// +/// The variant-exact semantics: +/// +/// - `install` inserts the key as a secondary (idempotent; a cross-cipher byte +/// twin of an already-present key is refused), +/// - `use` promotes the exact (variant + bytes) key to primary — a promote of the +/// current primary is a trivial success with no wire change, +/// - `remove` drops the exact secondary — refusing the current primary, +/// - `list` snapshots the keys and primary from the current state. +/// +/// A missing key on a keyed op, or a key absent from the ring, is refused with no +/// mutation. The caller handles the no-keyring-configured case; this helper always +/// receives a ring. +pub fn apply_key_request( + current: &Keyring, + op: KeyRequestOperation, + key: Option<&SecretKey>, +) -> KeyApplyOutcome { + let mut keyring = current.clone(); + let (response, mutated) = match (op, key) { + (KeyRequestOperation::Install, Some(key)) => { + // A same-variant re-install is the idempotent `insert_secondary` no-op, + // reported as success. A cross-cipher byte twin of an already-present key + // is refused to keep the byte-keyed rotation ops unambiguous. + if keyring_has_cross_cipher_twin(&keyring, key) { + (refused("cross-cipher key collision"), false) + } else { + keyring.insert_secondary(*key); + (success(), true) + } + } + (KeyRequestOperation::Use, Some(key)) => { + // Verify exact (variant + bytes) membership before the byte-keyed promote. + // Promoting the current primary is a trivial success with no wire change; a + // key absent from the live ring is refused with no mutation. + if keyring.primary_ref() == key { + (success(), false) + } else if keyring.secondaries().contains(key) { + match keyring.promote(key.as_bytes()) { + Ok(()) => (success(), true), + // Unreachable given the exact secondary membership just verified plus + // the collision-free invariant; handled fail-closed. + Err(_) => (refused("requested key is not installed"), false), + } + } else { + (refused("requested key is not installed"), false) + } + } + (KeyRequestOperation::Remove, Some(key)) => { + // Exact (variant + bytes) membership required, mirroring `use`. Removing the + // current primary is refused (operators promote a secondary first). + if keyring.primary_ref() == key { + ( + refused("cannot remove the primary key; promote a secondary first"), + false, + ) + } else if keyring.secondaries().contains(key) { + match keyring.remove_secondary(key.as_bytes()) { + Ok(()) => (success(), true), + // Unreachable given the exact secondary membership just verified plus + // the collision-free invariant; handled fail-closed. + Err(_) => (refused("requested key is not installed"), false), + } + } else { + (refused("requested key is not installed"), false) + } + } + (KeyRequestOperation::List, _) => { + let mut keys = Vec::with_capacity(1 + keyring.secondaries().len()); + keys.push(*keyring.primary_ref()); + keys.extend(keyring.secondaries().iter().copied()); + ( + KeyResponseArgs { + result: true, + primary_key: Some(*keyring.primary_ref()), + keys, + ..Default::default() + }, + false, + ) + } + (_, None) => ( + refused("key-management request missing its required key"), + false, + ), + }; + KeyApplyOutcome { + response, + rotated: if mutated { Some(keyring) } else { None }, + } +} + +/// A bare success response (`result = true`, empty message / keys). +#[inline] +fn success() -> KeyResponseArgs { + KeyResponseArgs { + result: true, + ..Default::default() + } +} + +/// A refusal response carrying a human-readable reason, leaving the ring unchanged. +#[inline] +fn refused(message: &str) -> KeyResponseArgs { + KeyResponseArgs { + result: false, + message: message.into(), + ..Default::default() + } +} diff --git a/serf-driver/src/keyring/tests.rs b/serf-driver/src/keyring/tests.rs new file mode 100644 index 00000000..631882e7 --- /dev/null +++ b/serf-driver/src/keyring/tests.rs @@ -0,0 +1,190 @@ +use super::{apply_key_request, keyring_carries_cross_cipher_twin}; +use memberlist_proto::{Keyring, SecretKey}; +use serf_proto::KeyRequestOperation; + +#[cfg(feature = "aes-gcm")] +fn aes(b: u8) -> SecretKey { + SecretKey::Aes256([b; 32]) +} + +#[cfg(feature = "chacha20-poly1305")] +fn chacha(b: u8) -> SecretKey { + SecretKey::ChaCha20Poly1305([b; 32]) +} + +// ── single-backend happy-path (aes-gcm) ─────────────────────────────────────── + +#[cfg(feature = "aes-gcm")] +#[test] +fn install_adds_secondary_and_rotates() { + let ring = Keyring::new(aes(1)); + let out = apply_key_request(&ring, KeyRequestOperation::Install, Some(&aes(2))); + assert!(out.response().result); + let rotated = out.rotated().expect("install mutates the ring"); + assert_eq!(rotated.primary_ref(), &aes(1)); + assert!(rotated.secondaries().contains(&aes(2))); +} + +#[cfg(feature = "aes-gcm")] +#[test] +fn install_existing_key_is_idempotent_success() { + let ring = Keyring::new(aes(1)); + let out = apply_key_request(&ring, KeyRequestOperation::Install, Some(&aes(1))); + assert!(out.response().result); + // The dup insert is dropped, so the rotated ring equals the original. + assert_eq!(out.rotated().expect("install reports a mutation"), &ring); +} + +#[cfg(feature = "aes-gcm")] +#[test] +fn use_promotes_exact_secondary() { + let mut ring = Keyring::new(aes(1)); + ring.insert_secondary(aes(2)); + let out = apply_key_request(&ring, KeyRequestOperation::Use, Some(&aes(2))); + assert!(out.response().result); + assert_eq!( + out + .rotated() + .expect("promote mutates the ring") + .primary_ref(), + &aes(2) + ); +} + +#[cfg(feature = "aes-gcm")] +#[test] +fn use_current_primary_is_trivial_success_without_rotation() { + let ring = Keyring::new(aes(1)); + let out = apply_key_request(&ring, KeyRequestOperation::Use, Some(&aes(1))); + assert!(out.response().result); + assert!(out.rotated().is_none()); +} + +#[cfg(feature = "aes-gcm")] +#[test] +fn use_absent_key_refused_and_unchanged() { + let ring = Keyring::new(aes(1)); + let out = apply_key_request(&ring, KeyRequestOperation::Use, Some(&aes(9))); + assert!(!out.response().result); + assert!(out.rotated().is_none()); +} + +#[cfg(feature = "aes-gcm")] +#[test] +fn remove_drops_exact_secondary() { + let mut ring = Keyring::new(aes(1)); + ring.insert_secondary(aes(2)); + let out = apply_key_request(&ring, KeyRequestOperation::Remove, Some(&aes(2))); + assert!(out.response().result); + let rotated = out.rotated().expect("remove mutates the ring"); + assert_eq!(rotated.primary_ref(), &aes(1)); + assert!(!rotated.secondaries().contains(&aes(2))); +} + +#[cfg(feature = "aes-gcm")] +#[test] +fn remove_primary_refused_and_unchanged() { + let ring = Keyring::new(aes(1)); + let out = apply_key_request(&ring, KeyRequestOperation::Remove, Some(&aes(1))); + assert!(!out.response().result); + assert!(out.rotated().is_none()); +} + +#[cfg(feature = "aes-gcm")] +#[test] +fn remove_absent_key_refused_and_unchanged() { + let ring = Keyring::new(aes(1)); + let out = apply_key_request(&ring, KeyRequestOperation::Remove, Some(&aes(9))); + assert!(!out.response().result); + assert!(out.rotated().is_none()); +} + +#[cfg(feature = "aes-gcm")] +#[test] +fn list_snapshots_state_without_rotation() { + let mut ring = Keyring::new(aes(1)); + ring.insert_secondary(aes(2)); + let out = apply_key_request(&ring, KeyRequestOperation::List, None); + assert!(out.response().result); + assert!(out.rotated().is_none()); + assert_eq!(out.response().primary_key, Some(aes(1))); + assert!(out.response().keys.contains(&aes(1))); + assert!(out.response().keys.contains(&aes(2))); +} + +#[cfg(feature = "aes-gcm")] +#[test] +fn keyed_op_missing_key_refused() { + let ring = Keyring::new(aes(1)); + let out = apply_key_request(&ring, KeyRequestOperation::Install, None); + assert!(!out.response().result); + assert!(out.rotated().is_none()); +} + +// ── dual-backend variant-exact + cross-cipher-collision regressions ──────────── + +#[cfg(all(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn use_wrong_variant_refused_ring_unchanged() { + // Ring holds an AES-256 key; a `use` of the SAME BYTES under a different cipher + // is not exact (variant + bytes) membership and is refused with no mutation. + let ring = Keyring::new(aes(1)); + let out = apply_key_request(&ring, KeyRequestOperation::Use, Some(&chacha(1))); + assert!(!out.response().result); + assert!(out.rotated().is_none()); +} + +#[cfg(all(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn remove_wrong_variant_refused_ring_unchanged() { + let mut ring = Keyring::new(aes(1)); + ring.insert_secondary(aes(2)); + // A remove of a ChaCha key byte-twinning the AES secondary is not exact + // membership: refused, ring intact. + let out = apply_key_request(&ring, KeyRequestOperation::Remove, Some(&chacha(2))); + assert!(!out.response().result); + assert!(out.rotated().is_none()); +} + +#[cfg(all(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn install_cross_cipher_twin_refused() { + // Installing a ChaCha key whose bytes twin the AES primary is refused so the + // coordinator's byte-keyed rotation ops stay unambiguous. + let ring = Keyring::new(aes(1)); + let out = apply_key_request(&ring, KeyRequestOperation::Install, Some(&chacha(1))); + assert!(!out.response().result); + assert_eq!( + out.response().message.as_str(), + "cross-cipher key collision" + ); + assert!(out.rotated().is_none()); +} + +#[cfg(all(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn install_distinct_cross_cipher_key_allowed() { + // A ChaCha key with bytes distinct from every AES key is a legitimate + // mixed-cipher migration secondary. + let ring = Keyring::new(aes(1)); + let out = apply_key_request(&ring, KeyRequestOperation::Install, Some(&chacha(2))); + assert!(out.response().result); + assert!( + out + .rotated() + .expect("install mutates the ring") + .secondaries() + .contains(&chacha(2)) + ); +} + +#[cfg(all(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn twin_detector_flags_cross_cipher_seed_ring() { + let mut twinned = Keyring::new(aes(1)); + twinned.insert_secondary(chacha(1)); + assert!(keyring_carries_cross_cipher_twin(&twinned)); + + let clean = Keyring::with_secondaries(aes(1), [aes(2), chacha(3)]); + assert!(!keyring_carries_cross_cipher_twin(&clean)); +} diff --git a/serf-driver/src/lib.rs b/serf-driver/src/lib.rs index 63d8d703..b1efa0e9 100644 --- a/serf-driver/src/lib.rs +++ b/serf-driver/src/lib.rs @@ -12,11 +12,22 @@ #![cfg_attr(docsrs, allow(unused_attributes))] pub mod error; +#[cfg(all(encryption, any(feature = "tcp", feature = "quic")))] +mod keyring; #[cfg(any(feature = "tcp", feature = "quic"))] mod observation; #[cfg(any(feature = "tcp", feature = "quic"))] mod snapshot; +#[cfg(all(encryption, any(feature = "tcp", feature = "quic")))] +#[cfg_attr( + docsrs, + doc(cfg(all( + any(feature = "aes-gcm", feature = "chacha20-poly1305"), + any(feature = "tcp", feature = "quic") + ))) +)] +pub use keyring::{KeyApplyOutcome, apply_key_request, keyring_carries_cross_cipher_twin}; #[cfg(any(feature = "tcp", feature = "quic"))] #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] pub use observation::observation_payload_bytes; diff --git a/serf-reactor/Cargo.toml b/serf-reactor/Cargo.toml index 6945e875..84a7d3d3 100644 --- a/serf-reactor/Cargo.toml +++ b/serf-reactor/Cargo.toml @@ -24,8 +24,12 @@ quic-rustls-ring = ["quic", "serf-proto/quic-rustls-ring", "serf-driver/quic-rus coordinates = ["serf-proto/coordinates", "serf-driver/coordinates"] # Encryption backends — forward to serf-proto (key-management messages) and to # memberlist-proto (the coordinator's gossip-encryption keyring + encrypt/decrypt). -aes-gcm = ["serf-proto/aes-gcm", "memberlist-proto/aes-gcm"] -chacha20-poly1305 = ["serf-proto/chacha20-poly1305", "memberlist-proto/chacha20-poly1305"] +aes-gcm = ["serf-proto/aes-gcm", "serf-driver/aes-gcm", "memberlist-proto/aes-gcm"] +chacha20-poly1305 = [ + "serf-proto/chacha20-poly1305", + "serf-driver/chacha20-poly1305", + "memberlist-proto/chacha20-poly1305", +] # Regex-backed tag-filter matching. tag-regex = ["serf-proto/tag-regex", "serf-driver/tag-regex"] # Emit `tracing` spans around the public driver operations. diff --git a/serf-reactor/src/delegate/mod.rs b/serf-reactor/src/delegate/mod.rs index 85e6d612..9da01f88 100644 --- a/serf-reactor/src/delegate/mod.rs +++ b/serf-reactor/src/delegate/mod.rs @@ -27,9 +27,7 @@ use std::{future::Future, sync::Arc}; use serf_proto::{event::QueryEvent, members::Member, typed::UserEventMessage}; #[cfg(encryption)] -use memberlist_proto::SecretKey; -#[cfg(encryption)] -use serf_proto::KeyResponseArgs; +use memberlist_proto::Keyring; /// Async observation hooks for serf membership events. /// @@ -166,19 +164,23 @@ pub trait Delegate: type Address; } -/// Synchronous delegate the driver implements to apply key-management operations -/// to its local keyring. +/// Observer the driver notifies after it rotates the LIVE wire keyring, so an +/// application can persist the new key material. /// -/// The driver's event loop calls the matching method when it receives a -/// [`serf_proto::event::Event::KeyRequest`] event, then forwards the result to -/// `StreamEndpoint::respond_key` / `QuicEndpoint::respond_key` so the response -/// is routed back to the originating node. +/// The wire keyring lives in the endpoint (the coordinator's `EncryptionOptions`), +/// and the driver is its single source of truth: on an inbound +/// [`Event::KeyRequest`](serf_proto::event::Event::KeyRequest) it read-modify-writes +/// that live keyring directly — install adds a secondary, use promotes the primary, +/// remove drops a secondary, every op variant-exact — and answers the originator +/// from the post-op live state via `respond_key`. This delegate does NOT author +/// those responses; it only OBSERVES a successful rotation, receiving the new live +/// [`Keyring`] so the application can persist it. A `list` and every refused or +/// no-op request do not fire it. /// -/// All methods are **synchronous and non-blocking**: key storage is the -/// driver's concern. If a key store requires async I/O, shadow the result -/// through pre-computed state the synchronous method can read. `Send + Sync + -/// 'static` because the driver holds it behind an `Arc` shared across worker -/// threads. +/// [`keyring_updated`](Self::keyring_updated) is **synchronous and non-blocking**: +/// it runs on the driver pump. If persistence needs async I/O, hand the ring off to +/// a channel the observer owns and drain it elsewhere. `Send + Sync + 'static` +/// because the driver holds it behind an `Arc` shared across worker threads. /// /// Requires the `aes-gcm` or `chacha20-poly1305` feature. #[cfg(encryption)] @@ -187,21 +189,14 @@ pub trait Delegate: doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] pub trait KeyringDelegate: Send + Sync + 'static { - /// Install a new key into the keyring. Returns the response args passed to - /// `respond_key`. - fn install(&self, key: SecretKey) -> KeyResponseArgs; - - /// Promote a key to the primary encryption key. Returns the response args - /// passed to `respond_key`. - fn use_key(&self, key: SecretKey) -> KeyResponseArgs; - - /// Remove a key from the keyring. Returns the response args passed to - /// `respond_key`. - fn remove(&self, key: SecretKey) -> KeyResponseArgs; - - /// List all installed keys and the current primary. Returns the response args - /// (with `keys` populated) passed to `respond_key`. - fn list(&self) -> KeyResponseArgs; + /// Called after a key-management request successfully rotated the live wire + /// keyring, with the new ring the gossip and reliable planes now encrypt under. + /// Not called for a `list` or any refused or no-op request. The default is a + /// no-op — the rotation is applied to the wire regardless; overriding this only + /// adds out-of-band persistence. + fn keyring_updated(&self, keyring: &Keyring) { + let _ = keyring; // Unused: default no-op; override to persist the rotation. + } } /// Async veto hook invoked by the driver on the join path before accepting diff --git a/serf-reactor/src/delegate/void.rs b/serf-reactor/src/delegate/void.rs index ed4ff7bb..0be2ab44 100644 --- a/serf-reactor/src/delegate/void.rs +++ b/serf-reactor/src/delegate/void.rs @@ -9,8 +9,6 @@ use super::{Delegate, MemberDelegate, MergeDelegate, QueryDelegate, UserEventDel #[cfg(encryption)] use super::KeyringDelegate; -#[cfg(encryption)] -use serf_proto::KeyResponseArgs; /// Zero-cost default observation delegate. Every hook is a no-op. /// @@ -101,14 +99,14 @@ where type Error = core::convert::Infallible; } -/// A keyring delegate that manages no key material. +/// A keyring delegate that persists nothing. /// -/// The default for nodes that do not participate in key management: every -/// operation reports `result = false` with an explanatory message and an empty -/// key list, so a cluster-wide key query records this node as not key-managed -/// rather than silently timing out. A node that DOES manage encryption keys -/// supplies its own [`KeyringDelegate`](super::KeyringDelegate) bound to the -/// keyring it shares with the transport's encryption configuration. +/// The default for nodes that do not need to observe key rotations. The driver +/// still applies every inbound key-management op to the live wire keyring and +/// answers from that live state; this delegate simply does not persist the +/// result. A node that wants to persist rotated key material supplies its own +/// [`KeyringDelegate`](super::KeyringDelegate), overriding +/// [`keyring_updated`](super::KeyringDelegate::keyring_updated). /// /// Requires the `aes-gcm` or `chacha20-poly1305` feature. #[cfg(encryption)] @@ -119,37 +117,4 @@ where pub struct VoidKeyringDelegate; #[cfg(encryption)] -impl VoidKeyringDelegate { - /// The response returned by every operation: not key-managed. - fn unmanaged() -> KeyResponseArgs { - KeyResponseArgs { - result: false, - message: "node has no keyring delegate configured".into(), - keys: Vec::new(), - primary_key: None, - } - } -} - -#[cfg(encryption)] -impl KeyringDelegate for VoidKeyringDelegate { - #[inline] - fn install(&self, _key: memberlist_proto::SecretKey) -> KeyResponseArgs { - Self::unmanaged() - } - - #[inline] - fn use_key(&self, _key: memberlist_proto::SecretKey) -> KeyResponseArgs { - Self::unmanaged() - } - - #[inline] - fn remove(&self, _key: memberlist_proto::SecretKey) -> KeyResponseArgs { - Self::unmanaged() - } - - #[inline] - fn list(&self) -> KeyResponseArgs { - Self::unmanaged() - } -} +impl KeyringDelegate for VoidKeyringDelegate {} diff --git a/serf-reactor/src/driver/quic/mod.rs b/serf-reactor/src/driver/quic/mod.rs index d55cbcce..ff56daca 100644 --- a/serf-reactor/src/driver/quic/mod.rs +++ b/serf-reactor/src/driver/quic/mod.rs @@ -89,7 +89,7 @@ use crate::{ shared::Shared, }; #[cfg(encryption)] -use serf_proto::{KeyRequestOperation, KeyResponseArgs, event::KeyRequest}; +use serf_proto::{KeyResponseArgs, event::KeyRequest}; /// Hard ceiling on the gossip-plane contribution to the per-recv UDP buffer — /// UDP's IPv4 wire payload is capped at 65507 bytes once the IP/UDP headers are @@ -961,13 +961,44 @@ where } #[cfg(encryption)] if let Event::KeyRequest(req) = ev { - let resp = apply_key_request(&*self.keyring, req); + let resp = self.apply_key_request_live(req); // Ignoring Err: `respond_key` fails only when the response cannot be routed; - // the key op has already applied locally. + // the key op has already applied to the live wire keyring. let _ = self.endpoint.respond_key(req, resp, Instant::now()); } } + /// Read-modify-write the endpoint's LIVE wire keyring for one inbound + /// [`KeyRequest`], returning the [`KeyResponseArgs`] built from the post-op live + /// state. + /// + /// The endpoint-facing wrapper over [`serf_driver::apply_key_request`]: it reads + /// the coordinator's live `encryption_options`, applies the op variant-exactly + /// against the live ring, and on a real mutation publishes the rotated ring back + /// via `set_encryption_options` — re-keying the gossip datagram plane (the QUIC + /// reliable path always skips, quinn encrypts the stream) — then notifies the + /// keyring observer for persistence. A node with no keyring configured answers + /// `result = false` and makes no wire change; a read-only `list` or a refused op + /// leaves the wire untouched. + #[cfg(encryption)] + fn apply_key_request_live(&mut self, req: &KeyRequest) -> KeyResponseArgs { + let mut encryption = self.endpoint.encryption_options().clone(); + let Some(current) = encryption.keyring() else { + return KeyResponseArgs { + result: false, + message: "no keyring configured on this node".into(), + ..Default::default() + }; + }; + let (resp, rotated) = serf_driver::apply_key_request(current, req.op(), req.key()).into_parts(); + if let Some(new_ring) = rotated { + encryption.set_keyring(new_ring.clone()); + self.endpoint.set_encryption_options(encryption); + self.keyring.keyring_updated(&new_ring); + } + resp + } + /// Reap await-result join waiters on the deadline timer (the reply terminal), /// then remove any waiter that has reached BOTH terminals (reply resolved AND /// `pending` empty), clearing its still-recorded ignore-join streams. @@ -1392,27 +1423,6 @@ fn complete_join_exchange( } } -/// Apply one inbound [`KeyRequest`] to the driver's keyring delegate, producing the -/// [`KeyResponseArgs`] the pump forwards to `respond_key`. -#[cfg(encryption)] -fn apply_key_request( - keyring: &dyn KeyringDelegate, - req: &KeyRequest, -) -> KeyResponseArgs { - match (req.op(), req.key()) { - (KeyRequestOperation::Install, Some(key)) => keyring.install(*key), - (KeyRequestOperation::Use, Some(key)) => keyring.use_key(*key), - (KeyRequestOperation::Remove, Some(key)) => keyring.remove(*key), - (KeyRequestOperation::List, _) => keyring.list(), - (_, None) => KeyResponseArgs { - result: false, - message: "key-management request missing its required key".into(), - keys: Vec::new(), - primary_key: None, - }, - } -} - /// The observation task: drains machine events off the pump, invokes the /// [`Delegate`] hooks, and forwards every serf event to the /// [`EventStream`](crate::EventStream). The forward is best-effort (a full queue diff --git a/serf-reactor/src/driver/stream/mod.rs b/serf-reactor/src/driver/stream/mod.rs index faff939c..d4ade24d 100644 --- a/serf-reactor/src/driver/stream/mod.rs +++ b/serf-reactor/src/driver/stream/mod.rs @@ -71,7 +71,7 @@ use crate::{ shared::Shared, }; #[cfg(encryption)] -use serf_proto::{KeyRequestOperation, KeyResponseArgs, event::KeyRequest}; +use serf_proto::{KeyResponseArgs, event::KeyRequest}; /// Hard ceiling on the per-recv UDP buffer — UDP's wire payload is capped at /// 65507 bytes once the IP/UDP headers are deducted, so a larger buffer just @@ -1246,13 +1246,43 @@ where } #[cfg(encryption)] if let Event::KeyRequest(req) = ev { - let resp = apply_key_request(&*self.keyring, req); + let resp = self.apply_key_request_live(req); // Ignoring Err: `respond_key` fails only when the response cannot be routed; - // the key op has already applied locally. + // the key op has already applied to the live wire keyring. let _ = self.endpoint.respond_key(req, resp, Instant::now()); } } + /// Read-modify-write the endpoint's LIVE wire keyring for one inbound + /// [`KeyRequest`], returning the [`KeyResponseArgs`] built from the post-op live + /// state. + /// + /// The endpoint-facing wrapper over [`serf_driver::apply_key_request`]: it reads + /// the coordinator's live `encryption_options`, applies the op variant-exactly + /// against the live ring, and on a real mutation publishes the rotated ring back + /// via `set_encryption_options` — so the gossip and reliable planes re-key in + /// lockstep — then notifies the keyring observer for persistence. A node with no + /// keyring configured answers `result = false` and makes no wire change; a + /// read-only `list` or a refused op leaves the wire untouched. + #[cfg(encryption)] + fn apply_key_request_live(&mut self, req: &KeyRequest) -> KeyResponseArgs { + let mut encryption = self.endpoint.encryption_options().clone(); + let Some(current) = encryption.keyring() else { + return KeyResponseArgs { + result: false, + message: "no keyring configured on this node".into(), + ..Default::default() + }; + }; + let (resp, rotated) = serf_driver::apply_key_request(current, req.op(), req.key()).into_parts(); + if let Some(new_ring) = rotated { + encryption.set_keyring(new_ring.clone()); + self.endpoint.set_encryption_options(encryption); + self.keyring.keyring_updated(&new_ring); + } + resp + } + /// Reap await-result join waiters on the deadline timer (the reply terminal), /// then remove any waiter that has reached BOTH terminals (reply resolved AND /// `pending` empty), clearing its still-recorded ignore-join streams. @@ -1831,27 +1861,6 @@ fn complete_join_exchange( } } -/// Apply one inbound [`KeyRequest`] to the driver's keyring delegate, producing -/// the [`KeyResponseArgs`] the pump forwards to `respond_key`. -#[cfg(encryption)] -fn apply_key_request( - keyring: &dyn KeyringDelegate, - req: &KeyRequest, -) -> KeyResponseArgs { - match (req.op(), req.key()) { - (KeyRequestOperation::Install, Some(key)) => keyring.install(*key), - (KeyRequestOperation::Use, Some(key)) => keyring.use_key(*key), - (KeyRequestOperation::Remove, Some(key)) => keyring.remove(*key), - (KeyRequestOperation::List, _) => keyring.list(), - (_, None) => KeyResponseArgs { - result: false, - message: "key-management request missing its required key".into(), - keys: Vec::new(), - primary_key: None, - }, - } -} - /// Accepts inbound TCP connections and forwards each to the pump, waking it after /// each enqueue. `accept` is async-only, so this cannot fold into the pump's poll; /// it stops promptly when the driver drops `shutdown_rx`'s sender, which cancels diff --git a/serf-reactor/src/quic/mod.rs b/serf-reactor/src/quic/mod.rs index 65ada43d..d9ed2838 100644 --- a/serf-reactor/src/quic/mod.rs +++ b/serf-reactor/src/quic/mod.rs @@ -262,6 +262,9 @@ where RES: Resolver
, AR: AdvertiseAddrResolver, { + // Refuse a seed keyring that carries a cross-cipher byte twin, before binding. + #[cfg(encryption)] + crate::transport::reject_cross_cipher_keyring(&options.encryption)?; let local_id = options.local_id.ok_or_else(|| { SerfError::Io(std::io::Error::new( ErrorKind::InvalidInput, diff --git a/serf-reactor/src/tcp/mod.rs b/serf-reactor/src/tcp/mod.rs index bb45d675..a80c6f15 100644 --- a/serf-reactor/src/tcp/mod.rs +++ b/serf-reactor/src/tcp/mod.rs @@ -229,6 +229,9 @@ where // Validate stream knobs that would deterministically break the backend BEFORE // binding any socket. options.stream.validate()?; + // Refuse a seed keyring that carries a cross-cipher byte twin, before binding. + #[cfg(encryption)] + crate::transport::reject_cross_cipher_keyring(&options.encryption)?; let local_id = options.local_id.ok_or_else(|| { SerfError::Io(std::io::Error::new( diff --git a/serf-reactor/src/tls/mod.rs b/serf-reactor/src/tls/mod.rs index 5c63572c..c814828e 100644 --- a/serf-reactor/src/tls/mod.rs +++ b/serf-reactor/src/tls/mod.rs @@ -336,6 +336,9 @@ where // zero `bridge_recv_buf_len` makes every bridge read return a false EOF) BEFORE // binding any socket. options.stream.validate()?; + // Refuse a seed keyring that carries a cross-cipher byte twin, before binding. + #[cfg(encryption)] + crate::transport::reject_cross_cipher_keyring(&options.encryption)?; let local_id = options.local_id.ok_or_else(|| { SerfError::Io(std::io::Error::new( diff --git a/serf-reactor/src/transport/mod.rs b/serf-reactor/src/transport/mod.rs index 719d4b13..29fd310a 100644 --- a/serf-reactor/src/transport/mod.rs +++ b/serf-reactor/src/transport/mod.rs @@ -170,3 +170,31 @@ pub(crate) fn post_bind_setup( validate_advertise_addr(advertise_addr)?; crate::os_seeded_std_rng() } + +/// Reject a construction-time encryption keyring that already carries a +/// cross-cipher byte twin — two keys sharing a raw byte value across different +/// cipher variants. +/// +/// The coordinator's byte-keyed rotation ops (`promote` / `remove_secondary`) +/// match on bytes alone, so such a ring would make every later key op ambiguous +/// and let a rotation promote or remove the wrong cipher's key. Each transport's +/// `Transport::new` calls this on its stored [`EncryptionOptions`] before binding +/// a socket, establishing the invariant — upheld thereafter by the drivers' live +/// key-op chokepoint — that the live keyring is cross-cipher-collision-free from +/// construction on. +/// +/// [`EncryptionOptions`]: memberlist_proto::EncryptionOptions +#[cfg(encryption)] +pub(crate) fn reject_cross_cipher_keyring( + encryption: &memberlist_proto::EncryptionOptions, +) -> Result<(), crate::SerfError> { + if let Some(keyring) = encryption.keyring() + && serf_driver::keyring_carries_cross_cipher_twin(keyring) + { + return Err(crate::SerfError::Io(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "encryption keyring carries a cross-cipher key collision", + ))); + } + Ok(()) +} From 3003b86720aa38995e5dc0a954334fe04acf17ad Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 10 Jul 2026 23:08:18 +1200 Subject: [PATCH 071/128] test(serf-reactor,serf-compio): prove key rotation reaches the wire Per-driver two-node rotations drive install/use/remove through the public commands and assert both nodes' LIVE endpoint keyrings rotate (primary moves to K2, the removed K1 is gone) via a recording KeyringDelegate plus a cluster-wide list_keys cross-check, and that traffic still flows after. The delegate proves the observer fires exactly once per mutation, never on list or a refused op. --- serf-compio/src/serf/tests.rs | 250 ++++++++++++++++++++- serf-reactor/tests/key_rotation.rs | 342 +++++++++++++++++++++++++++++ 2 files changed, 591 insertions(+), 1 deletion(-) create mode 100644 serf-reactor/tests/key_rotation.rs diff --git a/serf-compio/src/serf/tests.rs b/serf-compio/src/serf/tests.rs index 23873c71..92b55138 100644 --- a/serf-compio/src/serf/tests.rs +++ b/serf-compio/src/serf/tests.rs @@ -43,7 +43,7 @@ impl Resolver for EmptyResolver { } #[cfg(encryption)] -use crate::{EncryptionOptions, Keyring, SecretKey, VoidKeyringDelegate}; +use crate::{EncryptionOptions, Keyring, KeyringDelegate, SecretKey, VoidKeyringDelegate}; /// Build and spawn a TCP serf node bound to an ephemeral loopback port. async fn spawn_node(id: &str) -> Serf { @@ -897,3 +897,251 @@ async fn tcp_concurrent_dispatch_join_and_ignore_old_join_coexist() { b.shutdown().await.expect("cdj-b shuts down"); c.shutdown().await.expect("cdj-c shuts down"); } + +/// A [`KeyringDelegate`] that records, in order, every live keyring the driver +/// publishes through `keyring_updated`. The driver fires it only after it has +/// pushed the rotated ring to the endpoint via `set_encryption_options`, so the +/// recorded ring is exactly the ring the gossip and reliable planes now encrypt +/// under — the live-wire observable the rotation test asserts on. `!Send` behind an +/// `Rc`, matching the compio driver's single-threaded keyring delegate. +#[cfg(encryption)] +#[derive(Default)] +struct RecordingKeyring { + rings: core::cell::RefCell>, +} + +#[cfg(encryption)] +impl RecordingKeyring { + /// The keyrings observed so far, oldest first. + fn rings(&self) -> Vec { + self.rings.borrow().clone() + } +} + +#[cfg(encryption)] +impl KeyringDelegate for RecordingKeyring { + fn keyring_updated(&self, keyring: &Keyring) { + self.rings.borrow_mut().push(keyring.clone()); + } +} + +/// Build and spawn a TCP serf node on an ephemeral loopback port with `encryption` +/// as its keyring policy and `keyring` as its rotation observer. +#[cfg(encryption)] +async fn spawn_encrypted_node_with_keyring( + id: &str, + encryption: EncryptionOptions, + keyring: std::rc::Rc, +) -> Serf { + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(bind)) + .with_encryption(encryption); + Serf::new::, SocketAddrResolver, FirstAddrResolver, _, _>( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + gossip_rng().expect("seed gossip rng"), + keyring, + ) + .await + .expect("spawn serf node") +} + +/// Drive `events` until the originator's next `KeyResponse` surfaces (emitted when +/// the key query's deadline fires), draining any interleaved membership / gossip +/// events so a backlog cannot stall the stream. +#[cfg(encryption)] +async fn next_key_response(events: &mut S) -> serf_proto::event::KeyResponse +where + S: futures_util::Stream> + Unpin, +{ + compio::time::timeout(Duration::from_secs(20), async { + loop { + match events.next().await { + Some(Event::KeyResponse(kr)) => break kr, + Some(_) => {} + None => panic!("event stream closed before a KeyResponse"), + } + } + }) + .await + .expect("a KeyResponse within the timeout") +} + +/// Two encrypted nodes share primary K1, then A rotates the cluster to K2 via +/// `install_key` -> `use_key` -> `remove_key` over the compio single-threaded +/// stream driver. Each op propagates and every node applies it to its LIVE wire +/// keyring. The fail-on-revert check reads both nodes' recorded live rings (and +/// cross-checks cluster-wide via `list_keys`) and requires primary == K2 with K1 +/// gone — unreachable under the pre-fix shadow model; the observer must have fired +/// exactly once per mutation and not for the read-only `list` or the refused final +/// remove; finally a user event still propagates A -> B, proving the wire runs +/// under K2 on both planes. +#[cfg(encryption)] +#[compio::test] +async fn two_node_tcp_key_rotation_rotates_both_live_keyrings() { + use std::rc::Rc; + + let k1 = test_secret_key(0x11); + let k2 = test_secret_key(0x22); + + let rec_a = Rc::new(RecordingKeyring::default()); + let rec_b = Rc::new(RecordingKeyring::default()); + let enc = || EncryptionOptions::new().with_keyring(Keyring::new(k1)); + + let b = spawn_encrypted_node_with_keyring("rot-b", enc(), rec_b.clone()).await; + let a = spawn_encrypted_node_with_keyring("rot-a", enc(), rec_a.clone()).await; + let b_addr = b.advertise_address(); + + // Converge on a 2-member view under K1 (a real encrypted push/pull join). + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B over the encrypted reliable plane"); + compio::time::timeout(Duration::from_secs(20), async { + loop { + if a.num_members() == 2 && b.num_members() == 2 { + break; + } + compio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("both nodes converge to a 2-member cluster"); + + // Subscribe before issuing any key op so no KeyResponse races the subscription. + let mut a_events = a.events(); + let mut b_events = b.events(); + + // install K2: both nodes gain it as a secondary in their live ring. + a.install_key(k2).await.expect("install_key dispatched"); + let kr = next_key_response(&mut a_events).await; + assert!( + kr.num_resp >= 2, + "install_key must collect a response from BOTH nodes (num_resp={})", + kr.num_resp + ); + assert_eq!(kr.num_err, 0, "install_key must succeed on every node"); + + // use K2: both nodes promote it to primary. + a.use_key(k2).await.expect("use_key dispatched"); + let kr = next_key_response(&mut a_events).await; + assert_eq!(kr.num_err, 0, "use_key must succeed on every node"); + + // remove K1: both nodes drop the old key. + a.remove_key(k1).await.expect("remove_key dispatched"); + let kr = next_key_response(&mut a_events).await; + assert_eq!(kr.num_err, 0, "remove_key must succeed on every node"); + + // FAIL-ON-REVERT: each node's observer captured exactly the ring the driver + // published to its endpoint at each mutation, in order — install adds K2 as a + // secondary under K1, use promotes K2, remove drops K1. Under the pre-fix shadow + // model the coordinator keyring never rotated, so this sequence is unreachable. + for (name, rec) in [("rot-a", &rec_a), ("rot-b", &rec_b)] { + let rings = rec.rings(); + assert_eq!( + rings.len(), + 3, + "{name}: the observer fires exactly once per successful mutation" + ); + assert_eq!( + rings[0].primary_ref(), + &k1, + "{name}: install leaves K1 as primary" + ); + assert!( + rings[0].secondaries().contains(&k2), + "{name}: install adds K2 as a secondary" + ); + assert_eq!(rings[1].primary_ref(), &k2, "{name}: use promotes K2"); + assert_eq!( + rings[2].primary_ref(), + &k2, + "{name}: K2 stays primary after the remove" + ); + assert!( + !rings[2].secondaries().contains(&k1), + "{name}: the removed K1 is absent from the live keyring" + ); + } + + // Independent, endpoint-direct cluster cross-check: `list_keys` tallies every + // node's LIVE ring — K2 primary on BOTH nodes and K1 installed on none. + a.list_keys().await.expect("list_keys dispatched"); + let kr = next_key_response(&mut a_events).await; + assert_eq!( + kr.primary_keys.get(&k2).copied(), + Some(2), + "both nodes report K2 as their live primary" + ); + assert_eq!( + kr.keys.get(&k2).copied(), + Some(2), + "both nodes still hold K2 in their live ring" + ); + assert_eq!( + kr.keys.get(&k1), + None, + "the removed K1 is installed on no node" + ); + + // `list_keys` is read-only: it must NOT have fired the observer. + assert_eq!( + rec_a.rings().len(), + 3, + "list_keys does not fire the keyring observer" + ); + assert_eq!( + rec_b.rings().len(), + 3, + "list_keys does not fire the keyring observer" + ); + + // A refused op must not fire the observer either: removing the already-gone K1 is + // refused on every node, leaving both live rings — and the observer — untouched. + a.remove_key(k1).await.expect("remove_key dispatched"); + let kr = next_key_response(&mut a_events).await; + assert!( + kr.num_err >= 1, + "removing an absent key is refused (num_err={})", + kr.num_err + ); + assert_eq!( + rec_a.rings().len(), + 3, + "a refused op does not fire the keyring observer" + ); + assert_eq!( + rec_b.rings().len(), + 3, + "a refused op does not fire the keyring observer" + ); + + // Post-rotation traffic proof: a user event still crosses the wire, which now runs + // under K2 on both nodes — the reliable and gossip planes rotated with the keyring. + a.user_event("after-rotation", Bytes::from_static(b"payload"), false) + .await + .expect("user_event from a running node"); + let saw = compio::time::timeout(Duration::from_secs(20), async { + loop { + match b_events.next().await { + Some(Event::User(u)) if u.name.as_str() == "after-rotation" => break true, + Some(_) => {} + None => break false, + } + } + }) + .await + .expect("B observes the post-rotation user event within the timeout"); + assert!( + saw, + "a user event must still propagate A -> B after the rotation (wire under K2)" + ); + + a.shutdown().await.expect("rot-a shuts down"); + b.shutdown().await.expect("rot-b shuts down"); +} diff --git a/serf-reactor/tests/key_rotation.rs b/serf-reactor/tests/key_rotation.rs new file mode 100644 index 00000000..0e3730b2 --- /dev/null +++ b/serf-reactor/tests/key_rotation.rs @@ -0,0 +1,342 @@ +//! Key-management end-to-end over the reactor stream driver: two encrypted nodes +//! rotate their keyring across the cluster and BOTH nodes' LIVE wire keyrings +//! follow. +//! +//! The regression this guards: a driver that applies key ops to a caller-held +//! shadow reports a completed rotation the wire never sees. Node A drives +//! `install_key` -> `use_key` -> `remove_key` through the public commands; each op +//! propagates as a cluster query and every node applies it to the coordinator's +//! LIVE `EncryptionOptions` keyring, then answers from that post-op state. +//! +//! The rotation is observed two independent ways, both reading live state rather +//! than a shadow: each node's [`KeyringDelegate`] records the exact ring the driver +//! published to its endpoint, and a final `list_keys` tallies every node's live +//! ring cluster-wide. Both must show primary == K2 with K1 gone — unreachable under +//! the pre-fix shadow model, where the coordinator keyring stayed frozen at +//! construction K1. A post-rotation user event then still crosses the wire, proving +//! both planes now run under K2, and the observer's fire count proves it fired +//! exactly once per successful mutation and never for a `list` or a refused op. +//! +//! The scenario body is a runtime-generic `async fn ` helper, so the +//! same test runs as a `#[tokio::test]` cell over `TokioRuntime` and as a `_smol` +//! cell driven by `SmolRuntime::block_on`, mirroring the reactor's other real-node +//! suites. + +#![cfg(all( + feature = "tcp", + any(feature = "aes-gcm", feature = "chacha20-poly1305") +))] + +use core::time::Duration; +use std::{ + net::SocketAddr, + sync::{Arc, Mutex}, +}; + +use agnostic::Runtime; +use bytes::Bytes; +use futures_util::{Stream, StreamExt}; +use serf_proto::{ + event::{Event, KeyResponse}, + options::Options as SerfOptions, +}; +use serf_reactor::{ + EncryptionOptions, FirstAddrResolver, Keyring, KeyringDelegate, MaybeResolved, RuntimeOptions, + SecretKey, Serf, SocketAddrResolver, TcpTransportOptions, VoidDelegate, +}; +use smol_str::SmolStr; + +/// A reactor TCP node handle over the agnostic runtime `R`. +type Node = Serf; + +/// A [`KeyringDelegate`] that records, in order, every live keyring the driver +/// publishes through `keyring_updated`. The driver fires it only after it has +/// pushed the rotated ring to the endpoint via `set_encryption_options`, so the +/// recorded ring is exactly the ring the gossip and reliable planes now encrypt +/// under — the live-wire observable this suite asserts on. +#[derive(Default)] +struct RecordingKeyring { + rings: Mutex>, +} + +impl RecordingKeyring { + /// The keyrings observed so far, oldest first. + fn rings(&self) -> Vec { + self.rings.lock().expect("keyring log not poisoned").clone() + } +} + +impl KeyringDelegate for RecordingKeyring { + fn keyring_updated(&self, keyring: &Keyring) { + self + .rings + .lock() + .expect("keyring log not poisoned") + .push(keyring.clone()); + } +} + +/// A deterministic test secret key, selecting whichever AEAD cipher this build +/// compiled so the test works under either backend. +fn secret_key(fill: u8) -> SecretKey { + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes256([fill; 32]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = SecretKey::ChaCha20Poly1305([fill; 32]); + key +} + +/// Build and spawn a reactor TCP node on an ephemeral loopback port with +/// `encryption` as its keyring policy and `keyring` as its rotation observer. +async fn spawn_encrypted_node( + id: &str, + encryption: EncryptionOptions, + keyring: Arc, +) -> Node +where + R: Runtime, +{ + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(bind)) + .with_encryption(encryption); + Serf::::tcp( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + keyring, + ) + .await + .expect("spawn encrypted serf tcp node") +} + +/// Poll both nodes until each reports the full two-member cluster, or fail on a +/// generous timeout so a convergence regression surfaces as a timeout, not a hang. +async fn converge(a: &Node, b: &Node) +where + R: Runtime, +{ + R::timeout(Duration::from_secs(20), async { + loop { + if a.num_members() == 2 && b.num_members() == 2 { + break; + } + R::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("both nodes converge to a 2-member cluster"); +} + +/// Drive `events` until the originator's next `KeyResponse` surfaces (emitted when +/// the key query's deadline fires), draining any interleaved membership / gossip +/// events so a backlog cannot stall the stream. +async fn next_key_response(events: &mut S) -> KeyResponse +where + R: Runtime, + S: Stream> + Unpin + Send, +{ + R::timeout(Duration::from_secs(20), async { + loop { + match events.next().await { + Some(Event::KeyResponse(kr)) => break kr, + Some(_) => {} + None => panic!("event stream closed before a KeyResponse"), + } + } + }) + .await + .expect("a KeyResponse within the timeout") +} + +/// Two encrypted nodes share primary K1, then A rotates the cluster to K2 via +/// `install_key` -> `use_key` -> `remove_key`. Each op propagates and every node +/// applies it to its LIVE wire keyring. The fail-on-revert check reads both nodes' +/// recorded live rings (and cross-checks cluster-wide via `list_keys`) and requires +/// primary == K2 with K1 gone; the observer must have fired exactly once per +/// mutation and not for the read-only `list` or the refused final remove; finally a +/// user event still propagates A -> B, proving the wire now runs under K2. +async fn key_rotation_across_two_nodes_rotates_both_live_keyrings() +where + R: Runtime, +{ + let k1 = secret_key(0x11); + let k2 = secret_key(0x22); + + let rec_a = Arc::new(RecordingKeyring::default()); + let rec_b = Arc::new(RecordingKeyring::default()); + let enc = || EncryptionOptions::new().with_keyring(Keyring::new(k1)); + + let b = spawn_encrypted_node::("rot-b", enc(), rec_b.clone()).await; + let a = spawn_encrypted_node::("rot-a", enc(), rec_a.clone()).await; + let b_addr = b.advertise_address(); + + // Converge on a 2-member view under K1 (a real encrypted push/pull join). + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B over the encrypted reliable plane"); + converge(&a, &b).await; + + // Subscribe before issuing any key op so no KeyResponse races the subscription. + let mut a_events = a.events(); + let mut b_events = b.events(); + + // install K2: both nodes gain it as a secondary in their live ring. + a.install_key(k2).await.expect("install_key dispatched"); + let kr = next_key_response::(&mut a_events).await; + assert!( + kr.num_resp >= 2, + "install_key must collect a response from BOTH nodes (num_resp={})", + kr.num_resp + ); + assert_eq!(kr.num_err, 0, "install_key must succeed on every node"); + + // use K2: both nodes promote it to primary. + a.use_key(k2).await.expect("use_key dispatched"); + let kr = next_key_response::(&mut a_events).await; + assert_eq!(kr.num_err, 0, "use_key must succeed on every node"); + + // remove K1: both nodes drop the old key. + a.remove_key(k1).await.expect("remove_key dispatched"); + let kr = next_key_response::(&mut a_events).await; + assert_eq!(kr.num_err, 0, "remove_key must succeed on every node"); + + // FAIL-ON-REVERT: each node's observer captured exactly the ring the driver + // published to its endpoint at each mutation, in order — install adds K2 as a + // secondary under K1, use promotes K2, remove drops K1. Under the pre-fix shadow + // model the coordinator keyring never rotated, so this sequence is unreachable. + for (name, rec) in [("rot-a", &rec_a), ("rot-b", &rec_b)] { + let rings = rec.rings(); + assert_eq!( + rings.len(), + 3, + "{name}: the observer fires exactly once per successful mutation" + ); + assert_eq!( + rings[0].primary_ref(), + &k1, + "{name}: install leaves K1 as primary" + ); + assert!( + rings[0].secondaries().contains(&k2), + "{name}: install adds K2 as a secondary" + ); + assert_eq!(rings[1].primary_ref(), &k2, "{name}: use promotes K2"); + assert_eq!( + rings[2].primary_ref(), + &k2, + "{name}: K2 stays primary after the remove" + ); + assert!( + !rings[2].secondaries().contains(&k1), + "{name}: the removed K1 is absent from the live keyring" + ); + } + + // Independent, endpoint-direct cluster cross-check: `list_keys` tallies every + // node's LIVE ring — K2 primary on BOTH nodes and K1 installed on none. + a.list_keys().await.expect("list_keys dispatched"); + let kr = next_key_response::(&mut a_events).await; + assert_eq!( + kr.primary_keys.get(&k2).copied(), + Some(2), + "both nodes report K2 as their live primary" + ); + assert_eq!( + kr.keys.get(&k2).copied(), + Some(2), + "both nodes still hold K2 in their live ring" + ); + assert_eq!( + kr.keys.get(&k1), + None, + "the removed K1 is installed on no node" + ); + + // `list_keys` is read-only: it must NOT have fired the observer. + assert_eq!( + rec_a.rings().len(), + 3, + "list_keys does not fire the keyring observer" + ); + assert_eq!( + rec_b.rings().len(), + 3, + "list_keys does not fire the keyring observer" + ); + + // A refused op must not fire the observer either: removing the already-gone K1 is + // refused on every node, leaving both live rings — and the observer — untouched. + a.remove_key(k1).await.expect("remove_key dispatched"); + let kr = next_key_response::(&mut a_events).await; + assert!( + kr.num_err >= 1, + "removing an absent key is refused (num_err={})", + kr.num_err + ); + assert_eq!( + rec_a.rings().len(), + 3, + "a refused op does not fire the keyring observer" + ); + assert_eq!( + rec_b.rings().len(), + 3, + "a refused op does not fire the keyring observer" + ); + + // Post-rotation traffic proof: a user event still crosses the wire, which now runs + // under K2 on both nodes — the reliable and gossip planes rotated with the keyring. + a.user_event("after-rotation", Bytes::from_static(b"payload"), false) + .await + .expect("user_event from a running node"); + let saw = R::timeout(Duration::from_secs(20), async { + loop { + match b_events.next().await { + Some(Event::User(u)) if u.name.as_str() == "after-rotation" => break true, + Some(_) => {} + None => break false, + } + } + }) + .await + .expect("B observes the post-rotation user event within the timeout"); + assert!( + saw, + "a user event must still propagate A -> B after the rotation (wire under K2)" + ); + + a.shutdown().await.expect("rot-a shuts down"); + b.shutdown().await.expect("rot-b shuts down"); +} + +// The tokio cell: the runtime-generic scenario driven on tokio's multi-thread +// runtime. Gated on `tokio` so the `--test key_rotation -- smol` build can drop the +// `agnostic/tokio` code path. +#[cfg(feature = "tokio")] +mod tokio_cells { + use agnostic::tokio::TokioRuntime; + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn key_rotation_across_two_nodes_rotates_both_live_keyrings() { + super::key_rotation_across_two_nodes_rotates_both_live_keyrings::().await; + } +} + +// The smol cell: the identical scenario over `SmolRuntime`, driven by smol's +// `block_on`. `cargo test --test key_rotation -- smol` selects exactly this. +#[cfg(feature = "smol")] +mod smol_cells { + use agnostic::{RuntimeLite, smol::SmolRuntime}; + + #[test] + fn key_rotation_across_two_nodes_rotates_both_live_keyrings_smol() { + SmolRuntime::block_on( + super::key_rotation_across_two_nodes_rotates_both_live_keyrings::(), + ); + } +} From 7b40c4c06f746c715b7638c9803471734e36c678 Mon Sep 17 00:00:00 2001 From: al8n Date: Fri, 10 Jul 2026 23:23:02 +1200 Subject: [PATCH 072/128] fix(serf-driver): a duplicate install succeeds without republishing the ring Re-installing an already-present key was classified as a mutation, so the unchanged ring was pushed back and the persistence observer fired on a plain retry; an exact-membership check now reports success with no mutation, keeping keyring_updated to real rotations only. --- serf-driver/src/keyring/mod.rs | 12 ++++++++---- serf-driver/src/keyring/tests.rs | 15 ++++++++++++--- 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/serf-driver/src/keyring/mod.rs b/serf-driver/src/keyring/mod.rs index 94e9cbb7..b1f35072 100644 --- a/serf-driver/src/keyring/mod.rs +++ b/serf-driver/src/keyring/mod.rs @@ -103,7 +103,8 @@ pub fn keyring_carries_cross_cipher_twin(keyring: &Keyring) -> bool { /// /// The variant-exact semantics: /// -/// - `install` inserts the key as a secondary (idempotent; a cross-cipher byte +/// - `install` inserts the key as a secondary (an already-installed key is a +/// success with no mutation, so nothing is republished; a cross-cipher byte /// twin of an already-present key is refused), /// - `use` promotes the exact (variant + bytes) key to primary — a promote of the /// current primary is a trivial success with no wire change, @@ -121,11 +122,14 @@ pub fn apply_key_request( let mut keyring = current.clone(); let (response, mutated) = match (op, key) { (KeyRequestOperation::Install, Some(key)) => { - // A same-variant re-install is the idempotent `insert_secondary` no-op, - // reported as success. A cross-cipher byte twin of an already-present key - // is refused to keep the byte-keyed rotation ops unambiguous. + // A cross-cipher byte twin of an already-present key is refused to keep the + // byte-keyed rotation ops unambiguous. Installing a key that is already the + // primary or an installed secondary is a success WITHOUT mutation, so a + // retried install never republishes an unchanged ring to the observer. if keyring_has_cross_cipher_twin(&keyring, key) { (refused("cross-cipher key collision"), false) + } else if keyring.primary_ref() == key || keyring.secondaries().contains(key) { + (success(), false) } else { keyring.insert_secondary(*key); (success(), true) diff --git a/serf-driver/src/keyring/tests.rs b/serf-driver/src/keyring/tests.rs index 631882e7..d4ba6975 100644 --- a/serf-driver/src/keyring/tests.rs +++ b/serf-driver/src/keyring/tests.rs @@ -27,12 +27,21 @@ fn install_adds_secondary_and_rotates() { #[cfg(feature = "aes-gcm")] #[test] -fn install_existing_key_is_idempotent_success() { +fn install_existing_key_is_success_without_rotation() { + // Re-installing the current primary succeeds but mutates nothing, so no + // rotated ring is published and the persistence observer never fires on a + // retried install. let ring = Keyring::new(aes(1)); let out = apply_key_request(&ring, KeyRequestOperation::Install, Some(&aes(1))); assert!(out.response().result); - // The dup insert is dropped, so the rotated ring equals the original. - assert_eq!(out.rotated().expect("install reports a mutation"), &ring); + assert!(out.rotated().is_none()); + + // The same holds for a key already installed as a secondary. + let mut ring = Keyring::new(aes(1)); + ring.insert_secondary(aes(2)); + let out = apply_key_request(&ring, KeyRequestOperation::Install, Some(&aes(2))); + assert!(out.response().result); + assert!(out.rotated().is_none()); } #[cfg(feature = "aes-gcm")] From 3b202a626dcda490cbd9de2f7dea89203a51a7ea Mon Sep 17 00:00:00 2001 From: al8n Date: Sat, 11 Jul 2026 00:25:49 +1200 Subject: [PATCH 073/128] fix(serf-proto): transition to Shutdown on a lost conflict vote MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Losing the id-conflict vote emitted Event::Shutdown while the machine stayed Alive — the documented forced Alive→Shutdown transition (modelled on Go serf, whose loss branch shuts the serf layer down) was never performed, so every driver gated commands itself. The loss branch now sets SerfState::Shutdown before emitting; the chokepoints uniformly refuse commands and quiet ingress and timers (mirroring the memberlist post-leave contract); the buffered event still drains via poll_event; a won vote is untouched. Regressions cover each. --- serf-proto/src/endpoint/mod.rs | 106 ++++++++++++- serf-proto/src/endpoint/tests.rs | 252 ++++++++++++++++++++++++++++++- serf-proto/src/members/mod.rs | 11 ++ 3 files changed, 365 insertions(+), 4 deletions(-) diff --git a/serf-proto/src/endpoint/mod.rs b/serf-proto/src/endpoint/mod.rs index c6b735f0..609be9c0 100644 --- a/serf-proto/src/endpoint/mod.rs +++ b/serf-proto/src/endpoint/mod.rs @@ -515,6 +515,17 @@ pub enum Error { /// `bad_join_status` guard in serf-core `api.rs` `join()`. #[error("join called from invalid state: {0}")] BadJoinState(SerfState), + /// A command was issued after the serf machine shut down. + /// + /// Losing an id-conflict vote forces the machine to [`SerfState::Shutdown`] + /// (mirroring Go serf, whose conflict-loss branch calls `shutdown()`, which + /// sets the state). A shut-down machine is no longer a cluster participant, + /// so every command that would originate new work — user events, queries, tag + /// updates, query responses, and key-management issuance — is refused with + /// this error. The driver owns stopping I/O and delivering the buffered + /// [`Event::Shutdown`]; the machine only refuses to originate. + #[error("operation attempted after the serf machine shut down")] + Shutdown, /// The inner memberlist `leave()` returned an error. #[error("inner leave error: {0}")] InnerLeave(#[from] memberlist_proto::Error), @@ -901,6 +912,30 @@ where self.state } + /// The single post-[`Shutdown`](SerfState::Shutdown) command guard: `Ok(())` + /// while the machine is live, `Err(Error::Shutdown)` once it has shut down. + /// + /// Losing an id-conflict vote forces the machine to `SerfState::Shutdown` + /// (mirroring Go serf's conflict-loss `shutdown()`), after which every command + /// that originates new cluster work funnels its lifecycle check through here so + /// the contract is named in one place — the memberlist post-leave + /// `ensure_running` precedent applied to serf's terminal state. A shut-down + /// machine additionally goes inert on ingress and quiet on its timers; only the + /// already-buffered [`Event::Shutdown`] still drains via `poll_event`. The + /// driver owns stopping I/O and delivering that event. + /// + /// Gates only on `Shutdown`, not on every non-`Alive` state: `Leaving` and + /// `Left` keep originating (matching Go serf, where the layer keeps running + /// until `shutdown()`); a driver that wants a stricter post-leave policy + /// enforces it in its own command gate. + const fn ensure_not_shutdown(&self) -> Result<(), Error> { + if matches!(self.state, SerfState::Shutdown) { + Err(Error::Shutdown) + } else { + Ok(()) + } + } + /// The current member (SWIM membership) Lamport clock value. pub const fn member_time(&self) -> u64 { self.clock @@ -964,7 +999,15 @@ where /// Returns the minimum of serf's own periodic deadlines (reap, reconnect, /// queue-check, leave-broadcast, leave-complete, and pending-query closes). /// The composing super-machine folds in the coordinator's own deadline. + /// + /// A shut-down machine (lost id-conflict vote) schedules no wakeups: its + /// deadlines fire no work (`after_inner_timeout` is inert once Shutdown), so + /// surfacing them would spin the driver. Mirrors memberlist's `poll_timeout` + /// returning `None` once not Running. pub fn serf_poll_timeout(&self) -> Option { + if self.state.is_shutdown() { + return None; + } let query_min = self.pending_queries.iter().map(|pq| pq.deadline).min(); [ self.next_reap, @@ -1013,6 +1056,11 @@ where I: Clone, A: Clone, { + // A shut-down machine (lost id-conflict vote) is quiet on its timers: no + // snapshot resync, and `after_inner_timeout` fires no deadlines. + if self.state.is_shutdown() { + return; + } self.drain_now = now; if self.local_state_dirty { self.resync_local_state(t); @@ -1041,6 +1089,15 @@ where I: Clone, A: Clone, { + // A shut-down machine (lost id-conflict vote) fires no serf deadlines: no + // reap, reconnect, queue-check, query-close, or leave-completion, and its + // ingress drain is inert. Mirrors the memberlist post-leave `handle_timeout` + // early-out. The transition itself happens inside this method (a lost + // conflict close), so the gate quiets every *subsequent* tick, not the one + // that shut the machine down. + if self.state.is_shutdown() { + return; + } self.drain_now = now; // Step 1: drain all inner events produced by the tick through the serf sieve. @@ -1146,6 +1203,15 @@ where where T: Reliable, { + // Ingress chokepoint: a shut-down machine (lost id-conflict vote) is inert on + // ingress — no inbound inner event mutates serf state, rebroadcasts, or emits + // any event beyond the already-buffered Event::Shutdown. Gating the dispatch + // itself (before the match, every caller) mirrors the memberlist post-leave + // handle_packet gate, which returns early before its own message match once + // not Running. + if self.state.is_shutdown() { + return; + } use memberlist_proto::Event as IE; match ev { // ── membership ─────────────────────────────────────────────────────── @@ -1305,6 +1371,15 @@ where I: Clone + Data, A: Data, { + // Egress chokepoint: a shut-down machine (lost id-conflict vote) synthesises + // no push-pull snapshot. `fire_reap` can mark the snapshot dirty in the same + // tick that a lost conflict close shuts the machine down, so this guard keeps + // the deferred resync (in `drain_inner`) from pushing a fresh snapshot to the + // coordinator after shutdown; the driver owns tearing the coordinator down. + if self.state.is_shutdown() { + return; + } + // Gather status_ltimes from the membership store. // HashMap iteration order is arbitrary, so collect first then sort by the // stable encoded id bytes so two machines with identical membership always @@ -1449,6 +1524,9 @@ where { use buffa::Message as _; + // Refuse once the machine has shut down (lost id-conflict vote). + self.ensure_not_shutdown()?; + let pb_tags = tags_to_pb(&tags); let encoded = pb_tags.encode_to_vec(); // `Meta::try_from` only fails when the encoded size exceeds `Meta::MAX_SIZE` @@ -2471,6 +2549,9 @@ where where T: Reliable, { + // Refuse once the machine has shut down (lost id-conflict vote). + self.ensure_not_shutdown()?; + let name: smol_str::SmolStr = name.into(); let max_size = self.opts.max_user_event_size(); @@ -2771,6 +2852,10 @@ where I: Clone + Data, A: Clone + Data, { + // Refuse once the machine has shut down (lost id-conflict vote), before any + // RNG draw, clock read, or state mutation. + self.ensure_not_shutdown()?; + // Tag-regex pre-validation: compile-check every Filter::Tag pattern FIRST, // before any RNG draw, clock read, or state mutation. A broken pattern // returns Err with zero side effects — no RNG advance, no ltime stamp, no @@ -3241,6 +3326,10 @@ where I: Clone + Data, A: Clone + Data, { + // Refuse once the machine has shut down (lost id-conflict vote): answering an + // already-received query after shutdown is dead work. + self.ensure_not_shutdown()?; + // Look up the received-query entry for this token. let query_id = QueryId { ltime: token.ltime(), @@ -3368,6 +3457,10 @@ where I: Clone + Data, A: Clone + Data, { + // Refuse once the machine has shut down (lost id-conflict vote): answering an + // already-received key query after shutdown is dead work. + self.ensure_not_shutdown()?; + let query_id = QueryId { ltime: req.ltime, id: req.id, @@ -4546,6 +4639,11 @@ where I: Clone + Data, A: Clone + Data, { + // Refuse once the machine has shut down (lost id-conflict vote): the single + // chokepoint behind every key-management issuance (install / use / remove / + // list) and the conflict-resolution query itself. + self.ensure_not_shutdown()?; + // G8 / H8: stamp from the query clock; queries read the clock but do not // increment it. let ltime = LamportTime(self.query_clock); @@ -4851,7 +4949,13 @@ where // Won — the local node is the canonical holder. return; } - // We lost — the driver must shut this node down. + // We lost the vote. Perform serf's documented forced Alive/Leaving → + // Shutdown transition — Go serf's conflict-loss branch calls `shutdown()`, + // which sets the state — BEFORE emitting, so the event is born from an + // already-dead machine and the chokepoints (commands / ingress / timers) + // observe Shutdown for the rest of this drain. The driver remains + // responsible for stopping I/O and delivering this buffered event. + self.state = SerfState::Shutdown; self.pending_events.push_back(Event::Shutdown); } diff --git a/serf-proto/src/endpoint/tests.rs b/serf-proto/src/endpoint/tests.rs index d9469d03..0eb49030 100644 --- a/serf-proto/src/endpoint/tests.rs +++ b/serf-proto/src/endpoint/tests.rs @@ -2214,7 +2214,8 @@ fn far_future() -> memberlist_proto::Instant { #[test] fn conflict_win_does_not_shut_down() { - // Majority of responses agree → local node won → no Event::Shutdown. + // Majority of responses agree → local node won → no Event::Shutdown, and the + // machine stays Alive and fully functional (guards against over-eager gating). let mut e = ep(); let deadline = far_future(); let qid = e.test_register_conflict_query(deadline); @@ -2231,11 +2232,22 @@ fn conflict_win_does_not_shut_down() { e.poll_event().is_none(), "winning conflict must not emit Shutdown" ); + assert!( + e.state().is_alive(), + "winning conflict must leave the machine Alive" + ); + // A won vote leaves the command surface fully open. + assert!( + e.user_event("post-win", bytes::Bytes::new(), false).is_ok(), + "a won vote must not gate commands" + ); } #[test] -fn conflict_loss_emits_shutdown() { - // Minority of responses agree → local node lost → Event::Shutdown. +fn conflict_loss_transitions_to_shutdown_and_still_delivers_event() { + // Minority of responses agree → local node lost → the machine performs its + // documented forced Alive → Shutdown transition (Go serf's conflict-loss + // shutdown()) AND the buffered Event::Shutdown still drains via poll_event. let mut e = ep(); let deadline = far_future(); let qid = e.test_register_conflict_query(deadline); @@ -2245,15 +2257,249 @@ fn conflict_loss_emits_shutdown() { e.test_fold_conflict_response(qid, 201u32, false); e.test_fold_conflict_response(qid, 202u32, false); + assert!(e.state().is_alive(), "machine is Alive before the close"); + let past = memberlist_proto::Instant::ORIGIN + core::time::Duration::from_secs(3601); e.test_fire_due_query_closes(past); + // The transition happened at the close, before any event is drained: reverting + // the `self.state = Shutdown` in close_conflict_query leaves this Alive → fail. + assert!( + e.state().is_shutdown(), + "a lost conflict vote must transition the machine to Shutdown" + ); + + // Event delivery is PRESERVED: the already-buffered Event::Shutdown drains from + // the now-dead machine (poll_event stays functional post-Shutdown). let ev = e.poll_event().expect("conflict loss must emit an event"); assert!( matches!(ev, Event::Shutdown), "conflict loss must emit Event::Shutdown, got {:?}", ev ); + assert!( + e.state().is_shutdown(), + "the machine stays Shutdown after the event drains" + ); +} + +// ── post-Shutdown chokepoint contract ──────────────────────────────────────── +// +// A machine that lost an id-conflict vote transitions to `SerfState::Shutdown` +// and thereafter refuses commands, goes inert on ingress, and quiets its timers +// (mirroring the memberlist post-leave contract), while still draining the +// buffered `Event::Shutdown`. The tests below drive a REAL lost vote through the +// conflict-query scaffolding and sweep each chokepoint class. + +/// Drive the endpoint through a real lost id-conflict vote and drain the +/// resulting `Event::Shutdown`, leaving it in the terminal `Shutdown` state. +fn shut_down_via_lost_conflict(e: &mut StreamEndpoint) { + let qid = e.test_register_conflict_query(far_future()); + // 1 agree, 2 disagree → matching 1 < majority 2 → lost. + e.test_fold_conflict_response(qid, 200u32, true); + e.test_fold_conflict_response(qid, 201u32, false); + e.test_fold_conflict_response(qid, 202u32, false); + let past = memberlist_proto::Instant::ORIGIN + core::time::Duration::from_secs(3601); + e.test_fire_due_query_closes(past); + assert!( + matches!(e.poll_event(), Some(Event::Shutdown)), + "the buffered Event::Shutdown must drain from the shut-down machine" + ); + assert!(e.state().is_shutdown(), "the machine must be Shutdown"); +} + +#[test] +fn shutdown_refuses_originating_commands() { + let mut e = ep(); + // Register a live received-query token BEFORE shutdown so respond()'s refusal + // is proven to precede its received_queries lookup and deadline guard. + let token = e.test_register_received_query( + QueryId { + ltime: LamportTime::new(1), + id: 5, + }, + addr(1002), + far_future(), + ); + + shut_down_via_lost_conflict(&mut e); + + let now = memberlist_proto::Instant::ORIGIN; + let tags: Tags = [("role", "web")].into_iter().collect(); + + // Commands that originate cluster work funnel through ensure_not_shutdown. + assert!( + matches!( + e.user_event("x", bytes::Bytes::new(), false), + Err(Error::Shutdown) + ), + "user_event must be refused after shutdown" + ); + assert!( + matches!( + e.query("q", bytes::Bytes::new(), QueryParams::default(), now), + Err(Error::Shutdown) + ), + "query must be refused after shutdown" + ); + assert!( + matches!(e.set_tags(tags), Err(Error::Shutdown)), + "set_tags must be refused after shutdown" + ); + assert!( + matches!( + e.respond(&token, bytes::Bytes::new(), now), + Err(Error::Shutdown) + ), + "respond must be refused after shutdown, before the token lookup" + ); + + // The lifecycle commands keep their own pre-existing typed state errors — the + // transition alone already makes them refuse Shutdown. + assert!( + matches!(e.join(), Err(Error::BadJoinState(SerfState::Shutdown))), + "join keeps BadJoinState on a Shutdown machine" + ); + assert!( + matches!(e.leave(now), Err(Error::BadLeaveState(SerfState::Shutdown))), + "leave keeps BadLeaveState on a Shutdown machine" + ); + assert!( + matches!( + e.force_leave(2u32, false, now), + Err(Error::BadLeaveState(SerfState::Shutdown)) + ), + "force_leave keeps BadLeaveState on a Shutdown machine" + ); +} + +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn shutdown_refuses_key_management() { + use crate::event::{KeyRequest, KeyRequestOperation, KeyResponseArgs}; + use memberlist_proto::SecretKey; + + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes128([2u8; 16]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = SecretKey::ChaCha20Poly1305([2u8; 32]); + + let mut e = ep(); + // A KeyRequest token obtained before shutdown, to prove respond_key's refusal + // precedes its received_queries lookup. + let req = KeyRequest::::test_with_deadline( + KeyRequestOperation::List, + 7, + memberlist_proto::Node::new(2u32, addr(2000)), + None, + far_future(), + ); + + shut_down_via_lost_conflict(&mut e); + + let now = memberlist_proto::Instant::ORIGIN; + + // Every issuance funnels through internal_query → ensure_not_shutdown. + assert!( + matches!(e.list_keys(now), Err(Error::Shutdown)), + "list_keys must be refused after shutdown" + ); + assert!( + matches!(e.install_key(key, now), Err(Error::Shutdown)), + "install_key must be refused after shutdown" + ); + assert!( + matches!(e.use_key(key, now), Err(Error::Shutdown)), + "use_key must be refused after shutdown" + ); + assert!( + matches!(e.remove_key(key, now), Err(Error::Shutdown)), + "remove_key must be refused after shutdown" + ); + + // respond_key shares the ensure_not_shutdown gate. + let refused = e.respond_key( + &req, + KeyResponseArgs { + result: true, + message: smol_str::SmolStr::default(), + keys: vec![key], + primary_key: Some(key), + }, + now, + ); + assert!( + matches!(refused, Err(Error::Shutdown)), + "respond_key must be refused after shutdown" + ); +} + +#[test] +fn shutdown_ingress_is_inert() { + let mut e = ep(); + shut_down_via_lost_conflict(&mut e); + + let members_before = e.num_members(); + let event_time_before = e.event_time(); + + // A valid inbound user event that would normally advance the event clock and + // emit Event::User must mutate nothing and emit nothing on a Shutdown machine. + let serf_bytes = AnyMessage::::UserEvent(UserEventMessage { + ltime: 5.into(), + cc: false, + name: "post-shutdown".into(), + payload: bytes::Bytes::from_static(b"x"), + }) + .encode() + .unwrap(); + e.test_inject_user_packet(addr(1002), serf_bytes, memberlist_proto::Instant::ORIGIN); + + assert!( + e.poll_event().is_none(), + "ingress must emit nothing new after shutdown" + ); + assert_eq!( + e.event_time(), + event_time_before, + "ingress must not advance the event clock after shutdown" + ); + assert_eq!( + e.num_members(), + members_before, + "ingress must not change membership after shutdown" + ); +} + +#[test] +fn shutdown_timers_are_quiet() { + let mut e = ep(); + // A live endpoint schedules serf deadlines (reap / reconnect / queue-check). + assert!( + e.core_mut().serf_poll_timeout().is_some(), + "a live machine schedules serf deadlines" + ); + + shut_down_via_lost_conflict(&mut e); + + // A Shutdown machine schedules no serf wakeup (reverting the serf_poll_timeout + // gate surfaces the still-armed next_reap → this fails). + assert!( + e.core_mut().serf_poll_timeout().is_none(), + "a Shutdown machine must schedule no serf deadline" + ); + + // A timer tick far past every deadline fires no serf work: no reap, reconnect, + // query-close, or leave-completion, and its ingress drain is inert. + let far = memberlist_proto::Instant::ORIGIN + core::time::Duration::from_secs(86_400); + e.handle_timeout(far); + assert!( + e.poll_event().is_none(), + "a Shutdown machine must emit no serf event on a timer tick" + ); + assert!( + e.state().is_shutdown(), + "a Shutdown machine stays Shutdown across a timer tick" + ); } #[test] diff --git a/serf-proto/src/members/mod.rs b/serf-proto/src/members/mod.rs index 1048e34a..55c41fcd 100644 --- a/serf-proto/src/members/mod.rs +++ b/serf-proto/src/members/mod.rs @@ -91,6 +91,17 @@ pub enum SerfState { /// The endpoint has fully left the cluster. Left, /// The endpoint has been shut down (abnormal or forced). + /// + /// The machine performs this forced transition itself when it loses an + /// id-conflict vote (mirroring Go serf, whose conflict-loss branch calls + /// `shutdown()`). Once `Shutdown` the machine is terminal: it refuses every + /// command that would originate cluster work (with [`Error::Shutdown`]), goes + /// inert on ingress, and quiets its timers. Only the already-buffered + /// [`Event::Shutdown`] still drains via `poll_event`; the driver owns stopping + /// I/O and delivering that event. + /// + /// [`Error::Shutdown`]: crate::endpoint::Error::Shutdown + /// [`Event::Shutdown`]: crate::event::Event::Shutdown Shutdown, } From 8ea6ed6fed0b97f8b03edb62dba2824f98d91c4c Mon Sep 17 00:00:00 2001 From: al8n Date: Sat, 11 Jul 2026 00:54:33 +1200 Subject: [PATCH 074/128] fix(serf-proto): seal the Shutdown contract against replay and mid-tick work A Shutdown machine could still originate rejoin dials through the public snapshot-replay path, and a lost vote firing mid-way through the due-query loop let same-deadline closes emit after the terminal event and the rest of the pass keep pruning; load_snapshot now refuses on a Shutdown machine before any mutation, the due-query loop stops at the transition and that pass's remaining deadline work is skipped, so nothing follows Event::Shutdown; regressions cover both. --- serf-proto/src/endpoint/mod.rs | 30 +++- serf-proto/src/endpoint/tests.rs | 228 ++++++++++++++++++++++++-- serf-proto/src/quic_endpoint/mod.rs | 4 +- serf-proto/src/stream_endpoint/mod.rs | 8 +- 4 files changed, 251 insertions(+), 19 deletions(-) diff --git a/serf-proto/src/endpoint/mod.rs b/serf-proto/src/endpoint/mod.rs index 609be9c0..e959ee7e 100644 --- a/serf-proto/src/endpoint/mod.rs +++ b/serf-proto/src/endpoint/mod.rs @@ -1133,6 +1133,16 @@ where // Query-close: tally and expire any pending queries whose deadline elapsed. self.fire_due_query_closes(now); + // A lost id-conflict vote in the query-close pass transitions the machine to + // Shutdown. Nothing may follow the terminal Event::Shutdown, so skip the rest + // of this pass's deadline work (received-query prune, leave-complete/broadcast + // clears): it would prune protocol state or transition after the terminal + // event. `serf_poll_timeout` returns `None` once Shutdown, so the un-cleared + // leave deadlines never respin the driver. + if self.state.is_shutdown() { + return; + } + // Prune expired received-query tokens. Entries for which respond() succeeded // are removed there; this catches those whose deadline elapsed without a // respond() call (driver missed the response window). @@ -4918,6 +4928,13 @@ where self.close_key_query(pq); } } + // A lost conflict close transitions the machine to Shutdown mid-loop. + // Stop before closing any remaining due query so nothing is emitted + // after the terminal Event::Shutdown (e.g. a same-deadline KeyResponse): + // the delivery contract is that nothing follows Event::Shutdown. + if self.state.is_shutdown() { + break; + } // Do not advance i: swap_remove replaced index i with the last element. } else { i += 1; @@ -5306,15 +5323,25 @@ where /// /// The local state is marked dirty so the next push-pull egress ships the /// recovered clock state. + /// + /// Refuses with [`Error::Shutdown`] on a machine that lost an id-conflict vote, + /// before any mutation: replay is a public origination path (it advances the + /// Lamport clocks, dirties the snapshot, and dials every recorded peer), so a + /// terminated node must not be able to resurrect itself through it. pub(crate) fn load_snapshot( &mut self, t: &mut T, replay: crate::snapshot::ReplayResult, now: Instant, - ) where + ) -> Result<(), Error> + where T: Reliable, A: Clone, { + // Terminal-state gate: refuse before any mutation so a Shutdown conflict + // loser can neither advance its clocks nor originate rejoin dials. + self.ensure_not_shutdown()?; + // G5: advance the member clock to at least last_clock. // Whole-message drop gate: a corrupt or adversarially-crafted snapshot with // an unacceptable ltime must not advance the local clock. @@ -5362,6 +5389,7 @@ where t.start_push_pull(addr, PushPullKind::Join, now); self.drain_inner(t); } + Ok(()) } // ── PingCompleted handler (G9 both halves) ─────────────────────────────── diff --git a/serf-proto/src/endpoint/tests.rs b/serf-proto/src/endpoint/tests.rs index 0eb49030..12db1e73 100644 --- a/serf-proto/src/endpoint/tests.rs +++ b/serf-proto/src/endpoint/tests.rs @@ -2283,6 +2283,126 @@ fn conflict_loss_transitions_to_shutdown_and_still_delivers_event() { ); } +// ── same-tick conflict-loss ordering (mid-loop / mid-pass transition) ───────── +// +// When a due conflict close loses the vote and shuts the machine down partway +// through the timeout pass, the remaining due-query closes and the rest of the +// deadline work must not run: nothing may be produced after Event::Shutdown. + +#[test] +fn same_tick_conflict_loss_stops_remaining_due_conflict_query() { + // Two conflict queries (both losing) share a deadline; the received-query + // prune of this pass would evict an expired token. The first close transitions + // to Shutdown, so the loop must stop before the second close (only ONE + // Event::Shutdown) and the prune must not run (token retained). + let mut e = ep(); + let deadline = t_secs(10); + + // First losing conflict query (1 agree, 2 disagree → matching 1 < majority 2). + let cq1 = e.test_register_conflict_query(deadline); + e.test_fold_conflict_response(cq1, 200u32, true); + e.test_fold_conflict_response(cq1, 201u32, false); + e.test_fold_conflict_response(cq1, 202u32, false); + + // Bump the query clock so the second conflict query gets a distinct QueryId + // (test_register_conflict_query stamps the ltime from the query clock). + e.test_set_clocks(0, 0, 1); + let cq2 = e.test_register_conflict_query(deadline); + e.test_fold_conflict_response(cq2, 210u32, true); + e.test_fold_conflict_response(cq2, 211u32, false); + e.test_fold_conflict_response(cq2, 212u32, false); + + // An expired received-query token: the prune step, if reached, evicts it. + let _token = e.test_register_received_query( + QueryId { + ltime: LamportTime::new(1), + id: 5, + }, + addr(1002), + t_secs(1), + ); + assert_eq!( + e.test_received_queries_len(), + 1, + "the received-query token is present before the tick" + ); + + // One tick past the shared deadline drives the whole after_inner_timeout pass. + e.handle_timeout(t_secs(20)); + + let mut shutdowns = 0; + while let Some(ev) = e.poll_event() { + if matches!(ev, Event::Shutdown) { + shutdowns += 1; + } + } + // Reverting the mid-loop break closes the second losing conflict query too, + // emitting a second Event::Shutdown → this fails. + assert_eq!( + shutdowns, 1, + "only one Event::Shutdown: the loop must stop at the first conflict loss" + ); + // Reverting the after_inner_timeout early-return runs the prune → len 0 → fails. + assert_eq!( + e.test_received_queries_len(), + 1, + "the received-query prune must not run after the same-tick Shutdown" + ); +} + +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn same_tick_conflict_loss_stops_remaining_due_key_query() { + // A conflict query (losing) is registered FIRST and a key query shares its + // deadline. When the conflict close shuts the machine down mid-loop, the loop + // must stop before closing the key query, so no KeyResponse is ever enqueued + // after Event::Shutdown, and the received-query prune of the pass must not run. + let mut e = ep(); + let deadline = t_secs(10); + + let cq = e.test_register_conflict_query(deadline); + e.test_fold_conflict_response(cq, 200u32, true); + e.test_fold_conflict_response(cq, 201u32, false); + e.test_fold_conflict_response(cq, 202u32, false); + + // Key query at the SAME deadline, registered second (distinct QueryId: id 77). + let _kq = e.test_register_key_query(deadline); + + let _token = e.test_register_received_query( + QueryId { + ltime: LamportTime::new(1), + id: 5, + }, + addr(1002), + t_secs(1), + ); + assert_eq!(e.test_received_queries_len(), 1); + + e.handle_timeout(t_secs(20)); + + let mut events = vec![]; + while let Some(ev) = e.poll_event() { + events.push(ev); + } + // Event::Shutdown must be the LAST event, and no KeyResponse may appear at all. + // Reverting the mid-loop break closes the key query after the Shutdown → + // KeyResponse lands after Shutdown → both assertions fail. + assert!( + matches!(events.last(), Some(Event::Shutdown)), + "Event::Shutdown must be the last event delivered, got {events:?}" + ); + assert!( + !events.iter().any(|ev| matches!(ev, Event::KeyResponse(_))), + "no KeyResponse may follow the conflict-loss Shutdown, got {events:?}" + ); + // Reverting the after_inner_timeout early-return runs the prune → len 0 → fails. + assert_eq!( + e.test_received_queries_len(), + 1, + "the received-query prune must not run after the same-tick Shutdown" + ); +} + // ── post-Shutdown chokepoint contract ──────────────────────────────────────── // // A machine that lost an id-conflict vote transitions to `SerfState::Shutdown` @@ -2434,6 +2554,68 @@ fn shutdown_refuses_key_management() { ); } +#[test] +fn shutdown_refuses_load_snapshot_replay() { + // snapshot replay is a public origination path: on an Alive machine it advances + // the Lamport clocks, dirties the snapshot, and dials every recorded peer. A + // Shutdown conflict loser must refuse it with Error::Shutdown before any + // mutation — otherwise it could resurrect itself and rejoin the cluster. + let mut e = ep(); + shut_down_via_lost_conflict(&mut e); + + // Clear dirty so the no-mutation assertion is unambiguous, then snapshot every + // observable the replay would move. + e.test_clear_dirty(); + let member_before = e.member_time(); + let event_before = e.event_time(); + let query_before = e.query_time(); + let event_min_before = e.test_event_min_time(); + let query_min_before = e.test_query_min_time(); + + // A replay that WOULD advance all three clocks and dial peer 2 on an Alive node. + let replay = ReplayResult { + alive_nodes: vec![snapshot_node(1, 7946), snapshot_node(2, 1002)], + last_clock: 40.into(), + last_event_clock: 50.into(), + last_query_clock: 60.into(), + }; + assert!( + matches!( + e.load_snapshot(replay, memberlist_proto::Instant::ORIGIN), + Err(Error::Shutdown) + ), + "load_snapshot must refuse on a Shutdown machine" + ); + + // Reverting the ensure_not_shutdown gate advances these clocks, dirties the + // snapshot, and pushes the rejoin dial → each assertion below fails. + assert_eq!(e.member_time(), member_before, "member clock unchanged"); + assert_eq!(e.event_time(), event_before, "event clock unchanged"); + assert_eq!(e.query_time(), query_before, "query clock unchanged"); + assert_eq!( + e.test_event_min_time(), + event_min_before, + "event min_time unchanged" + ); + assert_eq!( + e.test_query_min_time(), + query_min_before, + "query min_time unchanged" + ); + assert!( + !e.test_is_dirty(), + "the refused replay must not dirty the snapshot" + ); + assert!( + e.test_rejoin_dials().is_empty(), + "a Shutdown machine must originate no rejoin dials" + ); + assert!( + e.poll_event().is_none(), + "the refused replay must enqueue no event" + ); +} + #[test] fn shutdown_ingress_is_inert() { let mut e = ep(); @@ -2651,7 +2833,8 @@ fn load_snapshot_sets_member_clock_to_last_clock() { last_event_clock: 0.into(), last_query_clock: 0.into(), }; - e.load_snapshot(r, memberlist_proto::Instant::ORIGIN); + e.load_snapshot(r, memberlist_proto::Instant::ORIGIN) + .unwrap(); // G5: member clock >= last_clock. assert!( e.member_time() >= 5, @@ -2668,7 +2851,8 @@ fn load_snapshot_sets_event_min_time_to_last_event_clock_plus_one() { last_event_clock: 7.into(), last_query_clock: 0.into(), }; - e.load_snapshot(r, memberlist_proto::Instant::ORIGIN); + e.load_snapshot(r, memberlist_proto::Instant::ORIGIN) + .unwrap(); // G5: event_buffer.min_time = last_event_clock + 1 = 8. assert_eq!( e.test_event_min_time(), @@ -2686,7 +2870,8 @@ fn load_snapshot_sets_query_min_time_to_last_query_clock_plus_one() { last_event_clock: 0.into(), last_query_clock: 9.into(), }; - e.load_snapshot(r, memberlist_proto::Instant::ORIGIN); + e.load_snapshot(r, memberlist_proto::Instant::ORIGIN) + .unwrap(); // G5: query_buffer.min_time = last_query_clock + 1 = 10. assert_eq!( e.test_query_min_time(), @@ -2709,7 +2894,8 @@ fn load_snapshot_skips_self_on_rejoin() { last_event_clock: 7.into(), last_query_clock: 9.into(), }; - e.load_snapshot(r, memberlist_proto::Instant::ORIGIN); + e.load_snapshot(r, memberlist_proto::Instant::ORIGIN) + .unwrap(); // Dials recorded via test_last_dial_addr: the reconnect should have been issued // for node 2 only (and none for self). let dialled = e.test_rejoin_dials(); @@ -2734,7 +2920,8 @@ fn load_snapshot_empty_alive_nodes_emits_no_dials() { last_event_clock: 2.into(), last_query_clock: 1.into(), }; - e.load_snapshot(r, memberlist_proto::Instant::ORIGIN); + e.load_snapshot(r, memberlist_proto::Instant::ORIGIN) + .unwrap(); let dialled = e.test_rejoin_dials(); assert!(dialled.is_empty(), "no dials for empty alive_nodes"); } @@ -2749,7 +2936,8 @@ fn load_snapshot_all_clocks_combined() { last_event_clock: 20.into(), last_query_clock: 30.into(), }; - e.load_snapshot(r, memberlist_proto::Instant::ORIGIN); + e.load_snapshot(r, memberlist_proto::Instant::ORIGIN) + .unwrap(); assert!(e.member_time() >= 10, "member_time >= last_clock"); assert_eq!(e.test_event_min_time(), 21, "event min_time = 20 + 1"); assert_eq!(e.test_query_min_time(), 31, "query min_time = 30 + 1"); @@ -2766,7 +2954,8 @@ fn load_snapshot_marks_local_state_dirty() { last_event_clock: 0.into(), last_query_clock: 0.into(), }; - e.load_snapshot(r, memberlist_proto::Instant::ORIGIN); + e.load_snapshot(r, memberlist_proto::Instant::ORIGIN) + .unwrap(); // Dirty flag must be set so the first push-pull egress carries the recovered clocks. assert!( e.test_is_dirty(), @@ -3097,7 +3286,8 @@ fn load_snapshot_advances_event_clock() { last_event_clock: 10.into(), last_query_clock: 0.into(), }; - e.load_snapshot(r, memberlist_proto::Instant::ORIGIN); + e.load_snapshot(r, memberlist_proto::Instant::ORIGIN) + .unwrap(); assert!( e.event_time() >= 10, "event_clock must be advanced to at least last_event_clock after load_snapshot, got {}", @@ -3116,7 +3306,8 @@ fn load_snapshot_advances_query_clock() { last_event_clock: 0.into(), last_query_clock: 20.into(), }; - e.load_snapshot(r, memberlist_proto::Instant::ORIGIN); + e.load_snapshot(r, memberlist_proto::Instant::ORIGIN) + .unwrap(); assert!( e.query_time() >= 20, "query_clock must be advanced to at least last_query_clock after load_snapshot, got {}", @@ -3135,7 +3326,8 @@ fn load_snapshot_event_clock_allows_new_events_above_floor() { last_event_clock: 10.into(), last_query_clock: 0.into(), }; - e.load_snapshot(r, memberlist_proto::Instant::ORIGIN); + e.load_snapshot(r, memberlist_proto::Instant::ORIGIN) + .unwrap(); // Drain any pending events from load_snapshot. while e.poll_event().is_some() {} // Issue a new user event — must succeed and be delivered above the floor. @@ -4018,7 +4210,8 @@ fn load_snapshot_max_clocks_are_ignored() { last_query_clock: LamportTime::new(u64::MAX), alive_nodes: vec![], }; - e.load_snapshot(replay, memberlist_proto::Instant::ORIGIN); + e.load_snapshot(replay, memberlist_proto::Instant::ORIGIN) + .unwrap(); assert_eq!( e.member_time(), @@ -6620,7 +6813,7 @@ fn load_snapshot_near_watermark_no_panic_integrity_floor() { }; let now = memberlist_proto::Instant::ORIGIN; // Must not panic. - e.load_snapshot(replay, now); + e.load_snapshot(replay, now).unwrap(); // Integrity floor: buffer floors are not 0 and not u64::MAX. // (They will be LTIME_MAX = saturating_add(1) of LTIME_MAX - 1.) @@ -6740,7 +6933,8 @@ fn load_snapshot_near_watermark_integrity_floor() { last_event_clock: LamportTime::new(LTIME_MAX - 1), last_query_clock: LamportTime::new(LTIME_MAX - 1), }; - e.load_snapshot(replay, memberlist_proto::Instant::ORIGIN); + e.load_snapshot(replay, memberlist_proto::Instant::ORIGIN) + .unwrap(); // Stored clocks = LTIME_MAX (degraded state); integrity floor: not 0, not u64::MAX. assert_ne!( @@ -6793,7 +6987,8 @@ fn load_snapshot_near_watermark_integrity_floor() { last_event_clock: LamportTime::new(LTIME_MAX - 2), last_query_clock: LamportTime::new(LTIME_MAX - 2), }; - e.load_snapshot(replay, memberlist_proto::Instant::ORIGIN); + e.load_snapshot(replay, memberlist_proto::Instant::ORIGIN) + .unwrap(); // stored clocks: witness(LTIME_MAX - 2) → LTIME_MAX - 1 (saturating_add(1)). assert_eq!( @@ -6838,7 +7033,8 @@ fn load_snapshot_near_watermark_integrity_floor() { last_event_clock: LamportTime::new(LTIME_MAX), last_query_clock: LamportTime::new(LTIME_MAX), }; - e.load_snapshot(replay, memberlist_proto::Instant::ORIGIN); + e.load_snapshot(replay, memberlist_proto::Instant::ORIGIN) + .unwrap(); // Clocks must not have advanced past their pre-snapshot values. assert_eq!( @@ -6953,7 +7149,7 @@ fn all_clock_derived_values_satisfy_integrity_floor_after_near_watermark_snapsho last_query_clock: LamportTime::new(LTIME_MAX - 1), }; let now = memberlist_proto::Instant::ORIGIN; - e.load_snapshot(replay, now); + e.load_snapshot(replay, now).unwrap(); // After witnessing LTIME_MAX - 1, stored clocks = LTIME_MAX - 1. // Integrity floor: no 0, no u64::MAX. diff --git a/serf-proto/src/quic_endpoint/mod.rs b/serf-proto/src/quic_endpoint/mod.rs index 2afdbcd3..cb7f32be 100644 --- a/serf-proto/src/quic_endpoint/mod.rs +++ b/serf-proto/src/quic_endpoint/mod.rs @@ -679,11 +679,13 @@ where } /// Forwards to [`Endpoint::load_snapshot`]. + /// + /// Refuses with [`Error::Shutdown`] on a machine that lost an id-conflict vote. pub fn load_snapshot( &mut self, replay: crate::snapshot::ReplayResult, now: Instant, - ) { + ) -> Result<(), Error> { self.core.load_snapshot(&mut self.transport, replay, now) } diff --git a/serf-proto/src/stream_endpoint/mod.rs b/serf-proto/src/stream_endpoint/mod.rs index a986d293..9feb410b 100644 --- a/serf-proto/src/stream_endpoint/mod.rs +++ b/serf-proto/src/stream_endpoint/mod.rs @@ -1289,7 +1289,13 @@ where } /// Forwards to [`Endpoint::load_snapshot`]. - pub fn load_snapshot(&mut self, replay: crate::snapshot::ReplayResult, now: Instant) + /// + /// Refuses with [`Error::Shutdown`] on a machine that lost an id-conflict vote. + pub fn load_snapshot( + &mut self, + replay: crate::snapshot::ReplayResult, + now: Instant, + ) -> Result<(), Error> where A: Clone, { From 439659e22fc7e7b960f23197dcbd3bd75b75f08e Mon Sep 17 00:00:00 2001 From: Al Liu Date: Sat, 11 Jul 2026 01:09:25 +1200 Subject: [PATCH 075/128] Modify codecov.yml ignore patterns Updated ignore patterns to include benches, tests, and examples. --- .codecov.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.codecov.yml b/.codecov.yml index d30a8506..453595f6 100644 --- a/.codecov.yml +++ b/.codecov.yml @@ -2,11 +2,10 @@ codecov: require_ci_to_pass: false ignore: - - serf-core/src/serf/base/tests - - serf-core/src/serf/base/tests.rs - - serf/test - - fuzz/ - - examples/ + - "**/benches/**" + - "**/tests/**" + - "**/examples/**" + - "**/tests.rs" coverage: status: From 13d4490669b9688f68d00236a7d01aa63ab41d11 Mon Sep 17 00:00:00 2001 From: Al Liu Date: Sat, 11 Jul 2026 20:23:35 +1200 Subject: [PATCH 076/128] test(serf-proto): port the legacy Go-parity unit suite (#79) --- serf-proto/src/endpoint/mod.rs | 18 +- serf-proto/src/endpoint/tests.rs | 254 ++++++++++++++++++++++++++ serf-proto/src/members/mod.rs | 14 ++ serf-proto/src/members/tests.rs | 29 +++ serf-proto/src/stream_endpoint/mod.rs | 6 + 5 files changed, 315 insertions(+), 6 deletions(-) diff --git a/serf-proto/src/endpoint/mod.rs b/serf-proto/src/endpoint/mod.rs index e959ee7e..3d3838eb 100644 --- a/serf-proto/src/endpoint/mod.rs +++ b/serf-proto/src/endpoint/mod.rs @@ -80,7 +80,7 @@ use crate::{ QueryResponse as QueryResponseEvent, }, framing::{encode_message, peek_frame_header}, - members::{IntentKind, Member, MemberState, MemberStatus, Members, SerfState}, + members::{IntentKind, Member, MemberState, MemberStatus, Members, SerfState, remove_old_member}, options::Options, typed::{ Filter, JoinMessage, LeaveMessage, PushPullMessage, QueryFlag, QueryMessage, @@ -1798,8 +1798,8 @@ where // Clear from failed/left lists when re-joining after failure or leave. if matches!(old_status, MemberStatus::Failed | MemberStatus::Left) { - self.members.failed_members.retain(|i| i != id); - self.members.left_members.retain(|i| i != id); + remove_old_member(&mut self.members.failed_members, id); + remove_old_member(&mut self.members.left_members, id); } // Membership changed — snapshot is stale. @@ -2065,7 +2065,7 @@ where }; let member = ms.member().clone(); // Move from failed_members to left_members. - self.members.failed_members.retain(|i| i != &id_clone); + remove_old_member(&mut self.members.failed_members, &id_clone); self.members.left_members.push(id_clone); self .pending_events @@ -2098,8 +2098,8 @@ where /// recent-intent entry so no stale reference to the forgotten node survives /// in any structure. fn prune_member(&mut self, id: &I) { - self.members.left_members.retain(|i| i != id); - self.members.failed_members.retain(|i| i != id); + remove_old_member(&mut self.members.left_members, id); + remove_old_member(&mut self.members.failed_members, id); self.members.recent_intents.remove(id); #[cfg(feature = "coordinates")] @@ -3882,6 +3882,12 @@ where self.members.states.get(&id).map(|ms| ms.status()) } + /// Expose the effective broadcast queue depth cap for assertions. + #[cfg(test)] + pub(crate) fn test_queue_max(&self) -> usize { + self.queue_max() + } + /// Return the `status_time` of member `id`, or `None` if unknown. #[cfg(test)] pub(crate) fn test_member_status_time(&self, id: I) -> Option diff --git a/serf-proto/src/endpoint/tests.rs b/serf-proto/src/endpoint/tests.rs index 12db1e73..3af771dc 100644 --- a/serf-proto/src/endpoint/tests.rs +++ b/serf-proto/src/endpoint/tests.rs @@ -49,6 +49,22 @@ fn ep() -> StreamEndpoint { e } +/// Build a serf `Endpoint` with caller-supplied [`Options`], for tests that +/// need non-default queue-depth / size-limit configuration. Otherwise +/// identical to [`ep`] (deterministic RNG seed, construction self-join drained). +fn ep_with_options(opts: Options) -> StreamEndpoint { + let inner_opts = EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); + let inner = memberlist_proto::Endpoint::new_at( + inner_opts, + memberlist_proto::Instant::ORIGIN, + SmallRng::seed_from_u64(0), + ); + let mut e = StreamEndpoint::new(coord(inner), opts); + let _ = e.poll_event(); + e +} + /// Build a serf `Endpoint` with coordinates enabled (for coordinate-gated tests). #[cfg(feature = "coordinates")] fn ep_with_coords() -> StreamEndpoint { @@ -6699,6 +6715,49 @@ mod key_request_responder { "Key field in Debug must show redacted: {debug_str}" ); } + + /// Mirrors the size-bound invariant behind Go serf's key-list response sizing + /// (`estimate_max_keys_in_list_key_response_factor`, + /// legacy/serf-core/src/serf/base/tests.rs): a key response is bounded by + /// `query_response_size_limit`. + /// + /// The legacy `key_list_response_with_correct_size` helper TRUNCATED the key + /// list to fit and set a "truncated" message. The Sans-I/O machine builds no + /// response itself — response construction (and any truncation) is a driver + /// concern — so `respond_key` instead REJECTS an over-limit key response via the + /// shared `respond_inner` size guard, returning `RespondTooLarge`. + #[test] + fn respond_key_oversized_key_list_is_rejected() { + let mut e = ep(); + let key = test_key(); + let q = make_key_query("_serf_list_keys", None); + e.test_handle_query(q); + let req = match e.poll_event().expect("must emit KeyRequest") { + Event::KeyRequest(kr) => kr, + other => panic!( + "expected KeyRequest, got {:?}", + core::mem::discriminant(&other) + ), + }; + // A key list far larger than the default query_response_size_limit (1024). + let keys = vec![key; 512]; + let err = e + .respond_key( + &req, + KeyResponseArgs { + result: true, + message: smol_str::SmolStr::default(), + keys, + primary_key: Some(key), + }, + memberlist_proto::Instant::ORIGIN, + ) + .expect_err("an over-limit key response must be rejected, not truncated"); + assert!( + matches!(err, Error::RespondTooLarge(_, _)), + "oversized key response must return RespondTooLarge, got {err:?}" + ); + } } // ── Lamport watermark boundary: integrity floor ─────────────────────────────── @@ -7688,3 +7747,198 @@ fn set_tags_before_join_does_not_emit_update_before_join() { ); } } + +// ── legacy Go-parity ports (bucket-A/bucket-B) ──────────────────────────────── + +/// Mirrors Go serf `serf_get_queue_max` +/// (legacy/serf/test/main/net/get_queue_max.rs → serf_core::tests::serf_get_queue_max): +/// the broadcast queue depth cap is the flat `max_queue_depth` when +/// `min_queue_depth == 0`; otherwise `max(min_queue_depth, 2 * num_members)`, so +/// the member-count branch takes precedence once `2 * num_members` exceeds the +/// configured floor. Asserted against the machine's own `queue_max()` and its +/// default constants, not the legacy runtime's fixed 4096/1024/200/202. +#[test] +fn queue_max_respects_min_queue_depth_and_member_count() { + // min_queue_depth == 0: the flat max_queue_depth applies regardless of members. + let mut e = ep(); + for id in 2u32..=101 { + e.test_seed_member(id, MemberStatus::Alive, LamportTime::new(0)); + } + assert_eq!( + e.test_queue_max(), + Options::new().max_queue_depth(), + "min_queue_depth == 0 must use the flat max_queue_depth" + ); + + // A min_queue_depth above 2 * num_members wins over the member-count branch + // (with ~101 members, 2 * 101 = 202 < 1024). + let mut e = ep_with_options(Options::new().with_min_queue_depth(1024)); + for id in 2u32..=101 { + e.test_seed_member(id, MemberStatus::Alive, LamportTime::new(0)); + } + assert_eq!( + e.test_queue_max(), + 1024, + "min_queue_depth must win when it exceeds 2 * num_members" + ); + + // A small min_queue_depth is overridden by the 2 * num_members branch. + let mut e = ep_with_options(Options::new().with_min_queue_depth(16)); + for id in 2u32..=101 { + e.test_seed_member(id, MemberStatus::Alive, LamportTime::new(0)); + } + let n = e.num_members(); + assert_eq!( + e.test_queue_max(), + 2 * n, + "2 * num_members must win when it exceeds min_queue_depth" + ); + + // Adjusting the node count scales the member-count branch by exactly 2. + e.test_seed_member(1000u32, MemberStatus::Alive, LamportTime::new(0)); + assert_eq!( + e.test_queue_max(), + 2 * (n + 1), + "the member-count branch must track num_members" + ); +} + +/// Mirrors Go serf `queries_conflict_same_name` +/// (legacy/serf-core/src/serf/base/tests.rs): an internal `_serf_conflict` query +/// is handled autonomously and must NOT surface to the app as `Event::Query`. +/// +/// The legacy `SerfQueries` event filter intercepted `InternalQueryEvent::Conflict` +/// before the app channel; the Sans-I/O machine intercepts `_serf_conflict` in +/// `handle_query` (it responds via `handle_conflict_query` and returns without +/// pushing `Event::Query`). +#[test] +fn conflict_query_never_surfaces_as_app_query() { + let mut e = ep(); + // A well-formed conflict query: payload is an exactly-encoded node id. + let id_bytes = (42u32).encode_to_bytes().unwrap(); + let q = QueryMessage { + ltime: LamportTime::new(3), + id: 77, + from: memberlist_proto::Node::new(99u32, "127.0.0.1:9999".parse().unwrap()), + filters: vec![], + flags: QueryFlag::empty(), + relay_factor: 0, + timeout: core::time::Duration::from_secs(5), + name: "_serf_conflict".into(), + payload: Bytes::from(id_bytes.to_vec()), + }; + e.test_handle_query(q); + assert!( + e.poll_event().is_none(), + "internal _serf_conflict query must never surface as Event::Query" + ); +} + +/// Mirrors Go serf `delegate_nodemeta` + `delegate_nodemeta_panic` +/// (legacy/serf-core/src/serf/base/tests/serf/delegate.rs): tags round-trip +/// through the node-meta byte encoding, and tags that exceed the meta size limit +/// are rejected. +/// +/// Adaptation: the legacy `delegate_nodemeta_panic` asserted a PANIC when the +/// encoded tags exceed the memberlist meta limit. The Sans-I/O `set_tags` +/// deliberately returns `Error::SetTagsMeta` instead of panicking (see its doc +/// comment: "Surface this as `SetTagsMeta` rather than panicking so the driver +/// can log and retry"). The invariant — over-limit tags are rejected — is +/// preserved with the machine's own limit (`Meta::MAX_SIZE`) and rejection +/// mechanism. +#[test] +fn set_tags_meta_round_trips_and_rejects_oversize() { + use crate::typed::Tags; + + let mut e = ep(); + + // Round-trip: a single `role=test` tag encodes into node meta and decodes back. + let tags: Tags = [("role", "test")].into_iter().collect(); + e.set_tags(tags) + .expect("set_tags must succeed on a live endpoint"); + let meta = e + .test_local_meta() + .expect("local node meta must be present after set_tags"); + let decoded = decode_tags_from_meta(meta.as_bytes()) + .expect("meta written by set_tags must decode as valid Tags"); + assert_eq!( + decoded.0.get("role").map(|s| s.as_str()), + Some("test"), + "role tag must round-trip through the node-meta byte encoding" + ); + + // Oversize: tags whose encoding exceeds the meta cap are rejected (not panicked). + let huge = "x".repeat(70_000); + let oversize: Tags = [("big", huge.as_str())].into_iter().collect(); + let err = e + .set_tags(oversize) + .expect_err("tags exceeding the meta size limit must be rejected"); + assert!( + matches!(err, Error::SetTagsMeta(_)), + "oversize tags must return SetTagsMeta, got {err:?}" + ); +} + +/// Mirrors Go serf `serf_query_size_limit_increased` +/// (legacy/serf-core/src/serf/base/tests/serf/event.rs): raising `query_size_limit` +/// admits a query that the default limit rejects. The companion +/// `query_size_limit_is_enforced` shows a ~1500-byte query is rejected at the +/// default limit (1024); here the same query succeeds once the limit is doubled. +#[test] +fn raised_query_size_limit_admits_larger_query() { + let default_limit = Options::new().query_size_limit(); + let mut e = ep_with_options(Options::new().with_query_size_limit(default_limit * 2)); + // A payload that exceeds the default limit but fits within the doubled limit. + let payload = bytes::Bytes::from(vec![0u8; 1500]); + let result = e.query( + "this is too large a query", + payload, + QueryParams::default(), + memberlist_proto::Instant::ORIGIN, + ); + assert!( + result.is_ok(), + "a query rejected at the default limit must succeed once the limit is doubled: {result:?}" + ); +} + +/// Mirrors Go serf `default_query` +/// (legacy/serf-core/src/serf/base/tests/serf/event.rs): the default query +/// parameters carry no filters, request no ack, and use no relay. +/// +/// The legacy test also asserts `timeout == gossip_interval * query_timeout_mult`. +/// That value is a DRIVER concern in the Sans-I/O split: the machine cannot read +/// the inner gossip_interval, so `query()` substitutes a placeholder base when the +/// timeout is the zero sentinel (see endpoint/mod.rs). The gossip-derived timeout +/// is covered by the reactor driver test `tcp_default_query_param_defaults`. This +/// asserts the machine-level default params, including the zero-timeout sentinel +/// that triggers the substitution. +#[test] +fn default_query_params_are_empty_no_ack_no_relay() { + let params = QueryParams::::default(); + assert!(params.filters.is_empty(), "default filters must be empty"); + assert!(!params.request_ack, "default must not request an ack"); + assert_eq!(params.relay_factor, 0, "default relay_factor must be zero"); + assert!( + params.timeout.is_zero(), + "default timeout must be the zero sentinel (the machine computes the effective value)" + ); +} + +/// Mirrors Go serf `serf_remove_failed_node_ourself` +/// (legacy/serf-core/src/serf/base/tests/serf/remove.rs): removing a node that is +/// not a member is a safe no-op — it returns Ok and does not fabricate a member. +/// +/// `remove_failed_node` maps to `force_leave(id, prune = false)` in the machine. +#[test] +fn force_leave_of_absent_node_is_ok_and_adds_no_member() { + let mut e = ep(); + let before = e.num_members(); + e.force_leave(999u32, false, memberlist_proto::Instant::ORIGIN) + .expect("force_leave of an absent node must be a safe no-op"); + assert_eq!( + e.num_members(), + before, + "force_leave of an absent node must not fabricate a member" + ); +} diff --git a/serf-proto/src/members/mod.rs b/serf-proto/src/members/mod.rs index 55c41fcd..93101659 100644 --- a/serf-proto/src/members/mod.rs +++ b/serf-proto/src/members/mod.rs @@ -413,5 +413,19 @@ where } } +/// Removes a node `id` from a reaper index list (`left_members` / +/// `failed_members`), retaining every other entry. +/// +/// Mirrors Go serf `base.rs` `remove_old_member`. The legacy helper retained +/// `MemberState`s by node id out of an `OneOrMore`; the Sans-I/O +/// machine tracks the reaper index lists as plain id `Vec`s (the full state +/// lives in `Members::states`), so this retains ids directly. +pub(crate) fn remove_old_member(old: &mut Vec, id: &I) +where + I: Eq, +{ + old.retain(|i| i != id); +} + #[cfg(test)] mod tests; diff --git a/serf-proto/src/members/tests.rs b/serf-proto/src/members/tests.rs index 4d95c463..a4e2bdac 100644 --- a/serf-proto/src/members/tests.rs +++ b/serf-proto/src/members/tests.rs @@ -179,3 +179,32 @@ fn recent_intent_kind_mismatch_returns_none() { Some(LamportTime::new(3)) ); } + +/// Mirrors Go serf `test_remove_old_member` +/// (legacy/serf-core/src/serf/base/tests/serf/remove.rs): removing a named node +/// from a reaper index list drops only that entry and retains the others. +/// +/// The legacy helper retained `MemberState`s by node id; the Sans-I/O machine +/// tracks the index lists as plain id `Vec`s, so this asserts the same invariant +/// against the id-list form used by `handle_node_join` reconcile and +/// `prune_member`. +#[test] +fn remove_old_member_drops_only_the_named_id() { + use smol_str::SmolStr; + + let mut old: Vec = vec!["foo".into(), "bar".into(), "baz".into()]; + remove_old_member(&mut old, &SmolStr::from("bar")); + assert_eq!(old.len(), 2); + assert!( + !old.contains(&SmolStr::from("bar")), + "named id must be removed" + ); + assert!( + old.contains(&SmolStr::from("foo")), + "other ids must be retained" + ); + assert!( + old.contains(&SmolStr::from("baz")), + "other ids must be retained" + ); +} diff --git a/serf-proto/src/stream_endpoint/mod.rs b/serf-proto/src/stream_endpoint/mod.rs index 9feb410b..0fbb69ad 100644 --- a/serf-proto/src/stream_endpoint/mod.rs +++ b/serf-proto/src/stream_endpoint/mod.rs @@ -697,6 +697,12 @@ where self.core.test_member_status(id) } + /// Forwards to [`Endpoint::test_queue_max`]. + #[cfg(test)] + pub(crate) fn test_queue_max(&self) -> usize { + self.core.test_queue_max() + } + /// Forwards to [`Endpoint::test_member_status_time`]. #[cfg(test)] pub(crate) fn test_member_status_time(&self, id: I) -> Option From cf4df860b0151a9956978d5758afa9d189375ce8 Mon Sep 17 00:00:00 2001 From: Al Liu Date: Sat, 11 Jul 2026 20:54:10 +1200 Subject: [PATCH 077/128] feat: event coalescing for the Sans-I/O serf (member + user), with bounded memory (#80) --- serf-compio/src/driver/quic/mod.rs | 41 +- serf-compio/src/driver/quic/tests.rs | 19 +- serf-compio/src/driver/stream/mod.rs | 46 +- serf-compio/src/driver/stream/tests.rs | 16 +- serf-compio/src/drop_counter/mod.rs | 50 + serf-compio/src/drop_counter/tests.rs | 27 + serf-compio/src/lib.rs | 2 + serf-compio/src/quic/mod.rs | 9 +- serf-compio/src/serf/mod.rs | 42 +- serf-compio/src/serf/tests.rs | 91 ++ serf-compio/src/tcp/mod.rs | 20 +- serf-compio/src/tls/mod.rs | 9 +- serf-compio/src/transport/runtime.rs | 13 +- serf-embassy/src/serf/mod.rs | 30 +- serf-embassy/src/shared/tests.rs | 47 + serf-embedded/src/engine/mod.rs | 28 +- serf-embedded/src/engine/tests.rs | 53 +- serf-proto/src/coalesce/drop_counter/mod.rs | 36 + serf-proto/src/coalesce/drop_counter/tests.rs | 21 + serf-proto/src/coalesce/mod.rs | 522 ++++++++++ serf-proto/src/coalesce/tests.rs | 976 ++++++++++++++++++ serf-proto/src/endpoint/mod.rs | 327 +++++- serf-proto/src/endpoint/serf_parity_tests.rs | 2 +- serf-proto/src/endpoint/tests.rs | 704 ++++++++++++- serf-proto/src/lib.rs | 5 + serf-proto/src/options/mod.rs | 133 +++ serf-proto/src/options/tests.rs | 104 +- serf-proto/src/quic_endpoint/mod.rs | 75 +- serf-proto/src/quic_endpoint/tests.rs | 64 +- serf-proto/src/stream_endpoint/mod.rs | 80 +- serf-proto/src/stream_endpoint/tests.rs | 2 +- serf-reactor/src/driver/quic/mod.rs | 17 +- serf-reactor/src/driver/quic/tests.rs | 16 +- serf-reactor/src/driver/stream/mod.rs | 15 +- serf-reactor/src/driver/stream/tests.rs | 32 +- serf-reactor/src/drop_counter/mod.rs | 72 ++ serf-reactor/src/drop_counter/tests.rs | 45 + serf-reactor/src/lib.rs | 2 + serf-reactor/src/quic/mod.rs | 9 +- serf-reactor/src/serf/mod.rs | 30 +- serf-reactor/src/serf/tests.rs | 131 +++ serf-reactor/src/shared/mod.rs | 30 +- serf-reactor/src/tcp/mod.rs | 20 +- serf-reactor/src/tls/mod.rs | 9 +- serf-reactor/src/transport/runtime.rs | 11 +- serf-smoltcp/src/serf/mod.rs | 27 +- serf-smoltcp/src/serf/tests.rs | 49 + serf-smoltcp/tests/key_management.rs | 9 +- serf-smoltcp/tests/user_event.rs | 2 +- 49 files changed, 3920 insertions(+), 200 deletions(-) create mode 100644 serf-compio/src/drop_counter/mod.rs create mode 100644 serf-compio/src/drop_counter/tests.rs create mode 100644 serf-proto/src/coalesce/drop_counter/mod.rs create mode 100644 serf-proto/src/coalesce/drop_counter/tests.rs create mode 100644 serf-proto/src/coalesce/mod.rs create mode 100644 serf-proto/src/coalesce/tests.rs create mode 100644 serf-reactor/src/drop_counter/mod.rs create mode 100644 serf-reactor/src/drop_counter/tests.rs diff --git a/serf-compio/src/driver/quic/mod.rs b/serf-compio/src/driver/quic/mod.rs index c703a781..9dca5919 100644 --- a/serf-compio/src/driver/quic/mod.rs +++ b/serf-compio/src/driver/quic/mod.rs @@ -15,6 +15,11 @@ //! handles communicate exclusively via the command channel and read state through //! the lock-free snapshot. The socket drops when the loop exits so the bound port //! is released before shutdown returns. +//! +//! The endpoint holds a write-capable [`CompioDropCounter`] over the same +//! `Rc>` a `Serf` handle reads through its read-only counterpart, so +//! the cumulative coalescer drop counts are observed directly with no publish +//! step — a shed on the pump is visible on the next handle read. #![cfg(feature = "quic")] @@ -58,6 +63,7 @@ use crate::{ observation_payload_bytes, yield_once, }, }, + drop_counter::CompioDropCounter, error::{JoinFailed, Result, SerfError}, snapshot::{SerfSnapshot, SnapshotCell}, }; @@ -154,7 +160,7 @@ impl PendingJoin { /// `(eid, peer, succeeded)` rather than the `Event` so it is callable without /// constructing a coordinator-internal `ExchangeCompleted`. fn complete_join_exchange( - endpoint: &mut QuicEndpoint, + endpoint: &mut QuicEndpoint, pending_joins: &mut Vec, eid: ExchangeId, peer: SocketAddr, @@ -283,7 +289,7 @@ fn recv_buf_len_for(gossip_mtu: usize, quic_max_udp_payload: u64) -> usize { /// happen here; reads happen via the published [`SerfSnapshot`]. #[allow(clippy::too_many_arguments)] pub(crate) async fn quic_driver_loop( - mut endpoint: QuicEndpoint, + mut endpoint: QuicEndpoint, gossip_socket: UdpSocket, // The quinn `EndpointConfig`'s accepted max UDP payload, read off the // `QuicOptions` in `QuicTransport::run` (the driver cannot reach the quinn @@ -720,7 +726,7 @@ fn reply_shutdown(c: Command) { /// `shutdown_reply` so the pump acks the caller only AFTER the socket drops in /// the post-loop cleanup. async fn dispatch_command( - endpoint: &mut QuicEndpoint, + endpoint: &mut QuicEndpoint, shutdown_reply: &mut Option>>, pending: &mut PendingCommands, leave_timeout: Duration, @@ -855,7 +861,7 @@ async fn dispatch_command( let name = cmd.name().clone(); let payload = cmd.payload().clone(); endpoint - .user_event(name, payload, cmd.coalesce) + .user_event(name, payload, cmd.coalesce, now) .map_err(SerfError::from) } else { Err(SerfError::NotRunning) @@ -893,7 +899,7 @@ async fn dispatch_command( } Command::SetTags(SetTagsCmd { tags, reply }) => { let res = if running { - endpoint.set_tags(tags).map_err(SerfError::from) + endpoint.set_tags(tags, now).map_err(SerfError::from) } else { Err(SerfError::NotRunning) }; @@ -977,7 +983,7 @@ async fn dispatch_command( /// (the main loop drops its `recv_fut` before invoking this), so the bounded drain /// is the sole builder of recv SQEs here. async fn fire_quic_timeout( - endpoint: &mut QuicEndpoint, + endpoint: &mut QuicEndpoint, gossip_socket: &UdpSocket, recv_buf_len: usize, driver_opts: RuntimeOptions, @@ -1044,7 +1050,10 @@ where /// cluster label is verified; with none built in the serf gossip plane carries no /// wire transforms, so the raw bytes ARE the label frame. A compound datagram is /// split into its ordered messages by `parse_messages`, each fed as a typed message. -fn drain_ingress(endpoint: &mut QuicEndpoint, label: &Option) -> bool +fn drain_ingress( + endpoint: &mut QuicEndpoint, + label: &Option, +) -> bool where I: memberlist_proto::Id + Clone, G: Rng, @@ -1092,7 +1101,7 @@ where /// `encode_outgoing_compound`), then — with an encryption backend built in — /// wrapped in the encryption layer (`encrypt_gossip`) before it hits the wire. async fn drain_transmits( - endpoint: &mut QuicEndpoint, + endpoint: &mut QuicEndpoint, gossip_socket: &UdpSocket, label: Option, ) -> bool @@ -1149,7 +1158,7 @@ where /// the coordinator queued, and send it on the shared UDP socket. These are /// already framed by quinn-proto, so no codec wrap is applied. async fn drain_quic_transmits( - endpoint: &mut QuicEndpoint, + endpoint: &mut QuicEndpoint, gossip_socket: &UdpSocket, ) -> bool where @@ -1178,7 +1187,7 @@ where /// is still delivered to subscribers before the main loop breaks. #[allow(clippy::too_many_arguments)] async fn drain_events( - endpoint: &mut QuicEndpoint, + endpoint: &mut QuicEndpoint, obs_tx: &mpsc::Sender>, observation_dropped: &Cell, obs_payload_bytes: &Cell, @@ -1286,7 +1295,7 @@ where /// teardown. #[allow(clippy::too_many_arguments)] async fn drain_outputs( - endpoint: &mut QuicEndpoint, + endpoint: &mut QuicEndpoint, gossip_socket: &UdpSocket, label: &Option, obs_tx: &mpsc::Sender>, @@ -1338,7 +1347,7 @@ where /// leaves the wire untouched. #[cfg(encryption)] fn apply_key_request_live( - endpoint: &mut QuicEndpoint, + endpoint: &mut QuicEndpoint, delegate: &dyn KeyringDelegate, req: &KeyRequest, ) -> KeyResponseArgs @@ -1422,7 +1431,7 @@ async fn observation_task( /// not merge. Mirrors the stream driver's `reap_pending_joins`; `swap_remove` is /// sound because `joins` has no ordering. async fn reap_pending_joins( - endpoint: &mut QuicEndpoint, + endpoint: &mut QuicEndpoint, pending_joins: &mut Vec, now: Instant, ) where @@ -1485,8 +1494,10 @@ fn min_pending_leave_deadline(pending_leave: &Option) -> Option(endpoint: &QuicEndpoint, snapshot: &SnapshotCell) -where +fn refresh_snapshot( + endpoint: &QuicEndpoint, + snapshot: &SnapshotCell, +) where I: memberlist_proto::Id + Clone, G: Rng, R: Rng + SeedableRng, diff --git a/serf-compio/src/driver/quic/tests.rs b/serf-compio/src/driver/quic/tests.rs index c57a6df6..2f8deea2 100644 --- a/serf-compio/src/driver/quic/tests.rs +++ b/serf-compio/src/driver/quic/tests.rs @@ -106,16 +106,29 @@ fn test_quic_options() -> QuicOptions { ) } +/// The driver endpoint, pinning the compio driver's shared drop-counter storage. +type DrvEndpoint = QuicEndpoint; + /// Build a standalone serf `QuicEndpoint` over a memberlist QUIC coordinator — /// no bound socket, no driver loop; just the composed machine, for driving the /// ingress surface directly. -fn build_endpoint() -> QuicEndpoint { +fn build_endpoint() -> DrvEndpoint { let local: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); let inner_opts = memberlist_proto::EndpointOptions::new(SmolStr::new("node"), local) .with_user_broadcast_tiers(NonZeroU8::new(3).expect("3 is nonzero")); let inner = memberlist_proto::Endpoint::new(inner_opts, StdRng::seed_from_u64(1)); let coord = memberlist_proto::QuicEndpoint::new(inner, test_quic_options()); - QuicEndpoint::new_with_rng(coord, SerfOptions::new(), StdRng::seed_from_u64(2)) + // These pump tests do not assert coalescer shed counts, so the write halves + // suffice; the read halves are unused here. + let (user_drop, _) = crate::drop_counter::drop_channel(); + let (member_drop, _) = crate::drop_counter::drop_channel(); + QuicEndpoint::new_with_rng_in( + coord, + SerfOptions::new(), + StdRng::seed_from_u64(2), + user_drop, + member_drop, + ) } /// The QUIC past-due drain must DECODE every drained gossip datagram before @@ -271,7 +284,7 @@ async fn fire_quic_timeout_drains_socket_before_handle_timeout() { /// `deadline` already in the past. Returns the join's `(StreamId, ExchangeId)` /// and the oneshot receiver the caller awaits. fn park_ignore_old_join( - endpoint: &mut QuicEndpoint, + endpoint: &mut DrvEndpoint, pending_joins: &mut Vec, deadline: Instant, ) -> (StreamId, ExchangeId, oneshot::Receiver) { diff --git a/serf-compio/src/driver/stream/mod.rs b/serf-compio/src/driver/stream/mod.rs index 0e6cc275..dae385c2 100644 --- a/serf-compio/src/driver/stream/mod.rs +++ b/serf-compio/src/driver/stream/mod.rs @@ -13,6 +13,11 @@ //! channel and read state through the lock-free snapshot. The listener and //! gossip socket are explicitly closed (awaited) when the loop exits so the //! bound ports are released before shutdown returns. +//! +//! The endpoint holds a write-capable [`CompioDropCounter`] over the same +//! `Rc>` a `Serf` handle reads through its read-only counterpart, so +//! the cumulative coalescer drop counts are observed directly with no publish +//! step — a shed on the pump is visible on the next handle read. use std::{ cell::Cell, @@ -66,6 +71,7 @@ use crate::{ observation_payload_bytes, yield_once, }, }, + drop_counter::CompioDropCounter, error::{JoinFailed, Result, SerfError}, snapshot::{SerfSnapshot, SnapshotCell}, }; @@ -164,7 +170,7 @@ impl PendingJoin { /// `(eid, peer, succeeded)` rather than the `Event` so it is callable without /// constructing a coordinator-internal `ExchangeCompleted`. fn complete_join_exchange( - endpoint: &mut StreamEndpoint, + endpoint: &mut StreamEndpoint, pending_joins: &mut Vec, eid: ExchangeId, peer: SocketAddr, @@ -386,7 +392,9 @@ const ENCRYPTED_WRAPPER_OVERHEAD: usize = 0; /// clamped at [`GOSSIP_RECV_BUF_MAX`]. Sizing to the inflated value keeps a /// configured `gossip_mtu` close to the historical default from being truncated /// once the encryption tag/nonce are added on the wire. -fn gossip_recv_buf_len(endpoint: &StreamEndpoint) -> usize +fn gossip_recv_buf_len( + endpoint: &StreamEndpoint, +) -> usize where I: memberlist_proto::Id + Clone, RT: StreamTransport, @@ -406,7 +414,7 @@ where /// endpoint happen here; reads happen via the published [`SerfSnapshot`]. #[allow(clippy::too_many_arguments)] pub(crate) async fn stream_driver_loop( - mut endpoint: StreamEndpoint, + mut endpoint: StreamEndpoint, gossip_socket: UdpSocket, listener: TcpListener, commands: Receiver>, @@ -993,7 +1001,7 @@ fn reply_shutdown(c: Command) { /// the post-loop cleanup. #[allow(clippy::too_many_arguments)] async fn dispatch_command( - endpoint: &mut StreamEndpoint, + endpoint: &mut StreamEndpoint, bridges: &mut HashMap, bridge_ready_tx: &Sender, stream_opts: StreamTransportOptions, @@ -1154,7 +1162,7 @@ async fn dispatch_command( let name = cmd.name().clone(); let payload = cmd.payload().clone(); endpoint - .user_event(name, payload, cmd.coalesce) + .user_event(name, payload, cmd.coalesce, now) .map_err(SerfError::from) } else { Err(SerfError::NotRunning) @@ -1192,7 +1200,7 @@ async fn dispatch_command( } Command::SetTags(SetTagsCmd { tags, reply }) => { let res = if running { - endpoint.set_tags(tags).map_err(SerfError::from) + endpoint.set_tags(tags, now).map_err(SerfError::from) } else { Err(SerfError::NotRunning) }; @@ -1266,7 +1274,7 @@ async fn dispatch_command( /// Route one bridge inbound message into the coordinator. fn dispatch_bridge_inbound( - endpoint: &mut StreamEndpoint, + endpoint: &mut StreamEndpoint, inbound: BridgeInbound, ) where I: memberlist_proto::Id + Clone, @@ -1312,7 +1320,7 @@ fn dispatch_bridge_inbound( /// compound datagram is split into its ordered messages by `parse_messages`, /// each fed as a typed message. fn dispatch_gossip( - endpoint: &mut StreamEndpoint, + endpoint: &mut StreamEndpoint, src: SocketAddr, datagram: &[u8], now: Instant, @@ -1455,7 +1463,7 @@ fn process_one_action( /// Drain every [`StreamAction`] the coordinator has queued. Returns `true` iff /// any action was processed. fn drain_actions( - endpoint: &mut StreamEndpoint, + endpoint: &mut StreamEndpoint, bridges: &mut HashMap, bridge_ready_tx: &Sender, stream_opts: StreamTransportOptions, @@ -1479,7 +1487,7 @@ where /// past a pending `Shutdown` / `Close` — the coordinator withholds the teardown /// for an exchange until its `poll_transport_transmit` queue is empty. fn drain_transport_transmits( - endpoint: &mut StreamEndpoint, + endpoint: &mut StreamEndpoint, bridges: &HashMap, ) -> bool where @@ -1508,7 +1516,7 @@ where /// `encode_outgoing_compound`), then — with an encryption backend built in — /// wrapped in the encryption layer (`encrypt_gossip`) before it hits the wire. async fn drain_transmits( - endpoint: &mut StreamEndpoint, + endpoint: &mut StreamEndpoint, gossip_socket: &UdpSocket, label: Option, ) -> bool @@ -1574,7 +1582,7 @@ where /// read-only `list` or a refused op leaves the wire untouched. #[cfg(encryption)] fn apply_key_request_live( - endpoint: &mut StreamEndpoint, + endpoint: &mut StreamEndpoint, delegate: &dyn KeyringDelegate, req: &KeyRequest, ) -> KeyResponseArgs @@ -1611,7 +1619,7 @@ where /// event is still delivered to subscribers before the main loop breaks. #[allow(clippy::too_many_arguments)] async fn drain_events( - endpoint: &mut StreamEndpoint, + endpoint: &mut StreamEndpoint, obs_tx: &mpsc::Sender>, observation_dropped: &Cell, obs_payload_bytes: &Cell, @@ -1720,7 +1728,7 @@ where /// break into teardown. #[allow(clippy::too_many_arguments)] async fn drain_outputs( - endpoint: &mut StreamEndpoint, + endpoint: &mut StreamEndpoint, bridges: &mut HashMap, bridge_ready_tx: &Sender, stream_opts: StreamTransportOptions, @@ -1821,7 +1829,7 @@ async fn observation_task( /// consumed is absent, so the clear removes only the streams whose exchange did /// not merge. `swap_remove` is sound because `joins` has no ordering. async fn reap_pending_joins( - endpoint: &mut StreamEndpoint, + endpoint: &mut StreamEndpoint, pending_joins: &mut Vec, now: Instant, ) where @@ -1897,7 +1905,7 @@ fn min_pending_leave_deadline(pending_leave: &Option) -> Option( - endpoint: &mut StreamEndpoint, + endpoint: &mut StreamEndpoint, bridges: &mut HashMap, bridge_inbound_tx: &mpsc::Sender, bridge_inbound_rx: &mut mpsc::Receiver, @@ -1987,7 +1995,7 @@ where /// snapshot — seeded at construction — stays current, and `SerfSnapshot::new` /// (which requires the local node) is never called with it absent. fn refresh_snapshot( - endpoint: &StreamEndpoint, + endpoint: &StreamEndpoint, snapshot: &SnapshotCell, ) where I: memberlist_proto::Id + Clone, @@ -2014,7 +2022,7 @@ fn refresh_snapshot( /// Route one [`BridgeReady`] — an outbound-dial result — into the coordinator, /// spawning a per-bridge byte-mover on success. fn handle_bridge_ready( - endpoint: &mut StreamEndpoint, + endpoint: &mut StreamEndpoint, bridges: &mut HashMap, bridge_inbound_tx: &mpsc::Sender, ready: BridgeReady, @@ -2067,7 +2075,7 @@ fn handle_bridge_ready( /// processed (a state-affecting event the caller treats as dirty). fn handle_accepted( accepted: io::Result<(TcpStream, SocketAddr)>, - endpoint: &mut StreamEndpoint, + endpoint: &mut StreamEndpoint, bridges: &mut HashMap, bridge_inbound_tx: &mpsc::Sender, stream_opts: StreamTransportOptions, diff --git a/serf-compio/src/driver/stream/tests.rs b/serf-compio/src/driver/stream/tests.rs index bee1e58f..1b201568 100644 --- a/serf-compio/src/driver/stream/tests.rs +++ b/serf-compio/src/driver/stream/tests.rs @@ -20,11 +20,15 @@ use rand::rngs::StdRng; use serf_proto::options::Options as SerfOptions; use smol_str::SmolStr; +/// The driver endpoint, pinning the compio driver's shared drop-counter storage. +type DrvEndpoint = + StreamEndpoint; + /// Build a standalone plain-TCP serf `StreamEndpoint` over a memberlist stream /// coordinator — no bound socket, no driver loop; just the composed machine, for /// driving the gossip ingress surface directly. Mirrors the coordinator the TCP /// transport's `run` builds. -fn build_endpoint() -> StreamEndpoint { +fn build_endpoint() -> DrvEndpoint { let local: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); let inner_opts = EndpointOptions::new(SmolStr::new("node"), local) .with_user_broadcast_tiers(NonZeroU8::new(3).expect("3 is nonzero")); @@ -35,10 +39,16 @@ fn build_endpoint() -> StreamEndpoint::new_with_rng( + // These pump tests do not assert coalescer shed counts, so the write halves + // suffice; the read halves are unused here. + let (user_drop, _) = crate::drop_counter::drop_channel(); + let (member_drop, _) = crate::drop_counter::drop_channel(); + StreamEndpoint::::new_with_rng_in( coord, SerfOptions::new(), StdRng::seed_from_u64(2), + user_drop, + member_drop, ) } @@ -151,7 +161,7 @@ async fn fire_timeout_with_drain_drains_socket_before_handle_timeout() { /// `deadline` already in the past. Returns the join's `(StreamId, ExchangeId)` /// and the oneshot receiver the caller awaits. fn park_ignore_old_join( - endpoint: &mut StreamEndpoint, + endpoint: &mut DrvEndpoint, pending_joins: &mut Vec, deadline: Instant, ) -> ( diff --git a/serf-compio/src/drop_counter/mod.rs b/serf-compio/src/drop_counter/mod.rs new file mode 100644 index 00000000..c61b0d89 --- /dev/null +++ b/serf-compio/src/drop_counter/mod.rs @@ -0,0 +1,50 @@ +//! The shared coalescer-drop counter split into a write-capable half held by the +//! driver-owned endpoint and a read-only half held by the handle's `Shared`. +//! +//! Both wrap the SAME `Rc>` (minted once by [`drop_channel`]). compio's +//! driver is single-threaded and `!Send`, so an `Rc>` — atomics-free — +//! is the right backing: the endpoint increments it on the pump and a `Serf` +//! handle clone reads it on the same executor thread, with no publish step. The +//! reader exposes no mutator, so a handle has no type-level path to write. + +use std::{cell::Cell, rc::Rc}; + +use serf_proto::DropCounter; + +/// The write-capable half, moved into the serf endpoint the driver pumps. +pub(crate) struct CompioDropCounter(Rc>); + +impl DropCounter for CompioDropCounter { + #[inline] + fn incr_saturating(&mut self) { + self.0.set(self.0.get().saturating_add(1)); + } + + #[inline] + fn get(&self) -> u64 { + self.0.get() + } +} + +/// The read-only half, kept on the handle's `Shared`. Exposes only a load, so a +/// `Serf` clone can never write the counter. +pub(crate) struct DropReader(Rc>); + +impl DropReader { + /// The endpoint's current cumulative shed count. + #[inline] + pub(crate) fn get(&self) -> u64 { + self.0.get() + } +} + +/// Mint one shared counter as a `(writer, reader)` pair over a single backing +/// cell, so the endpoint's writes and the handle's reads cannot accidentally +/// address two separate allocations. +pub(crate) fn drop_channel() -> (CompioDropCounter, DropReader) { + let cell = Rc::new(Cell::new(0u64)); + (CompioDropCounter(cell.clone()), DropReader(cell)) +} + +#[cfg(test)] +mod tests; diff --git a/serf-compio/src/drop_counter/tests.rs b/serf-compio/src/drop_counter/tests.rs new file mode 100644 index 00000000..8367f729 --- /dev/null +++ b/serf-compio/src/drop_counter/tests.rs @@ -0,0 +1,27 @@ +use super::*; + +use serf_proto::DropCounter; + +#[test] +fn incr_saturates_at_max() { + let cell = Rc::new(Cell::new(u64::MAX - 1)); + let mut w = CompioDropCounter(cell); + w.incr_saturating(); + assert_eq!(w.get(), u64::MAX); + // A further increment must saturate, not wrap back to zero. + w.incr_saturating(); + assert_eq!(w.get(), u64::MAX); +} + +#[test] +fn reader_observes_writer_over_the_shared_cell() { + let (mut w, r) = drop_channel(); + assert_eq!(r.get(), 0); + w.incr_saturating(); + w.incr_saturating(); + assert_eq!( + r.get(), + 2, + "the read half observes the write half's increments over one cell" + ); +} diff --git a/serf-compio/src/lib.rs b/serf-compio/src/lib.rs index 223ae7ed..1bd35a8e 100644 --- a/serf-compio/src/lib.rs +++ b/serf-compio/src/lib.rs @@ -41,6 +41,8 @@ mod command; #[cfg(any(feature = "tcp", feature = "quic"))] mod delegate; mod driver; +#[cfg(any(feature = "tcp", feature = "quic"))] +mod drop_counter; mod error; mod events; mod resolver; diff --git a/serf-compio/src/quic/mod.rs b/serf-compio/src/quic/mod.rs index 8587631c..41955c25 100644 --- a/serf-compio/src/quic/mod.rs +++ b/serf-compio/src/quic/mod.rs @@ -329,10 +329,17 @@ where // Serf's core RNG is seeded from its own OS-drawn entropy (`self.serf_rng`), // independent of the coordinator's gossip RNG, so two nodes never share the // query-ID / relay-selection stream. - let endpoint = serf_proto::QuicEndpoint::::new_with_rng( + let endpoint = serf_proto::QuicEndpoint::< + Self::Id, + G, + StdRng, + crate::drop_counter::CompioDropCounter, + >::new_with_rng_in( coord, runtime.serf_options, self.serf_rng, + runtime.user_drop, + runtime.member_drop, ); crate::driver::quic::quic_driver_loop::( diff --git a/serf-compio/src/serf/mod.rs b/serf-compio/src/serf/mod.rs index 3d654f86..f6cbaf05 100644 --- a/serf-compio/src/serf/mod.rs +++ b/serf-compio/src/serf/mod.rs @@ -42,6 +42,7 @@ use crate::{ }, delegate::Delegate, driver::options::RuntimeOptions, + drop_counter::DropReader, error::{JoinFailed, Result, SerfError}, events::EventStream, resolver::{AdvertiseAddrResolver, Resolver}, @@ -67,6 +68,12 @@ struct Shared { /// at the bounded internal observation channel when the delegate dispatch /// loop falls behind. Monotonically increasing. observation_dropped: Rc>, + /// Read-only view of the endpoint's cumulative user-coalescer drop count, over + /// the SAME cell the endpoint's writer increments. The driver owns the + /// endpoint, so a handle reads the shed count here with no publish step. + coalesced_user_events_dropped: DropReader, + /// Read-only view of the endpoint's cumulative member-coalescer drop count. + coalesced_member_events_dropped: DropReader, snapshot: SnapshotCell, shutdown_flag: Rc>, local_id: I, @@ -177,6 +184,11 @@ where flume::bounded::>(runtime_options.event_queue_cap()); let events_dropped = Rc::new(Cell::new(0u64)); let observation_dropped = Rc::new(Cell::new(0u64)); + // Mint the two shed counters as (writer, reader) pairs: the driver injects the + // writers into the endpoint, the handle keeps the readers, both over the same + // backing cell so no publish step exists. + let (user_drop_writer, user_drop_reader) = crate::drop_counter::drop_channel(); + let (member_drop_writer, member_drop_reader) = crate::drop_counter::drop_channel(); let shutdown_flag = Rc::new(Cell::new(false)); let snapshot: SnapshotCell = Rc::new(RefCell::new(Rc::new(initial_snapshot( &local_id, advertise, @@ -198,6 +210,8 @@ where events_tx, events_dropped, observation_dropped, + user_drop_writer, + member_drop_writer, snapshot.clone(), shutdown_flag.clone(), runtime_options, @@ -216,6 +230,8 @@ where events_rx, events_dropped: events_dropped_handle, observation_dropped: observation_dropped_handle, + coalesced_user_events_dropped: user_drop_reader, + coalesced_member_events_dropped: member_drop_reader, snapshot, shutdown_flag, local_id, @@ -363,6 +379,30 @@ where self.shared.observation_dropped.get() } + /// Cumulative number of coalescing user events the driver's endpoint shed + /// because its user coalescer was at the configured buffered-volume cap + /// (`Options::max_coalesced_user_events`) since this node started. + /// + /// Republished by the driver pump each iteration. Lifetime total, saturating, + /// and never cleared by a flush; always `0` when user coalescing is disabled. A + /// non-zero value indicates the coalescer is shedding load: raise + /// `Options::max_coalesced_user_events` or slow the user-event source. + #[inline] + pub fn coalesced_user_events_dropped(&self) -> u64 { + self.shared.coalesced_user_events_dropped.get() + } + + /// Cumulative number of member changes the driver's endpoint shed because its + /// member coalescer was at its per-window cardinality cap since this node + /// started. + /// + /// Republished by the driver pump each iteration. Lifetime total, saturating; + /// always `0` when member coalescing is disabled. + #[inline] + pub fn coalesced_member_events_dropped(&self) -> u64 { + self.shared.coalesced_member_events_dropped.get() + } + /// Subscribe to the serf [`Event`] stream. Multiple subscribers round-robin /// (the channel is MPMC, not broadcast). #[inline] @@ -371,7 +411,7 @@ where } /// Send `cmd` to the driver, failing fast if the node has shut down. - fn send(&self, cmd: Command) -> Result<()> { + pub(crate) fn send(&self, cmd: Command) -> Result<()> { if self.shared.shutdown_flag.get() { return Err(SerfError::Shutdown); } diff --git a/serf-compio/src/serf/tests.rs b/serf-compio/src/serf/tests.rs index 92b55138..321eb07e 100644 --- a/serf-compio/src/serf/tests.rs +++ b/serf-compio/src/serf/tests.rs @@ -518,6 +518,97 @@ async fn tcp_events_dropped_counter_observable_under_backpressure() { ); } +/// Build a TCP serf node with a custom `SerfOptions` (runtime options at defaults). +async fn spawn_node_with_serf_options(id: &str, serf_options: SerfOptions) -> Serf { + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(bind)); + Serf::new::, SocketAddrResolver, FirstAddrResolver, _, _>( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + serf_options, + gossip_rng().expect("seed gossip rng"), + #[cfg(encryption)] + std::rc::Rc::new(VoidKeyringDelegate), + ) + .await + .expect("spawn serf node") +} + +/// A single node with user coalescing enabled and a small buffered-volume cap sheds +/// every distinct-named coalescing user event issued past the cap through the public +/// `user_event` command path, and the cumulative drop count surfaces on the public +/// `coalesced_user_events_dropped` accessor — the endpoint counter is otherwise +/// unreachable once the driver moves the endpoint into the detached pump. +#[compio::test] +async fn tcp_coalesced_user_events_dropped_observable() { + let cap = core::num::NonZeroUsize::new(4).unwrap(); + let serf_opts = SerfOptions::new() + .with_user_coalesce_period(Duration::from_secs(10)) + .with_user_quiescent_period(Duration::from_secs(2)) + .with_max_coalesced_user_events(Some(cap)); + let a = spawn_node_with_serf_options("coalesce-a", serf_opts).await; + + assert_eq!( + a.coalesced_user_events_dropped(), + 0, + "no drops before any user event is issued" + ); + + // Issue distinct-named coalescing user events past the cap. Each is buffered by + // name in the open window; every name past the cap is shed and counted. + let n: u32 = 20; + for i in 0..n { + a.user_event(format!("evt-{i}"), Bytes::new(), true) + .await + .expect("user event dispatched"); + } + + // Each `user_event().await` returned only after the pump processed that command + // and incremented the shared shed cell, so the handle read is already current on + // the same executor thread with no publish step. + let dropped = a.coalesced_user_events_dropped(); + a.shutdown().await.expect("coalesce-a shuts down"); + + assert_eq!( + dropped, + u64::from(n) - cap.get() as u64, + "every distinct-named cc event past the cap is counted on the public handle (got {dropped})" + ); +} + +/// After exactly one shed through the public `user_event` path the handle getter +/// returns at least one. A construction typo that wired the handle's reader to a +/// different cell than the endpoint's writer would leave this a permanent zero, so +/// the aliasing bug fails loudly here rather than silently reporting no drops. +#[compio::test] +async fn coalesced_drop_aliasing_guard() { + // A cap of one: the second distinct-named coalescing event is shed. + let cap = core::num::NonZeroUsize::new(1).unwrap(); + let serf_opts = SerfOptions::new() + .with_user_coalesce_period(Duration::from_secs(10)) + .with_user_quiescent_period(Duration::from_secs(2)) + .with_max_coalesced_user_events(Some(cap)); + let a = spawn_node_with_serf_options("coalesce-alias", serf_opts).await; + + a.user_event("first".to_string(), Bytes::new(), true) + .await + .expect("first user event dispatched"); + a.user_event("second".to_string(), Bytes::new(), true) + .await + .expect("second user event dispatched"); + + assert!( + a.coalesced_user_events_dropped() >= 1, + "the handle observes the endpoint's shed; a mis-wired reader would read a permanent 0" + ); + a.shutdown().await.expect("coalesce-alias shuts down"); +} + /// `join_many` over two seeds — one reachable (node B), one a blackhole port — /// returns only the reached seed's address. The reachable exchange succeeds and /// the blackhole exchange fails fast; once both terminate the call resolves diff --git a/serf-compio/src/tcp/mod.rs b/serf-compio/src/tcp/mod.rs index d274e46c..b126e14a 100644 --- a/serf-compio/src/tcp/mod.rs +++ b/serf-compio/src/tcp/mod.rs @@ -353,12 +353,20 @@ where // Serf's core RNG is seeded from its own OS-drawn entropy (`self.serf_rng`), // independent of the coordinator's gossip RNG, so two nodes never share the // query-ID / relay-selection stream. - let endpoint = - serf_proto::StreamEndpoint::::new_with_rng( - coord, - runtime.serf_options, - self.serf_rng, - ); + let endpoint = serf_proto::StreamEndpoint::< + Self::Id, + SocketAddr, + RawRecords, + G, + StdRng, + crate::drop_counter::CompioDropCounter, + >::new_with_rng_in( + coord, + runtime.serf_options, + self.serf_rng, + runtime.user_drop, + runtime.member_drop, + ); crate::driver::stream::stream_driver_loop::( endpoint, diff --git a/serf-compio/src/tls/mod.rs b/serf-compio/src/tls/mod.rs index 3a7effa6..847bf258 100644 --- a/serf-compio/src/tls/mod.rs +++ b/serf-compio/src/tls/mod.rs @@ -445,7 +445,14 @@ where Labeled, G, StdRng, - >::new_with_rng(coord, runtime.serf_options, self.serf_rng); + crate::drop_counter::CompioDropCounter, + >::new_with_rng_in( + coord, + runtime.serf_options, + self.serf_rng, + runtime.user_drop, + runtime.member_drop, + ); crate::driver::stream::stream_driver_loop::, D, G, StdRng>( endpoint, diff --git a/serf-compio/src/transport/runtime.rs b/serf-compio/src/transport/runtime.rs index 84eca6bd..e5d42d5e 100644 --- a/serf-compio/src/transport/runtime.rs +++ b/serf-compio/src/transport/runtime.rs @@ -13,8 +13,8 @@ use flume::{Receiver, Sender}; use serf_proto::{event::Event, options::Options as SerfOptions}; use crate::{ - command::Command, delegate::Delegate, driver::options::RuntimeOptions, snapshot::SnapshotCell, - transport::Transport, + command::Command, delegate::Delegate, driver::options::RuntimeOptions, + drop_counter::CompioDropCounter, snapshot::SnapshotCell, transport::Transport, }; #[cfg(encryption)] @@ -49,6 +49,11 @@ where /// Counter for events dropped at the delegate observation channel when the /// delegate fell behind — may include unrecoverable app-data. pub(crate) observation_dropped: Rc>, + /// The write half of the user-coalescer shed counter, injected into the + /// endpoint by `T::run` so its increments land in the cell the handle reads. + pub(crate) user_drop: CompioDropCounter, + /// The write half of the member-coalescer shed counter. + pub(crate) member_drop: CompioDropCounter, pub(crate) snapshot: SnapshotCell, pub(crate) shutdown_flag: Rc>, pub(crate) driver_options: RuntimeOptions, @@ -74,6 +79,8 @@ where events_tx: Sender>, events_dropped: Rc>, observation_dropped: Rc>, + user_drop: CompioDropCounter, + member_drop: CompioDropCounter, snapshot: SnapshotCell, shutdown_flag: Rc>, driver_options: RuntimeOptions, @@ -86,6 +93,8 @@ where events_tx, events_dropped, observation_dropped, + user_drop, + member_drop, snapshot, shutdown_flag, driver_options, diff --git a/serf-embassy/src/serf/mod.rs b/serf-embassy/src/serf/mod.rs index fb0525a1..26cadd07 100644 --- a/serf-embassy/src/serf/mod.rs +++ b/serf-embassy/src/serf/mod.rs @@ -634,6 +634,30 @@ where self.shared.events_dropped() } + /// Cumulative count of coalescing user events the engine's user coalescer shed + /// because its buffered volume was at the configured cap. + /// + /// Lifetime total, saturating, and never cleared by a flush or reset. Reads `0` + /// when user coalescing is disabled. + #[inline] + pub fn coalesced_user_events_dropped(&self) -> u64 { + self.shared.engine.borrow().coalesced_user_events_dropped() + } + + /// Cumulative count of member changes the engine's member coalescer shed + /// because its per-window map was at its cardinality cap. + /// + /// Lifetime total, saturating, and never cleared. Reads `0` when member + /// coalescing is disabled. + #[inline] + pub fn coalesced_member_events_dropped(&self) -> u64 { + self + .shared + .engine + .borrow() + .coalesced_member_events_dropped() + } + /// Announce the local node's join intent and await the result: resolve each seed, /// dispatch an await-result join on the engine, and await its outcome. /// @@ -800,11 +824,12 @@ where if self.shared.is_shutdown() { return Err(OpError::Shutdown); } + let now = time::now(); let r = self .shared .engine .borrow_mut() - .user_event(name, payload, coalesce); + .user_event(name, payload, coalesce, now); self.shared.wake_pump(); r.map_err(OpError::from) } @@ -867,7 +892,8 @@ where if self.shared.is_shutdown() { return Err(OpError::Shutdown); } - let r = self.shared.engine.borrow_mut().set_tags(tags); + let now = time::now(); + let r = self.shared.engine.borrow_mut().set_tags(tags, now); self.shared.wake_pump(); r.map_err(OpError::from) } diff --git a/serf-embassy/src/shared/tests.rs b/serf-embassy/src/shared/tests.rs index fcb99da2..e945f880 100644 --- a/serf-embassy/src/shared/tests.rs +++ b/serf-embassy/src/shared/tests.rs @@ -73,3 +73,50 @@ fn drained_shutdown_event_poisons_and_buffers() { "the terminal Event::Shutdown must be buffered for poll_event" ); } + +/// With user coalescing enabled and a small buffered-volume cap, distinct-named +/// coalescing user events fed past the cap are shed by the engine's user coalescer, +/// and the running total surfaces through the same engine borrow the public +/// `coalesced_user_events_dropped` handle accessor reads. +#[test] +fn coalesced_user_events_dropped_surfaces_overflow() { + let cap = core::num::NonZeroUsize::new(4).unwrap(); + let advertise = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)), 7946); + let engine = SerfEngine::::try_new_at( + Options::new() + .with_port(7946) + .with_close_timeout(Duration::from_secs(10)), + TransformOptions::default(), + EndpointOptions::new(SmolStr::new("test"), advertise), + SerfOptions::new() + .with_user_coalesce_period(Duration::from_secs(10)) + .with_user_quiescent_period(Duration::from_secs(2)) + .with_max_coalesced_user_events(Some(cap)), + at(), + SmallRng::seed_from_u64(42), + ) + .expect("a routable single-node configuration constructs"); + let shared = Shared::new(engine, advertise); + assert_eq!(shared.engine.borrow().coalesced_user_events_dropped(), 0); + + let n: u32 = 20; + for i in 0..n { + shared + .engine + .borrow_mut() + .user_event( + SmolStr::from(alloc::format!("evt-{i}")), + bytes::Bytes::from_static(b"p"), + true, + at(), + ) + .expect("a coalescing user event is accepted while running"); + } + + assert_eq!( + shared.engine.borrow().coalesced_user_events_dropped(), + u64::from(n) - cap.get() as u64, + "every distinct-named coalescing event past the cap is shed and counted" + ); + assert_eq!(shared.engine.borrow().coalesced_member_events_dropped(), 0); +} diff --git a/serf-embedded/src/engine/mod.rs b/serf-embedded/src/engine/mod.rs index bb4e4027..6c043954 100644 --- a/serf-embedded/src/engine/mod.rs +++ b/serf-embedded/src/engine/mod.rs @@ -975,6 +975,27 @@ where self.events_dropped } + /// Cumulative count of coalescing user events the endpoint's user coalescer + /// shed because its buffered volume was at the configured cap. + /// + /// Lifetime total, saturating, and never cleared by a flush or reset. Reads `0` + /// when user coalescing is disabled. Forwards the endpoint counter unchanged — + /// the engine adds no coalescer of its own. + #[inline] + pub fn coalesced_user_events_dropped(&self) -> u64 { + self.endpoint.coalesced_user_events_dropped() + } + + /// Cumulative count of member changes the endpoint's member coalescer shed + /// because its per-window map was at its cardinality cap. + /// + /// Lifetime total, saturating, and never cleared. Reads `0` when member + /// coalescing is disabled. + #[inline] + pub fn coalesced_member_events_dropped(&self) -> u64 { + self.endpoint.coalesced_member_events_dropped() + } + /// Fold one machine event into the await-result join it terminates, if any. /// /// A push/pull `ExchangeCompleted` whose `eid` was bound to a join at its @@ -1337,8 +1358,9 @@ where name: impl Into, payload: Bytes, coalesce: bool, + now: Instant, ) -> Result<(), SerfError> { - self.endpoint.user_event(name, payload, coalesce) + self.endpoint.user_event(name, payload, coalesce, now) } /// Issue a cluster-wide query, returning its [`QueryId`]. @@ -1369,8 +1391,8 @@ where /// Replace the local node's tags, re-advertising them via the coordinator and /// refreshing the local member in the membership store. - pub fn set_tags(&mut self, tags: Tags) -> Result<(), SerfError> { - self.endpoint.set_tags(tags) + pub fn set_tags(&mut self, tags: Tags, now: Instant) -> Result<(), SerfError> { + self.endpoint.set_tags(tags, now) } /// Issue a cluster-wide `install_key` query to add `key` to every node's diff --git a/serf-embedded/src/engine/tests.rs b/serf-embedded/src/engine/tests.rs index c68490f7..75e8cdab 100644 --- a/serf-embedded/src/engine/tests.rs +++ b/serf-embedded/src/engine/tests.rs @@ -249,10 +249,61 @@ fn user_event_accepted_while_running() { let now = Instant::from_origin(Duration::from_secs(86_400)); engine.start(now); engine - .user_event("deploy", Bytes::from_static(b"v2"), false) + .user_event("deploy", Bytes::from_static(b"v2"), false, now) .expect("a user event is accepted while the node is running"); } +/// With user coalescing enabled and a small buffered-volume cap, a flood of +/// distinct-named coalescing user events fed past the cap is shed and counted, and +/// the running total surfaces through the engine's `coalesced_user_events_dropped` +/// forward (the endpoint counter reachable through the engine handle). +#[test] +fn coalesced_user_events_dropped_surfaces_overflow() { + let cap = core::num::NonZeroUsize::new(4).unwrap(); + let cfg = Options::new() + .with_port(7946) + .with_close_timeout(Duration::from_secs(10)); + let ep_cfg = EndpointOptions::new(SmolStr::new("test"), node_addr(7946)); + let serf_opts = SerfOptions::new() + .with_user_coalesce_period(Duration::from_secs(10)) + .with_user_quiescent_period(Duration::from_secs(2)) + .with_max_coalesced_user_events(Some(cap)); + let now = Instant::from_origin(Duration::from_secs(86_400)); + let mut engine: SerfEngine = SerfEngine::try_new_at( + cfg, + TransformOptions::default(), + ep_cfg, + serf_opts, + now, + test_rng(), + ) + .expect("valid configuration must construct without error"); + engine.start(now); + assert_eq!( + engine.coalesced_user_events_dropped(), + 0, + "no drops before any user event is fed" + ); + + let n: u32 = 20; + for i in 0..n { + engine + .user_event(format!("evt-{i}"), Bytes::from_static(b"p"), true, now) + .expect("a coalescing user event is accepted while running"); + } + + assert_eq!( + engine.coalesced_user_events_dropped(), + u64::from(n) - cap.get() as u64, + "every distinct-named coalescing event past the cap is shed and counted" + ); + assert_eq!( + engine.coalesced_member_events_dropped(), + 0, + "member coalescing is disabled, so its drop counter stays zero" + ); +} + /// `leave` transitions the endpoint out of `Alive`, and a subsequent `join` is /// rejected (serf announces its own join intent only from `Alive`). #[test] diff --git a/serf-proto/src/coalesce/drop_counter/mod.rs b/serf-proto/src/coalesce/drop_counter/mod.rs new file mode 100644 index 00000000..f66493f3 --- /dev/null +++ b/serf-proto/src/coalesce/drop_counter/mod.rs @@ -0,0 +1,36 @@ +//! The [`DropCounter`] seam the coalescer increments and a driver exposes to the +//! application. +//! +//! The pure crate stores the two coalescer shed counts behind this trait rather +//! than as bare `u64` fields, so an async driver can supply a shared, +//! read-observable backing (an atomic or a `Cell`) that the handle reads WITHOUT +//! copying the endpoint's value out each pump iteration. The single-owner default +//! is a plain `u64`, which keeps the endpoint atomics-free and `Send + Sync`. + +/// A monotone, saturating drop counter the coalescer increments and a driver +/// exposes to the application. +/// +/// Implemented by the pure crate only for `u64` (the single-owner default); each +/// async driver supplies its own shared, read-observable backing so the handle +/// observes increments without a copy. +pub trait DropCounter { + /// Increment by one, saturating at `u64::MAX`. + fn incr_saturating(&mut self); + /// The current cumulative count. + fn get(&self) -> u64; +} + +impl DropCounter for u64 { + #[inline] + fn incr_saturating(&mut self) { + *self = self.saturating_add(1); + } + + #[inline] + fn get(&self) -> u64 { + *self + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-proto/src/coalesce/drop_counter/tests.rs b/serf-proto/src/coalesce/drop_counter/tests.rs new file mode 100644 index 00000000..31e73a26 --- /dev/null +++ b/serf-proto/src/coalesce/drop_counter/tests.rs @@ -0,0 +1,21 @@ +use super::DropCounter; + +#[test] +fn u64_incr_counts_from_zero() { + let mut c: u64 = 0; + assert_eq!(c.get(), 0); + c.incr_saturating(); + c.incr_saturating(); + c.incr_saturating(); + assert_eq!(c.get(), 3); +} + +#[test] +fn u64_incr_saturates_at_max() { + let mut c: u64 = u64::MAX - 1; + c.incr_saturating(); + assert_eq!(c.get(), u64::MAX); + // A further increment must not wrap back to zero. + c.incr_saturating(); + assert_eq!(c.get(), u64::MAX); +} diff --git a/serf-proto/src/coalesce/mod.rs b/serf-proto/src/coalesce/mod.rs new file mode 100644 index 00000000..4e5d3e16 --- /dev/null +++ b/serf-proto/src/coalesce/mod.rs @@ -0,0 +1,522 @@ +//! Pure, timer-driven event coalescing for the serf [`crate::endpoint::Endpoint`]. +//! +//! Go/legacy serf batches rapid membership and user events into time windows +//! before delivering them to the application, collapsing flapping into a single +//! grouped observation. The reference implementation +//! (`serf-core/src/coalesce.rs` + `coalesce/{member,user}.rs`) runs the window as +//! an async task fed by a channel; this port re-expresses the exact same +//! algorithm as a synchronous, allocation-backed state machine owned by the +//! endpoint, so `now` is threaded in (no clock reads) and every driver inherits +//! coalescing without spawning a task. +//! +//! Two deadlines drive a flush (mirroring the reference `coalesceLoop`): +//! * the **coalesce** window — the maximum batch delay, armed once on the first +//! buffered event of a batch and never pushed forward; and +//! * the **quiescent** window — the flush-after-quiet delay, re-armed on every +//! buffered event. +//! +//! Whichever fires first flushes the coalesced batch and disarms the window. +//! The endpoint folds [`MemberEventCoalescer::flush_deadline`] / +//! [`UserEventCoalescer::flush_deadline`] into its `serf_poll_timeout` and calls +//! `flush` from `after_inner_timeout` when the deadline is due. + +use core::{hash::Hash, time::Duration}; + +use std::{collections::VecDeque, vec::Vec}; + +use memberlist_proto::Instant; +use smol_str::SmolStr; + +use crate::{ + FxHashMap, LamportTime, Tags, UserEventMessage, + event::{Event, MemberEvent, MemberEventKind}, + members::{Member, MemberStatus}, +}; + +pub(crate) mod drop_counter; + +pub use drop_counter::DropCounter; + +// ── CoalesceWindow ──────────────────────────────────────────────────────────── + +/// The two-deadline flush timing shared by the member and user coalescers. +/// +/// * `coalesce_deadline` is the maximum batch window: set once on the first +/// buffered event of a batch and never advanced, so a steady stream of events +/// still flushes at most `coalesce_period` after the batch opened. +/// * `quiescent_deadline` is the flush-after-quiet window: re-armed on every +/// buffered event, so a batch flushes `quiescent_period` after the last event +/// once the stream goes quiet. +/// +/// The binding flush deadline is the minimum of the two; both are `None` while +/// the window holds no buffered events. +struct CoalesceWindow { + coalesce_period: Duration, + quiescent_period: Duration, + coalesce_deadline: Option, + quiescent_deadline: Option, +} + +impl CoalesceWindow { + const fn new(coalesce_period: Duration, quiescent_period: Duration) -> Self { + Self { + coalesce_period, + quiescent_period, + coalesce_deadline: None, + quiescent_deadline: None, + } + } + + /// Arm the window for a newly buffered event: open the max-window on the first + /// event of a batch, and (re)start the quiescent timer on every event. + fn arm(&mut self, now: Instant) { + if self.coalesce_deadline.is_none() { + self.coalesce_deadline = Some(now + self.coalesce_period); + } + self.quiescent_deadline = Some(now + self.quiescent_period); + } + + /// The binding flush deadline (`min(coalesce, quiescent)`), or `None` when the + /// window is disarmed (no buffered events). + fn deadline(&self) -> Option { + match (self.coalesce_deadline, self.quiescent_deadline) { + (Some(c), Some(q)) => Some(c.min(q)), + (c, None) => c, + (None, q) => q, + } + } + + /// Whether the flush deadline has elapsed at `now`. + fn due(&self, now: Instant) -> bool { + self.deadline().is_some_and(|d| now >= d) + } + + /// Disarm the window after a flush (or drop). + fn reset(&mut self) { + self.coalesce_deadline = None; + self.quiescent_deadline = None; + } +} + +// ── MemberEventCoalescer ──────────────────────────────────────────────────────── + +/// Hard cap on the number of distinct nodes buffered in a single member-coalesce +/// window. +/// +/// The `latest` map holds one entry per node id observed while the window is +/// open; a burst of membership churn across many distinct ids would otherwise +/// grow it without bound before the window closes. At the cap a change for a +/// NEW id is dropped (and counted), while an id already buffered still updates in +/// place, so the collapse of an in-progress node stays exact. A normal cluster +/// never approaches the limit — only an adversarial fan-out of distinct ids is +/// bounded. Mirrors the inbound-query overflow cap on the endpoint. +const MAX_COALESCED_MEMBER_EVENTS: usize = 2048; + +/// The latest buffered observation of a node within an open coalesce window. +struct LatestMember { + kind: MemberEventKind, + member: Member, +} + +/// The full observable state last EMITTED for a node, kept across flushes so a +/// repeat that changed nothing is suppressed while ANY observable change — a new +/// address, new tags, or a new status — re-emits. A node can fail and rejoin at +/// the same address within one window with different tags (its `Member` is +/// rebuilt from freshly decoded metadata); comparing the whole observed state +/// rather than just the kind and address means that changed metadata is +/// delivered instead of collapsed away. +struct LastEmitted { + kind: MemberEventKind, + addr: A, + tags: Tags, + status: MemberStatus, +} + +/// Coalesces membership changes over a time window, collapsing rapid transitions +/// of a node to its latest observed status. +/// +/// Ports serf-core `coalesce/member.rs` `MemberEventCoalescer`: +/// * `latest` holds the LATEST `(kind, member)` per node id within the open +/// window, so a rapid join → leave → join for one node collapses to its final +/// observation. +/// * `last` remembers the last EMITTED observable state per node id ACROSS +/// flushes, so a repeated identical observation is suppressed — except +/// [`MemberEventKind::Update`], which always re-emits. The stored state is the +/// full `(kind, address, tags, status)`, so a rejoin at a new address OR a +/// same-address rejoin whose tags or status changed re-emits, and consumers +/// never retain stale member metadata. +/// +/// A flush groups the surviving observations by kind and emits one +/// [`Event::Member`] batch per kind. +pub(crate) struct MemberEventCoalescer +where + I: Eq + Hash, +{ + window: CoalesceWindow, + latest: FxHashMap>, + last: FxHashMap>, +} + +impl MemberEventCoalescer +where + I: Eq + Hash + Clone, +{ + /// Construct a member coalescer over the given windows. + pub(crate) fn new(coalesce_period: Duration, quiescent_period: Duration) -> Self { + Self { + window: CoalesceWindow::new(coalesce_period, quiescent_period), + latest: FxHashMap::default(), + last: FxHashMap::default(), + } + } + + /// Buffer a batch of member changes of one `kind`, keyed by node id (latest + /// wins), and arm the flush window. + /// + /// A change for a not-yet-buffered id dropped by the cardinality cap increments + /// `drops` (the endpoint-owned member shed counter); the counter is never read + /// back by any protocol path. + pub(crate) fn feed( + &mut self, + kind: MemberEventKind, + members: Vec>, + now: Instant, + drops: &mut D, + ) where + D: DropCounter, + { + let mut admitted = false; + for member in members { + let id = member.node().id_ref().clone(); + // A terminal Reap forgets the node's last-emitted status at feed time: a + // Reap overwritten by a rejoin Join later in the same window would never + // reach a flush-time eviction, leaving a stale `last[id]` that suppresses + // the genuinely-new Join. This eviction is UNCONDITIONAL and runs before + // the cardinality gate below: a Reap dropped by an overflowing window must + // still forget the id, or the stale suppression entry outlives it. + if kind == MemberEventKind::Reap { + self.last.remove(&id); + } + // Bound the per-window map: a change for a not-yet-buffered id is dropped + // once the map is at capacity, while an id already buffered still updates + // in place (its collapse must stay exact). Only genuinely-new ids can grow + // the map, so only they are gated. + if !self.latest.contains_key(&id) && self.latest.len() >= MAX_COALESCED_MEMBER_EVENTS { + drops.incr_saturating(); + continue; + } + self.latest.insert(id, LatestMember { kind, member }); + admitted = true; + } + // Arm only when at least one member was buffered this call: a fully-rejected + // batch must not extend the quiescent window. + if admitted { + self.window.arm(now); + } + } + + /// The next flush deadline while the window is open, else `None`. + pub(crate) fn flush_deadline(&self) -> Option { + self.window.deadline() + } + + /// Whether the flush deadline has elapsed at `now`. + pub(crate) fn due(&self, now: Instant) -> bool { + self.window.due(now) + } + + /// Drop the buffered batch and disarm the window without emitting anything. + /// + /// Used when the machine transitions to `Shutdown` (a lost id-conflict vote): + /// the reference implementation abandons its coalescer goroutine on shutdown, + /// so the not-yet-flushed batch is discarded rather than delivered. The `last` + /// suppression history is retained (it is harmless once the machine is dead). + pub(crate) fn reset(&mut self) { + self.latest.clear(); + self.window.reset(); + } + + /// The number of ids currently held in the cross-flush suppression map. + /// + /// Bounded by live membership because `flush` evicts a node's id on its + /// terminal `Reap`. + #[cfg(test)] + pub(crate) fn last_len(&self) -> usize { + self.last.len() + } + + /// The number of distinct ids currently buffered in the open window. + #[cfg(test)] + pub(crate) fn latest_len(&self) -> usize { + self.latest.len() + } + + /// Whether the cross-flush suppression map currently holds `id`. + #[cfg(test)] + pub(crate) fn last_contains(&self, id: &I) -> bool { + self.last.contains_key(id) + } +} + +impl MemberEventCoalescer +where + I: Eq + Hash + Clone, + A: Clone + PartialEq, +{ + /// Emit the coalesced member events into `out` and disarm the window. + /// + /// Groups the surviving observations by kind, suppressing a node whose full + /// observable state (kind, address, tags, and status) is unchanged since the + /// last flush (except `Update`, which always re-emits), and emits one + /// [`Event::Member`] batch per surviving kind. The `last` map persists across + /// flushes so the suppression is stateful. + /// + /// **Eviction rule (bounds `last` to live membership):** a node's id is + /// forgotten from `last` at FEED time the instant a [`MemberEventKind::Reap`] + /// is buffered for it, and a `Reap` is never recorded on flush. `Reap` is the + /// terminal removal-from-membership signal — the node is gone from the + /// endpoint's `states`, so retaining it would grow `last` without bound as + /// distinct ids churn through join → … → reap (the reference serf + /// implementation leaks here). `last` stays keyed by id — one entry per id — + /// so pairing the address into the value leaves that live-membership bound + /// intact. Evicting at feed time (rather than on flush) also keeps a `Reap` + /// that is overwritten by a rejoin `Join` within the same window from leaving a + /// stale `last[id]` that would wrongly suppress the genuinely-new `Join`. + /// Forgetting a reaped id is correct regardless: its later re-join is a + /// genuinely new member, and a `Join` differs from the absent entry so it + /// re-emits. Every non-`Reap` kind can still transition, so it is retained to + /// suppress a repeated identical observation. + pub(crate) fn flush(&mut self, out: &mut VecDeque>) { + // At most five kinds, so a linear-probed Vec is cheaper than a hash map and + // avoids requiring `Hash` on the public `MemberEventKind`. + let mut grouped: Vec<(MemberEventKind, Vec>)> = Vec::new(); + for (id, latest) in self.latest.drain() { + let addr = latest.member.node().addr_ref().clone(); + let tags = latest.member.tags().clone(); + let status = latest.member.status(); + if let Some(prev) = self.last.get(&id) { + // Suppress only a genuinely unchanged observation — same kind AND the + // same full observable state (address, tags, status). A rejoin at a new + // address, or a same-address rejoin whose tags or status changed, must + // still emit so consumers never retain stale member metadata. An Update + // always re-emits. + if prev.kind == latest.kind + && prev.addr == addr + && prev.tags == tags + && prev.status == status + && latest.kind != MemberEventKind::Update + { + continue; + } + } + // A Reap is never recorded in `last`: `feed` already evicted the id, and + // storing it would grow `last` without bound as ids churn through + // join → … → reap. A forgotten reaped id re-emits its later Join correctly + // (Join differs from the absent entry). Every non-Reap kind is retained to + // suppress a repeated identical observation. + if latest.kind != MemberEventKind::Reap { + self.last.insert( + id, + LastEmitted { + kind: latest.kind, + addr, + tags, + status, + }, + ); + } + match grouped.iter_mut().find(|(k, _)| *k == latest.kind) { + Some((_, members)) => members.push(latest.member), + None => grouped.push((latest.kind, std::vec![latest.member])), + } + } + for (kind, members) in grouped { + out.push_back(Event::Member(MemberEvent::new(kind, members))); + } + self.window.reset(); + } +} + +// ── UserEventCoalescer ────────────────────────────────────────────────────────── + +/// The newest Lamport generation of a named user event within an open window. +struct LatestUserEvents { + ltime: LamportTime, + events: Vec, +} + +/// Coalesces user events over a time window, keeping only the newest Lamport +/// generation per event name. +/// +/// Ports serf-core `coalesce/user.rs` `UserEventCoalescer`: for each event +/// `name`, buffer the events carrying the highest `ltime` seen so far — a newer +/// ltime clears the older buffer, an equal ltime accumulates (distinct payloads +/// at the same generation are all delivered), and an older ltime is dropped. +/// +/// Only events that opted into coalescing (`UserEventMessage::cc == true`) are +/// fed here; the endpoint passes non-coalescing user events straight through, +/// mirroring the reference coalescer's `handle` predicate. +pub(crate) struct UserEventCoalescer { + window: CoalesceWindow, + events: FxHashMap, + /// Upper bound on `buffered` (the total buffered event volume). `None` + /// disables the bound. + cap: Option, + /// Running sum of every buffered event count across `events`. A flush emits + /// one [`Event::User`] per buffered event, so this equals both the true + /// buffered volume and the maximum flush burst — the quantity the `cap` + /// bounds (not the per-name key count). + buffered: usize, +} + +impl UserEventCoalescer { + /// Construct a user coalescer over the given windows, bounding the total + /// buffered event volume to `cap` (`None` disables the bound). + pub(crate) fn new( + coalesce_period: Duration, + quiescent_period: Duration, + cap: Option, + ) -> Self { + Self { + window: CoalesceWindow::new(coalesce_period, quiescent_period), + events: FxHashMap::default(), + cap, + buffered: 0, + } + } + + /// Buffer a coalescing user event (dedup by name, newest ltime wins) and arm + /// the flush window. + /// + /// The total buffered event volume is bounded by `cap`: an admission that + /// would grow it past the cap is dropped and counted, EXCEPT a newer + /// generation for a buffered name, which clears that name's older buffer first + /// (net change `<= 0`) and so is always admitted. An older generation is + /// dropped as normal dedup and is NOT counted. The window is armed only when + /// an event is admitted, so a rejected or dropped-older event never extends + /// the quiescent timer. + /// + /// A volume-cap drop increments `drops` (the endpoint-owned user shed counter); + /// an older-generation dedup drop does not. The counter is never read back by + /// any protocol path. + pub(crate) fn feed(&mut self, event: UserEventMessage, now: Instant, drops: &mut D) + where + D: DropCounter, + { + let cap = self.cap.map_or(usize::MAX, core::num::NonZeroUsize::get); + let ltime = event.ltime; + let admitted = match self.events.get_mut(&event.name) { + None => { + if self.buffered >= cap { + drops.incr_saturating(); + false + } else { + self.events.insert( + event.name.clone(), + LatestUserEvents { + ltime, + events: std::vec![event], + }, + ); + self.buffered += 1; + true + } + } + Some(latest) => { + if latest.ltime < ltime { + // A newer generation supersedes the buffered one, so the net change is + // `1 - old_len <= 0` and this is always admitted regardless of the cap. + // Replace the buffer with a fresh single-element vector rather than + // clearing it in place: `Vec::clear` retains the old capacity, so a name + // repeatedly filled toward the cap and then superseded would keep a + // large allocation while `buffered` reads low — retained memory the + // count-based cap cannot see (a per-name `Theta(cap)` leak). Assigning a + // new vector frees the old allocation. + self.buffered -= latest.events.len(); + latest.ltime = ltime; + latest.events = std::vec![event]; + self.buffered += 1; + true + } else if latest.ltime == ltime { + // Same generation: keep both (e.g. distinct payloads at one ltime), + // subject to the cap. + if self.buffered >= cap { + drops.incr_saturating(); + false + } else { + latest.events.push(event); + self.buffered += 1; + true + } + } else { + // Older generation: drop (normal dedup, not counted). + false + } + } + }; + if admitted { + self.window.arm(now); + } + } + + /// The next flush deadline while the window is open, else `None`. + pub(crate) fn flush_deadline(&self) -> Option { + self.window.deadline() + } + + /// Whether the flush deadline has elapsed at `now`. + pub(crate) fn due(&self, now: Instant) -> bool { + self.window.due(now) + } + + /// Emit the coalesced user events into `out` and disarm the window. + pub(crate) fn flush(&mut self, out: &mut VecDeque>) { + for (_, latest) in self.events.drain() { + for event in latest.events { + out.push_back(Event::User(event)); + } + } + // The map is now empty, so the buffered-volume invariant resets to zero + // (a live-content reset, not a decrement). + self.buffered = 0; + self.window.reset(); + } + + /// Drop the buffered batch and disarm the window without emitting anything + /// (see [`MemberEventCoalescer::reset`]). + pub(crate) fn reset(&mut self) { + self.events.clear(); + self.buffered = 0; + self.window.reset(); + } + + /// The total buffered event volume (the running sum bounded by `cap`). + #[cfg(test)] + pub(crate) fn buffered(&self) -> usize { + self.buffered + } + + /// The number of distinct event names currently buffered. + #[cfg(test)] + pub(crate) fn distinct_names(&self) -> usize { + self.events.len() + } + + /// The true sum of buffered payload counts across every name — the invariant + /// `buffered` must equal. + #[cfg(test)] + pub(crate) fn live_payload_count(&self) -> usize { + self.events.values().map(|l| l.events.len()).sum() + } + + /// The total ALLOCATED capacity retained across every name's buffer — bounded + /// alongside `buffered`, since a superseded buffer must free its allocation + /// rather than retain it. + #[cfg(test)] + pub(crate) fn retained_capacity(&self) -> usize { + self.events.values().map(|l| l.events.capacity()).sum() + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-proto/src/coalesce/tests.rs b/serf-proto/src/coalesce/tests.rs new file mode 100644 index 00000000..334de0e5 --- /dev/null +++ b/serf-proto/src/coalesce/tests.rs @@ -0,0 +1,976 @@ +use core::{net::SocketAddr, num::NonZeroUsize, time::Duration}; + +use std::{collections::VecDeque, vec, vec::Vec}; + +use memberlist_proto::{Instant, Node}; +use smol_str::SmolStr; + +use crate::{ + LamportTime, UserEventMessage, + event::{Event, MemberEventKind}, + members::{Member, MemberStatus}, + typed::Tags, +}; + +use super::{MAX_COALESCED_MEMBER_EVENTS, MemberEventCoalescer, UserEventCoalescer}; + +// ── helpers ──────────────────────────────────────────────────────────────────── + +fn secs(n: u64) -> Duration { + Duration::from_secs(n) +} + +fn addr() -> SocketAddr { + "127.0.0.1:8080".parse().unwrap() +} + +fn member(id: u32) -> Member { + Member::new(Node::new(id, addr()), Tags::new(), MemberStatus::None) +} + +fn member_tagged(id: u32, role: &str) -> Member { + Member::new( + Node::new(id, addr()), + Tags::from_iter([("role", role)]), + MemberStatus::None, + ) +} + +fn uev(name: &str, ltime: u64, payload: &str) -> UserEventMessage { + UserEventMessage { + ltime: LamportTime::new(ltime), + cc: true, + name: name.into(), + payload: bytes::Bytes::copy_from_slice(payload.as_bytes()), + } +} + +/// Drain a flushed queue into `(kind, member-ids)` groups for order-independent +/// assertions (the flush groups by kind but does not sort). +fn member_groups(out: VecDeque>) -> Vec<(MemberEventKind, Vec)> { + out + .into_iter() + .map(|ev| match ev { + Event::Member(me) => { + let mut ids: Vec = me.members().iter().map(|m| *m.node().id_ref()).collect(); + ids.sort_unstable(); + (me.kind(), ids) + } + other => panic!("expected Event::Member, got {other:?}"), + }) + .collect() +} + +fn find_group( + groups: &[(MemberEventKind, Vec)], + kind: MemberEventKind, +) -> Option<&(MemberEventKind, Vec)> { + groups.iter().find(|(k, _)| *k == kind) +} + +fn user_events(out: VecDeque>) -> Vec { + out + .into_iter() + .map(|ev| match ev { + Event::User(u) => u, + other => panic!("expected Event::User, got {other:?}"), + }) + .collect() +} + +// ── window timing ─────────────────────────────────────────────────────────────── + +#[test] +fn empty_member_window_has_no_deadline() { + let c = MemberEventCoalescer::::new(secs(10), secs(2)); + assert_eq!(c.flush_deadline(), None); + assert!(!c.due(Instant::ORIGIN)); + assert!(!c.due(Instant::ORIGIN + secs(1000))); +} + +#[test] +fn empty_user_window_has_no_deadline() { + let c = UserEventCoalescer::new(secs(10), secs(2), None); + assert_eq!(c.flush_deadline(), None); + assert!(!c.due(Instant::ORIGIN)); +} + +#[test] +fn quiescent_binds_before_coalesce_on_a_single_event() { + let mut drops = 0u64; + // quiescent (2s) < coalesce (10s): a lone event flushes at first + quiescent. + let mut c = MemberEventCoalescer::::new(secs(10), secs(2)); + let t0 = Instant::ORIGIN + secs(1); + c.feed(MemberEventKind::Join, vec![member(1)], t0, &mut drops); + assert_eq!(c.flush_deadline(), Some(t0 + secs(2))); + assert!(!c.due(t0 + secs(1))); + assert!(c.due(t0 + secs(2))); +} + +#[test] +fn coalesce_caps_the_maximum_batch_delay() { + let mut drops = 0u64; + // A steady stream keeps re-arming the quiescent timer, but the coalesce + // max-window is armed once on the first event and never advances, so the flush + // deadline can never exceed first_event + coalesce_period. + let mut c = MemberEventCoalescer::::new(secs(10), secs(8)); + let t0 = Instant::ORIGIN; + c.feed(MemberEventKind::Join, vec![member(1)], t0, &mut drops); + // First arm: min(t0 + 10, t0 + 8) = t0 + 8. + assert_eq!(c.flush_deadline(), Some(t0 + secs(8))); + // A later event re-arms quiescent to t0 + 13, but coalesce stays t0 + 10. + c.feed( + MemberEventKind::Join, + vec![member(2)], + t0 + secs(5), + &mut drops, + ); + assert_eq!(c.flush_deadline(), Some(t0 + secs(10))); +} + +#[test] +fn flush_disarms_the_window() { + let mut drops = 0u64; + let mut c = MemberEventCoalescer::::new(secs(10), secs(2)); + let t0 = Instant::ORIGIN; + c.feed(MemberEventKind::Join, vec![member(1)], t0, &mut drops); + assert!(c.flush_deadline().is_some()); + let mut out = VecDeque::new(); + c.flush(&mut out); + assert_eq!(c.flush_deadline(), None, "window disarms after a flush"); + assert!(!c.due(t0 + secs(1000))); +} + +// ── member dedup (ports serf-core coalesce/member.rs) ──────────────────────────── + +#[test] +fn member_flush_collapses_to_latest_status_per_node() { + let mut drops = 0u64; + // Ports `test_member_event_coealesce_basic`: rapid transitions per node + // collapse to the final status; the flush groups survivors by kind. + let mut c = MemberEventCoalescer::::new(secs(10), secs(2)); + let t0 = Instant::ORIGIN; + // node 1: Join then Leave -> Leave wins. + c.feed(MemberEventKind::Join, vec![member(1)], t0, &mut drops); + c.feed(MemberEventKind::Leave, vec![member(1)], t0, &mut drops); + // node 2: Leave. + c.feed(MemberEventKind::Leave, vec![member(2)], t0, &mut drops); + // node 3: Update then Update -> the latter (role=bar) wins. + c.feed( + MemberEventKind::Update, + vec![member_tagged(3, "foo")], + t0, + &mut drops, + ); + c.feed( + MemberEventKind::Update, + vec![member_tagged(3, "bar")], + t0, + &mut drops, + ); + // node 4: Reap. + c.feed(MemberEventKind::Reap, vec![member(4)], t0, &mut drops); + + let mut out = VecDeque::new(); + c.flush(&mut out); + let groups = member_groups(out); + + assert_eq!(groups.len(), 3, "Leave, Update, Reap (no Join): {groups:?}"); + assert_eq!( + find_group(&groups, MemberEventKind::Leave).map(|(_, ids)| ids.as_slice()), + Some([1u32, 2].as_slice()), + "node 1 (join->leave) and node 2 collapse into one Leave batch" + ); + assert_eq!( + find_group(&groups, MemberEventKind::Reap).map(|(_, ids)| ids.as_slice()), + Some([4u32].as_slice()) + ); + assert!( + find_group(&groups, MemberEventKind::Join).is_none(), + "node 1's Join is superseded by its Leave" + ); +} + +#[test] +fn member_update_carries_the_latest_tags() { + let mut drops = 0u64; + let mut c = MemberEventCoalescer::::new(secs(10), secs(2)); + let t0 = Instant::ORIGIN; + c.feed( + MemberEventKind::Update, + vec![member_tagged(3, "foo")], + t0, + &mut drops, + ); + c.feed( + MemberEventKind::Update, + vec![member_tagged(3, "bar")], + t0, + &mut drops, + ); + + let mut out = VecDeque::new(); + c.flush(&mut out); + assert_eq!(out.len(), 1); + match out.pop_front().unwrap() { + Event::Member(me) => { + assert_eq!(me.kind(), MemberEventKind::Update); + assert_eq!(me.members().len(), 1); + assert_eq!( + me.members()[0].tags().0.get("role").map(|s| s.as_str()), + Some("bar") + ); + } + other => panic!("expected Update, got {other:?}"), + } +} + +#[test] +fn member_update_always_re_emits_across_flushes() { + let mut drops = 0u64; + // Ports `test_member_event_coalesce_tag_update`: a second Update for a node is + // NOT suppressed even though the last emitted kind was already Update, because + // its tags may have changed. + let mut c = MemberEventCoalescer::::new(secs(10), secs(2)); + let t0 = Instant::ORIGIN; + + c.feed( + MemberEventKind::Update, + vec![member_tagged(1, "foo")], + t0, + &mut drops, + ); + let mut out = VecDeque::new(); + c.flush(&mut out); + assert_eq!(out.len(), 1, "first Update delivered"); + + c.feed( + MemberEventKind::Update, + vec![member_tagged(1, "bar")], + t0 + secs(1), + &mut drops, + ); + let mut out2 = VecDeque::new(); + c.flush(&mut out2); + assert_eq!(out2.len(), 1, "second Update re-emitted, not suppressed"); + match out2.pop_front().unwrap() { + Event::Member(me) => { + assert_eq!(me.kind(), MemberEventKind::Update); + assert_eq!( + me.members()[0].tags().0.get("role").map(|s| s.as_str()), + Some("bar") + ); + } + other => panic!("expected Update, got {other:?}"), + } +} + +#[test] +fn member_repeated_same_status_is_suppressed() { + let mut drops = 0u64; + // A non-Update kind unchanged since the last flush is suppressed (the node is + // not re-announced). Only Update is exempt from this suppression. + let mut c = MemberEventCoalescer::::new(secs(10), secs(2)); + let t0 = Instant::ORIGIN; + + c.feed(MemberEventKind::Failed, vec![member(1)], t0, &mut drops); + let mut out = VecDeque::new(); + c.flush(&mut out); + assert_eq!(out.len(), 1, "first Failed delivered"); + + c.feed( + MemberEventKind::Failed, + vec![member(1)], + t0 + secs(1), + &mut drops, + ); + let mut out2 = VecDeque::new(); + c.flush(&mut out2); + assert!( + out2.is_empty(), + "repeated Failed for the same node is suppressed" + ); +} + +#[test] +fn reap_then_rejoin_in_one_window_still_emits_the_join() { + let mut drops = 0u64; + // A prior window records `last[id] = Join`. Within a single later window the + // node is Reaped and then rejoins (Join) before the window closes. The Reap is + // forgotten from `last` at feed time, so the stale `last[id] == Join` can no + // longer suppress the rejoin — the Join must still be delivered. + let mut c = MemberEventCoalescer::::new(secs(10), secs(2)); + let t0 = Instant::ORIGIN; + + // Window 1: a plain Join records `last[1] = Join`. + c.feed(MemberEventKind::Join, vec![member(1)], t0, &mut drops); + let mut out = VecDeque::new(); + c.flush(&mut out); + assert_eq!(member_groups(out), vec![(MemberEventKind::Join, vec![1])]); + + // Window 2: Reap then a rejoin Join for the same id, within one window. + let t1 = t0 + secs(5); + c.feed(MemberEventKind::Reap, vec![member(1)], t1, &mut drops); + c.feed(MemberEventKind::Join, vec![member(1)], t1, &mut drops); + let mut out2 = VecDeque::new(); + c.flush(&mut out2); + let groups = member_groups(out2); + + assert_eq!( + find_group(&groups, MemberEventKind::Join).map(|(_, ids)| ids.as_slice()), + Some([1u32].as_slice()), + "the rejoin Join must not be suppressed by the pre-Reap last[id]: {groups:?}" + ); +} + +#[test] +fn member_reap_evicts_suppression_entry_bounding_last_to_live_membership() { + let mut drops = 0u64; + // The cross-flush `last` map must track only LIVE membership: a node whose + // flushed terminal event is `Reap` is gone from membership for good, so its id + // is evicted. Churning many DISTINCT ids through join → reap must therefore + // NOT grow `last` — reverting the eviction makes it grow to the total ids ever + // seen and this assertion fails. + let mut c = MemberEventCoalescer::::new(secs(10), secs(2)); + + // Three long-lived members that join once and never leave: they stay in `last`. + for id in 0..3u32 { + c.feed( + MemberEventKind::Join, + vec![member(id)], + Instant::ORIGIN, + &mut drops, + ); + } + let mut out = VecDeque::new(); + c.flush(&mut out); + assert_eq!(c.last_len(), 3, "three live members are tracked"); + + // Churn 1000 distinct transient ids through join (flush) → reap (flush). + let mut t = Instant::ORIGIN; + for id in 100..1100u32 { + c.feed(MemberEventKind::Join, vec![member(id)], t, &mut drops); + let mut j = VecDeque::new(); + c.flush(&mut j); + c.feed(MemberEventKind::Reap, vec![member(id)], t, &mut drops); + let mut r = VecDeque::new(); + c.flush(&mut r); + t += secs(1); + } + + assert_eq!( + c.last_len(), + 3, + "after 1000 join→reap cycles `last` still tracks only the 3 live members, \ + not the 1000 reaped ids (reverting Reap-eviction grows it to 1003)" + ); +} + +#[test] +fn address_change_rejoin_is_not_suppressed() { + let mut drops = 0u64; + // A prior window records `last[id] = (Join, addr_a)`. When the node then leaves + // addr_a and rejoins at addr_b within a later window, `latest[id]` collapses to + // Join(addr_b). Suppression keys on (kind, address), so even though the last + // emitted kind was also Join the changed address defeats it and the move must + // be delivered — consumers would otherwise retain the stale addr_a. + let addr_a: SocketAddr = "127.0.0.1:5000".parse().unwrap(); + let addr_b: SocketAddr = "127.0.0.1:6000".parse().unwrap(); + let at = |a: SocketAddr| Member::new(Node::new(7u32, a), Tags::new(), MemberStatus::None); + + let mut c = MemberEventCoalescer::::new(secs(10), secs(2)); + let t0 = Instant::ORIGIN; + + // Window 1: Join at addr_a records `last[7] = (Join, addr_a)`. + c.feed(MemberEventKind::Join, vec![at(addr_a)], t0, &mut drops); + let mut out = VecDeque::new(); + c.flush(&mut out); + assert_eq!(member_groups(out), vec![(MemberEventKind::Join, vec![7])]); + + // Window 2: Leave(addr_a) then a rejoin Join(addr_b), collapsing to Join(addr_b). + let t1 = t0 + secs(5); + c.feed(MemberEventKind::Leave, vec![at(addr_a)], t1, &mut drops); + c.feed(MemberEventKind::Join, vec![at(addr_b)], t1, &mut drops); + let mut out2 = VecDeque::new(); + c.flush(&mut out2); + + let joined: Vec<(u32, SocketAddr)> = out2 + .iter() + .filter_map(|ev| match ev { + Event::Member(me) if me.kind() == MemberEventKind::Join => Some( + me.members() + .iter() + .map(|m| (*m.node().id_ref(), *m.node().addr_ref())), + ), + _ => None, + }) + .flatten() + .collect(); + assert_eq!( + joined, + vec![(7u32, addr_b)], + "the rejoin at a new address must re-emit, carrying addr_b" + ); + + // Window 3: a plain same-address repeat is STILL suppressed — the dedup is + // intact and only a changed address defeats it. + let t2 = t1 + secs(5); + c.feed(MemberEventKind::Join, vec![at(addr_b)], t2, &mut drops); + let mut out3 = VecDeque::new(); + c.flush(&mut out3); + assert!( + out3.is_empty(), + "a repeat Join at the unchanged address is still suppressed" + ); +} + +#[test] +fn same_address_rejoin_with_changed_tags_is_not_suppressed() { + let mut drops = 0u64; + // A node joins with one set of tags, then fails and rejoins at the SAME address + // within a later window carrying DIFFERENT tags (its member is rebuilt from + // freshly decoded metadata). The window collapses Failed → Join, and the last + // emitted kind and address are both unchanged — but the tags changed, so the + // Join must still emit or consumers retain the stale tags. Suppression compares + // the full observable state, not just kind and address. + let addr: SocketAddr = "127.0.0.1:5000".parse().unwrap(); + let tagged = |role: &str| { + Member::new( + Node::new(7u32, addr), + Tags::from_iter([("role", role)]), + MemberStatus::None, + ) + }; + + let mut c = MemberEventCoalescer::::new(secs(10), secs(2)); + let t0 = Instant::ORIGIN; + + // Window 1: Join with role=web records the last emitted state. + c.feed(MemberEventKind::Join, vec![tagged("web")], t0, &mut drops); + let mut out = VecDeque::new(); + c.flush(&mut out); + assert_eq!(member_groups(out), vec![(MemberEventKind::Join, vec![7])]); + + // Window 2: Failed then a same-address rejoin Join with role=db, collapsing to + // Join(role=db). + let t1 = t0 + secs(5); + c.feed(MemberEventKind::Failed, vec![tagged("web")], t1, &mut drops); + c.feed(MemberEventKind::Join, vec![tagged("db")], t1, &mut drops); + let mut out2 = VecDeque::new(); + c.flush(&mut out2); + + let joined_tags: Vec<(u32, Option)> = out2 + .iter() + .filter_map(|ev| match ev { + Event::Member(me) if me.kind() == MemberEventKind::Join => Some( + me.members() + .iter() + .map(|m| (*m.node().id_ref(), m.tags().0.get("role").cloned())), + ), + _ => None, + }) + .flatten() + .collect(); + assert_eq!( + joined_tags, + vec![(7u32, Some(SmolStr::new("db")))], + "the same-address rejoin with changed tags must re-emit, carrying the new tags" + ); + + // Window 3: a genuinely identical repeat (same kind, address, AND tags) is still + // suppressed — the dedup is intact and only an observable change defeats it. + let t2 = t1 + secs(5); + c.feed(MemberEventKind::Join, vec![tagged("db")], t2, &mut drops); + let mut out3 = VecDeque::new(); + c.flush(&mut out3); + assert!( + out3.is_empty(), + "a repeat Join at the unchanged address AND tags is still suppressed" + ); +} + +#[test] +fn member_reset_drops_buffer_without_emitting() { + let mut drops = 0u64; + let mut c = MemberEventCoalescer::::new(secs(10), secs(2)); + let t0 = Instant::ORIGIN; + c.feed(MemberEventKind::Join, vec![member(1)], t0, &mut drops); + assert!(c.flush_deadline().is_some()); + + c.reset(); + assert_eq!(c.flush_deadline(), None, "reset disarms the window"); + + let mut out = VecDeque::new(); + c.flush(&mut out); + assert!(out.is_empty(), "a reset coalescer flushes nothing"); +} + +// ── user dedup (ports serf-core coalesce/user.rs) ──────────────────────────────── + +#[test] +fn user_flush_keeps_newest_generation_per_name() { + let mut drops = 0u64; + // Ports `test_user_event_coalesce_basic`: foo@1 then foo@2 keeps only foo@2; + // bar@2(test1) then bar@2(test2) keeps both (same generation). + let mut c = UserEventCoalescer::new(secs(10), secs(2), None); + let t0 = Instant::ORIGIN; + c.feed(uev("foo", 1, ""), t0, &mut drops); + c.feed(uev("foo", 2, ""), t0, &mut drops); + c.feed(uev("bar", 2, "test1"), t0, &mut drops); + c.feed(uev("bar", 2, "test2"), t0, &mut drops); + + let mut out = VecDeque::new(); + c.flush::(&mut out); + let events = user_events(out); + + let foo: Vec<_> = events.iter().filter(|e| e.name == "foo").collect(); + assert_eq!(foo.len(), 1, "only the newest foo generation survives"); + assert_eq!(foo[0].ltime, LamportTime::new(2)); + + let mut bar_payloads: Vec<&[u8]> = events + .iter() + .filter(|e| e.name == "bar") + .map(|e| e.payload.as_ref()) + .collect(); + bar_payloads.sort_unstable(); + assert_eq!( + bar_payloads, + vec![b"test1".as_slice(), b"test2".as_slice()], + "both same-generation bar payloads survive" + ); +} + +#[test] +fn user_older_generation_is_dropped() { + let mut drops = 0u64; + let mut c = UserEventCoalescer::new(secs(10), secs(2), None); + let t0 = Instant::ORIGIN; + c.feed(uev("foo", 5, "new"), t0, &mut drops); + c.feed(uev("foo", 3, "old"), t0, &mut drops); + + let mut out = VecDeque::new(); + c.flush::(&mut out); + let events = user_events(out); + assert_eq!(events.len(), 1); + assert_eq!(events[0].ltime, LamportTime::new(5)); + assert_eq!(events[0].payload.as_ref(), b"new"); +} + +#[test] +fn user_reset_drops_buffer_without_emitting() { + let mut drops = 0u64; + let mut c = UserEventCoalescer::new(secs(10), secs(2), None); + c.feed(uev("foo", 1, "x"), Instant::ORIGIN, &mut drops); + assert!(c.flush_deadline().is_some()); + + c.reset(); + assert_eq!(c.flush_deadline(), None); + let mut out = VecDeque::new(); + c.flush::(&mut out); + assert!(out.is_empty()); +} + +// ── user buffered-volume cap (total-sum bound, not per-key) ────────────────────── + +#[test] +fn user_distinct_name_flood_is_capped_and_counted() { + let mut drops = 0u64; + // A flood of DISTINCT names at advancing generations pins the buffered volume + // at the cap; every event past it is dropped and counted, and a flush emits at + // most `cap` user events. + const CAP: usize = 1024; + const N: usize = 3000; + let mut c = UserEventCoalescer::new(secs(10), secs(2), NonZeroUsize::new(CAP)); + let t0 = Instant::ORIGIN; + for i in 0..N { + c.feed(uev(&format!("evt-{i}"), i as u64 + 1, "p"), t0, &mut drops); + } + assert_eq!(c.buffered(), CAP, "buffered is pinned at the cap"); + assert_eq!( + c.distinct_names(), + CAP, + "one buffered event per surviving distinct name" + ); + assert_eq!( + drops, + (N - CAP) as u64, + "every event past the cap is counted as dropped" + ); + + let mut out = VecDeque::new(); + c.flush::(&mut out); + assert!(out.len() <= CAP, "a flush emits at most `cap` events"); + assert_eq!(out.len(), CAP, "exactly the buffered volume is emitted"); + assert_eq!(c.buffered(), 0, "flush resets the buffered invariant"); + assert_eq!( + drops, + (N - CAP) as u64, + "the drop counter is cumulative and survives the flush" + ); +} + +#[test] +fn user_equal_ltime_payload_flood_is_capped_by_total_volume() { + let mut drops = 0u64; + // A single name accumulating many DISTINCT payloads at ONE generation must be + // bounded by the SAME total-volume cap: the bound is the running sum across the + // map, not a per-key constant. + const CAP: usize = 1024; + const M: usize = 3000; + let mut c = UserEventCoalescer::new(secs(10), secs(2), NonZeroUsize::new(CAP)); + let t0 = Instant::ORIGIN; + for i in 0..M { + c.feed(uev("burst", 5, &format!("p{i}")), t0, &mut drops); + } + assert_eq!( + c.distinct_names(), + 1, + "all payloads share the one name/generation" + ); + assert_eq!( + c.buffered(), + CAP, + "the single name's accumulated payloads are bounded by the total-volume cap" + ); + assert_eq!(drops, (M - CAP) as u64, "the overflow is counted"); +} + +#[test] +fn superseding_a_name_frees_its_old_buffer_allocation() { + let mut drops = 0u64; + // The memory cap counts live events, but a superseded buffer must also FREE + // its allocation: `Vec::clear` retains capacity, so a name filled toward the + // cap and then superseded would keep a large allocation the count-based cap + // cannot see. Repeatedly filling and superseding distinct names would then + // retain quadratic memory while `buffered` and the drop counter stay low. + const CAP: usize = 1024; + const FILL: usize = 512; + let mut c = UserEventCoalescer::new(secs(10), secs(2), NonZeroUsize::new(CAP)); + let t0 = Instant::ORIGIN; + + // Accumulate many payloads at generation 1 for one name, then supersede to a + // single event at generation 2. + for i in 0..FILL { + c.feed(uev("burst", 1, &format!("p{i}")), t0, &mut drops); + } + assert_eq!(c.buffered(), FILL); + assert!( + c.retained_capacity() >= FILL, + "the buffer grew to hold the flood" + ); + + c.feed(uev("burst", 2, "new"), t0, &mut drops); + assert_eq!( + c.buffered(), + 1, + "supersession collapses the buffered volume to one" + ); + assert!( + c.retained_capacity() <= 8, + "superseding must release the old buffer's allocation; retained {} slots", + c.retained_capacity() + ); +} + +#[test] +fn user_newer_generation_for_existing_key_admitted_at_cap() { + let mut drops = 0u64; + // At the cap, a NEWER generation for an already-buffered key must still be + // admitted: it clears that key's older payloads first (net change <= 0), so no + // drop is counted and the map now carries only the new generation. + const CAP: usize = 1024; + let mut c = UserEventCoalescer::new(secs(10), secs(2), NonZeroUsize::new(CAP)); + let t0 = Instant::ORIGIN; + for i in 0..CAP { + c.feed(uev(&format!("evt-{i}"), 1, "old"), t0, &mut drops); + } + assert_eq!(c.buffered(), CAP); + assert_eq!(drops, 0); + + // A newer generation for an existing key, at saturation. + c.feed(uev("evt-5", 2, "new"), t0, &mut drops); + assert_eq!( + c.buffered(), + CAP, + "superseding an existing key keeps the volume at the cap" + ); + assert_eq!(drops, 0, "a superseding admit is never a drop"); + + let mut out = VecDeque::new(); + c.flush::(&mut out); + let events = user_events(out); + let five: Vec<_> = events.iter().filter(|e| e.name == "evt-5").collect(); + assert_eq!( + five.len(), + 1, + "only the newest generation of evt-5 survives" + ); + assert_eq!(five[0].ltime, LamportTime::new(2)); + assert_eq!(five[0].payload.as_ref(), b"new"); +} + +#[test] +fn user_unsaturated_collapse_matches_pre_cap_behavior() { + let mut drops = 0u64; + // Well below the cap, the collapse is byte-identical to the pre-cap coalescer: + // newest generation per name wins, same-generation payloads accumulate, and + // nothing is dropped. Guards against the cap logic perturbing normal traffic. + let mut c = UserEventCoalescer::new(secs(10), secs(2), NonZeroUsize::new(1024)); + let t0 = Instant::ORIGIN; + c.feed(uev("foo", 1, ""), t0, &mut drops); + c.feed(uev("foo", 2, ""), t0, &mut drops); + c.feed(uev("bar", 2, "test1"), t0, &mut drops); + c.feed(uev("bar", 2, "test2"), t0, &mut drops); + assert_eq!(drops, 0, "an unsaturated workload drops nothing"); + assert_eq!(c.buffered(), c.live_payload_count()); + + let mut out = VecDeque::new(); + c.flush::(&mut out); + let events = user_events(out); + + let foo: Vec<_> = events.iter().filter(|e| e.name == "foo").collect(); + assert_eq!(foo.len(), 1, "only the newest foo generation survives"); + assert_eq!(foo[0].ltime, LamportTime::new(2)); + + let mut bar_payloads: Vec<&[u8]> = events + .iter() + .filter(|e| e.name == "bar") + .map(|e| e.payload.as_ref()) + .collect(); + bar_payloads.sort_unstable(); + assert_eq!( + bar_payloads, + vec![b"test1".as_slice(), b"test2".as_slice()], + "both same-generation bar payloads survive" + ); +} + +#[test] +fn user_buffered_invariant_holds_under_randomized_interleave() { + let mut drops = 0u64; + // `buffered` must equal the true sum of live payload counts after every feed, + // through any interleave of insert / supersede / accumulate / drop-older; a + // flush and a reset both zero it. + let mut c = UserEventCoalescer::new(secs(10), secs(2), NonZeroUsize::new(256)); + // A small deterministic LCG drives a reproducible mix of the four branches. + let mut state: u64 = 0x9e37_79b9_7f4a_7c15; + let mut next = || { + state = state + .wrapping_mul(6364136223846793005) + .wrapping_add(1442695040888963407); + state + }; + let t0 = Instant::ORIGIN; + for step in 0..5000u64 { + let r = next(); + let name_idx = r % 32; + let ltime = (r >> 8) % 8; + c.feed( + uev(&format!("k{name_idx}"), ltime, "p"), + t0 + secs(step % 3), + &mut drops, + ); + assert_eq!( + c.buffered(), + c.live_payload_count(), + "buffered must track the live payload sum at step {step}" + ); + } + let mut out = VecDeque::new(); + c.flush::(&mut out); + assert_eq!(c.buffered(), 0, "flush zeroes buffered"); + c.feed(uev("k0", 1, "p"), t0, &mut drops); + c.reset(); + assert_eq!(c.buffered(), 0, "reset zeroes buffered"); +} + +// ── arm-on-admit-only (a rejected event must not extend the window) ────────────── + +#[test] +fn user_rejected_feed_does_not_arm_the_window() { + let mut drops = 0u64; + const CAP: usize = 4; + let mut c = UserEventCoalescer::new(secs(10), secs(2), NonZeroUsize::new(CAP)); + let t0 = Instant::ORIGIN; + + // An empty map always admits and arms. + c.feed(uev("a", 1, "p"), t0, &mut drops); + assert_eq!( + c.flush_deadline(), + Some(t0 + secs(2)), + "the first event arms the window" + ); + + // Saturate the buffered volume. + for i in 1..CAP { + c.feed(uev(&format!("n{i}"), 1, "p"), t0 + secs(1), &mut drops); + } + assert_eq!(c.buffered(), CAP); + let deadline_before = c.flush_deadline(); + + // A rejected feed (a new name at the cap) must NOT re-arm the quiescent window. + c.feed(uev("overflow", 9, "p"), t0 + secs(5), &mut drops); + assert_eq!(drops, 1); + assert_eq!( + c.flush_deadline(), + deadline_before, + "a rejected event must not extend the quiescent deadline" + ); +} + +// ── member cardinality cap (MAX_COALESCED_MEMBER_EVENTS) ───────────────────────── + +#[test] +fn member_reap_eviction_runs_before_the_cardinality_gate() { + let mut drops = 0u64; + // The unconditional `last.remove` on a Reap must run even when that same Reap is + // rejected by the cardinality cap; otherwise a stale suppression entry outlives + // the reaped node and wrongly suppresses its later rejoin. + const ABSENT_ID: u32 = 9_000_000; + let mut c = MemberEventCoalescer::::new(secs(10), secs(2)); + let t0 = Instant::ORIGIN; + + // Record `last[ABSENT_ID] = (Join, addr)` via a flushed Join, leaving `latest` + // empty afterwards. + c.feed( + MemberEventKind::Join, + vec![member(ABSENT_ID)], + t0, + &mut drops, + ); + let mut out = VecDeque::new(); + c.flush(&mut out); + assert!( + c.last_contains(&ABSENT_ID), + "the Join is recorded in `last`" + ); + assert_eq!(c.latest_len(), 0, "the window drained"); + + // Saturate `latest` with distinct OTHER ids. + for id in 0..MAX_COALESCED_MEMBER_EVENTS as u32 { + c.feed(MemberEventKind::Join, vec![member(id)], t0, &mut drops); + } + assert_eq!(c.latest_len(), MAX_COALESCED_MEMBER_EVENTS); + + // A Reap for ABSENT_ID: absent from the saturated map, so the insert is rejected + // and counted — but the `last.remove` must still run. + c.feed( + MemberEventKind::Reap, + vec![member(ABSENT_ID)], + t0, + &mut drops, + ); + assert_eq!( + c.latest_len(), + MAX_COALESCED_MEMBER_EVENTS, + "the Reap insert was rejected by the cap" + ); + assert_eq!(drops, 1, "the rejected Reap is counted"); + assert!( + !c.last_contains(&ABSENT_ID), + "the unconditional Reap eviction ran before the cap rejected the insert" + ); + + // Drain the saturated window; ABSENT_ID never entered `latest`, so this flush + // leaves `last` still lacking it. + let mut drained = VecDeque::new(); + c.flush(&mut drained); + assert_eq!(c.latest_len(), 0); + assert!(!c.last_contains(&ABSENT_ID)); + + // A later rejoin Join now lands in an empty window and must NOT be suppressed. + c.feed( + MemberEventKind::Join, + vec![member(ABSENT_ID)], + t0, + &mut drops, + ); + let mut out2 = VecDeque::new(); + c.flush(&mut out2); + let groups = member_groups(out2); + assert_eq!( + find_group(&groups, MemberEventKind::Join).map(|(_, ids)| ids.as_slice()), + Some([ABSENT_ID].as_slice()), + "the rejoin must re-emit, not be suppressed by a stale last entry: {groups:?}" + ); +} + +#[test] +fn member_cardinality_cap_bounds_distinct_ids() { + let mut drops = 0u64; + let mut c = MemberEventCoalescer::::new(secs(10), secs(2)); + let t0 = Instant::ORIGIN; + + // Fill to capacity with distinct new ids. + for id in 0..MAX_COALESCED_MEMBER_EVENTS as u32 { + c.feed(MemberEventKind::Join, vec![member(id)], t0, &mut drops); + } + assert_eq!(c.latest_len(), MAX_COALESCED_MEMBER_EVENTS); + assert_eq!(drops, 0); + + // Further DISTINCT ids overflow the cap and are counted. + const OVERFLOW: u32 = 100; + for i in 0..OVERFLOW { + c.feed( + MemberEventKind::Join, + vec![member(MAX_COALESCED_MEMBER_EVENTS as u32 + i)], + t0, + &mut drops, + ); + } + assert_eq!( + c.latest_len(), + MAX_COALESCED_MEMBER_EVENTS, + "the map stays at the cap" + ); + assert_eq!(drops, OVERFLOW as u64, "each overflow id is counted"); + + // An EXISTING id always updates in place, never rejected. + c.feed(MemberEventKind::Leave, vec![member(0)], t0, &mut drops); + assert_eq!( + c.latest_len(), + MAX_COALESCED_MEMBER_EVENTS, + "an existing-id update does not grow the map" + ); + assert_eq!( + drops, OVERFLOW as u64, + "an existing-id update is not a drop" + ); +} + +#[test] +fn member_rejected_feed_does_not_arm_the_window() { + let mut drops = 0u64; + let mut c = MemberEventCoalescer::::new(secs(10), secs(2)); + let t0 = Instant::ORIGIN; + + // An empty map always admits and arms. + c.feed(MemberEventKind::Join, vec![member(0)], t0, &mut drops); + assert_eq!(c.flush_deadline(), Some(t0 + secs(2))); + + // Saturate `latest` with distinct ids. + for id in 1..MAX_COALESCED_MEMBER_EVENTS as u32 { + c.feed( + MemberEventKind::Join, + vec![member(id)], + t0 + secs(1), + &mut drops, + ); + } + assert_eq!(c.latest_len(), MAX_COALESCED_MEMBER_EVENTS); + let deadline_before = c.flush_deadline(); + + // A rejected feed (a new id at the cap) must NOT re-arm the window. + c.feed( + MemberEventKind::Join, + vec![member(9_000_000)], + t0 + secs(5), + &mut drops, + ); + assert_eq!(drops, 1); + assert_eq!( + c.flush_deadline(), + deadline_before, + "a rejected member event must not extend the quiescent deadline" + ); +} diff --git a/serf-proto/src/endpoint/mod.rs b/serf-proto/src/endpoint/mod.rs index 3d3838eb..b3078ccd 100644 --- a/serf-proto/src/endpoint/mod.rs +++ b/serf-proto/src/endpoint/mod.rs @@ -75,6 +75,7 @@ use crate::event::{KeyRequest as KeyRequestEvent, KeyRequestOperation, KeyRespon use crate::{ AnyMessage, ConflictResponseMessage, EncodeError, LamportTime, MessageType, bridge::{tags_from_pb, tags_to_pb, user_event_to_pb}, + coalesce::{DropCounter, MemberEventCoalescer, UserEventCoalescer}, event::{ DialPassthrough, Event, MemberEvent, MemberEventKind, QueryAck, QueryEvent, QueryResponse as QueryResponseEvent, @@ -672,9 +673,16 @@ fn next_ltime(clock: &mut u64) -> u64 { /// serf's own selection draws (relay picks, reconnect probabilistic gate, /// coordinate jitter, query id generation) use `self.rng`; the inner /// memberlist `Endpoint`'s gossip uses its own independently-seeded `R`. -pub struct Endpoint +/// +/// `D` is the [`DropCounter`] storage for the two coalescer shed counts +/// (default: a plain `u64`, keeping the machine atomics-free and `Send + Sync`). +/// An async driver injects its own shared, read-observable backing via +/// [`new_with_rng_in`](Self::new_with_rng_in) so its detached handle reads the +/// shed count without the endpoint publishing a copy. +pub struct Endpoint where I: Eq + core::hash::Hash, + D: DropCounter, { /// serf configuration knobs. opts: Options, @@ -739,6 +747,30 @@ where received_queries: crate::FxHashMap>, /// serf-level events queued for the driver to drain via `poll_event`. pending_events: VecDeque>, + /// Member-event coalescer, or `None` when member coalescing is disabled + /// (either period is zero — the default). + /// + /// When `Some`, membership events are fed here at their emission sites instead + /// of pushed straight to `pending_events`; the batch is flushed into + /// `pending_events` once the coalescer's window closes (`after_inner_timeout`). + /// When `None`, every membership event passes straight through unchanged — + /// the exact behaviour of a machine built with the default (disabled) options. + member_coalescer: Option>, + /// User-event coalescer, or `None` when user coalescing is disabled (either + /// user period is zero — the default). + /// + /// Only user events that opted in (`UserEventMessage::cc == true`) are fed + /// here; a non-coalescing user event passes straight through even when the + /// coalescer is enabled (mirrors the legacy coalescer's `handle` predicate). + user_coalescer: Option, + /// Cumulative user-coalescer shed count, incremented in `emit_user` when the + /// user coalescer drops an event at its volume cap. Held always-present (out + /// of the `Option` coalescer) so the count survives `reset`/`flush` and a + /// driver-injected backing stays wired for the endpoint's whole lifetime. + user_drop: D, + /// Cumulative member-coalescer shed count, incremented in `emit_member` when + /// the member coalescer drops a change at its cardinality cap. + member_drop: D, /// The most recent directed-send (address, bytes) produced by /// `handle_relay` or `relay_response`. /// @@ -775,6 +807,13 @@ where /// Callers that do not have a meaningful `now` (e.g., `poll_event` called /// after a prior handle call) use the last latched value. drain_now: Instant, + /// Monotonically non-decreasing processing clock for coalescer window + /// scheduling. `drain_now` carries protocol arrival time, which is NOT + /// monotonic — reliable ingress can be processed after a newer command yet + /// carry an earlier `received_at`. Feeding that raw value to the coalescer + /// would move an active window's quiescent deadline backward and flush it + /// prematurely, so the coalescer arms from this max-clamped clock instead. + coalesce_now: Instant, /// Dirty flag for the push-pull local-state snapshot (H6). /// /// Set whenever any of the three Lamport clocks, member status-ltimes, @@ -824,10 +863,11 @@ where // ── construction + cheap accessors ──────────────────────────────────────────── -impl Endpoint +impl Endpoint where I: Clone + Eq + core::hash::Hash, R: SeedableRng, + D: DropCounter, { /// Construct a serf `Endpoint` core using `opts` for serf-level knobs and /// `rng` as serf's own injected selection entropy. @@ -839,7 +879,27 @@ where /// /// `rng` is **separate** from the coordinator's `R`. Seed it from the /// driver's own entropy source; do not share the same `R` instance. - pub fn new_with_rng(opts: Options, rng: R) -> Self { + /// + /// The two coalescer shed counters start at `D::default()` (`0` for the + /// default `u64`). A driver that must observe the counts from a detached + /// handle injects a shared backing via + /// [`new_with_rng_in`](Self::new_with_rng_in) instead. + pub fn new_with_rng(opts: Options, rng: R) -> Self + where + D: Default, + { + Self::new_with_rng_in(opts, rng, D::default(), D::default()) + } + + /// Construct a serf `Endpoint` core injecting the two coalescer shed counters + /// `user_drop` / `member_drop`. + /// + /// An async driver mints a shared, read-observable backing (an atomic or a + /// `Cell`), keeps a read-only clone on its handle, and passes the write-capable + /// clones here so the handle observes every coalescer shed WITHOUT the endpoint + /// publishing a copy each pump iteration. The single-owner drivers use the + /// `u64` default via [`new_with_rng`](Self::new_with_rng). + pub fn new_with_rng_in(opts: Options, rng: R, user_drop: D, member_drop: D) -> Self { // Arm the first reap/reconnect/queue-check deadlines relative to the ORIGIN instant. // The driver calls handle_timeout(now) and the deadlines fire when now >= deadline. let first_reap = Instant::ORIGIN + opts.reap_interval(); @@ -848,6 +908,21 @@ where let event_buf_size = opts.event_buffer_size(); let query_buf_size = opts.query_buffer_size(); + // Enable each coalescer iff its (period > 0 && quiescent > 0), else None + // (disabled) — the default, which yields exact passthrough at the emission + // sites so a machine built with default options behaves identically to one + // with no coalescer at all. + let member_coalescer = opts + .member_coalesce_enabled() + .then(|| MemberEventCoalescer::new(opts.coalesce_period(), opts.quiescent_period())); + let user_coalescer = opts.user_coalesce_enabled().then(|| { + UserEventCoalescer::new( + opts.user_coalesce_period(), + opts.user_quiescent_period(), + opts.max_coalesced_user_events(), + ) + }); + #[cfg(feature = "coordinates")] let coord_client: Option> = if opts.disable_coordinates() { @@ -876,7 +951,12 @@ where pending_queries: Vec::new(), received_queries: crate::FxHashMap::default(), pending_events: VecDeque::new(), + member_coalescer, + user_coalescer, + user_drop, + member_drop, drain_now: Instant::ORIGIN, + coalesce_now: Instant::ORIGIN, // The snapshot starts dirty so the first push-pull always ships a fresh // body even if no explicit API call has been made yet. local_state_dirty: true, @@ -901,7 +981,10 @@ where /// Suitable for tests and environments where determinism or an explicit seed /// is acceptable. Production drivers should use `new_with_rng` and seed from /// a cryptographically-secure source. - pub fn new(opts: Options) -> Self { + pub fn new(opts: Options) -> Self + where + D: Default, + { Self::new_with_rng(opts, R::seed_from_u64(0)) } @@ -956,6 +1039,31 @@ where self.members.states.len() } + /// Cumulative count of coalescing user events dropped because the user + /// coalescer's buffered volume was at its configured cap + /// ([`Options::max_coalesced_user_events`](crate::options::Options::max_coalesced_user_events)). + /// + /// Lifetime total, saturating, and never cleared — a flush or a `reset` does + /// not reset it. Returns `0` when user coalescing is disabled. + pub fn coalesced_user_events_dropped(&self) -> u64 { + self.user_drop.get() + } + + /// Cumulative count of member changes dropped because the member coalescer's + /// per-window map was at its cardinality cap. + /// + /// Lifetime total, saturating, and never cleared. Returns `0` when member + /// coalescing is disabled. + pub fn coalesced_member_events_dropped(&self) -> u64 { + self.member_drop.get() + } + + /// Number of coalesced events currently waiting in the flush queue to be + /// drained by [`poll_event`](Self::poll_event). + pub fn pending_events_len(&self) -> usize { + self.pending_events.len() + } + /// A snapshot of every tracked member (alive, leaving, left, or failed within /// the reap window) as owned [`Member`](crate::members::Member) values, for a /// driver's observable membership view published after each membership change. @@ -975,11 +1083,12 @@ where // ── poll API (requires full Id + Data bounds for inner delegation) ───────────── -impl Endpoint +impl Endpoint where I: Id + Clone, A: CheapClone + Data + PartialEq + Clone + 'static, R: Rng + SeedableRng, + D: DropCounter, { /// Drain one serf event. /// @@ -1009,6 +1118,16 @@ where return None; } let query_min = self.pending_queries.iter().map(|pq| pq.deadline).min(); + // Fold in each enabled coalescer's flush deadline so the driver wakes to + // flush a buffered member/user batch on time. + let member_flush = self + .member_coalescer + .as_ref() + .and_then(|c| c.flush_deadline()); + let user_flush = self + .user_coalescer + .as_ref() + .and_then(|c| c.flush_deadline()); [ self.next_reap, self.next_reconnect, @@ -1016,6 +1135,8 @@ where self.leave_broadcast_deadline, self.leave_complete_deadline, query_min, + member_flush, + user_flush, ] .into_iter() .flatten() @@ -1167,6 +1288,33 @@ where self.leave_broadcast_deadline = None; } } + + // Flush any coalescer whose window has closed, delivering the coalesced batch + // via `pending_events`. A member/user event fed earlier this tick (during + // `drain_inner` / `fire_reap`) arms a future deadline, so it is NOT flushed + // now — only a window armed on a PRIOR tick that has since elapsed flushes + // here. Placed after the Shutdown gate above: a machine that lost its + // conflict vote this tick has already dropped its buffered batch (in + // `close_conflict_query`) and returned early, so nothing is flushed after the + // terminal Event::Shutdown. + self.flush_due_coalescers(now); + } + + /// Flush each enabled coalescer whose window has closed at `now` into + /// `pending_events`. + fn flush_due_coalescers(&mut self, now: Instant) { + let now = self.coalesce_now.max(now); + self.coalesce_now = now; + if let Some(c) = self.member_coalescer.as_mut() { + if c.due(now) { + c.flush(&mut self.pending_events); + } + } + if let Some(c) = self.user_coalescer.as_mut() { + if c.due(now) { + c.flush(&mut self.pending_events); + } + } } /// Drop received-query tokens that are strictly past their response deadline. @@ -1526,7 +1674,7 @@ where /// /// Returns [`Error::SetTagsMeta`] if the encoded tag map exceeds /// `Meta::MAX_SIZE` or the coordinator's configured `meta_max_size`. - pub(crate) fn set_tags(&mut self, t: &mut T, tags: Tags) -> Result<(), Error> + pub(crate) fn set_tags(&mut self, t: &mut T, tags: Tags, now: Instant) -> Result<(), Error> where T: Reliable, I: Clone, @@ -1534,6 +1682,11 @@ where { use buffa::Message as _; + // Latch the command's instant so the coordinator's resulting `NodeUpdated`, + // drained synchronously at the end of this call, arms the member coalescer + // from live `now` rather than a stale `drain_now`. + self.drain_now = now; + // Refuse once the machine has shut down (lost id-conflict vote). self.ensure_not_shutdown()?; @@ -1563,6 +1716,12 @@ where *ms.member_mut() = Member::new(node, tags, status); } + // Emit the resulting NodeUpdated synchronously under the freshly latched + // `now`, mirroring how `user_event` processes its event inline. Deferring it + // to a later `poll_event` drain would let an intervening ingress or timeout + // overwrite `drain_now`, arming the member coalescer from the wrong instant. + self.drain_inner(t); + Ok(()) } @@ -1728,6 +1887,61 @@ where } } + // ── event emission (coalesce-or-passthrough) ────────────────────────────── + + /// Emit a batch of membership changes of one `kind`. + /// + /// When the member coalescer is enabled the batch is fed into it (buffered, + /// deduped to the latest status per node, and flushed later once its window + /// closes in `after_inner_timeout`); otherwise it is pushed straight to + /// `pending_events` — the exact passthrough a machine with member coalescing + /// disabled (the default) performs. The feed is armed at `self.drain_now`, + /// the freshest instant the machine has latched. + fn emit_member(&mut self, kind: MemberEventKind, members: Vec>) { + let now = self.coalesce_now.max(self.drain_now); + self.coalesce_now = now; + if let Some(c) = self.member_coalescer.as_mut() { + // The window may have elapsed while the driver was busy and has not yet + // fired the overdue flush timer. Flush the completed batch before the new + // event mutates it — otherwise a feed after the deadline would overwrite a + // due observation and extend the window, merging two separate windows and + // dropping the earlier one. + if c.due(now) { + c.flush(&mut self.pending_events); + } + c.feed(kind, members, now, &mut self.member_drop); + } else { + self + .pending_events + .push_back(Event::Member(MemberEvent::new(kind, members))); + } + } + + /// Emit a user event. + /// + /// A coalescing user event (`cc == true`) is fed to the user coalescer when it + /// is enabled; every other case (a non-coalescing event, or the coalescer + /// disabled) passes straight through to `pending_events`. Mirrors the legacy + /// coalescer's `handle` predicate (`CrateEvent::User(e) => e.cc()`): only + /// coalescable user events are buffered. + fn emit_user(&mut self, msg: UserEventMessage) { + let now = self.coalesce_now.max(self.drain_now); + self.coalesce_now = now; + if msg.cc { + if let Some(c) = self.user_coalescer.as_mut() { + // Flush an elapsed-but-not-yet-fired window before the new event mutates + // it (see emit_member): a newer generation fed after the deadline would + // otherwise supersede and drop a due earlier generation. + if c.due(now) { + c.flush(&mut self.pending_events); + } + c.feed(msg, now, &mut self.user_drop); + return; + } + } + self.pending_events.push_back(Event::User(msg)); + } + // ── member-status FSM handlers ─────────────────────────────────────────── /// Handle an inner `NodeJoined` event. @@ -1807,12 +2021,7 @@ where // Always emit Member(Join). let member = self.members.states[id].member().clone(); - self - .pending_events - .push_back(Event::Member(MemberEvent::new( - MemberEventKind::Join, - vec![member], - ))); + self.emit_member(MemberEventKind::Join, vec![member]); } /// Handle an inner `NodeLeft` event. @@ -1854,9 +2063,7 @@ where // Membership changed — snapshot is stale. self.mark_local_state_dirty(); - self - .pending_events - .push_back(Event::Member(MemberEvent::new(event_kind, vec![member]))); + self.emit_member(event_kind, vec![member]); } /// Handle an inner `NodeUpdated` event. @@ -1894,12 +2101,7 @@ where // unaffected and a resync would be wasted work — and `set_tags` queues // exactly this event via `update_meta` on every local tag change. - self - .pending_events - .push_back(Event::Member(MemberEvent::new( - MemberEventKind::Update, - vec![member], - ))); + self.emit_member(MemberEventKind::Update, vec![member]); } /// Handle a gossiped join intent (`JoinMessage`). @@ -2067,12 +2269,7 @@ where // Move from failed_members to left_members. remove_old_member(&mut self.members.failed_members, &id_clone); self.members.left_members.push(id_clone); - self - .pending_events - .push_back(Event::Member(MemberEvent::new( - MemberEventKind::Leave, - vec![member], - ))); + self.emit_member(MemberEventKind::Leave, vec![member]); self.mark_local_state_dirty(); if prune { self.prune_member(id); @@ -2111,12 +2308,7 @@ where } if let Some(ms) = self.members.states.remove(id) { - self - .pending_events - .push_back(Event::Member(MemberEvent::new( - MemberEventKind::Reap, - vec![ms.member().clone()], - ))); + self.emit_member(MemberEventKind::Reap, vec![ms.member().clone()]); } self.mark_local_state_dirty(); } @@ -2168,12 +2360,7 @@ where } self.coord_cache.remove(&id); } - self - .pending_events - .push_back(Event::Member(MemberEvent::new( - MemberEventKind::Reap, - vec![ms.member().clone()], - ))); + self.emit_member(MemberEventKind::Reap, vec![ms.member().clone()]); } // Do not increment i — swap_remove moved the last element here. } else { @@ -2203,12 +2390,7 @@ where } self.coord_cache.remove(&id); } - self - .pending_events - .push_back(Event::Member(MemberEvent::new( - MemberEventKind::Reap, - vec![ms.member().clone()], - ))); + self.emit_member(MemberEventKind::Reap, vec![ms.member().clone()]); } } else { i += 1; @@ -2384,6 +2566,11 @@ where I: Clone, A: Clone, { + // Latch the command's instant so any coalesced member event this leave + // reaches arms its window from live `now`, consistent with the ingress and + // timeout paths and with `force_leave`. + self.drain_now = now; + match self.state { SerfState::Left => return Ok(()), // idempotent SerfState::Leaving | SerfState::Shutdown => { @@ -2469,6 +2656,10 @@ where I: Clone, A: Clone, { + // Latch the command's instant so the coalesced Leave/Reap this force-leave + // reaches arms its window from live `now`, not a stale `drain_now`. + self.drain_now = now; + if self.state == SerfState::Shutdown { return Err(Error::BadLeaveState(self.state)); } @@ -2555,10 +2746,17 @@ where name: impl Into, payload: bytes::Bytes, coalesce: bool, + now: Instant, ) -> Result<(), Error> where T: Reliable, { + // Latch the command's instant so the coalescer arms from live `now` when + // this event feeds it (mirrors the ingress/timeout paths). Without this a + // coalescing event issued after an idle gap would arm from a stale + // `drain_now` and flush immediately, defeating the batching window. + self.drain_now = now; + // Refuse once the machine has shut down (lost id-conflict vote). self.ensure_not_shutdown()?; @@ -2660,8 +2858,9 @@ where } self.mark_local_state_dirty(); - // First sight — emit to the driver. - self.pending_events.push_back(Event::User(msg)); + // First sight — emit to the driver (coalesced when enabled and the event + // opted into coalescing). + self.emit_user(msg); true } @@ -4433,6 +4632,25 @@ where self.drain_now = now; } + /// The member coalescer's current flush deadline (test adapter), unpolluted by + /// the periodic serf deadlines that `serf_poll_timeout` folds in. + #[cfg(all(test, feature = "tcp"))] + pub(crate) fn test_member_flush_deadline(&self) -> Option { + self + .member_coalescer + .as_ref() + .and_then(|c| c.flush_deadline()) + } + + /// The user coalescer's current flush deadline (test adapter). + #[cfg(all(test, feature = "tcp"))] + pub(crate) fn test_user_flush_deadline(&self) -> Option { + self + .user_coalescer + .as_ref() + .and_then(|c| c.flush_deadline()) + } + /// Return the `QueryId` of the last pending query entry (test adapter). #[cfg(test)] pub(crate) fn test_last_query_id(&self) -> Option { @@ -4978,6 +5196,18 @@ where // already-dead machine and the chokepoints (commands / ingress / timers) // observe Shutdown for the rest of this drain. The driver remains // responsible for stopping I/O and delivering this buffered event. + // + // Drop each coalescer's not-yet-flushed batch here, before the terminal + // Event::Shutdown: Go serf's `shutdown()` tears down the coalescer goroutine, + // abandoning its buffered events rather than delivering them, and the + // delivery contract forbids emitting anything after Event::Shutdown. Ingress + // is already inert post-Shutdown, so nothing can refill them. + if let Some(c) = self.member_coalescer.as_mut() { + c.reset(); + } + if let Some(c) = self.user_coalescer.as_mut() { + c.reset(); + } self.state = SerfState::Shutdown; self.pending_events.push_back(Event::Shutdown); } @@ -5547,9 +5777,10 @@ pub(crate) fn coord_ack_payload(coord: crate::typed::Coordinate) -> Bytes { // ── Coordinate public accessors ─────────────────────────────────────────────── -impl Endpoint +impl Endpoint where I: Clone + Eq + core::hash::Hash, + D: DropCounter, { /// Return the local node's current Vivaldi coordinate. /// diff --git a/serf-proto/src/endpoint/serf_parity_tests.rs b/serf-proto/src/endpoint/serf_parity_tests.rs index 8af97966..c876c1b5 100644 --- a/serf-proto/src/endpoint/serf_parity_tests.rs +++ b/serf-proto/src/endpoint/serf_parity_tests.rs @@ -606,7 +606,7 @@ fn three_tiers_drain_intent_then_query_then_event() { e.test_enqueue_query_broadcast(Bytes::from_static(b"query-bytes")); // user_event enqueues on the event tier (rank 2). - e.user_event("ev", Bytes::from_static(b"event"), false) + e.user_event("ev", Bytes::from_static(b"event"), false, Instant::ORIGIN) .expect("user_event must succeed"); // All three tiers should be populated (user_broadcast_queue_len = total across all tiers). diff --git a/serf-proto/src/endpoint/tests.rs b/serf-proto/src/endpoint/tests.rs index 3af771dc..ed40574b 100644 --- a/serf-proto/src/endpoint/tests.rs +++ b/serf-proto/src/endpoint/tests.rs @@ -12,6 +12,17 @@ use memberlist_proto::{EndpointOptions, RawRecords, SeedableRng, SmallRng, strea /// The plain-TCP record layer the unit-test coordinators run over. type TestTransport = RawRecords; +/// The default `u64` drop-counter storage keeps the endpoint free of interior +/// mutability, so the machine stays `Send + Sync` exactly as before the counter +/// became a generic parameter. A future reintroduction of an interior-mutable +/// default would break this and fail the build here. +#[test] +fn default_endpoint_is_send_and_sync() { + fn assert_send_sync() {} + assert_send_sync::>(); + assert_send_sync::>(); +} + /// Wrap a raw membership [`memberlist_proto::Endpoint`] into the plain-TCP /// reliable coordinator the serf `StreamEndpoint` composes with. /// @@ -887,8 +898,13 @@ fn reap_deadline_fires_via_handle_timeout() { #[test] fn user_event_increments_event_clock_and_emits_locally() { let mut e = ep(); - e.user_event("deploy", bytes::Bytes::from_static(b"v2"), false) - .unwrap(); + e.user_event( + "deploy", + bytes::Bytes::from_static(b"v2"), + false, + Instant::ORIGIN, + ) + .unwrap(); // Clock is incremented after stamping; local event at ltime=0 → clock now 1. assert_eq!(e.event_time(), 1); let ev = e.poll_event().expect("local user event must be pending"); @@ -951,7 +967,7 @@ fn oversized_user_event_is_rejected() { // 1024 bytes payload, well over the 512-byte limit. let big = bytes::Bytes::from(vec![0u8; 1024]); assert!( - e.user_event("big", big, false).is_err(), + e.user_event("big", big, false, Instant::ORIGIN).is_err(), "oversized user event must return Err" ); } @@ -959,8 +975,13 @@ fn oversized_user_event_is_rejected() { #[test] fn user_event_broadcast_is_queued_at_event_tier() { let mut e = ep(); - e.user_event("ship", bytes::Bytes::from_static(b"ok"), false) - .unwrap(); + e.user_event( + "ship", + bytes::Bytes::from_static(b"ok"), + false, + Instant::ORIGIN, + ) + .unwrap(); // After queuing, the user-broadcast queue must be non-empty (event tier = rank 2). assert!( e.user_broadcast_queue_len() > 0, @@ -1246,7 +1267,8 @@ fn mutating_event_clock_via_user_event_marks_state_dirty() { let mut e = ep(); e.test_clear_dirty(); // user_event increments the event clock → must mark dirty. - e.user_event("x", bytes::Bytes::new(), false).unwrap(); + e.user_event("x", bytes::Bytes::new(), false, Instant::ORIGIN) + .unwrap(); assert!(e.test_is_dirty(), "user_event must mark local_state dirty"); } @@ -1625,7 +1647,7 @@ fn query_emits_on_query_tier_broadcast_queue() { fn invalid_tag_regex_does_not_advance_rng() { // Helper that builds a serf Endpoint with a specified u64 seed so both // endpoints start with exactly the same RNG state. - let make_ep = |seed: u64| { + let make_ep = |seed: u64| -> StreamEndpoint { let inner_opts = EndpointOptions::new( 1u32, "127.0.0.1:7946".parse::().unwrap(), @@ -2254,7 +2276,8 @@ fn conflict_win_does_not_shut_down() { ); // A won vote leaves the command surface fully open. assert!( - e.user_event("post-win", bytes::Bytes::new(), false).is_ok(), + e.user_event("post-win", bytes::Bytes::new(), false, Instant::ORIGIN) + .is_ok(), "a won vote must not gate commands" ); } @@ -2466,7 +2489,7 @@ fn shutdown_refuses_originating_commands() { // Commands that originate cluster work funnel through ensure_not_shutdown. assert!( matches!( - e.user_event("x", bytes::Bytes::new(), false), + e.user_event("x", bytes::Bytes::new(), false, Instant::ORIGIN), Err(Error::Shutdown) ), "user_event must be refused after shutdown" @@ -2479,7 +2502,7 @@ fn shutdown_refuses_originating_commands() { "query must be refused after shutdown" ); assert!( - matches!(e.set_tags(tags), Err(Error::Shutdown)), + matches!(e.set_tags(tags, Instant::ORIGIN), Err(Error::Shutdown)), "set_tags must be refused after shutdown" ); assert!( @@ -3347,8 +3370,13 @@ fn load_snapshot_event_clock_allows_new_events_above_floor() { // Drain any pending events from load_snapshot. while e.poll_event().is_some() {} // Issue a new user event — must succeed and be delivered above the floor. - e.user_event("post-snap", bytes::Bytes::from_static(b"ok"), false) - .expect("user_event after load_snapshot must succeed"); + e.user_event( + "post-snap", + bytes::Bytes::from_static(b"ok"), + false, + Instant::ORIGIN, + ) + .expect("user_event after load_snapshot must succeed"); let ev = e .poll_event() .expect("user_event after load_snapshot must be delivered"); @@ -4440,7 +4468,7 @@ fn join_intent_max_minus_one_ltime_is_dropped() { "status_time must not be updated to u64::MAX-1" ); // After the bad ingress a subsequent valid local user_event must still work. - e.user_event("ok", bytes::Bytes::new(), false) + e.user_event("ok", bytes::Bytes::new(), false, Instant::ORIGIN) .expect("user_event must succeed after rejected intent"); } @@ -4507,7 +4535,7 @@ fn user_event_max_minus_one_ltime_is_dropped() { "no event must be emitted for u64::MAX-1 user event" ); // A subsequent valid user_event() must still work. - e.user_event("ok", bytes::Bytes::new(), false) + e.user_event("ok", bytes::Bytes::new(), false, Instant::ORIGIN) .expect("user_event must succeed after rejected event"); } @@ -4800,7 +4828,7 @@ fn zero_event_buffer_size_does_not_panic_on_first_event() { StreamEndpoint::new(coord(inner), opts); // Must NOT panic. - e.user_event("test", bytes::Bytes::new(), false) + e.user_event("test", bytes::Bytes::new(), false, Instant::ORIGIN) .expect("user_event must not panic when event_buffer_size was 0"); } @@ -4930,7 +4958,7 @@ fn after_high_clock_local_user_event_still_works() { }; let _ = e.test_handle_user_event(msg); // Now emit a local user_event via user_event() — next_ltime clamps and advances. - let result = e.user_event("local", bytes::Bytes::new(), false); + let result = e.user_event("local", bytes::Bytes::new(), false, Instant::ORIGIN); assert!(result.is_ok(), "user_event must succeed even at high clock"); } @@ -6783,7 +6811,7 @@ fn next_ltime_integrity_floor_near_watermark() { // next_ltime stamps LTIME_MAX - 1 and stores LTIME_MAX. The user_event is // emitted locally (it passes the min_time floor of 0); it then enters the // event ring. - let result = e.user_event("probe", bytes::Bytes::new(), false); + let result = e.user_event("probe", bytes::Bytes::new(), false, Instant::ORIGIN); assert!( result.is_ok(), "user_event must succeed with event_clock at LTIME_MAX-1: {result:?}" @@ -6929,7 +6957,7 @@ fn load_snapshot_near_watermark_no_panic_integrity_floor() { assert_ne!(e.query_time(), u64::MAX, "query clock must not be u64::MAX"); // user_event() must not panic (returns Ok even in the degraded state). - let ue_result = e.user_event("near-max", bytes::Bytes::new(), false); + let ue_result = e.user_event("near-max", bytes::Bytes::new(), false, Instant::ORIGIN); assert!( ue_result.is_ok(), "user_event must not panic/error after near-watermark snapshot: {ue_result:?}" @@ -7030,7 +7058,12 @@ fn load_snapshot_near_watermark_integrity_floor() { ); // user_event must not panic. - let stamp_result = e.user_event("after-snapshot", bytes::Bytes::new(), false); + let stamp_result = e.user_event( + "after-snapshot", + bytes::Bytes::new(), + false, + Instant::ORIGIN, + ); assert!( stamp_result.is_ok(), "user_event must not panic after LTIME_MAX-1 snapshot" @@ -7571,7 +7604,7 @@ fn push_pull_near_watermark_event_floor_integrity_and_delivery() { // witness(event_clock, LTIME_MAX-1) → event_clock = LTIME_MAX - 1. // next_ltime stamps LTIME_MAX - 1; handle_user_event drops if stamp < min_time // (LTIME_MAX - 1). LTIME_MAX - 1 is NOT < LTIME_MAX - 1, so the event is kept. - e.user_event("post-join", bytes::Bytes::new(), false) + e.user_event("post-join", bytes::Bytes::new(), false, Instant::ORIGIN) .expect("user_event must succeed after near-watermark push-pull"); assert!( matches!(e.poll_event(), Some(Event::User(_))), @@ -7593,7 +7626,7 @@ fn set_tags_round_trips_via_local_meta() { let mut e = ep(); let tags: Tags = [("role", "web"), ("dc", "us-east-1")].into_iter().collect(); - e.set_tags(tags.clone()) + e.set_tags(tags.clone(), Instant::ORIGIN) .expect("set_tags must succeed on a live endpoint"); let meta = e @@ -7628,7 +7661,7 @@ fn set_tags_local_member_state_is_observable_without_poll_event() { e.test_seed_member(1u32, MemberStatus::Alive, LamportTime::new(0)); let tags: Tags = [("role", "db")].into_iter().collect(); - e.set_tags(tags.clone()) + e.set_tags(tags.clone(), Instant::ORIGIN) .expect("set_tags must succeed on a live endpoint"); let local_tags = e.test_local_tags(); @@ -7655,7 +7688,7 @@ fn set_tags_does_not_materialize_absent_local_member() { let mut e = ep(); let tags: Tags = [("env", "staging")].into_iter().collect(); - e.set_tags(tags.clone()) + e.set_tags(tags.clone(), Instant::ORIGIN) .expect("set_tags must succeed when the local node is in members.states"); // set_tags must have updated the existing local member's tags in-place. @@ -7696,7 +7729,8 @@ fn set_tags_does_not_mark_local_state_dirty() { assert!(!e.test_is_dirty(), "resync must clear the dirty flag"); let tags: Tags = [("dc", "us-west-2")].into_iter().collect(); - e.set_tags(tags).expect("set_tags must succeed"); + e.set_tags(tags, Instant::ORIGIN) + .expect("set_tags must succeed"); assert!( !e.test_is_dirty(), @@ -7719,7 +7753,8 @@ fn set_tags_before_join_does_not_emit_update_before_join() { let mut e = ep(); let tags: Tags = [("role", "cache")].into_iter().collect(); - e.set_tags(tags).expect("set_tags must succeed"); + e.set_tags(tags, Instant::ORIGIN) + .expect("set_tags must succeed"); // Drive the serf tick so inner events (NodeUpdated) are drained. e.handle_timeout(t_secs(1)); @@ -7748,6 +7783,621 @@ fn set_tags_before_join_does_not_emit_update_before_join() { } } +// ── event coalescing (member + user) ───────────────────────────────────────── +// +// The endpoint owns a member + a user coalescer, each enabled iff its +// (coalesce_period > 0 && quiescent_period > 0). When enabled, membership / +// coalescing user events are buffered at their emission sites and flushed via +// the machine's own poll_timeout / handle_timeout window; when disabled (the +// default) every event passes straight through unchanged. + +/// Build a serf endpoint with MEMBER coalescing enabled over the given windows. +/// +/// Coalescing is enabled, so the construction self-join is buffered in the +/// coalescer; drain it into the coalescer (`poll_event`) then flush its window +/// (`handle_timeout`) so each test starts from an empty coalescer. +fn ep_member_coalescing_at( + coalesce: core::time::Duration, + quiescent: core::time::Duration, +) -> StreamEndpoint { + let inner_opts = EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); + let inner = memberlist_proto::Endpoint::new_at( + inner_opts, + memberlist_proto::Instant::ORIGIN, + SmallRng::seed_from_u64(0), + ); + let opts = Options::new() + .with_coalesce_period(coalesce) + .with_quiescent_period(quiescent); + let mut e = StreamEndpoint::new(coord(inner), opts); + let _ = e.poll_event(); + e.handle_timeout(memberlist_proto::Instant::ORIGIN + quiescent); + while e.poll_event().is_some() {} + e +} + +fn ep_member_coalescing() -> StreamEndpoint { + ep_member_coalescing_at( + core::time::Duration::from_secs(10), + core::time::Duration::from_secs(2), + ) +} + +/// Build a serf endpoint with USER coalescing enabled (member coalescing off, so +/// the construction self-join is delivered immediately as usual). +fn ep_user_coalescing() -> StreamEndpoint { + let inner_opts = EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); + let inner = memberlist_proto::Endpoint::new_at( + inner_opts, + memberlist_proto::Instant::ORIGIN, + SmallRng::seed_from_u64(0), + ); + let opts = Options::new() + .with_user_coalesce_period(core::time::Duration::from_secs(10)) + .with_user_quiescent_period(core::time::Duration::from_secs(2)); + let mut e = StreamEndpoint::new(coord(inner), opts); + let _ = e.poll_event(); + e +} + +fn member_ids(ev: &Event) -> Vec { + match ev { + Event::Member(me) => { + let mut ids: Vec = me.members().iter().map(|m| *m.node().id_ref()).collect(); + ids.sort_unstable(); + ids + } + other => panic!("expected Event::Member, got {other:?}"), + } +} + +#[test] +fn coalescing_disabled_delivers_member_events_immediately() { + // Default options: coalescing disabled → exact passthrough (the pre-coalescing + // behavior every other test relies on). + let mut e = ep(); + e.test_inner_node_joined(2, t_secs(1)); + let ev = e + .poll_event() + .expect("member event delivered immediately when disabled"); + assert!(matches!(ev, Event::Member(ref me) if me.kind() == MemberEventKind::Join)); + assert_eq!(member_ids(&ev), vec![2]); + assert!(e.poll_event().is_none()); +} + +#[test] +fn coalescing_disabled_delivers_user_events_immediately() { + // A coalescing (cc == true) user event still passes straight through when the + // user coalescer is disabled (default). + let mut e = ep(); + e.user_event( + "deploy", + bytes::Bytes::from_static(b"v2"), + true, + Instant::ORIGIN, + ) + .unwrap(); + let ev = e + .poll_event() + .expect("user event delivered immediately when disabled"); + assert!(matches!(ev, Event::User(ref u) if u.name == "deploy")); +} + +#[test] +fn member_coalescing_batches_rapid_joins_into_one_flush() { + let mut e = ep_member_coalescing(); + // Two joins within the window are buffered, not delivered immediately. + e.test_inner_node_joined(2, t_secs(5)); + e.test_inner_node_joined(3, t_secs(5)); + assert!( + e.poll_event().is_none(), + "joins are buffered by the coalescer, not delivered immediately" + ); + // The flush deadline (last event + quiescent = 2s) surfaces in serf_poll_timeout. + assert_eq!( + e.core_mut().serf_poll_timeout(), + Some(t_secs(7)), + "the coalescer flush deadline appears in serf_poll_timeout" + ); + // Firing the window delivers ONE coalesced Join batch carrying both nodes. + e.handle_timeout(t_secs(7)); + let ev = e + .poll_event() + .expect("one coalesced batch at the flush deadline"); + assert!(matches!(ev, Event::Member(ref me) if me.kind() == MemberEventKind::Join)); + assert_eq!(member_ids(&ev), vec![2, 3], "both joins in ONE batch"); + assert!(e.poll_event().is_none(), "exactly one batch delivered"); +} + +#[test] +fn member_coalescing_collapses_transitions_to_latest_status() { + let mut e = ep_member_coalescing(); + // node 2 joins then immediately fails within the same window. + e.test_inner_node_joined(2, t_secs(5)); + e.test_inner_node_left(2, t_secs(5)); // Alive -> Failed + assert!(e.poll_event().is_none(), "transitions buffered"); + + e.handle_timeout(t_secs(7)); + let mut kinds = Vec::new(); + while let Some(ev) = e.poll_event() { + if let Event::Member(me) = ev { + kinds.push(me.kind()); + } + } + assert_eq!( + kinds, + vec![MemberEventKind::Failed], + "join collapses into the final Failed status: {kinds:?}" + ); +} + +#[test] +fn member_coalescing_coalesce_cap_bounds_a_busy_stream() { + // coalesce cap 3s, quiescent 2s: a stream of events every 1s keeps re-arming + // the quiescent timer, but the flush deadline can never exceed first + 3s. + let mut e = ep_member_coalescing_at( + core::time::Duration::from_secs(3), + core::time::Duration::from_secs(2), + ); + e.test_inner_node_joined(2, t_secs(5)); // cap = t8, quiescent = t7 + assert_eq!(e.core_mut().serf_poll_timeout(), Some(t_secs(7))); + e.test_inner_node_joined(3, t_secs(6)); // quiescent -> t8, cap still t8 + assert_eq!(e.core_mut().serf_poll_timeout(), Some(t_secs(8))); + e.test_inner_node_joined(4, t_secs(7)); // quiescent -> t9, but cap t8 binds + assert_eq!( + e.core_mut().serf_poll_timeout(), + Some(t_secs(8)), + "the coalesce cap (first event + 3s) bounds the busy stream" + ); + // The cap flush delivers all three joins in one batch. + e.handle_timeout(t_secs(8)); + let ev = e.poll_event().expect("cap flush delivers the batch"); + assert_eq!(member_ids(&ev), vec![2, 3, 4]); +} + +#[test] +fn overdue_member_window_flushes_before_a_late_same_id_event() { + // A window can be due (its deadline elapsed) yet not flushed, because the + // driver drained ready ingress before firing the overdue timer. A late event + // for the same node must NOT overwrite the due observation: the machine flushes + // the completed window first, then opens a fresh one. + let mut e = ep_member_coalescing(); // coalesce 10s, quiescent 2s + e.test_inner_node_joined(2, t_secs(5)); // Join(2): window due at t7 + assert!(e.poll_event().is_none(), "the join is buffered"); + + // Node 2 fails at t8 — PAST the t7 deadline, but before handle_timeout fires. + e.test_inner_node_left(2, t_secs(8)); // Alive -> Failed + + // The overdue Join window is flushed by the late feed, before the Failed can + // overwrite it. + let ev = e + .poll_event() + .expect("the overdue Join batch is flushed by the late feed"); + assert!(matches!(ev, Event::Member(ref me) if me.kind() == MemberEventKind::Join)); + assert_eq!(member_ids(&ev), vec![2]); + assert!( + e.poll_event().is_none(), + "the Failed is buffered in a fresh window, not delivered yet" + ); + + // The fresh window (t8 + 2s quiescent) delivers the Failed separately. + e.handle_timeout(t_secs(10)); + let ev = e + .poll_event() + .expect("the fresh window delivers the Failed"); + assert!(matches!(ev, Event::Member(ref me) if me.kind() == MemberEventKind::Failed)); + assert_eq!(member_ids(&ev), vec![2]); +} + +#[test] +fn backward_ingress_after_a_newer_event_does_not_backdate_the_window() { + // The coalescer schedules on a monotonic clock, so a delayed reliable-ingress + // member event carrying an earlier arrival time — processed after a newer + // command — must not move an active window's deadline into the past. + let mut e = ep_member_coalescing(); // coalesce 10s, quiescent 2s + e.test_inner_node_joined(2, t_secs(100)); // arms the window; deadline 100 + 2 = 102 + assert_eq!(e.core_mut().test_member_flush_deadline(), Some(t_secs(102))); + + // A join carrying an EARLIER arrival time, processed after the newer join. + e.test_inner_node_joined(3, t_secs(50)); + assert_eq!( + e.core_mut().test_member_flush_deadline(), + Some(t_secs(102)), + "an older interposed ingress event must not backdate the active window" + ); + assert!( + e.poll_event().is_none(), + "both joins remain buffered — the deadline was not backdated into the past" + ); +} + +#[test] +fn startup_self_join_coalesces_from_the_scheduling_instant() { + // The coordinator queues the local self-join during construction, before any + // live-time entry point has run. `start_scheduling` — the driver's first call, + // made with its live clock — must fold that queued join under that instant: + // deferring it to a later un-latched `poll_event` would arm the coalescing + // window at the machine's origin, already overdue, flushing the self join + // immediately instead of batching it with the startup membership changes. + let inner_opts = EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); + let inner = memberlist_proto::Endpoint::new_at( + inner_opts, + memberlist_proto::Instant::ORIGIN, + SmallRng::seed_from_u64(0), + ); + let opts = Options::new() + .with_coalesce_period(core::time::Duration::from_secs(10)) + .with_quiescent_period(core::time::Duration::from_secs(2)); + let mut e: StreamEndpoint = + StreamEndpoint::new(coord(inner), opts); + + // The driver arms the schedulers at its live clock; the queued self-join is + // folded here, opening the member window at t100. + e.start_scheduling(t_secs(100)); + assert!( + e.poll_event().is_none(), + "the startup self join is buffered in the window, not delivered immediately" + ); + assert_eq!( + e.core_mut().test_member_flush_deadline(), + Some(t_secs(102)), + "the startup window arms from the scheduling instant, not the origin" + ); + + // A node joining within the startup window batches with the self join. + e.test_inner_node_joined(2, t_secs(101)); + e.handle_timeout(t_secs(103)); + let ev = e + .poll_event() + .expect("one coalesced batch for the startup window"); + assert!(matches!(ev, Event::Member(ref me) if me.kind() == MemberEventKind::Join)); + assert_eq!( + member_ids(&ev), + vec![1, 2], + "the self join batches with the startup-window join" + ); + assert!(e.poll_event().is_none()); +} + +#[test] +fn overdue_user_window_flushes_before_a_late_newer_generation() { + // The user coalescer keeps only the newest generation per name. A newer + // generation fed after the window is due must not silently supersede a due + // earlier generation: the machine flushes the elapsed window first. + let mut e = ep_user_coalescing(); // user coalesce 10s, quiescent 2s + e.user_event("foo", bytes::Bytes::from_static(b"gen1"), true, t_secs(5)) + .unwrap(); // window due at t7 + assert!(e.poll_event().is_none(), "the first generation is buffered"); + + // A newer "foo" at t8 — PAST the t7 deadline, before handle_timeout fires. + e.user_event("foo", bytes::Bytes::from_static(b"gen2"), true, t_secs(8)) + .unwrap(); + + // The earlier generation is flushed by the late feed, before the newer one + // supersedes it. + let ev = e + .poll_event() + .expect("the overdue earlier generation is flushed by the late feed"); + match ev { + Event::User(u) => { + assert_eq!(u.name, "foo"); + assert_eq!(u.payload, bytes::Bytes::from_static(b"gen1")); + } + other => panic!("expected Event::User, got {other:?}"), + } + assert!(e.poll_event().is_none(), "the newer generation is buffered"); + + // The fresh window delivers the newer generation. + e.handle_timeout(t_secs(10)); + let ev = e + .poll_event() + .expect("the fresh window delivers the newer generation"); + match ev { + Event::User(u) => { + assert_eq!(u.name, "foo"); + assert_eq!(u.payload, bytes::Bytes::from_static(b"gen2")); + } + other => panic!("expected Event::User, got {other:?}"), + } +} + +#[test] +fn user_coalescing_batches_cc_events_and_passes_non_cc_through() { + let mut e = ep_user_coalescing(); + + // The command's `now` arms the window directly — no manual `test_set_drain_now`. + // A non-coalescing user event passes straight through even when enabled. + e.user_event("plain", bytes::Bytes::from_static(b"a"), false, t_secs(5)) + .unwrap(); + assert!( + matches!(e.poll_event(), Some(Event::User(u)) if u.name == "plain"), + "a non-cc user event passes through immediately" + ); + + // A coalescing user event is buffered; a newer generation supersedes it. + e.user_event("cc", bytes::Bytes::from_static(b"v1"), true, t_secs(5)) + .unwrap(); + assert!(e.poll_event().is_none(), "cc user event buffered"); + e.user_event("cc", bytes::Bytes::from_static(b"v2"), true, t_secs(6)) + .unwrap(); + assert!(e.poll_event().is_none()); + + // The user flush deadline (last event + quiescent = 2s) surfaces in poll_timeout. + assert_eq!(e.core_mut().serf_poll_timeout(), Some(t_secs(8))); + e.handle_timeout(t_secs(8)); + + let mut delivered = Vec::new(); + while let Some(ev) = e.poll_event() { + if let Event::User(u) = ev { + delivered.push(u); + } + } + assert_eq!( + delivered.len(), + 1, + "one coalesced user event: {delivered:?}" + ); + assert_eq!(delivered[0].name, "cc"); + assert_eq!( + delivered[0].payload.as_ref(), + b"v2", + "only the newest generation survives" + ); +} + +#[test] +fn coalesced_member_batch_dropped_on_midwindow_shutdown() { + // A membership batch buffered mid-window is DROPPED when a lost id-conflict + // vote shuts the machine down (Go serf abandons the coalescer on shutdown). + // Nothing may follow the terminal Event::Shutdown. + let mut e = ep_member_coalescing(); + e.test_inner_node_joined(2, t_secs(5)); + assert!(e.poll_event().is_none(), "join buffered mid-window"); + + // A conflict query whose deadline coincides with the flush window; the vote + // is lost (1 agree, 2 disagree). + let qid = e.test_register_conflict_query(t_secs(7)); + e.test_fold_conflict_response(qid, 200u32, true); + e.test_fold_conflict_response(qid, 201u32, false); + e.test_fold_conflict_response(qid, 202u32, false); + + // Driving the tick closes the conflict (lost) → Shutdown; the buffered batch + // is dropped before the terminal event, not flushed after it. + e.handle_timeout(t_secs(7)); + + let mut events = Vec::new(); + while let Some(ev) = e.poll_event() { + events.push(ev); + } + assert_eq!( + events.len(), + 1, + "only Event::Shutdown drains — the buffered member batch is dropped: {events:?}" + ); + assert!(matches!(events[0], Event::Shutdown)); + assert!(e.state().is_shutdown()); + + // Ticking past the former flush deadline delivers nothing more. + e.handle_timeout(t_secs(20)); + assert!( + e.poll_event().is_none(), + "no coalesced batch may surface after Event::Shutdown" + ); +} + +#[test] +fn user_event_after_idle_gap_arms_window_from_live_now() { + // Regression (coalescer armed from a STALE command time): a coalescing user + // event issued as a COMMAND after an idle gap must arm its window from the + // command's live `now`, not from a stale `drain_now` (which command paths did + // not refresh). `ep_user_coalescing` leaves `drain_now` at ORIGIN; issue the + // event far in the future WITHOUT touching `drain_now`. + let mut e = ep_user_coalescing(); // user coalesce 10s, quiescent 2s + + e.user_event("cc", bytes::Bytes::from_static(b"v1"), true, t_secs(100)) + .unwrap(); + + // Buffered, NOT flushed immediately. + assert!( + e.poll_event().is_none(), + "the coalescing user event is buffered, not delivered immediately" + ); + // A FULL quiescent window applies from live now: 100 + 2 = 102 — a future + // deadline, not one near ORIGIN. Reverting the now-threading arms at ORIGIN + // (deadline t2) and this assertion fails. + assert_eq!( + e.core_mut().test_user_flush_deadline(), + Some(t_secs(102)), + "the user coalesce window must arm from the command's live now" + ); + // A tick within the window does not flush (a stale-armed window would have + // been past-due and flushed here). + e.handle_timeout(t_secs(101)); + assert!( + e.poll_event().is_none(), + "still buffered within the live window" + ); + // The window closes at 102, delivering exactly the coalesced event. + e.handle_timeout(t_secs(102)); + assert!( + matches!(e.poll_event(), Some(Event::User(u)) if u.name == "cc"), + "the coalesced user event flushes when its live-armed window closes" + ); +} + +#[test] +fn set_tags_after_idle_gap_arms_member_window_from_live_now() { + // Regression (member side): `set_tags` emits a `Member(Update)` via the + // coordinator's `NodeUpdated`, which a later `poll_event` drains WITHOUT + // latching `drain_now`. The member window must arm from `set_tags`'s live + // `now`, not the stale `drain_now` left by a prior tick. + use crate::typed::Tags; + + let mut e = ep_member_coalescing(); // member coalesce 10s, quiescent 2s + + let tags: Tags = [("role", "web")].into_iter().collect(); + e.set_tags(tags, t_secs(100)) + .expect("set_tags must succeed"); + + // Drain the coordinator's NodeUpdated into the member coalescer: buffered, not + // delivered immediately. + assert!( + e.poll_event().is_none(), + "the Member(Update) is buffered by the member coalescer, not delivered immediately" + ); + // A FULL quiescent window applies from live now: 100 + 2 = 102. Reverting the + // now-threading arms from the stale drain_now (t2, from the helper's self-join + // flush) → deadline t4 → this assertion fails. + assert_eq!( + e.core_mut().test_member_flush_deadline(), + Some(t_secs(102)), + "the member coalesce window must arm from set_tags's live now" + ); + e.handle_timeout(t_secs(101)); + assert!( + e.poll_event().is_none(), + "still buffered within the live window" + ); + e.handle_timeout(t_secs(102)); + assert!( + matches!(e.poll_event(), Some(Event::Member(me)) if me.kind() == MemberEventKind::Update), + "the coalesced Member(Update) flushes when its live-armed window closes" + ); +} + +#[test] +fn set_tags_emits_its_node_updated_synchronously_under_the_command_latch() { + // `set_tags` arms the member coalescer from its OWN command instant by + // draining the coordinator's resulting `NodeUpdated` synchronously. A drain + // captured at an EARLIER instant that lands afterwards must therefore NOT + // re-time the member window: the batch already belongs to the set_tags + // instant, so the older drain finds nothing left to arm and cannot pull the + // flush deadline backwards (which would flush the batch early). + use crate::typed::Tags; + + let mut e = ep_member_coalescing(); // member coalesce 10s, quiescent 2s + + let tags: Tags = [("role", "web")].into_iter().collect(); + e.set_tags(tags, t_secs(100)) + .expect("set_tags must succeed"); + + // Interpose an ingress drain captured at an EARLIER instant than the command. + // With the synchronous drain in `set_tags` the NodeUpdated is already + // consumed, so this older drain has nothing to feed the coalescer. + e.test_drain_after_ingress(t_secs(50)); + + // The window is armed at set_tags's live now (100 + quiescent 2 = 102), NOT the + // earlier ingress instant (50 + 2 = 52). Dropping the synchronous drain lets + // the older ingress arm the window at 52 and this assertion fails. + assert_eq!( + e.core_mut().test_member_flush_deadline(), + Some(t_secs(102)), + "the member window is armed at the set_tags instant, not the later older ingress drain" + ); +} + +/// Build a serf endpoint with USER coalescing enabled and a small buffered-volume +/// cap, for the overflow-through-the-endpoint tests. +fn ep_user_coalescing_capped( + cap: core::num::NonZeroUsize, +) -> StreamEndpoint { + let inner_opts = EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); + let inner = memberlist_proto::Endpoint::new_at( + inner_opts, + memberlist_proto::Instant::ORIGIN, + SmallRng::seed_from_u64(0), + ); + let opts = Options::new() + .with_user_coalesce_period(core::time::Duration::from_secs(10)) + .with_user_quiescent_period(core::time::Duration::from_secs(2)) + .with_max_coalesced_user_events(Some(cap)); + let mut e = StreamEndpoint::new(coord(inner), opts); + let _ = e.poll_event(); + e +} + +#[test] +fn coalescing_disabled_dropped_accessors_return_zero() { + // With coalescing disabled (the default) the coalescers are absent: events pass + // straight through and the drop counters read zero, so the observability + // accessors never conflate passthrough with a bounded/overflowing coalescer. + let mut e = ep(); + while e.poll_event().is_some() {} // drain any construction events + assert_eq!(e.coalesced_user_events_dropped(), 0); + assert_eq!(e.coalesced_member_events_dropped(), 0); + assert_eq!(e.pending_events_len(), 0); + + e.test_inner_node_joined(2, t_secs(1)); + e.user_event("d", bytes::Bytes::from_static(b"v"), true, t_secs(1)) + .unwrap(); + + let mut saw_join = false; + let mut saw_user = false; + while let Some(ev) = e.poll_event() { + match ev { + Event::Member(me) if me.kind() == MemberEventKind::Join => saw_join = true, + Event::User(u) if u.name == "d" => saw_user = true, + _ => {} + } + } + assert!(saw_join, "the join passes straight through when disabled"); + assert!( + saw_user, + "the cc user event passes straight through when disabled" + ); + assert_eq!(e.coalesced_user_events_dropped(), 0); + assert_eq!(e.coalesced_member_events_dropped(), 0); + assert_eq!(e.pending_events_len(), 0, "nothing left buffered"); +} + +#[test] +fn user_coalescing_flood_through_endpoint_bounds_burst_and_counts_drops() { + // Through the full endpoint: a flood of distinct coalescing user-event names is + // bounded by the configured cap. The drop counter surfaces via the public + // accessor, and the flush burst never exceeds the cap. + let cap = core::num::NonZeroUsize::new(16).unwrap(); + let mut e = ep_user_coalescing_capped(cap); + + let n: u32 = 200; + for i in 0..n { + e.user_event( + format!("evt-{i}"), + bytes::Bytes::from_static(b"p"), + true, + t_secs(5), + ) + .unwrap(); + } + assert!( + e.poll_event().is_none(), + "all coalescing user events are buffered, not surfaced immediately" + ); + assert_eq!( + e.coalesced_user_events_dropped(), + (n as u64) - cap.get() as u64, + "every event past the cap is counted via the public accessor" + ); + + // Firing the window delivers at most `cap` user events. + e.handle_timeout(t_secs(7)); + let mut users = 0usize; + while let Some(ev) = e.poll_event() { + if matches!(ev, Event::User(_)) { + users += 1; + } + } + assert_eq!(users, cap.get(), "the flush burst is bounded by the cap"); + // The drop counter is cumulative and survives the flush. + assert_eq!( + e.coalesced_user_events_dropped(), + (n as u64) - cap.get() as u64 + ); +} + // ── legacy Go-parity ports (bucket-A/bucket-B) ──────────────────────────────── /// Mirrors Go serf `serf_get_queue_max` @@ -7854,7 +8504,7 @@ fn set_tags_meta_round_trips_and_rejects_oversize() { // Round-trip: a single `role=test` tag encodes into node meta and decodes back. let tags: Tags = [("role", "test")].into_iter().collect(); - e.set_tags(tags) + e.set_tags(tags, memberlist_proto::Instant::ORIGIN) .expect("set_tags must succeed on a live endpoint"); let meta = e .test_local_meta() @@ -7871,7 +8521,7 @@ fn set_tags_meta_round_trips_and_rejects_oversize() { let huge = "x".repeat(70_000); let oversize: Tags = [("big", huge.as_str())].into_iter().collect(); let err = e - .set_tags(oversize) + .set_tags(oversize, memberlist_proto::Instant::ORIGIN) .expect_err("tags exceeding the meta size limit must be rejected"); assert!( matches!(err, Error::SetTagsMeta(_)), diff --git a/serf-proto/src/lib.rs b/serf-proto/src/lib.rs index 4f10ba22..5f770ac1 100644 --- a/serf-proto/src/lib.rs +++ b/serf-proto/src/lib.rs @@ -97,6 +97,11 @@ pub use coordinate_client::{ CoordinateClient, CoordinateClientStats, CoordinateError, CoordinateOptions, }; +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) mod coalesce; +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub use coalesce::DropCounter; #[cfg(any(feature = "tcp", feature = "quic"))] #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] pub mod endpoint; diff --git a/serf-proto/src/options/mod.rs b/serf-proto/src/options/mod.rs index 4429bf6f..3c92ac73 100644 --- a/serf-proto/src/options/mod.rs +++ b/serf-proto/src/options/mod.rs @@ -48,6 +48,12 @@ pub struct Options { user_coalesce_period: Duration, /// Same as `quiescent_period` but for user events only. user_quiescent_period: Duration, + /// Upper bound on the total buffered user-event volume in the user coalescer; + /// `None` disables the bound. The coalescer keeps only the newest Lamport + /// generation per name, so a modest cap covers legitimate traffic while + /// bounding an adversarial flood of distinct names (or distinct payloads at a + /// single generation) that would otherwise grow the buffer without limit. + max_coalesced_user_events: Option, // ── buffers ─────────────────────────────────────────────────────────────── /// Number of user-event slots in the dedup ring buffer. @@ -93,6 +99,11 @@ pub struct Options { } impl Options { + /// Default upper bound on the total buffered user-event volume in the user + /// coalescer (see [`Options::max_coalesced_user_events`]). + pub const DEFAULT_MAX_COALESCED_USER_EVENTS: core::num::NonZeroUsize = + core::num::NonZeroUsize::new(1024).unwrap(); + /// Returns a new `Options` with all defaults as specified in Go serf /// `options.go` and the legacy `serf-core/src/options.rs` port. pub fn new() -> Self { @@ -109,6 +120,7 @@ impl Options { quiescent_period: Duration::ZERO, user_coalesce_period: Duration::ZERO, user_quiescent_period: Duration::ZERO, + max_coalesced_user_events: Some(Self::DEFAULT_MAX_COALESCED_USER_EVENTS), event_buffer_size: 512, query_buffer_size: 512, max_user_event_size: 512, @@ -188,6 +200,26 @@ impl Options { self.user_quiescent_period } + /// Upper bound on the total buffered user-event volume in the user coalescer + /// (`None` = unbounded). Defaults to + /// [`DEFAULT_MAX_COALESCED_USER_EVENTS`](Options::DEFAULT_MAX_COALESCED_USER_EVENTS). + pub const fn max_coalesced_user_events(&self) -> Option { + self.max_coalesced_user_events + } + + /// Whether member-event coalescing is enabled: both the coalesce and quiescent + /// periods are non-zero. Mirrors the legacy `serf-core/src/serf/base.rs` + /// enable gate (`coalesce_period > 0 && quiescent_period > 0`). + pub const fn member_coalesce_enabled(&self) -> bool { + !self.coalesce_period.is_zero() && !self.quiescent_period.is_zero() + } + + /// Whether user-event coalescing is enabled: both the user coalesce and user + /// quiescent periods are non-zero. Mirrors the legacy enable gate. + pub const fn user_coalesce_enabled(&self) -> bool { + !self.user_coalesce_period.is_zero() && !self.user_quiescent_period.is_zero() + } + /// Number of slots in the user-event dedup ring buffer. pub const fn event_buffer_size(&self) -> usize { self.event_buffer_size @@ -322,24 +354,60 @@ impl Options { self } + /// Sets `coalesce_period` in place. + pub fn set_coalesce_period(&mut self, v: Duration) -> &mut Self { + self.coalesce_period = v; + self + } + /// Sets `quiescent_period`. pub fn with_quiescent_period(mut self, v: Duration) -> Self { self.quiescent_period = v; self } + /// Sets `quiescent_period` in place. + pub fn set_quiescent_period(&mut self, v: Duration) -> &mut Self { + self.quiescent_period = v; + self + } + /// Sets `user_coalesce_period`. pub fn with_user_coalesce_period(mut self, v: Duration) -> Self { self.user_coalesce_period = v; self } + /// Sets `user_coalesce_period` in place. + pub fn set_user_coalesce_period(&mut self, v: Duration) -> &mut Self { + self.user_coalesce_period = v; + self + } + /// Sets `user_quiescent_period`. pub fn with_user_quiescent_period(mut self, v: Duration) -> Self { self.user_quiescent_period = v; self } + /// Sets `user_quiescent_period` in place. + pub fn set_user_quiescent_period(&mut self, v: Duration) -> &mut Self { + self.user_quiescent_period = v; + self + } + + /// Sets `max_coalesced_user_events` (`None` disables the bound). + pub fn with_max_coalesced_user_events(mut self, v: Option) -> Self { + self.max_coalesced_user_events = v; + self + } + + /// Sets `max_coalesced_user_events` in place (`None` disables the bound). + pub fn set_max_coalesced_user_events(&mut self, v: Option) -> &mut Self { + self.max_coalesced_user_events = v; + self + } + /// Sets `event_buffer_size`. pub fn with_event_buffer_size(mut self, v: usize) -> Self { self.event_buffer_size = v; @@ -425,6 +493,71 @@ impl Options { self.disable_coordinates = v; self } + + /// Validates the coalescing configuration. + /// + /// When a coalescing pair is enabled (both periods non-zero) the quiescent + /// period must be strictly less than the coalesce period: the quiescent window + /// is the "went quiet" fast-flush, and the coalesce period is the + /// maximum-delay cap. If quiescent `>=` coalesce the quiescent window can + /// never bind — almost always a misconfiguration. This mirrors the semantics + /// documented on the legacy `serf-core/src/options.rs` period fields. + /// + /// Returns `Ok(())` when coalescing is disabled or the invariant holds for + /// every enabled pair. The Sans-I/O [`Endpoint`](crate::endpoint::Endpoint) + /// construction is infallible and tolerates any configuration (its flush + /// deadline is always `min(coalesce, quiescent)`); a driver that wants to + /// reject a nonsensical configuration up front calls this. + pub fn validate(&self) -> Result<(), InvalidOptions> { + if self.member_coalesce_enabled() && self.quiescent_period >= self.coalesce_period { + return Err(InvalidOptions::MemberCoalesce(CoalesceConfig { + coalesce_period: self.coalesce_period, + quiescent_period: self.quiescent_period, + })); + } + if self.user_coalesce_enabled() && self.user_quiescent_period >= self.user_coalesce_period { + return Err(InvalidOptions::UserCoalesce(CoalesceConfig { + coalesce_period: self.user_coalesce_period, + quiescent_period: self.user_quiescent_period, + })); + } + Ok(()) + } +} + +/// The coalescing periods that failed [`Options::validate`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct CoalesceConfig { + /// The configured coalesce (maximum-delay) period. + pub coalesce_period: Duration, + /// The configured quiescent (flush-after-quiet) period. + pub quiescent_period: Duration, +} + +impl core::fmt::Display for CoalesceConfig { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + write!( + f, + "quiescent_period ({:?}) must be strictly less than coalesce_period ({:?})", + self.quiescent_period, self.coalesce_period + ) + } +} + +/// Error returned by [`Options::validate`] for a self-contradictory coalescing +/// configuration (an enabled quiescent period not strictly less than its +/// coalesce period). +#[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] +#[non_exhaustive] +pub enum InvalidOptions { + /// The member-event quiescent period is not strictly less than the + /// member-event coalesce period while member coalescing is enabled. + #[error("member-event coalescing: {0}")] + MemberCoalesce(CoalesceConfig), + /// The user-event quiescent period is not strictly less than the user-event + /// coalesce period while user coalescing is enabled. + #[error("user-event coalescing: {0}")] + UserCoalesce(CoalesceConfig), } impl Default for Options { diff --git a/serf-proto/src/options/tests.rs b/serf-proto/src/options/tests.rs index 2987afca..a5d6f0e0 100644 --- a/serf-proto/src/options/tests.rs +++ b/serf-proto/src/options/tests.rs @@ -1,6 +1,6 @@ use core::time::Duration; -use super::Options; +use super::{InvalidOptions, Options}; #[test] fn options_defaults_match_legacy() { @@ -64,6 +64,12 @@ fn options_all_defaults() { assert_eq!(o.quiescent_period(), Duration::ZERO); assert_eq!(o.user_coalesce_period(), Duration::ZERO); assert_eq!(o.user_quiescent_period(), Duration::ZERO); + // The user-coalescer volume cap defaults to 1024 (bounded, not disabled). + assert_eq!( + o.max_coalesced_user_events(), + Some(Options::DEFAULT_MAX_COALESCED_USER_EVENTS) + ); + assert_eq!(Options::DEFAULT_MAX_COALESCED_USER_EVENTS.get(), 1024); // Buffers assert_eq!(o.event_buffer_size(), 512); assert_eq!(o.query_buffer_size(), 512); @@ -115,3 +121,99 @@ fn query_timeout_mult_zero_is_clamped_to_one() { "with_query_timeout_mult(0) must be clamped to 1" ); } + +// ── coalescing config ──────────────────────────────────────────────────────── + +#[test] +fn coalescing_disabled_by_default() { + let o = Options::new(); + assert!( + !o.member_coalesce_enabled(), + "member coalescing off by default" + ); + assert!(!o.user_coalesce_enabled(), "user coalescing off by default"); +} + +#[test] +fn coalescing_enabled_only_when_both_periods_are_non_zero() { + // A single non-zero period does NOT enable coalescing (mirrors the legacy gate). + let only_coalesce = Options::new().with_coalesce_period(Duration::from_secs(10)); + assert!(!only_coalesce.member_coalesce_enabled()); + let only_quiescent = Options::new().with_quiescent_period(Duration::from_secs(2)); + assert!(!only_quiescent.member_coalesce_enabled()); + + let both = Options::new() + .with_coalesce_period(Duration::from_secs(10)) + .with_quiescent_period(Duration::from_secs(2)); + assert!(both.member_coalesce_enabled()); + + let user_both = Options::new() + .with_user_coalesce_period(Duration::from_secs(10)) + .with_user_quiescent_period(Duration::from_secs(2)); + assert!(user_both.user_coalesce_enabled()); +} + +#[test] +fn max_coalesced_user_events_builder_and_disable() { + use core::num::NonZeroUsize; + + // The `with_` builder overrides the default. + let bounded = Options::new().with_max_coalesced_user_events(NonZeroUsize::new(64)); + assert_eq!(bounded.max_coalesced_user_events(), NonZeroUsize::new(64)); + + // `None` disables the bound. + let unbounded = Options::new().with_max_coalesced_user_events(None); + assert_eq!(unbounded.max_coalesced_user_events(), None); + + // The in-place setter matches the `with_` builder. + let mut o = Options::new(); + o.set_max_coalesced_user_events(NonZeroUsize::new(64)); + assert_eq!(o.max_coalesced_user_events(), NonZeroUsize::new(64)); +} + +#[test] +fn set_period_builders_match_with_builders() { + let mut o = Options::new(); + o.set_coalesce_period(Duration::from_secs(10)) + .set_quiescent_period(Duration::from_secs(2)) + .set_user_coalesce_period(Duration::from_secs(30)) + .set_user_quiescent_period(Duration::from_secs(5)); + assert_eq!(o.coalesce_period(), Duration::from_secs(10)); + assert_eq!(o.quiescent_period(), Duration::from_secs(2)); + assert_eq!(o.user_coalesce_period(), Duration::from_secs(30)); + assert_eq!(o.user_quiescent_period(), Duration::from_secs(5)); +} + +#[test] +fn validate_accepts_disabled_and_well_ordered_periods() { + // Disabled coalescing is always valid. + assert!(Options::new().validate().is_ok()); + // quiescent strictly less than coalesce is valid. + let ok = Options::new() + .with_coalesce_period(Duration::from_secs(10)) + .with_quiescent_period(Duration::from_secs(2)) + .with_user_coalesce_period(Duration::from_secs(30)) + .with_user_quiescent_period(Duration::from_secs(5)); + assert!(ok.validate().is_ok()); +} + +#[test] +fn validate_rejects_quiescent_not_less_than_coalesce() { + // Member: quiescent == coalesce is rejected. + let member_bad = Options::new() + .with_coalesce_period(Duration::from_secs(5)) + .with_quiescent_period(Duration::from_secs(5)); + assert!(matches!( + member_bad.validate(), + Err(InvalidOptions::MemberCoalesce(_)) + )); + + // User: quiescent > coalesce is rejected. + let user_bad = Options::new() + .with_user_coalesce_period(Duration::from_secs(2)) + .with_user_quiescent_period(Duration::from_secs(10)); + assert!(matches!( + user_bad.validate(), + Err(InvalidOptions::UserCoalesce(_)) + )); +} diff --git a/serf-proto/src/quic_endpoint/mod.rs b/serf-proto/src/quic_endpoint/mod.rs index cb7f32be..6b0be980 100644 --- a/serf-proto/src/quic_endpoint/mod.rs +++ b/serf-proto/src/quic_endpoint/mod.rs @@ -41,6 +41,7 @@ use memberlist_proto::{ use smol_str::SmolStr; use crate::{ + DropCounter, endpoint::{Endpoint, Error, QueryId, QueryParams}, event::{Event, QueryEvent}, members::{Member, SerfState}, @@ -74,12 +75,13 @@ use crate::event::KeyResponseArgs; /// type to `SocketAddr` too. #[cfg(feature = "quic")] #[cfg_attr(docsrs, doc(cfg(feature = "quic")))] -pub struct QuicEndpoint +pub struct QuicEndpoint where I: Eq + core::hash::Hash, + D: DropCounter, { /// The serf-logic core, holding all serf state and no transport reference. - core: Endpoint, + core: Endpoint, /// The memberlist QUIC coordinator serf drives through the `Reliable` seam. /// Holds the single membership `Endpoint` and the quinn endpoint. transport: Coordinator, @@ -87,28 +89,53 @@ where #[cfg(feature = "quic")] #[cfg_attr(docsrs, doc(cfg(feature = "quic")))] -impl QuicEndpoint +impl QuicEndpoint where I: Clone + Eq + core::hash::Hash, R: SeedableRng, + D: DropCounter, { /// Construct a `QuicEndpoint` from a memberlist QUIC coordinator `transport`, /// serf `opts`, and serf's own injected `rng`. /// /// `rng` is **separate** from the coordinator's RNG `G`; seed it from the /// driver's own entropy source. - pub fn new_with_rng(transport: Coordinator, opts: Options, rng: R) -> Self { + pub fn new_with_rng(transport: Coordinator, opts: Options, rng: R) -> Self + where + D: Default, + { Self { core: Endpoint::new_with_rng(opts, rng), transport, } } + /// Construct a `QuicEndpoint` injecting the two coalescer shed counters, for a + /// driver that shares them with a detached handle. + /// + /// Forwards `user_drop` / `member_drop` into + /// [`Endpoint::new_with_rng_in`](crate::endpoint::Endpoint::new_with_rng_in). + pub fn new_with_rng_in( + transport: Coordinator, + opts: Options, + rng: R, + user_drop: D, + member_drop: D, + ) -> Self { + Self { + core: Endpoint::new_with_rng_in(opts, rng, user_drop, member_drop), + transport, + } + } + /// Convenience constructor that seeds serf's `R` with a zero seed. /// /// Suitable for tests and deterministic environments. Production drivers /// should use `new_with_rng` and seed from a cryptographically-secure source. - pub fn new(transport: Coordinator, opts: Options) -> Self { + pub fn new(transport: Coordinator, opts: Options) -> Self + where + D: Default, + { Self::new_with_rng(transport, opts, R::seed_from_u64(0)) } } @@ -123,11 +150,12 @@ where #[cfg(feature = "quic")] #[cfg_attr(docsrs, doc(cfg(feature = "quic")))] -impl QuicEndpoint +impl QuicEndpoint where I: Id + Clone, G: Rng, R: Rng + SeedableRng, + D: DropCounter, { /// Feed one inbound UDP datagram from `from` into the coordinator. /// @@ -242,8 +270,16 @@ where /// The driver calls this once at loop entry; without it the coordinator's /// `next_probe` / `next_gossip` / `next_pushpull` stay unset and failure /// detection, dissemination, and anti-entropy never run. + /// + /// Also folds the inner events the coordinator queued during construction — + /// the local self-join in particular — into serf state under the driver's + /// live `now`. Deferring that drain to a later un-latched `poll_event` would + /// process the self-join at the machine's origin instant, so a coalescing + /// window it opens would be armed already-overdue and flush immediately + /// instead of batching the startup membership changes. pub fn start_scheduling(&mut self, now: Instant) { self.transport.start_scheduling(now); + self.core.drain_after_ingress(&mut self.transport, now); } /// Initiate an outbound push-pull dial to `peer`, then sieve the resulting @@ -567,6 +603,21 @@ where self.core.num_members() } + /// Forwards to [`Endpoint::coalesced_user_events_dropped`]. + pub fn coalesced_user_events_dropped(&self) -> u64 { + self.core.coalesced_user_events_dropped() + } + + /// Forwards to [`Endpoint::coalesced_member_events_dropped`]. + pub fn coalesced_member_events_dropped(&self) -> u64 { + self.core.coalesced_member_events_dropped() + } + + /// Forwards to [`Endpoint::pending_events_len`]. + pub fn pending_events_len(&self) -> usize { + self.core.pending_events_len() + } + /// Forwards to [`Endpoint::poll_event`]. pub fn poll_event(&mut self) -> Option> { self.core.poll_event(&mut self.transport) @@ -586,11 +637,11 @@ where /// # Errors /// /// Returns [`Error::SetTagsMeta`] if the encoded tags exceed the metadata cap. - pub fn set_tags(&mut self, tags: Tags) -> Result<(), Error> + pub fn set_tags(&mut self, tags: Tags, now: Instant) -> Result<(), Error> where I: Clone, { - self.core.set_tags(&mut self.transport, tags) + self.core.set_tags(&mut self.transport, tags, now) } /// Forwards to [`Endpoint::leave`]. @@ -615,10 +666,11 @@ where name: impl Into, payload: bytes::Bytes, coalesce: bool, + now: Instant, ) -> Result<(), Error> { self .core - .user_event(&mut self.transport, name, payload, coalesce) + .user_event(&mut self.transport, name, payload, coalesce, now) } /// Forwards to [`Endpoint::query`]. @@ -713,11 +765,12 @@ where #[cfg(all(test, feature = "quic"))] #[allow(dead_code)] -impl QuicEndpoint +impl QuicEndpoint where I: Id + Clone, G: Rng, R: Rng + SeedableRng, + D: DropCounter, { /// Forwards to [`Endpoint::handle_node_join_intent`]. pub(crate) fn handle_node_join_intent(&mut self, ltime: LamportTime, id: &I, now: Instant) -> bool @@ -1274,7 +1327,7 @@ where /// Mutable access to the serf-logic core, for tests that manipulate its /// private state directly. - pub(crate) fn core_mut(&mut self) -> &mut Endpoint { + pub(crate) fn core_mut(&mut self) -> &mut Endpoint { &mut self.core } diff --git a/serf-proto/src/quic_endpoint/tests.rs b/serf-proto/src/quic_endpoint/tests.rs index 4de72431..4e4ac26e 100644 --- a/serf-proto/src/quic_endpoint/tests.rs +++ b/serf-proto/src/quic_endpoint/tests.rs @@ -27,7 +27,12 @@ use rustls::{ }; use rustls_pki_types::{CertificateDer, PrivateKeyDer}; -use crate::{QuicEndpoint, members::MemberStatus, options::Options}; +use crate::{ + QuicEndpoint, + event::{Event, MemberEventKind}, + members::MemberStatus, + options::Options, +}; fn sa(port: u16) -> SocketAddr { format!("127.0.0.1:{port}").parse().unwrap() @@ -163,7 +168,7 @@ fn constructs_alive_with_zero_clocks() { fn user_event_marks_local_state_dirty() { let mut e = ep(1, 7946); e.test_clear_dirty(); - e.user_event("deploy", Bytes::from_static(b"v2"), false) + e.user_event("deploy", Bytes::from_static(b"v2"), false, Instant::ORIGIN) .expect("user_event on an alive endpoint"); assert!( e.test_is_dirty(), @@ -181,6 +186,61 @@ fn handle_packet_with_garbage_bytes_is_a_noop() { ); } +#[test] +fn startup_self_join_coalesces_from_the_scheduling_instant() { + // The coordinator queues the local self-join during construction, before any + // live-time entry point has run. `start_scheduling` — the driver's first call, + // made with its live clock — must fold that queued join under that instant: + // deferring it to a later un-latched `poll_event` would arm the coalescing + // window at the machine's origin, already overdue, flushing the self join + // immediately instead of holding it for the configured window. + let inner_opts = EndpointOptions::new(1u32, sa(7946)) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); + let inner = + memberlist_proto::Endpoint::new_at(inner_opts, Instant::ORIGIN, SmallRng::seed_from_u64(0)); + let coord = memberlist_proto::QuicEndpoint::::with_quinn_rng_seed( + inner, + test_quic_options(), + Some([0x5au8; 32]), + ); + let opts = Options::new() + .with_coalesce_period(Duration::from_secs(10)) + .with_quiescent_period(Duration::from_secs(2)); + let mut e: QuicEndpoint = QuicEndpoint::new(coord, opts); + + // The driver arms the schedulers at its live clock; the queued self-join is + // folded here, opening the member window at t100 with its quiescent deadline + // at t102. + let t = |s: u64| Instant::ORIGIN + Duration::from_secs(s); + e.start_scheduling(t(100)); + assert!( + e.poll_event().is_none(), + "the startup self join is buffered in the window, not delivered immediately" + ); + + // Before the window's own deadline nothing flushes — a window armed at the + // origin would be long overdue here and would flush the join early. + e.handle_timeout(t(101)); + assert!( + e.poll_event().is_none(), + "the startup window holds until the deadline armed from the scheduling instant" + ); + + // At the deadline the self join is delivered. + e.handle_timeout(t(102)); + let ev = e + .poll_event() + .expect("the startup window flushes at its own deadline"); + match ev { + Event::Member(me) => { + assert_eq!(me.kind(), MemberEventKind::Join); + let ids: Vec = me.members().iter().map(|m| *m.node().id_ref()).collect(); + assert_eq!(ids, vec![1], "the self join is delivered from the window"); + } + other => panic!("expected the coalesced self join, got {other:?}"), + } +} + /// A reconnect dial against a failed member targets the failed peer's address. /// On QUIC the coordinator IS the driver and dials itself (sieving its own /// `DialRequested` into a private queue), so the observable serf-side fact is diff --git a/serf-proto/src/stream_endpoint/mod.rs b/serf-proto/src/stream_endpoint/mod.rs index 0fbb69ad..b1302e95 100644 --- a/serf-proto/src/stream_endpoint/mod.rs +++ b/serf-proto/src/stream_endpoint/mod.rs @@ -37,6 +37,7 @@ use memberlist_proto::{ use smol_str::SmolStr; use crate::{ + DropCounter, endpoint::{Endpoint, Error, QueryId, QueryParams}, event::{Event, QueryEvent}, members::{Member, SerfState}, @@ -71,13 +72,14 @@ use crate::event::KeyResponseArgs; /// `Labeled` for TLS. #[cfg(feature = "tcp")] #[cfg_attr(docsrs, doc(cfg(feature = "tcp")))] -pub struct StreamEndpoint +pub struct StreamEndpoint where I: Eq + core::hash::Hash, RT: StreamTransport, + D: DropCounter, { /// The serf-logic core, holding all serf state and no transport reference. - core: Endpoint, + core: Endpoint, /// The memberlist reliable coordinator serf drives through the `Reliable` /// seam. Holds the single membership `Endpoint`. transport: Coordinator, @@ -85,29 +87,54 @@ where #[cfg(feature = "tcp")] #[cfg_attr(docsrs, doc(cfg(feature = "tcp")))] -impl StreamEndpoint +impl StreamEndpoint where I: Clone + Eq + core::hash::Hash, RT: StreamTransport, R: SeedableRng, + D: DropCounter, { /// Construct a `StreamEndpoint` from a memberlist reliable coordinator /// `transport`, serf `opts`, and serf's own injected `rng`. /// /// `rng` is **separate** from the coordinator's RNG `G`; seed it from the /// driver's own entropy source. - pub fn new_with_rng(transport: Coordinator, opts: Options, rng: R) -> Self { + pub fn new_with_rng(transport: Coordinator, opts: Options, rng: R) -> Self + where + D: Default, + { Self { core: Endpoint::new_with_rng(opts, rng), transport, } } + /// Construct a `StreamEndpoint` injecting the two coalescer shed counters, for + /// a driver that shares them with a detached handle. + /// + /// Forwards `user_drop` / `member_drop` into + /// [`Endpoint::new_with_rng_in`](crate::endpoint::Endpoint::new_with_rng_in). + pub fn new_with_rng_in( + transport: Coordinator, + opts: Options, + rng: R, + user_drop: D, + member_drop: D, + ) -> Self { + Self { + core: Endpoint::new_with_rng_in(opts, rng, user_drop, member_drop), + transport, + } + } + /// Convenience constructor that seeds serf's `R` with a zero seed. /// /// Suitable for tests and deterministic environments. Production drivers /// should use `new_with_rng` and seed from a cryptographically-secure source. - pub fn new(transport: Coordinator, opts: Options) -> Self { + pub fn new(transport: Coordinator, opts: Options) -> Self + where + D: Default, + { Self::new_with_rng(transport, opts, R::seed_from_u64(0)) } } @@ -120,13 +147,14 @@ where #[cfg(feature = "tcp")] #[cfg_attr(docsrs, doc(cfg(feature = "tcp")))] -impl StreamEndpoint +impl StreamEndpoint where I: Id + Clone, A: CheapClone + Data + PartialEq + Clone + 'static, RT: StreamTransport, G: Rng, R: Rng + SeedableRng, + D: DropCounter, { /// Feed one decoded unreliable memberlist `Message` into the /// coordinator, then sieve the resulting inner events into serf. @@ -282,8 +310,16 @@ where /// The driver calls this once at loop entry; without it the coordinator's /// `next_probe` / `next_gossip` / `next_pushpull` stay unset and failure /// detection, dissemination, and anti-entropy never run. + /// + /// Also folds the inner events the coordinator queued during construction — + /// the local self-join in particular — into serf state under the driver's + /// live `now`. Deferring that drain to a later un-latched `poll_event` would + /// process the self-join at the machine's origin instant, so a coalescing + /// window it opens would be armed already-overdue and flush immediately + /// instead of batching the startup membership changes. pub fn start_scheduling(&mut self, now: Instant) { self.transport.start_scheduling(now); + self.core.drain_after_ingress(&mut self.transport, now); } /// Initiate an outbound push-pull dial to `peer`, then sieve the resulting @@ -508,6 +544,21 @@ where self.core.num_members() } + /// Forwards to [`Endpoint::coalesced_user_events_dropped`]. + pub fn coalesced_user_events_dropped(&self) -> u64 { + self.core.coalesced_user_events_dropped() + } + + /// Forwards to [`Endpoint::coalesced_member_events_dropped`]. + pub fn coalesced_member_events_dropped(&self) -> u64 { + self.core.coalesced_member_events_dropped() + } + + /// Forwards to [`Endpoint::pending_events_len`]. + pub fn pending_events_len(&self) -> usize { + self.core.pending_events_len() + } + /// Forwards to [`Endpoint::poll_event`]. pub fn poll_event(&mut self) -> Option> { self.core.poll_event(&mut self.transport) @@ -528,12 +579,12 @@ where /// # Errors /// /// Returns [`Error::SetTagsMeta`] if the encoded tags exceed the metadata cap. - pub fn set_tags(&mut self, tags: Tags) -> Result<(), Error> + pub fn set_tags(&mut self, tags: Tags, now: Instant) -> Result<(), Error> where I: Clone, A: Clone, { - self.core.set_tags(&mut self.transport, tags) + self.core.set_tags(&mut self.transport, tags, now) } /// Forwards to [`Endpoint::handle_node_join_intent`]. @@ -586,10 +637,11 @@ where name: impl Into, payload: bytes::Bytes, coalesce: bool, + now: Instant, ) -> Result<(), Error> { self .core - .user_event(&mut self.transport, name, payload, coalesce) + .user_event(&mut self.transport, name, payload, coalesce, now) } /// Forwards to [`Endpoint::handle_user_event`]. @@ -1091,6 +1143,14 @@ where self.core.test_set_drain_now(now) } + /// Forwards to [`Endpoint::drain_after_ingress`], latching `now` and sieving + /// the coordinator's pending inner events — the interposed-ingress seam for a + /// test that asserts a command's effect is not re-timed by a later drain. + #[cfg(test)] + pub(crate) fn test_drain_after_ingress(&mut self, now: Instant) { + self.core.drain_after_ingress(&mut self.transport, now) + } + /// Forwards to [`Endpoint::test_last_query_id`]. #[cfg(test)] pub(crate) fn test_last_query_id(&self) -> Option { @@ -1380,7 +1440,7 @@ where /// Mutable access to the serf-logic core, for tests that manipulate its /// private state directly. #[cfg(test)] - pub(crate) fn core_mut(&mut self) -> &mut Endpoint { + pub(crate) fn core_mut(&mut self) -> &mut Endpoint { &mut self.core } diff --git a/serf-proto/src/stream_endpoint/tests.rs b/serf-proto/src/stream_endpoint/tests.rs index 6cb729a3..95451c1a 100644 --- a/serf-proto/src/stream_endpoint/tests.rs +++ b/serf-proto/src/stream_endpoint/tests.rs @@ -60,7 +60,7 @@ fn constructs_alive_with_zero_clocks() { fn user_event_marks_local_state_dirty() { let mut e = ep(1, 7946); e.test_clear_dirty(); - e.user_event("deploy", Bytes::from_static(b"v2"), false) + e.user_event("deploy", Bytes::from_static(b"v2"), false, Instant::ORIGIN) .expect("user_event on an alive endpoint"); assert!( e.test_is_dirty(), diff --git a/serf-reactor/src/driver/quic/mod.rs b/serf-reactor/src/driver/quic/mod.rs index ff56daca..501923ab 100644 --- a/serf-reactor/src/driver/quic/mod.rs +++ b/serf-reactor/src/driver/quic/mod.rs @@ -85,6 +85,7 @@ use crate::{ options::RuntimeOptions, shared::{ExchangeId, dispatch_event_delegate, observation_payload_bytes}, }, + drop_counter::ReactorDropCounter, error::{JoinFailed, Result, SerfError}, shared::Shared, }; @@ -277,12 +278,12 @@ impl PendingLeave { /// a lost id-conflict `Event::Shutdown`, or the last handle dropped). pub(crate) struct QuicDriver where - // Structurally required: `endpoint` names `QuicEndpoint`, whose struct + // Structurally required: `endpoint` names `QuicEndpoint`, whose struct // declares `I: Eq + Hash`. I: core::hash::Hash + Eq, R: Runtime, { - endpoint: QuicEndpoint, + endpoint: QuicEndpoint, /// The shared UDP socket carrying QUIC packets AND plain-UDP gossip. `Option` so /// the shutdown branch can drop it (releasing the bound port) BEFORE acking; /// `Some` for the running lifetime, taken only during teardown. @@ -352,7 +353,7 @@ where /// the observation hand-off, and the recv-buffer inputs. #[allow(clippy::too_many_arguments)] pub(crate) fn new( - endpoint: QuicEndpoint, + endpoint: QuicEndpoint, socket: ::UdpSocket, quic_max_udp_payload: u64, shared: Arc>, @@ -543,7 +544,7 @@ where let payload = cmd.payload().clone(); self .endpoint - .user_event(name, payload, cmd.coalesce) + .user_event(name, payload, cmd.coalesce, now) .map_err(SerfError::from) } else { Err(SerfError::NotRunning) @@ -583,7 +584,7 @@ where } Command::SetTags(SetTagsCmd { tags, reply }) => { let res = if running { - self.endpoint.set_tags(tags).map_err(SerfError::from) + self.endpoint.set_tags(tags, now).map_err(SerfError::from) } else { Err(SerfError::NotRunning) }; @@ -1144,7 +1145,7 @@ where R: Runtime, G: rand::Rng + Unpin, SR: rand::Rng + SeedableRng + Unpin, - QuicEndpoint: Unpin, + QuicEndpoint: Unpin, { type Output = (); @@ -1389,7 +1390,7 @@ where /// a slow delegate cannot delay it), and once fully done clear the still-recorded /// ignore-join streams and reap the waiter. fn complete_join_exchange( - endpoint: &mut QuicEndpoint, + endpoint: &mut QuicEndpoint, pending_joins: &mut Vec, eid: ExchangeId, peer: SocketAddr, @@ -1465,7 +1466,7 @@ async fn observation_task( /// then build the [`QuicDriver`] future. The caller (`Transport::run`) awaits it. #[allow(clippy::too_many_arguments)] pub(crate) fn spawn_quic_driver( - mut endpoint: QuicEndpoint, + mut endpoint: QuicEndpoint, socket: ::UdpSocket, quic_max_udp_payload: u64, shared: Arc>, diff --git a/serf-reactor/src/driver/quic/tests.rs b/serf-reactor/src/driver/quic/tests.rs index 031b4898..788a49ce 100644 --- a/serf-reactor/src/driver/quic/tests.rs +++ b/serf-reactor/src/driver/quic/tests.rs @@ -245,12 +245,24 @@ mod gate { let quic_max_udp_payload = quic_config.endpoint_ref().get_max_udp_payload_size(); let inner = Endpoint::new(inner_opts, StdRng::seed_from_u64(0)); let coord = Coordinator::new(inner, quic_config); - let endpoint = QuicEndpoint::::new_with_rng( + // These pump tests do not assert coalescer shed counts, so the write halves + // suffice for the endpoint; the read halves are unused here. + let (user_drop, _) = crate::drop_counter::drop_channel(); + let (member_drop, _) = crate::drop_counter::drop_channel(); + let endpoint = QuicEndpoint::::new_with_rng_in( coord, SerfOptions::new(), StdRng::seed_from_u64(1), + user_drop, + member_drop, ); - let shared = Arc::new(Shared::new(initial_snapshot("drv", advertise))); + let (_, user_drop_reader) = crate::drop_counter::drop_channel(); + let (_, member_drop_reader) = crate::drop_counter::drop_channel(); + let shared = Arc::new(Shared::new( + initial_snapshot("drv", advertise), + user_drop_reader, + member_drop_reader, + )); let obs_payload_bytes = Arc::new(AtomicU64::new(0)); let (obs_tx, obs_rx) = flume::unbounded(); let driver = QuicDriver::::new( diff --git a/serf-reactor/src/driver/stream/mod.rs b/serf-reactor/src/driver/stream/mod.rs index d4ade24d..837aa50c 100644 --- a/serf-reactor/src/driver/stream/mod.rs +++ b/serf-reactor/src/driver/stream/mod.rs @@ -67,6 +67,7 @@ use crate::{ options::{RuntimeOptions, StreamTransportOptions}, shared::{ExchangeId, dispatch_event_delegate, observation_payload_bytes}, }, + drop_counter::ReactorDropCounter, error::{JoinFailed, Result, SerfError}, shared::Shared, }; @@ -324,7 +325,7 @@ where R: Runtime, T: StreamTransport, { - endpoint: StreamEndpoint, + endpoint: StreamEndpoint, /// Unreliable gossip datagrams. `Option` so the shutdown branch can drop it /// (releasing the bound UDP port) BEFORE acking; `Some` for the running /// lifetime, taken only during teardown. @@ -451,7 +452,7 @@ where /// state, the observation hand-off, and the accept task's channels/handle. #[allow(clippy::too_many_arguments)] pub(crate) fn new( - endpoint: StreamEndpoint, + endpoint: StreamEndpoint, socket: ::UdpSocket, shared: Arc>, obs_tx: Sender>, @@ -703,7 +704,7 @@ where let payload = cmd.payload().clone(); self .endpoint - .user_event(name, payload, cmd.coalesce) + .user_event(name, payload, cmd.coalesce, now) .map_err(SerfError::from) } else { Err(SerfError::NotRunning) @@ -743,7 +744,7 @@ where } Command::SetTags(SetTagsCmd { tags, reply }) => { let res = if running { - self.endpoint.set_tags(tags).map_err(SerfError::from) + self.endpoint.set_tags(tags, now).map_err(SerfError::from) } else { Err(SerfError::NotRunning) }; @@ -1444,7 +1445,7 @@ where T::Options: Unpin, G: rand::Rng + Unpin, SR: rand::Rng + SeedableRng + Unpin, - StreamEndpoint: Unpin, + StreamEndpoint: Unpin, { type Output = (); @@ -1826,7 +1827,7 @@ where /// hand-off, so a slow delegate cannot delay it), and once fully done clear the /// still-recorded ignore-join streams and reap the waiter. fn complete_join_exchange( - endpoint: &mut StreamEndpoint, + endpoint: &mut StreamEndpoint, pending_joins: &mut Vec, eid: ExchangeId, peer: SocketAddr, @@ -1974,7 +1975,7 @@ async fn observation_task( /// caller (`Transport::run`) awaits the returned future. #[allow(clippy::too_many_arguments)] pub(crate) fn spawn_stream_driver( - mut endpoint: StreamEndpoint, + mut endpoint: StreamEndpoint, gossip_socket: ::UdpSocket, listener: ::TcpListener, shared: Arc>, diff --git a/serf-reactor/src/driver/stream/tests.rs b/serf-reactor/src/driver/stream/tests.rs index 039af298..2374a450 100644 --- a/serf-reactor/src/driver/stream/tests.rs +++ b/serf-reactor/src/driver/stream/tests.rs @@ -31,6 +31,11 @@ const DRIVER_ADDR: &str = "127.0.0.1:7946"; type TestDriver = StreamDriver; +/// The driver's serf endpoint, pinning the reactor's shared drop-counter storage +/// the [`StreamDriver`] requires. +type DrvEndpoint = + StreamEndpoint; + fn sa(s: &str) -> SocketAddr { s.parse().expect("loopback addr") } @@ -51,10 +56,7 @@ const DEFAULT_TEST_STREAM_TIMEOUT: Duration = Duration::from_secs(10); /// `advertise`, mirroring the production construction (memberlist inner endpoint → /// reliable coordinator → serf super-machine). Seeded deterministically; the /// initial local `NodeJoined` self-event is drained so a caller starts clean. -fn build_endpoint( - id: &str, - advertise: SocketAddr, -) -> StreamEndpoint { +fn build_endpoint(id: &str, advertise: SocketAddr) -> DrvEndpoint { build_endpoint_with_stream_timeout(id, advertise, DEFAULT_TEST_STREAM_TIMEOUT) } @@ -64,7 +66,7 @@ fn build_endpoint_with_stream_timeout( id: &str, advertise: SocketAddr, stream_timeout: Duration, -) -> StreamEndpoint { +) -> DrvEndpoint { let inner_opts = EndpointOptions::new(SmolStr::new(id), advertise) .with_user_broadcast_tiers(NonZeroU8::new(3).expect("3 is nonzero")) .with_stream_timeout(stream_timeout); @@ -75,7 +77,17 @@ fn build_endpoint_with_stream_timeout( Box::new(|_addr: &SocketAddr| None), Box::new(|addr: &SocketAddr| *addr), ); - let mut e = StreamEndpoint::new(coord, SerfOptions::new()); + // These pump-ordering tests do not assert coalescer shed counts, so the write + // halves suffice; the read halves are unused here. + let (user_drop, _) = crate::drop_counter::drop_channel(); + let (member_drop, _) = crate::drop_counter::drop_channel(); + let mut e = StreamEndpoint::new_with_rng_in( + coord, + SerfOptions::new(), + SmallRng::seed_from_u64(0), + user_drop, + member_drop, + ); while e.poll_event().is_some() {} e } @@ -136,7 +148,13 @@ async fn build_driver_with_stream_timeout( .await .expect("bind gossip socket"); let endpoint = build_endpoint_with_stream_timeout("drv", sa(DRIVER_ADDR), stream_timeout); - let shared = Arc::new(Shared::new(initial_snapshot("drv", sa(DRIVER_ADDR)))); + let (_, user_drop_reader) = crate::drop_counter::drop_channel(); + let (_, member_drop_reader) = crate::drop_counter::drop_channel(); + let shared = Arc::new(Shared::new( + initial_snapshot("drv", sa(DRIVER_ADDR)), + user_drop_reader, + member_drop_reader, + )); let obs_payload_bytes = Arc::new(AtomicU64::new(0)); let (obs_tx, obs_rx) = flume::unbounded(); let (accepted_tx, accepted_rx) = flume::bounded(ACCEPT_CAP); diff --git a/serf-reactor/src/drop_counter/mod.rs b/serf-reactor/src/drop_counter/mod.rs new file mode 100644 index 00000000..32aca831 --- /dev/null +++ b/serf-reactor/src/drop_counter/mod.rs @@ -0,0 +1,72 @@ +//! The shared coalescer-drop counter split into a write-capable half held by the +//! driver-owned endpoint and a read-only half held by the handle's [`Shared`]. +//! +//! Both wrap the SAME `Arc` (minted once by [`drop_channel`]), so the +//! endpoint's increment is observed by a `Serf` handle clone on any worker +//! without the driver copying the value out each pump iteration. The reader +//! exposes no mutator, so a handle has no type-level path to write — the endpoint +//! is the structural sole writer. +//! +//! [`Shared`]: crate::shared::Shared + +use std::sync::{ + Arc, + atomic::{AtomicU64, Ordering}, +}; + +use serf_proto::DropCounter; + +/// The write-capable half, moved into the serf endpoint the driver pumps. It is +/// the sole writer of its backing atomic. +pub(crate) struct ReactorDropCounter(Arc); + +impl DropCounter for ReactorDropCounter { + #[inline] + fn incr_saturating(&mut self) { + // A saturating compare-and-swap rather than a wrapping `fetch_add`: the shed + // path is low-frequency, so the retry cost is negligible, and the counter + // stops at `u64::MAX` instead of wrapping to zero. Robust to any future writer + // topology even though the endpoint is the sole writer today. + let mut cur = self.0.load(Ordering::Relaxed); + loop { + let next = cur.saturating_add(1); + match self + .0 + .compare_exchange_weak(cur, next, Ordering::Relaxed, Ordering::Relaxed) + { + Ok(_) => break, + Err(actual) => cur = actual, + } + } + } + + #[inline] + fn get(&self) -> u64 { + self.0.load(Ordering::Relaxed) + } +} + +/// The read-only half, kept on the handle's [`Shared`]. Exposes only a load, so a +/// `Serf` clone can never write the counter. +/// +/// [`Shared`]: crate::shared::Shared +pub(crate) struct DropReader(Arc); + +impl DropReader { + /// The endpoint's current cumulative shed count. + #[inline] + pub(crate) fn get(&self) -> u64 { + self.0.load(Ordering::Relaxed) + } +} + +/// Mint one shared counter as a `(writer, reader)` pair over a single backing +/// atomic, so the endpoint's writes and the handle's reads cannot accidentally +/// address two separate allocations. +pub(crate) fn drop_channel() -> (ReactorDropCounter, DropReader) { + let arc = Arc::new(AtomicU64::new(0)); + (ReactorDropCounter(arc.clone()), DropReader(arc)) +} + +#[cfg(test)] +mod tests; diff --git a/serf-reactor/src/drop_counter/tests.rs b/serf-reactor/src/drop_counter/tests.rs new file mode 100644 index 00000000..352d22fa --- /dev/null +++ b/serf-reactor/src/drop_counter/tests.rs @@ -0,0 +1,45 @@ +use super::*; + +use serf_proto::DropCounter; + +#[test] +fn incr_saturates_at_max() { + let arc = Arc::new(AtomicU64::new(u64::MAX - 1)); + let mut w = ReactorDropCounter(arc); + w.incr_saturating(); + assert_eq!(w.get(), u64::MAX); + // A further increment must saturate, not wrap the compare-and-swap back to zero. + w.incr_saturating(); + assert_eq!(w.get(), u64::MAX); +} + +#[test] +fn reader_observes_writer_over_the_shared_atomic() { + let (mut w, r) = drop_channel(); + assert_eq!(r.get(), 0); + w.incr_saturating(); + w.incr_saturating(); + w.incr_saturating(); + assert_eq!( + r.get(), + 3, + "the read half observes the write half's increments over one atomic" + ); +} + +/// The reactor endpoint carries a `Send + Sync` shared-atomic counter, so the +/// detached pump future stays spawnable on a multi-thread runtime. +#[test] +fn reactor_endpoint_is_send_and_sync() { + fn assert_send_sync() {} + assert_send_sync::(); + assert_send_sync::(); + assert_send_sync::< + serf_proto::endpoint::Endpoint< + u32, + std::net::SocketAddr, + rand::rngs::StdRng, + ReactorDropCounter, + >, + >(); +} diff --git a/serf-reactor/src/lib.rs b/serf-reactor/src/lib.rs index d1b33021..8ca54e2f 100644 --- a/serf-reactor/src/lib.rs +++ b/serf-reactor/src/lib.rs @@ -53,6 +53,8 @@ mod command; #[cfg(any(feature = "tcp", feature = "quic"))] mod delegate; mod driver; +#[cfg(any(feature = "tcp", feature = "quic"))] +mod drop_counter; mod error; mod events; #[cfg(feature = "quic")] diff --git a/serf-reactor/src/quic/mod.rs b/serf-reactor/src/quic/mod.rs index d9ed2838..75057009 100644 --- a/serf-reactor/src/quic/mod.rs +++ b/serf-reactor/src/quic/mod.rs @@ -390,10 +390,17 @@ where // Serf's core RNG is seeded from its own OS-drawn entropy (`self.serf_rng`), // independent of the coordinator's gossip RNG, so two nodes never share the // query-ID / relay-selection stream. - let endpoint = serf_proto::QuicEndpoint::::new_with_rng( + let endpoint = serf_proto::QuicEndpoint::< + Self::Id, + G, + StdRng, + crate::drop_counter::ReactorDropCounter, + >::new_with_rng_in( coord, runtime.serf_options, self.serf_rng, + runtime.user_drop, + runtime.member_drop, ); let driver = crate::driver::quic::spawn_quic_driver::( diff --git a/serf-reactor/src/serf/mod.rs b/serf-reactor/src/serf/mod.rs index ba24f432..38e208ce 100644 --- a/serf-reactor/src/serf/mod.rs +++ b/serf-reactor/src/serf/mod.rs @@ -182,7 +182,16 @@ where let (events_tx, events_rx) = flume::bounded::>(runtime_options.event_queue_cap()); - let shared = Arc::new(Shared::new(initial_snapshot(&local_id, advertise))); + // Mint the two shed counters as (writer, reader) pairs: the driver injects the + // writers into the endpoint, the handle reads the readers, both over the same + // backing atomic so no publish step exists. + let (user_drop_writer, user_drop_reader) = crate::drop_counter::drop_channel(); + let (member_drop_writer, member_drop_reader) = crate::drop_counter::drop_channel(); + let shared = Arc::new(Shared::new( + initial_snapshot(&local_id, advertise), + user_drop_reader, + member_drop_reader, + )); let runtime = TransportRuntime::::new( delegate, @@ -190,6 +199,8 @@ where events_tx, runtime_options, serf_options, + user_drop_writer, + member_drop_writer, #[cfg(encryption)] keyring, ); @@ -618,6 +629,23 @@ impl Serf { self.shared.observation_dropped() } + /// The cumulative count of coalescing user events the driver's endpoint shed + /// because its user coalescer was at the configured buffered-volume cap + /// (`Options::max_coalesced_user_events`). Lifetime total, saturating; always + /// `0` when user coalescing is disabled. + #[must_use] + pub fn coalesced_user_events_dropped(&self) -> u64 { + self.shared.coalesced_user_events_dropped() + } + + /// The cumulative count of member changes the driver's endpoint shed because + /// its member coalescer was at its per-window cardinality cap. Lifetime total, + /// saturating; always `0` when member coalescing is disabled. + #[must_use] + pub fn coalesced_member_events_dropped(&self) -> u64 { + self.shared.coalesced_member_events_dropped() + } + /// The cumulative count of gossip payloads sent over the QUIC datagram plane /// (a datagram queued onto the peer's pooled, TLS-protected connection) rather /// than the plain-UDP fallback. Always `0` on the stream transports and on a diff --git a/serf-reactor/src/serf/tests.rs b/serf-reactor/src/serf/tests.rs index b5dc1c48..ef1e2dcd 100644 --- a/serf-reactor/src/serf/tests.rs +++ b/serf-reactor/src/serf/tests.rs @@ -72,6 +72,137 @@ async fn spawn_node(id: &str) -> Node { .expect("spawn serf node") } +/// Build and spawn a reactor TCP node with a custom `SerfOptions` (runtime options +/// at defaults). +async fn spawn_node_with_serf_options(id: &str, serf_options: SerfOptions) -> Node { + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(bind)); + Serf::::tcp( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + serf_options, + #[cfg(encryption)] + std::sync::Arc::new(crate::VoidKeyringDelegate), + ) + .await + .expect("spawn serf node") +} + +/// A single node with user coalescing enabled and a small buffered-volume cap sheds +/// every distinct-named coalescing user event issued past the cap through the public +/// `user_event` command path, and the cumulative drop count surfaces on the public +/// `coalesced_user_events_dropped` accessor — the endpoint counter is otherwise +/// unreachable once the driver moves the endpoint into the detached pump. +#[tokio::test] +async fn tcp_coalesced_user_events_dropped_observable() { + let cap = core::num::NonZeroUsize::new(4).unwrap(); + let serf_opts = SerfOptions::new() + .with_user_coalesce_period(Duration::from_secs(10)) + .with_user_quiescent_period(Duration::from_secs(2)) + .with_max_coalesced_user_events(Some(cap)); + let a = spawn_node_with_serf_options("coalesce-a", serf_opts).await; + + assert_eq!( + a.coalesced_user_events_dropped(), + 0, + "no drops before any user event is issued" + ); + + // Issue distinct-named coalescing user events past the cap. Each is buffered by + // name in the open window; every name past the cap is shed and counted. + let n: u32 = 20; + for i in 0..n { + a.user_event(format!("evt-{i}"), bytes::Bytes::new(), true) + .await + .expect("user event dispatched"); + } + + // Each `user_event().await` returned only after the pump processed that command + // and incremented the shared shed counter, so the handle read is already current + // with no publish step or extra wake. + let dropped = a.coalesced_user_events_dropped(); + a.shutdown().await.expect("coalesce-a shuts down"); + + assert_eq!( + dropped, + u64::from(n) - cap.get() as u64, + "every distinct-named cc event past the cap is counted on the public handle (got {dropped})" + ); +} + +/// On a multi-threaded runtime a `Serf` clone read on a DIFFERENT worker than the +/// pump observes a coalescer shed the instant the `user_event` reply resolves: the +/// handle's reader and the endpoint's writer share one atomic, so there is no +/// publish step for the read to lag behind. Repeated so a regression that +/// reintroduced a copied-out mirror would be reliably caught. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn coalesced_drop_visible_cross_thread_after_reply() { + let cap = core::num::NonZeroUsize::new(4).unwrap(); + let n: u32 = 20; + for trial in 0..24 { + let serf_opts = SerfOptions::new() + .with_user_coalesce_period(Duration::from_secs(10)) + .with_user_quiescent_period(Duration::from_secs(2)) + .with_max_coalesced_user_events(Some(cap)); + let a = spawn_node_with_serf_options(&format!("coalesce-xthread-{trial}"), serf_opts).await; + + // Issue distinct-named coalescing events past the cap; await each reply so the + // pump has processed and shed it. + for i in 0..n { + a.user_event(format!("evt-{i}"), bytes::Bytes::new(), true) + .await + .expect("user event dispatched"); + } + + // Read the shed count from a CLONE on a spawned task — a different worker than + // the pump — immediately after the replies, with no sleep and no publish. + let b = a.clone(); + let observed = tokio::spawn(async move { b.coalesced_user_events_dropped() }) + .await + .expect("read task joins"); + + a.shutdown().await.expect("coalesce-xthread shuts down"); + assert_eq!( + observed, + u64::from(n) - cap.get() as u64, + "a clone on another worker observes the shed count with no publish step" + ); + } +} + +/// After exactly one shed through the public `user_event` path the handle getter +/// returns at least one. A construction typo that wired the handle's reader to a +/// different atomic than the endpoint's writer would leave this a permanent zero, +/// so the aliasing bug fails loudly here rather than silently reporting no drops. +#[tokio::test] +async fn coalesced_drop_aliasing_guard() { + // A cap of one: the second distinct-named coalescing event is shed. + let cap = core::num::NonZeroUsize::new(1).unwrap(); + let serf_opts = SerfOptions::new() + .with_user_coalesce_period(Duration::from_secs(10)) + .with_user_quiescent_period(Duration::from_secs(2)) + .with_max_coalesced_user_events(Some(cap)); + let a = spawn_node_with_serf_options("coalesce-alias", serf_opts).await; + + a.user_event("first".to_string(), bytes::Bytes::new(), true) + .await + .expect("first user event dispatched"); + a.user_event("second".to_string(), bytes::Bytes::new(), true) + .await + .expect("second user event dispatched"); + + assert!( + a.coalesced_user_events_dropped() >= 1, + "the handle observes the endpoint's shed; a mis-wired reader would read a permanent 0" + ); + a.shutdown().await.expect("coalesce-alias shuts down"); +} + /// Build VALID TCP transport options paired with a deliberately invalid /// `runtime`, and assert `Serf::tcp` rejects it with [`SerfError::InvalidOption`] /// — before binding a socket or spawning the detached driver — rather than diff --git a/serf-reactor/src/shared/mod.rs b/serf-reactor/src/shared/mod.rs index 47155ef6..3dd53b63 100644 --- a/serf-reactor/src/shared/mod.rs +++ b/serf-reactor/src/shared/mod.rs @@ -23,7 +23,7 @@ use std::{ use arc_swap::ArcSwap; use flume::{Receiver, Sender}; -use crate::command::Command; +use crate::{command::Command, drop_counter::DropReader}; use serf_driver::SerfSnapshot; /// The lock-guarded part of [`Shared`]: the command queue handles push onto, the @@ -48,6 +48,12 @@ pub(crate) struct Shared { events_dropped: AtomicU64, /// Observation-channel drops (a slow delegate; may lose application data). observation_dropped: AtomicU64, + /// Read-only view of the endpoint's cumulative user-coalescer drop count. The + /// driver owns the endpoint, so a handle reads the shed count here — over the + /// SAME atomic the endpoint's writer increments, so no publish step exists. + coalesced_user_events_dropped: DropReader, + /// Read-only view of the endpoint's cumulative member-coalescer drop count. + coalesced_member_events_dropped: DropReader, /// Cumulative gossip payloads that rode the QUIC datagram plane (a /// `DatagramSendStatus::Queued`), as opposed to the plain-UDP fallback. Zero on /// the stream transports and on a QUIC endpoint in `UnreliableTransport::Udp` @@ -72,7 +78,15 @@ pub(crate) struct Shared { impl Shared { /// Builds the shared state around an initial published snapshot, with one live /// handle. - pub(crate) fn new(initial: SerfSnapshot) -> Self { + /// + /// `coalesced_user_events_dropped` / `coalesced_member_events_dropped` are the + /// read-only halves of the shed counters; the driver holds the write halves + /// inside the endpoint, over the same backing atomics. + pub(crate) fn new( + initial: SerfSnapshot, + coalesced_user_events_dropped: DropReader, + coalesced_member_events_dropped: DropReader, + ) -> Self { let (shutdown_complete_tx, shutdown_complete_rx) = flume::bounded(0); Self { inner: Mutex::new(Inner:: { @@ -83,6 +97,8 @@ impl Shared { snapshot: ArcSwap::from_pointee(initial), events_dropped: AtomicU64::new(0), observation_dropped: AtomicU64::new(0), + coalesced_user_events_dropped, + coalesced_member_events_dropped, datagrams_sent: AtomicU64::new(0), shutdown: AtomicBool::new(false), handles: AtomicUsize::new(1), @@ -162,6 +178,16 @@ impl Shared { self.observation_dropped.load(Ordering::Relaxed) } + /// The endpoint's live cumulative user-coalescer drop count. + pub(crate) fn coalesced_user_events_dropped(&self) -> u64 { + self.coalesced_user_events_dropped.get() + } + + /// The endpoint's live cumulative member-coalescer drop count. + pub(crate) fn coalesced_member_events_dropped(&self) -> u64 { + self.coalesced_member_events_dropped.get() + } + /// The cumulative count of gossip payloads sent over the QUIC datagram plane. pub(crate) fn datagrams_sent(&self) -> u64 { self.datagrams_sent.load(Ordering::Relaxed) diff --git a/serf-reactor/src/tcp/mod.rs b/serf-reactor/src/tcp/mod.rs index a80c6f15..04d6b5d4 100644 --- a/serf-reactor/src/tcp/mod.rs +++ b/serf-reactor/src/tcp/mod.rs @@ -378,12 +378,20 @@ where coord.set_encryption_options(self.encryption); // Serf's core RNG is seeded from its own OS-drawn entropy (`self.serf_rng`), // independent of the coordinator's gossip RNG. - let endpoint = - serf_proto::StreamEndpoint::::new_with_rng( - coord, - runtime.serf_options, - self.serf_rng, - ); + let endpoint = serf_proto::StreamEndpoint::< + Self::Id, + SocketAddr, + RawRecords, + G, + StdRng, + crate::drop_counter::ReactorDropCounter, + >::new_with_rng_in( + coord, + runtime.serf_options, + self.serf_rng, + runtime.user_drop, + runtime.member_drop, + ); let driver = crate::driver::stream::spawn_stream_driver::( endpoint, diff --git a/serf-reactor/src/tls/mod.rs b/serf-reactor/src/tls/mod.rs index c814828e..7af2f3ec 100644 --- a/serf-reactor/src/tls/mod.rs +++ b/serf-reactor/src/tls/mod.rs @@ -507,7 +507,14 @@ where Labeled, G, StdRng, - >::new_with_rng(coord, runtime.serf_options, self.serf_rng); + crate::drop_counter::ReactorDropCounter, + >::new_with_rng_in( + coord, + runtime.serf_options, + self.serf_rng, + runtime.user_drop, + runtime.member_drop, + ); let driver = crate::driver::stream::spawn_stream_driver::, D, G, StdRng>( diff --git a/serf-reactor/src/transport/runtime.rs b/serf-reactor/src/transport/runtime.rs index e0d76377..507fcb35 100644 --- a/serf-reactor/src/transport/runtime.rs +++ b/serf-reactor/src/transport/runtime.rs @@ -14,7 +14,7 @@ use std::{net::SocketAddr, sync::Arc}; use flume::Sender; use serf_proto::{event::Event, options::Options as SerfOptions}; -use crate::{driver::options::RuntimeOptions, shared::Shared}; +use crate::{driver::options::RuntimeOptions, drop_counter::ReactorDropCounter, shared::Shared}; #[cfg(encryption)] use crate::delegate::KeyringDelegate; @@ -35,6 +35,11 @@ pub struct TransportRuntime { pub(crate) events_tx: Sender>, pub(crate) driver_options: RuntimeOptions, pub(crate) serf_options: SerfOptions, + /// The write half of the user-coalescer shed counter, injected into the + /// endpoint by `T::run` so its increments land in the atomic the handle reads. + pub(crate) user_drop: ReactorDropCounter, + /// The write half of the member-coalescer shed counter. + pub(crate) member_drop: ReactorDropCounter, /// The driver's keyring delegate, applied to inbound key-management requests. /// Present only under an encryption backend. #[cfg(encryption)] @@ -52,6 +57,8 @@ impl TransportRuntime { events_tx: Sender>, driver_options: RuntimeOptions, serf_options: SerfOptions, + user_drop: ReactorDropCounter, + member_drop: ReactorDropCounter, #[cfg(encryption)] keyring: Arc, ) -> Self { Self { @@ -60,6 +67,8 @@ impl TransportRuntime { events_tx, driver_options, serf_options, + user_drop, + member_drop, #[cfg(encryption)] keyring, } diff --git a/serf-smoltcp/src/serf/mod.rs b/serf-smoltcp/src/serf/mod.rs index d2114c11..94005669 100644 --- a/serf-smoltcp/src/serf/mod.rs +++ b/serf-smoltcp/src/serf/mod.rs @@ -783,6 +783,26 @@ where .saturating_add(self.app_events_dropped) } + /// Cumulative count of coalescing user events the engine's user coalescer shed + /// because its buffered volume was at the configured cap. + /// + /// Lifetime total, saturating, and never cleared by a flush or reset. Reads `0` + /// when user coalescing is disabled. + #[inline] + pub fn coalesced_user_events_dropped(&self) -> u64 { + self.engine.coalesced_user_events_dropped() + } + + /// Cumulative count of member changes the engine's member coalescer shed + /// because its per-window map was at its cardinality cap. + /// + /// Lifetime total, saturating, and never cleared. Reads `0` when member + /// coalescing is disabled. + #[inline] + pub fn coalesced_member_events_dropped(&self) -> u64 { + self.engine.coalesced_member_events_dropped() + } + /// The local node's id. #[inline] pub fn local_id(&self) -> I { @@ -890,8 +910,9 @@ where name: impl Into, payload: bytes::Bytes, coalesce: bool, + now: Instant, ) -> Result<(), SerfError> { - self.engine.user_event(name, payload, coalesce) + self.engine.user_event(name, payload, coalesce, now) } /// Issue a cluster-wide query, returning its [`QueryId`]. Responders observe it as @@ -919,8 +940,8 @@ where /// Replace the local node's tags, re-advertising them and refreshing the local /// member in the membership store. - pub fn set_tags(&mut self, tags: Tags) -> Result<(), SerfError> { - self.engine.set_tags(tags) + pub fn set_tags(&mut self, tags: Tags, now: Instant) -> Result<(), SerfError> { + self.engine.set_tags(tags, now) } /// Issue a cluster-wide `install_key` query to add `key` to every node's keyring. diff --git a/serf-smoltcp/src/serf/tests.rs b/serf-smoltcp/src/serf/tests.rs index cc4968a6..8ce54441 100644 --- a/serf-smoltcp/src/serf/tests.rs +++ b/serf-smoltcp/src/serf/tests.rs @@ -239,6 +239,55 @@ fn single_pump_over_cap_drop_is_counted() { ); } +/// With user coalescing enabled and a small buffered-volume cap, distinct-named +/// coalescing user events fed through the public command path past the cap are shed +/// by the engine's user coalescer, and the running total surfaces through the +/// public `coalesced_user_events_dropped` forward. +#[test] +fn coalesced_user_events_dropped_surfaces_overflow() { + let cap = core::num::NonZeroUsize::new(4).unwrap(); + let mut dev = NullDevice; + let serf_opts = SerfOptions::new() + .with_user_coalesce_period(core::time::Duration::from_secs(10)) + .with_user_quiescent_period(core::time::Duration::from_secs(2)) + .with_max_coalesced_user_events(Some(cap)); + let mut node = Serf::::try_new( + Options::new(), + ip_iface(1), + TransformOptions::default(), + EndpointOptions::new( + SmolStr::new("a"), + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)), 7946), + ), + serf_opts, + &SocketAddrResolver, + &mut dev, + now(), + ) + .expect("a valid configuration constructs"); + node.start(now()); + assert_eq!(node.coalesced_user_events_dropped(), 0); + + let n: u32 = 20; + for i in 0..n { + node + .user_event( + SmolStr::from(std::format!("evt-{i}")), + bytes::Bytes::from_static(b"p"), + true, + now(), + ) + .expect("a coalescing user event is accepted while running"); + } + + assert_eq!( + node.coalesced_user_events_dropped(), + u64::from(n) - cap.get() as u64, + "every distinct-named coalescing event past the cap is shed and counted" + ); + assert_eq!(node.coalesced_member_events_dropped(), 0); +} + #[test] fn advertise_not_local_is_rejected() { let mut dev = NullDevice; diff --git a/serf-smoltcp/tests/key_management.rs b/serf-smoltcp/tests/key_management.rs index 41c47998..ec4d55ea 100644 --- a/serf-smoltcp/tests/key_management.rs +++ b/serf-smoltcp/tests/key_management.rs @@ -223,8 +223,13 @@ fn key_rotation_across_two_nodes_rotates_both_live_keyrings() { // Post-rotation traffic proof: a user event still crosses the wire, which now runs // under K2 on both nodes — the reliable and gossip planes rotated with the keyring. - a.user_event("after-rotation", Bytes::from_static(b"payload"), false) - .expect("user_event from a running node"); + a.user_event( + "after-rotation", + Bytes::from_static(b"payload"), + false, + clk.now(), + ) + .expect("user_event from a running node"); let mut b_saw_user = false; for _ in 0..BUDGET { let na = a.poll(clk.now(), &mut da); diff --git a/serf-smoltcp/tests/user_event.rs b/serf-smoltcp/tests/user_event.rs index f7f2db8a..7ca2338d 100644 --- a/serf-smoltcp/tests/user_event.rs +++ b/serf-smoltcp/tests/user_event.rs @@ -77,7 +77,7 @@ fn user_event_propagates_across_the_gossip_plane() { assert!(joined, "nodes did not converge before the user event"); // A broadcasts a user event; drive gossip until B observes it. - a.user_event("greeting", Bytes::from_static(b"hello"), false) + a.user_event("greeting", Bytes::from_static(b"hello"), false, clk.now()) .expect("queue user event from a running node"); let mut b_saw_user = false; From 9bb960e4cea34859568dbfc069fd6130c7b7020f Mon Sep 17 00:00:00 2001 From: Al Liu Date: Sat, 11 Jul 2026 22:56:23 +1200 Subject: [PATCH 078/128] Per-member reconnect-timeout override (ReconnectDelegate) (#81) --- serf-compio/src/quic/mod.rs | 3 +- serf-compio/src/quic/tests.rs | 4 + serf-compio/src/serf/mod.rs | 2 + serf-compio/src/serf/tests.rs | 6 + serf-compio/src/tcp/mod.rs | 3 +- serf-compio/src/tls/mod.rs | 3 +- serf-compio/src/tls/tests.rs | 3 + serf-compio/src/transport/runtime.rs | 6 + serf-embassy/src/lib.rs | 5 +- serf-embassy/src/serf/mod.rs | 15 +++ serf-embedded/src/engine/mod.rs | 14 +++ serf-embedded/src/lib.rs | 2 +- serf-proto/src/endpoint/mod.rs | 89 ++++++++++--- serf-proto/src/endpoint/tests.rs | 139 +++++++++++++++++++++ serf-proto/src/lib.rs | 5 + serf-proto/src/quic_endpoint/mod.rs | 37 ++++++ serf-proto/src/reconnect_delegate/mod.rs | 33 +++++ serf-proto/src/reconnect_delegate/tests.rs | 42 +++++++ serf-proto/src/stream_endpoint/mod.rs | 38 ++++++ serf-reactor/src/quic/mod.rs | 3 +- serf-reactor/src/serf/mod.rs | 14 +++ serf-reactor/src/serf/tests.rs | 3 + serf-reactor/src/tcp/mod.rs | 3 +- serf-reactor/src/tls/mod.rs | 3 +- serf-reactor/src/transport/runtime.rs | 6 + serf-reactor/tests/key_rotation.rs | 1 + serf-reactor/tests/quic.rs | 3 + serf-reactor/tests/tcp.rs | 97 ++++++++++++++ serf-reactor/tests/tls.rs | 1 + serf-smoltcp/src/lib.rs | 5 +- serf-smoltcp/src/serf/mod.rs | 12 ++ 31 files changed, 574 insertions(+), 26 deletions(-) create mode 100644 serf-proto/src/reconnect_delegate/mod.rs create mode 100644 serf-proto/src/reconnect_delegate/tests.rs diff --git a/serf-compio/src/quic/mod.rs b/serf-compio/src/quic/mod.rs index 41955c25..2bce0526 100644 --- a/serf-compio/src/quic/mod.rs +++ b/serf-compio/src/quic/mod.rs @@ -340,7 +340,8 @@ where self.serf_rng, runtime.user_drop, runtime.member_drop, - ); + ) + .with_reconnect_delegate(runtime.reconnect_delegate); crate::driver::quic::quic_driver_loop::( endpoint, diff --git a/serf-compio/src/quic/tests.rs b/serf-compio/src/quic/tests.rs index ff678e3c..cf50fc8f 100644 --- a/serf-compio/src/quic/tests.rs +++ b/serf-compio/src/quic/tests.rs @@ -198,6 +198,7 @@ async fn try_spawn_node_at( RuntimeOptions::new(), SerfOptions::new(), gossip_rng().expect("seed gossip rng"), + None, #[cfg(encryption)] std::rc::Rc::new(VoidKeyringDelegate), ) @@ -224,6 +225,7 @@ async fn assert_quic_new_rejects(runtime: RuntimeOptions) { runtime, SerfOptions::new(), gossip_rng().expect("seed gossip rng"), + None, #[cfg(encryption)] std::rc::Rc::new(VoidKeyringDelegate), ) @@ -545,6 +547,7 @@ async fn spawn_encrypted_node(id: &str, encryption: EncryptionOptions) -> Serf Serf>>, #[cfg(encryption)] keyring: Rc, ) -> core::result::Result where @@ -216,6 +217,7 @@ where shutdown_flag.clone(), runtime_options, serf_options, + reconnect_delegate, #[cfg(encryption)] keyring, ); diff --git a/serf-compio/src/serf/tests.rs b/serf-compio/src/serf/tests.rs index 321eb07e..da3d623d 100644 --- a/serf-compio/src/serf/tests.rs +++ b/serf-compio/src/serf/tests.rs @@ -67,6 +67,7 @@ async fn try_spawn_node_at(id: &str, bind: SocketAddr) -> Result, RuntimeOptions::new(), SerfOptions::new(), gossip_rng().expect("seed gossip rng"), + None, #[cfg(encryption)] std::rc::Rc::new(VoidKeyringDelegate), ) @@ -183,6 +184,7 @@ async fn assert_tcp_new_rejects(runtime: RuntimeOptions) { runtime, SerfOptions::new(), gossip_rng().expect("seed gossip rng"), + None, #[cfg(encryption)] std::rc::Rc::new(VoidKeyringDelegate), ) @@ -316,6 +318,7 @@ async fn spawn_encrypted_node(id: &str, encryption: EncryptionOptions) -> Serf S runtime_options, SerfOptions::new(), gossip_rng().expect("seed gossip rng"), + None, #[cfg(encryption)] std::rc::Rc::new(VoidKeyringDelegate), ) @@ -532,6 +536,7 @@ async fn spawn_node_with_serf_options(id: &str, serf_options: SerfOptions) -> Se RuntimeOptions::new(), serf_options, gossip_rng().expect("seed gossip rng"), + None, #[cfg(encryption)] std::rc::Rc::new(VoidKeyringDelegate), ) @@ -1037,6 +1042,7 @@ async fn spawn_encrypted_node_with_keyring( RuntimeOptions::new(), SerfOptions::new(), gossip_rng().expect("seed gossip rng"), + None, keyring, ) .await diff --git a/serf-compio/src/tcp/mod.rs b/serf-compio/src/tcp/mod.rs index b126e14a..dd2bc02f 100644 --- a/serf-compio/src/tcp/mod.rs +++ b/serf-compio/src/tcp/mod.rs @@ -366,7 +366,8 @@ where self.serf_rng, runtime.user_drop, runtime.member_drop, - ); + ) + .with_reconnect_delegate(runtime.reconnect_delegate); crate::driver::stream::stream_driver_loop::( endpoint, diff --git a/serf-compio/src/tls/mod.rs b/serf-compio/src/tls/mod.rs index 847bf258..255f9791 100644 --- a/serf-compio/src/tls/mod.rs +++ b/serf-compio/src/tls/mod.rs @@ -452,7 +452,8 @@ where self.serf_rng, runtime.user_drop, runtime.member_drop, - ); + ) + .with_reconnect_delegate(runtime.reconnect_delegate); crate::driver::stream::stream_driver_loop::, D, G, StdRng>( endpoint, diff --git a/serf-compio/src/tls/tests.rs b/serf-compio/src/tls/tests.rs index 5ce83d03..b309e73e 100644 --- a/serf-compio/src/tls/tests.rs +++ b/serf-compio/src/tls/tests.rs @@ -127,6 +127,7 @@ async fn try_spawn_node_at(id: &str, bind: SocketAddr) -> Result, RuntimeOptions::new(), SerfOptions::new(), gossip_rng().expect("seed gossip rng"), + None, #[cfg(encryption)] std::rc::Rc::new(VoidKeyringDelegate), ) @@ -176,6 +177,7 @@ async fn tls_new_rejects_zero_observation_channel() { RuntimeOptions::new().with_observation_channel(Channel::Bounded(0)), SerfOptions::new(), gossip_rng().expect("seed gossip rng"), + None, #[cfg(encryption)] std::rc::Rc::new(VoidKeyringDelegate), ) @@ -410,6 +412,7 @@ async fn spawn_encrypted_node(id: &str, encryption: EncryptionOptions) -> Serf>, pub(crate) driver_options: RuntimeOptions, pub(crate) serf_options: SerfOptions, + /// Optional per-member reconnect-timeout override (Go serf `ReconnectDelegate`), + /// installed into the endpoint by `T::run` before the endpoint moves into the + /// detached pump. `None` keeps the flat configured reap timeouts. + pub(crate) reconnect_delegate: Option>>, /// The driver's keyring delegate, applied to inbound key-management requests. /// Present only under an encryption backend. #[cfg(encryption)] @@ -85,6 +89,7 @@ where shutdown_flag: Rc>, driver_options: RuntimeOptions, serf_options: SerfOptions, + reconnect_delegate: Option>>, #[cfg(encryption)] keyring: Rc, ) -> Self { Self { @@ -99,6 +104,7 @@ where shutdown_flag, driver_options, serf_options, + reconnect_delegate, #[cfg(encryption)] keyring, } diff --git a/serf-embassy/src/lib.rs b/serf-embassy/src/lib.rs index eecc1b62..f63147ab 100644 --- a/serf-embassy/src/lib.rs +++ b/serf-embassy/src/lib.rs @@ -55,8 +55,9 @@ pub use memberlist_proto::{EndpointOptions, Instant, Node, Rng, SeedableRng, Sma // transform config, the await-result join types, and serf's config / error types. pub use serf_embedded::{ AliveDelegate, DEFAULT_EVENT_BUFFER_CAP, Event, JoinFailed, JoinId, LabelError, - MAX_RESOLVED_ADDRS_PER_SEED, MaybeOwned, MaybeResolved, MergeDelegate, ReachedSet, ResolvedAddrs, - SerfError, SerfOptions, TransformOptions, socket_addr_is_routable, + MAX_RESOLVED_ADDRS_PER_SEED, MaybeOwned, MaybeResolved, MergeDelegate, ReachedSet, + ReconnectDelegate, ResolvedAddrs, SerfError, SerfOptions, TransformOptions, + socket_addr_is_routable, }; // serf's own protocol types named in the command signatures. pub use serf_proto::{ diff --git a/serf-embassy/src/serf/mod.rs b/serf-embassy/src/serf/mod.rs index 26cadd07..94be1c6c 100644 --- a/serf-embassy/src/serf/mod.rs +++ b/serf-embassy/src/serf/mod.rs @@ -898,6 +898,21 @@ where r.map_err(OpError::from) } + /// Install (or clear) the per-member reconnect-timeout override + /// [`ReconnectDelegate`](crate::ReconnectDelegate) on the serf engine. + /// + /// `None` (the default) keeps the flat configured reap timeouts. + pub fn set_reconnect_delegate( + &self, + delegate: Option>>, + ) { + self + .shared + .engine + .borrow_mut() + .set_reconnect_delegate(delegate); + } + /// Issue a cluster-wide `install_key` query to add `key` to every node's keyring. #[cfg(encryption)] #[cfg_attr( diff --git a/serf-embedded/src/engine/mod.rs b/serf-embedded/src/engine/mod.rs index 6c043954..e265c2a9 100644 --- a/serf-embedded/src/engine/mod.rs +++ b/serf-embedded/src/engine/mod.rs @@ -750,6 +750,20 @@ where self.plane.listener = Some(c); } + /// Install (or clear) the per-member reconnect-timeout override + /// [`ReconnectDelegate`](serf_proto::ReconnectDelegate) on the serf endpoint. + /// + /// `None` (the default) keeps the flat configured reap timeouts; a delegate + /// lets the driver lengthen or shorten the reaper's per-member window (Go serf + /// `ReconnectDelegate`). + #[inline] + pub fn set_reconnect_delegate( + &mut self, + delegate: Option>>, + ) { + self.endpoint.set_reconnect_delegate(delegate); + } + /// The configured local port (gossip + reliable listener both bind it). #[inline] pub fn port(&self) -> u16 { diff --git a/serf-embedded/src/lib.rs b/serf-embedded/src/lib.rs index fb92381f..6c2330e5 100644 --- a/serf-embedded/src/lib.rs +++ b/serf-embedded/src/lib.rs @@ -81,4 +81,4 @@ pub use memberlist_embedded::{EncryptionOptions, Keyring, SecretKey}; /// serf's own [`Endpoint`](serf_proto::endpoint::Endpoint) configuration, distinct /// from the memberlist-layer engine [`Options`]. pub use serf_proto::options::Options as SerfOptions; -pub use serf_proto::{endpoint::Error as SerfError, event::Event}; +pub use serf_proto::{ReconnectDelegate, endpoint::Error as SerfError, event::Event}; diff --git a/serf-proto/src/endpoint/mod.rs b/serf-proto/src/endpoint/mod.rs index b3078ccd..f75cd24c 100644 --- a/serf-proto/src/endpoint/mod.rs +++ b/serf-proto/src/endpoint/mod.rs @@ -73,7 +73,7 @@ use crate::KeyRequestMessage; #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] use crate::event::{KeyRequest as KeyRequestEvent, KeyRequestOperation, KeyResponseArgs}; use crate::{ - AnyMessage, ConflictResponseMessage, EncodeError, LamportTime, MessageType, + AnyMessage, ConflictResponseMessage, EncodeError, LamportTime, MessageType, ReconnectDelegate, bridge::{tags_from_pb, tags_to_pb, user_event_to_pb}, coalesce::{DropCounter, MemberEventCoalescer, UserEventCoalescer}, event::{ @@ -771,6 +771,14 @@ where /// Cumulative member-coalescer shed count, incremented in `emit_member` when /// the member coalescer drops a change at its cardinality cap. member_drop: D, + /// Optional per-member override for the reaper's reconnect / tombstone + /// timeout (Go serf `ReconnectDelegate`). + /// + /// Consulted by `fire_reap` for every failed and left member it considers; + /// `None` is the noop (the flat configured timeouts apply unchanged). Boxed + /// `dyn` carries `Send + Sync` from the trait's supertraits, so the endpoint + /// keeps its auto-traits for the multi-threaded drivers. + reconnect_delegate: Option>>, /// The most recent directed-send (address, bytes) produced by /// `handle_relay` or `relay_response`. /// @@ -955,6 +963,7 @@ where user_coalescer, user_drop, member_drop, + reconnect_delegate: None, drain_now: Instant::ORIGIN, coalesce_now: Instant::ORIGIN, // The snapshot starts dirty so the first push-pull always ships a fresh @@ -1081,6 +1090,39 @@ where } } +// ── reconnect delegate (minimal bounds: a plain injected-field setter) ──────── + +impl Endpoint +where + I: Eq + core::hash::Hash, + D: DropCounter, +{ + /// Install (or clear) the per-member reconnect-timeout override + /// [`ReconnectDelegate`], consuming builder form. + /// + /// `None` (the default) is the noop: the flat configured `reconnect_timeout` + /// / `tombstone_timeout` apply to every member in the reaper. + #[must_use] + pub fn with_reconnect_delegate( + mut self, + delegate: Option>>, + ) -> Self { + self.reconnect_delegate = delegate; + self + } + + /// Install (or clear) the per-member reconnect-timeout override + /// [`ReconnectDelegate`]. + /// + /// `None` restores the default flat timeouts. + pub fn set_reconnect_delegate( + &mut self, + delegate: Option>>, + ) { + self.reconnect_delegate = delegate; + } +} + // ── poll API (requires full Id + Data bounds for inner delegation) ───────────── impl Endpoint @@ -2338,16 +2380,25 @@ where let tombstone_timeout = self.opts.tombstone_timeout(); let intent_timeout = self.opts.recent_intent_timeout(); - // Reap failed members whose leave_time > reconnect_timeout. + // Reap failed members whose leave_time > reconnect_timeout, honoring the + // per-member override: Go serf's `reap!` (base.rs ~521-553) consults the + // `ReconnectDelegate` here with `reconnect_timeout` as the base. The + // delegate and the membership store are disjoint fields, so their shared + // borrows compose. let mut i = 0; while i < self.members.failed_members.len() { let id = self.members.failed_members[i].clone(); - let expired = self - .members - .states - .get(&id) - .and_then(|ms| ms.leave_time()) - .is_some_and(|lt| now.duration_since(lt) > reconnect_timeout); + let expired = match self.members.states.get(&id) { + Some(ms) => { + let timeout = match &self.reconnect_delegate { + Some(d) => d.reconnect_timeout(ms.member(), reconnect_timeout), + None => reconnect_timeout, + }; + ms.leave_time() + .is_some_and(|lt| now.duration_since(lt) > timeout) + } + None => false, + }; if expired { self.members.failed_members.swap_remove(i); if let Some(ms) = self.members.states.remove(&id) { @@ -2368,16 +2419,24 @@ where } } - // Reap left (tombstone) members whose leave_time > tombstone_timeout. + // Reap left (tombstone) members whose leave_time > tombstone_timeout, + // honoring the same per-member override: Go serf shares one `reap!` pass + // for the left list, passing `tombstone_timeout` as the base to the very + // same `ReconnectDelegate` (base.rs ~568-569). let mut i = 0; while i < self.members.left_members.len() { let id = self.members.left_members[i].clone(); - let expired = self - .members - .states - .get(&id) - .and_then(|ms| ms.leave_time()) - .is_some_and(|lt| now.duration_since(lt) > tombstone_timeout); + let expired = match self.members.states.get(&id) { + Some(ms) => { + let timeout = match &self.reconnect_delegate { + Some(d) => d.reconnect_timeout(ms.member(), tombstone_timeout), + None => tombstone_timeout, + }; + ms.leave_time() + .is_some_and(|lt| now.duration_since(lt) > timeout) + } + None => false, + }; if expired { self.members.left_members.swap_remove(i); if let Some(ms) = self.members.states.remove(&id) { diff --git a/serf-proto/src/endpoint/tests.rs b/serf-proto/src/endpoint/tests.rs index ed40574b..89839d55 100644 --- a/serf-proto/src/endpoint/tests.rs +++ b/serf-proto/src/endpoint/tests.rs @@ -893,6 +893,145 @@ fn reap_deadline_fires_via_handle_timeout() { ); } +// ── ReconnectDelegate: per-member reap-timeout override (Go serf `reap!`) ────── + +/// Build a serf `StreamEndpoint` with caller-supplied serf `opts`, so a test can +/// set the flat reap timeouts. Mirrors [`ep`] otherwise. +fn ep_with_opts(opts: Options) -> StreamEndpoint { + let inner_opts = EndpointOptions::new(1u32, "127.0.0.1:7946".parse().unwrap()) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); + let inner = memberlist_proto::Endpoint::new_at( + inner_opts, + memberlist_proto::Instant::ORIGIN, + SmallRng::seed_from_u64(0), + ); + let mut e = StreamEndpoint::new(coord(inner), opts); + // Drain the construction NodeJoined(self) so tests start from a clean queue. + let _ = e.poll_event(); + e +} + +/// A reconnect delegate that overrides the reap timeout for one target member id +/// and passes every other member through the configured base timeout unchanged. +struct OverrideOne { + target: u32, + override_timeout: core::time::Duration, +} + +impl ReconnectDelegate for OverrideOne { + fn reconnect_timeout( + &self, + member: &Member, + timeout: core::time::Duration, + ) -> core::time::Duration { + if *member.node().id_ref() == self.target { + self.override_timeout + } else { + timeout + } + } +} + +#[test] +fn reconnect_delegate_overrides_the_failed_reap_timeout() { + // Flat reconnect_timeout is long (30s); the delegate shortens member 2 to 1s. + let opts = Options::new().with_reconnect_timeout(core::time::Duration::from_secs(30)); + let mut e = ep_with_opts(opts); + e.set_reconnect_delegate(Some(Box::new(OverrideOne { + target: 2, + override_timeout: core::time::Duration::from_secs(1), + }))); + + let t0 = memberlist_proto::Instant::ORIGIN; + seed_failed(&mut e, 2, "127.0.0.1:1002".parse().unwrap(), t0); + seed_failed(&mut e, 3, "127.0.0.1:1003".parse().unwrap(), t0); + + // Past t0 + 1s (override) + reap_interval (15s): only member 2 has crossed its + // shortened window; member 3 is still inside the flat 30s window. + e.handle_timeout(t0 + core::time::Duration::from_secs(1 + 15)); + assert_eq!( + e.test_member_status(2), + None, + "the overridden member (1s window) must be reaped at the early tick" + ); + assert_eq!( + e.test_member_status(3), + Some(MemberStatus::Failed), + "the passthrough member (flat 30s) must survive the early tick" + ); + let reaped_early = core::iter::from_fn(|| e.poll_event()) + .any(|ev| matches!(ev, Event::Member(ref me) if me.kind() == MemberEventKind::Reap)); + assert!( + reaped_early, + "a Member(Reap) event must fire for the overridden member" + ); + + // Past the flat 30s window (+ reap_interval): member 3 reaps too. + e.handle_timeout(t0 + core::time::Duration::from_secs(30 + 15 + 1)); + assert_eq!( + e.test_member_status(3), + None, + "the passthrough member must reap once past the flat reconnect_timeout" + ); +} + +#[test] +fn reconnect_delegate_overrides_the_left_reap_timeout() { + // Same shape on the LEFT list, proving the second reap loop consults the very + // same delegate with tombstone_timeout as the base (Go serf's shared reap). + let opts = Options::new().with_tombstone_timeout(core::time::Duration::from_secs(30)); + let mut e = ep_with_opts(opts); + e.set_reconnect_delegate(Some(Box::new(OverrideOne { + target: 2, + override_timeout: core::time::Duration::from_secs(1), + }))); + + let t0 = memberlist_proto::Instant::ORIGIN; + e.test_seed_left_member_by_status(2, LamportTime::new(3), t0); + e.test_seed_left_member_by_status(3, LamportTime::new(3), t0); + + e.handle_timeout(t0 + core::time::Duration::from_secs(1 + 15)); + assert_eq!( + e.test_member_status(2), + None, + "the overridden left member (1s window) must be reaped at the early tick" + ); + assert_eq!( + e.test_member_status(3), + Some(MemberStatus::Left), + "the passthrough left member (flat 30s) must survive the early tick" + ); + + e.handle_timeout(t0 + core::time::Duration::from_secs(30 + 15 + 1)); + assert_eq!( + e.test_member_status(3), + None, + "the passthrough left member must reap once past the flat tombstone_timeout" + ); +} + +#[test] +fn no_delegate_keeps_the_flat_timeouts() { + // With no delegate (default None) both members obey the flat reconnect_timeout, + // proving None is exact passthrough — the reaper is unchanged from today. + let opts = Options::new().with_reconnect_timeout(core::time::Duration::from_secs(30)); + let mut e = ep_with_opts(opts); + + let t0 = memberlist_proto::Instant::ORIGIN; + seed_failed(&mut e, 2, "127.0.0.1:1002".parse().unwrap(), t0); + seed_failed(&mut e, 3, "127.0.0.1:1003".parse().unwrap(), t0); + + // Early tick (16s): neither member has crossed the flat 30s window. + e.handle_timeout(t0 + core::time::Duration::from_secs(1 + 15)); + assert_eq!(e.test_member_status(2), Some(MemberStatus::Failed)); + assert_eq!(e.test_member_status(3), Some(MemberStatus::Failed)); + + // Past 30s (+ reap_interval): both reap together under the same flat timeout. + e.handle_timeout(t0 + core::time::Duration::from_secs(30 + 15 + 1)); + assert_eq!(e.test_member_status(2), None); + assert_eq!(e.test_member_status(3), None); +} + // ── Task 2.1: event ring-buffer dedup + event-clock + event broadcast tier ──── #[test] diff --git a/serf-proto/src/lib.rs b/serf-proto/src/lib.rs index 5f770ac1..a18a08be 100644 --- a/serf-proto/src/lib.rs +++ b/serf-proto/src/lib.rs @@ -103,6 +103,11 @@ pub(crate) mod coalesce; #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] pub use coalesce::DropCounter; #[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) mod reconnect_delegate; +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub use reconnect_delegate::ReconnectDelegate; +#[cfg(any(feature = "tcp", feature = "quic"))] #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] pub mod endpoint; #[cfg(any(feature = "tcp", feature = "quic"))] diff --git a/serf-proto/src/quic_endpoint/mod.rs b/serf-proto/src/quic_endpoint/mod.rs index 6b0be980..8cd5daf6 100644 --- a/serf-proto/src/quic_endpoint/mod.rs +++ b/serf-proto/src/quic_endpoint/mod.rs @@ -140,6 +140,43 @@ where } } +// ── reconnect delegate (minimal bounds: forwards a plain field setter) ──────── + +#[cfg(feature = "quic")] +#[cfg_attr(docsrs, doc(cfg(feature = "quic")))] +impl QuicEndpoint +where + I: Eq + core::hash::Hash, + D: DropCounter, +{ + /// Install (or clear) the per-member reconnect-timeout override + /// [`ReconnectDelegate`](crate::ReconnectDelegate), consuming builder form. + /// + /// Forwards to + /// [`Endpoint::with_reconnect_delegate`](crate::endpoint::Endpoint::with_reconnect_delegate); + /// `None` (the default) keeps the flat configured timeouts. + #[must_use] + pub fn with_reconnect_delegate( + mut self, + delegate: Option>>, + ) -> Self { + self.core.set_reconnect_delegate(delegate); + self + } + + /// Install (or clear) the per-member reconnect-timeout override + /// [`ReconnectDelegate`](crate::ReconnectDelegate). + /// + /// Forwards to + /// [`Endpoint::set_reconnect_delegate`](crate::endpoint::Endpoint::set_reconnect_delegate). + pub fn set_reconnect_delegate( + &mut self, + delegate: Option>>, + ) { + self.core.set_reconnect_delegate(delegate); + } +} + // ── transport-level driver surface + serf commands ───────────────────────────── // // The driver-surface methods reach the coordinator (`transport`) directly — the diff --git a/serf-proto/src/reconnect_delegate/mod.rs b/serf-proto/src/reconnect_delegate/mod.rs new file mode 100644 index 00000000..7b06b8c3 --- /dev/null +++ b/serf-proto/src/reconnect_delegate/mod.rs @@ -0,0 +1,33 @@ +//! The [`ReconnectDelegate`] hook: a per-member override for the reaper's +//! reconnect / tombstone timeout. +//! +//! Unlike serf's other delegates (async, driver-side observers and vetoes), this +//! hook fires inside the machine's own autonomous reap tick, where the driver is +//! not in the path — and it is pure and synchronous. So it lives in the machine +//! as an injected pure dependency rather than on a driver, mirroring Go serf's +//! `ReconnectDelegate`, which the reaper consults per member inside its shared +//! reap pass (`legacy/serf-core/src/serf/base.rs`, the `reap!` macro). + +use core::time::Duration; + +use crate::members::Member; + +#[cfg(test)] +mod tests; + +/// Overrides the reap timeout for individual members. +/// +/// Consulted by the reaper for every member it considers: the failed list +/// (base timeout = `reconnect_timeout`) and the left list (base timeout = +/// `tombstone_timeout`), mirroring Go serf's per-member override applied +/// inside its shared reap pass. Return `timeout` unchanged to keep the +/// configured value, or a per-member override (e.g. a longer window for known +/// slow-to-return nodes). +/// +/// The hook is synchronous and runs inside the machine's reap tick; it must +/// not block. `Send + Sync` are supertraits so a boxed delegate leaves the +/// endpoint's auto-traits intact for the multi-threaded drivers. +pub trait ReconnectDelegate: Send + Sync { + /// The reap timeout to use for `member`, given the configured base `timeout`. + fn reconnect_timeout(&self, member: &Member, timeout: Duration) -> Duration; +} diff --git a/serf-proto/src/reconnect_delegate/tests.rs b/serf-proto/src/reconnect_delegate/tests.rs new file mode 100644 index 00000000..b96928a4 --- /dev/null +++ b/serf-proto/src/reconnect_delegate/tests.rs @@ -0,0 +1,42 @@ +use core::{net::SocketAddr, time::Duration}; + +use super::ReconnectDelegate; +use crate::{ + Tags, + members::{Member, MemberStatus}, +}; + +// A boxed delegate must stay `Send + Sync` so it can live inside `Endpoint` +// without stripping the endpoint's auto-traits for the multi-threaded drivers. +#[test] +fn boxed_delegate_is_send_sync_and_object_safe() { + fn assert_send_sync() + where + T: Send + Sync + ?Sized, + { + } + assert_send_sync::>(); + assert_send_sync::>>(); +} + +/// A delegate that returns a fixed override regardless of the base timeout. +struct FixedTimeout(Duration); + +impl ReconnectDelegate for FixedTimeout { + fn reconnect_timeout(&self, _member: &Member, _timeout: Duration) -> Duration { + self.0 + } +} + +#[test] +fn concrete_delegate_boxes_and_overrides() { + let d: Box> = + Box::new(FixedTimeout(Duration::from_secs(1))); + let node = memberlist_proto::Node::new(1u32, "127.0.0.1:1".parse::().unwrap()); + let member = Member::new(node, Tags::new(), MemberStatus::Alive); + // The override is returned verbatim, ignoring the configured base timeout. + assert_eq!( + d.reconnect_timeout(&member, Duration::from_secs(30)), + Duration::from_secs(1) + ); +} diff --git a/serf-proto/src/stream_endpoint/mod.rs b/serf-proto/src/stream_endpoint/mod.rs index b1302e95..dbba94ac 100644 --- a/serf-proto/src/stream_endpoint/mod.rs +++ b/serf-proto/src/stream_endpoint/mod.rs @@ -139,6 +139,44 @@ where } } +// ── reconnect delegate (minimal bounds: forwards a plain field setter) ──────── + +#[cfg(feature = "tcp")] +#[cfg_attr(docsrs, doc(cfg(feature = "tcp")))] +impl StreamEndpoint +where + I: Eq + core::hash::Hash, + RT: StreamTransport, + D: DropCounter, +{ + /// Install (or clear) the per-member reconnect-timeout override + /// [`ReconnectDelegate`](crate::ReconnectDelegate), consuming builder form. + /// + /// Forwards to + /// [`Endpoint::with_reconnect_delegate`](crate::endpoint::Endpoint::with_reconnect_delegate); + /// `None` (the default) keeps the flat configured timeouts. + #[must_use] + pub fn with_reconnect_delegate( + mut self, + delegate: Option>>, + ) -> Self { + self.core.set_reconnect_delegate(delegate); + self + } + + /// Install (or clear) the per-member reconnect-timeout override + /// [`ReconnectDelegate`](crate::ReconnectDelegate). + /// + /// Forwards to + /// [`Endpoint::set_reconnect_delegate`](crate::endpoint::Endpoint::set_reconnect_delegate). + pub fn set_reconnect_delegate( + &mut self, + delegate: Option>>, + ) { + self.core.set_reconnect_delegate(delegate); + } +} + // ── transport-level driver surface ──────────────────────────────────────────── // // These reach the coordinator (`transport`) directly — the `Reliable` seam diff --git a/serf-reactor/src/quic/mod.rs b/serf-reactor/src/quic/mod.rs index 75057009..761bebf8 100644 --- a/serf-reactor/src/quic/mod.rs +++ b/serf-reactor/src/quic/mod.rs @@ -401,7 +401,8 @@ where self.serf_rng, runtime.user_drop, runtime.member_drop, - ); + ) + .with_reconnect_delegate(runtime.reconnect_delegate); let driver = crate::driver::quic::spawn_quic_driver::( endpoint, diff --git a/serf-reactor/src/serf/mod.rs b/serf-reactor/src/serf/mod.rs index 38e208ce..ae31e920 100644 --- a/serf-reactor/src/serf/mod.rs +++ b/serf-reactor/src/serf/mod.rs @@ -157,6 +157,7 @@ where runtime_options: RuntimeOptions, serf_options: SerfOptions, gossip_rng: G, + reconnect_delegate: Option>>, #[cfg(encryption)] keyring: Arc, ) -> core::result::Result where @@ -201,6 +202,7 @@ where serf_options, user_drop_writer, member_drop_writer, + reconnect_delegate, #[cfg(encryption)] keyring, ); @@ -259,6 +261,7 @@ where delegate: D, runtime_options: RuntimeOptions, serf_options: SerfOptions, + reconnect_delegate: Option>>, #[cfg(encryption)] keyring: Arc, ) -> Result where @@ -274,6 +277,7 @@ where runtime_options, serf_options, crate::gossip_rng()?, + reconnect_delegate, #[cfg(encryption)] keyring, ) @@ -291,6 +295,7 @@ where runtime_options: RuntimeOptions, serf_options: SerfOptions, gossip_rng: G, + reconnect_delegate: Option>>, #[cfg(encryption)] keyring: Arc, ) -> Result where @@ -307,6 +312,7 @@ where runtime_options, serf_options, gossip_rng, + reconnect_delegate, #[cfg(encryption)] keyring, ) @@ -357,6 +363,7 @@ where delegate: D, runtime_options: RuntimeOptions, serf_options: SerfOptions, + reconnect_delegate: Option>>, #[cfg(encryption)] keyring: Arc, ) -> Result where @@ -372,6 +379,7 @@ where runtime_options, serf_options, crate::gossip_rng()?, + reconnect_delegate, #[cfg(encryption)] keyring, ) @@ -389,6 +397,7 @@ where runtime_options: RuntimeOptions, serf_options: SerfOptions, gossip_rng: G, + reconnect_delegate: Option>>, #[cfg(encryption)] keyring: Arc, ) -> Result where @@ -405,6 +414,7 @@ where runtime_options, serf_options, gossip_rng, + reconnect_delegate, #[cfg(encryption)] keyring, ) @@ -454,6 +464,7 @@ where delegate: D, runtime_options: RuntimeOptions, serf_options: SerfOptions, + reconnect_delegate: Option>>, #[cfg(encryption)] keyring: Arc, ) -> Result where @@ -469,6 +480,7 @@ where runtime_options, serf_options, crate::gossip_rng()?, + reconnect_delegate, #[cfg(encryption)] keyring, ) @@ -486,6 +498,7 @@ where runtime_options: RuntimeOptions, serf_options: SerfOptions, gossip_rng: G, + reconnect_delegate: Option>>, #[cfg(encryption)] keyring: Arc, ) -> Result where @@ -502,6 +515,7 @@ where runtime_options, serf_options, gossip_rng, + reconnect_delegate, #[cfg(encryption)] keyring, ) diff --git a/serf-reactor/src/serf/tests.rs b/serf-reactor/src/serf/tests.rs index ef1e2dcd..35254879 100644 --- a/serf-reactor/src/serf/tests.rs +++ b/serf-reactor/src/serf/tests.rs @@ -59,6 +59,7 @@ async fn try_spawn_node_at(id: &str, bind: SocketAddr) -> Result::new(), RuntimeOptions::new(), SerfOptions::new(), + None, #[cfg(encryption)] std::sync::Arc::new(crate::VoidKeyringDelegate), ) @@ -86,6 +87,7 @@ async fn spawn_node_with_serf_options(id: &str, serf_options: SerfOptions) -> No VoidDelegate::::new(), RuntimeOptions::new(), serf_options, + None, #[cfg(encryption)] std::sync::Arc::new(crate::VoidKeyringDelegate), ) @@ -219,6 +221,7 @@ async fn assert_tcp_new_rejects(runtime: RuntimeOptions) { VoidDelegate::::new(), runtime, SerfOptions::new(), + None, #[cfg(encryption)] std::sync::Arc::new(crate::VoidKeyringDelegate), ) diff --git a/serf-reactor/src/tcp/mod.rs b/serf-reactor/src/tcp/mod.rs index 04d6b5d4..e1f89baf 100644 --- a/serf-reactor/src/tcp/mod.rs +++ b/serf-reactor/src/tcp/mod.rs @@ -391,7 +391,8 @@ where self.serf_rng, runtime.user_drop, runtime.member_drop, - ); + ) + .with_reconnect_delegate(runtime.reconnect_delegate); let driver = crate::driver::stream::spawn_stream_driver::( endpoint, diff --git a/serf-reactor/src/tls/mod.rs b/serf-reactor/src/tls/mod.rs index 7af2f3ec..568129ac 100644 --- a/serf-reactor/src/tls/mod.rs +++ b/serf-reactor/src/tls/mod.rs @@ -514,7 +514,8 @@ where self.serf_rng, runtime.user_drop, runtime.member_drop, - ); + ) + .with_reconnect_delegate(runtime.reconnect_delegate); let driver = crate::driver::stream::spawn_stream_driver::, D, G, StdRng>( diff --git a/serf-reactor/src/transport/runtime.rs b/serf-reactor/src/transport/runtime.rs index 507fcb35..91b1bd5a 100644 --- a/serf-reactor/src/transport/runtime.rs +++ b/serf-reactor/src/transport/runtime.rs @@ -40,6 +40,10 @@ pub struct TransportRuntime { pub(crate) user_drop: ReactorDropCounter, /// The write half of the member-coalescer shed counter. pub(crate) member_drop: ReactorDropCounter, + /// Optional per-member reconnect-timeout override (Go serf `ReconnectDelegate`), + /// installed into the endpoint by `T::run` before the endpoint moves into the + /// detached pump. `None` keeps the flat configured reap timeouts. + pub(crate) reconnect_delegate: Option>>, /// The driver's keyring delegate, applied to inbound key-management requests. /// Present only under an encryption backend. #[cfg(encryption)] @@ -59,6 +63,7 @@ impl TransportRuntime { serf_options: SerfOptions, user_drop: ReactorDropCounter, member_drop: ReactorDropCounter, + reconnect_delegate: Option>>, #[cfg(encryption)] keyring: Arc, ) -> Self { Self { @@ -69,6 +74,7 @@ impl TransportRuntime { serf_options, user_drop, member_drop, + reconnect_delegate, #[cfg(encryption)] keyring, } diff --git a/serf-reactor/tests/key_rotation.rs b/serf-reactor/tests/key_rotation.rs index 0e3730b2..faa9f0e7 100644 --- a/serf-reactor/tests/key_rotation.rs +++ b/serf-reactor/tests/key_rotation.rs @@ -108,6 +108,7 @@ where VoidDelegate::::new(), RuntimeOptions::new(), SerfOptions::new(), + None, keyring, ) .await diff --git a/serf-reactor/tests/quic.rs b/serf-reactor/tests/quic.rs index 9c56024d..ba18ed8e 100644 --- a/serf-reactor/tests/quic.rs +++ b/serf-reactor/tests/quic.rs @@ -158,6 +158,7 @@ where VoidDelegate::::new(), RuntimeOptions::new(), SerfOptions::new(), + None, #[cfg(encryption)] std::sync::Arc::new(VoidKeyringDelegate), ) @@ -459,6 +460,7 @@ where VoidDelegate::::new(), RuntimeOptions::new(), SerfOptions::new(), + None, #[cfg(encryption)] std::sync::Arc::new(VoidKeyringDelegate), ) @@ -514,6 +516,7 @@ where VoidDelegate::::new(), RuntimeOptions::new(), SerfOptions::new(), + None, std::sync::Arc::new(VoidKeyringDelegate), ) .await diff --git a/serf-reactor/tests/tcp.rs b/serf-reactor/tests/tcp.rs index 88f45395..f24f0d1a 100644 --- a/serf-reactor/tests/tcp.rs +++ b/serf-reactor/tests/tcp.rs @@ -37,6 +37,27 @@ use smol_str::SmolStr; /// A reactor TCP node handle over the agnostic runtime `R`. type Node = Serf; +/// A [`ReconnectDelegate`](serf_proto::ReconnectDelegate) that forces an immediate +/// reap (zero timeout) for one target member id and passes every other member +/// through the configured base timeout unchanged. +struct ReapImmediately { + target: SmolStr, +} + +impl serf_proto::ReconnectDelegate for ReapImmediately { + fn reconnect_timeout( + &self, + member: &serf_proto::members::Member, + base: Duration, + ) -> Duration { + if member.node().id_ref() == &self.target { + Duration::ZERO + } else { + base + } + } +} + /// Build and spawn a reactor TCP node on an ephemeral loopback port through the /// ergonomic `Serf::tcp` constructor. async fn spawn_node(id: &str) -> Node @@ -54,6 +75,7 @@ where VoidDelegate::::new(), RuntimeOptions::new(), SerfOptions::new(), + None, #[cfg(encryption)] std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), ) @@ -263,6 +285,70 @@ where b.shutdown().await.expect("lv-b shuts down"); } +/// The construction-time `reconnect_delegate` is installed into the endpoint and +/// consulted by the reaper: node A carries a delegate that zeroes node B's LEFT +/// tombstone while A's flat `tombstone_timeout` stays at 24h. After B leaves +/// gracefully, A drops back to a single member — which can only happen if the +/// driver installed the delegate AND the reaper consulted it (the flat 24h +/// timeout would otherwise hold B for the whole test). Proves the reactor +/// constructor wiring end-to-end. +async fn reconnect_delegate_reaps_left_member() +where + R: Runtime, +{ + let b = spawn_node::("rd-b").await; + let b_addr = b.advertise_address(); + + // A: a long flat tombstone (so a default reap never fires within the test) with + // fast reap ticks, plus a delegate overriding ONLY B's tombstone to zero. + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new("rd-a")) + .with_advertise_addr(MaybeResolved::Resolved(bind)); + let serf_opts = SerfOptions::new() + .with_reap_interval(Duration::from_millis(100)) + .with_tombstone_timeout(Duration::from_secs(86_400)); + let a = Serf::::tcp( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + serf_opts, + Some(Box::new(ReapImmediately { + target: SmolStr::new("rd-b"), + })), + #[cfg(encryption)] + std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), + ) + .await + .expect("spawn serf tcp node A with a reconnect delegate"); + + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + converge(&a, &b).await; + + // B leaves gracefully: A moves B to its LEFT tombstone list. With the delegate + // zeroing B's tombstone, A's next reap tick drops B — the 2-member cluster + // returns to 1. Without the delegate consult, A would hold B for the flat 24h. + b.leave().await.expect("B leaves the cluster"); + + R::timeout(Duration::from_secs(20), async { + loop { + if a.num_members() == 1 { + break; + } + R::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("A reaps the left member B early via the reconnect-delegate override"); + + a.shutdown().await.expect("rd-a shuts down"); + b.shutdown().await.expect("rd-b shuts down"); +} + /// After a two-node join, the snapshot read-forwarders on the joined node reflect /// the two-member cluster: `members` returns both nodes, `local_member` / `local_id` /// return this node, `state` is `Alive`, `advertise_node` composes id + advertise, @@ -415,6 +501,7 @@ where VoidDelegate::::new(), RuntimeOptions::new(), SerfOptions::new(), + None, std::sync::Arc::new(VoidKeyringDelegate), ) .await @@ -559,6 +646,11 @@ mod tokio_cells { super::leave_emits_left_cluster::().await; } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn reconnect_delegate_reaps_left_member() { + super::reconnect_delegate_reaps_left_member::().await; + } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn snapshot_forwarders_reflect_joined_cluster() { super::snapshot_forwarders_reflect_joined_cluster::().await; @@ -616,6 +708,11 @@ mod smol_cells { SmolRuntime::block_on(super::leave_emits_left_cluster::()); } + #[test] + fn reconnect_delegate_reaps_left_member_smol() { + SmolRuntime::block_on(super::reconnect_delegate_reaps_left_member::()); + } + #[test] fn snapshot_forwarders_reflect_joined_cluster_smol() { SmolRuntime::block_on(super::snapshot_forwarders_reflect_joined_cluster::< diff --git a/serf-reactor/tests/tls.rs b/serf-reactor/tests/tls.rs index b21ae281..3d709301 100644 --- a/serf-reactor/tests/tls.rs +++ b/serf-reactor/tests/tls.rs @@ -132,6 +132,7 @@ where VoidDelegate::::new(), RuntimeOptions::new(), SerfOptions::new(), + None, #[cfg(encryption)] std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), ) diff --git a/serf-smoltcp/src/lib.rs b/serf-smoltcp/src/lib.rs index e8be1ef8..cc76b9ed 100644 --- a/serf-smoltcp/src/lib.rs +++ b/serf-smoltcp/src/lib.rs @@ -40,8 +40,9 @@ pub use memberlist_proto::{EndpointOptions, Instant}; // transform config, the await-result join types, and serf's config / error types. pub use serf_embedded::{ AliveDelegate, DEFAULT_EVENT_BUFFER_CAP, Event, JoinFailed, JoinId, LabelError, - MAX_RESOLVED_ADDRS_PER_SEED, MaybeOwned, MaybeResolved, MergeDelegate, ReachedSet, ResolvedAddrs, - SerfError, SerfOptions, TransformOptions, socket_addr_is_routable, + MAX_RESOLVED_ADDRS_PER_SEED, MaybeOwned, MaybeResolved, MergeDelegate, ReachedSet, + ReconnectDelegate, ResolvedAddrs, SerfError, SerfOptions, TransformOptions, + socket_addr_is_routable, }; // serf's own protocol types named in the command signatures. pub use serf_proto::{ diff --git a/serf-smoltcp/src/serf/mod.rs b/serf-smoltcp/src/serf/mod.rs index 94005669..42eccc5f 100644 --- a/serf-smoltcp/src/serf/mod.rs +++ b/serf-smoltcp/src/serf/mod.rs @@ -755,6 +755,18 @@ where self.engine.start(now); } + /// Install (or clear) the per-member reconnect-timeout override + /// [`ReconnectDelegate`](crate::ReconnectDelegate) on the serf engine. + /// + /// `None` (the default) keeps the flat configured reap timeouts. + #[inline] + pub fn set_reconnect_delegate( + &mut self, + delegate: Option>>, + ) { + self.engine.set_reconnect_delegate(delegate); + } + /// serf's current lifecycle state. #[inline] pub fn state(&self) -> SerfState { From dec12909dbc9ea82e7ac63a57010a96bae7bcc34 Mon Sep 17 00:00:00 2001 From: Al Liu Date: Sat, 11 Jul 2026 23:15:44 +1200 Subject: [PATCH 079/128] feat: configurable user-event size ceiling, validated at every driver construction (#82) --- serf-compio/src/serf/mod.rs | 8 +- serf-compio/src/serf/tests.rs | 32 ++++++++ serf-embassy/src/error/mod.rs | 15 +++- serf-embassy/src/lib.rs | 2 +- serf-embassy/src/serf/mod.rs | 6 ++ serf-embedded/src/engine/mod.rs | 32 +++++--- serf-embedded/src/engine/tests.rs | 35 +++++++- serf-embedded/src/error/mod.rs | 55 +++++++++++++ serf-embedded/src/error/tests.rs | 33 ++++++++ serf-embedded/src/lib.rs | 9 ++- serf-proto/src/endpoint/mod.rs | 5 +- serf-proto/src/options/mod.rs | 127 ++++++++++++++++++++++++++---- serf-proto/src/options/tests.rs | 101 ++++++++++++++++++++++++ serf-reactor/src/serf/mod.rs | 9 ++- serf-reactor/src/serf/tests.rs | 30 +++++++ serf-smoltcp/src/error/mod.rs | 43 +++++++--- serf-smoltcp/src/error/tests.rs | 22 ++++-- serf-smoltcp/src/lib.rs | 2 +- serf-smoltcp/src/serf/mod.rs | 16 +++- serf-smoltcp/src/serf/tests.rs | 50 ++++++++++++ 20 files changed, 577 insertions(+), 55 deletions(-) create mode 100644 serf-embedded/src/error/mod.rs create mode 100644 serf-embedded/src/error/tests.rs diff --git a/serf-compio/src/serf/mod.rs b/serf-compio/src/serf/mod.rs index 58c3d5ed..cf11fd54 100644 --- a/serf-compio/src/serf/mod.rs +++ b/serf-compio/src/serf/mod.rs @@ -43,7 +43,7 @@ use crate::{ delegate::Delegate, driver::options::RuntimeOptions, drop_counter::DropReader, - error::{JoinFailed, Result, SerfError}, + error::{InvalidOption, JoinFailed, Result, SerfError}, events::EventStream, resolver::{AdvertiseAddrResolver, Resolver}, snapshot::{SerfSnapshot, SnapshotCell}, @@ -170,6 +170,12 @@ where // `event_queue_cap` would make the event-stream channel a rendezvous the // non-blocking forward can never deposit into, dropping every event. runtime_options.validate()?; + // Reject a serf-level configuration a driver cannot honor (an over-ceiling + // `max_user_event_size`, or a self-contradictory coalescing pair) at the same + // early stage, before any socket is bound or the detached driver is spawned. + serf_options + .validate() + .map_err(|e| SerfError::InvalidOption(InvalidOption::new("serf_options", e.to_string())))?; // Cache the join deadline on the handle BEFORE `runtime_options` is moved // into the driver bundle, so each await-result join can stamp its absolute diff --git a/serf-compio/src/serf/tests.rs b/serf-compio/src/serf/tests.rs index da3d623d..b7486eb8 100644 --- a/serf-compio/src/serf/tests.rs +++ b/serf-compio/src/serf/tests.rs @@ -209,6 +209,38 @@ async fn tcp_new_rejects_zero_event_queue_cap() { assert_tcp_new_rejects(RuntimeOptions::new().with_event_queue_cap(0)).await; } +/// An over-ceiling `max_user_event_size` in the serf options is rejected by +/// `Serf::new` at construction — before binding a socket or spawning the driver — +/// rather than returning `Ok` and later dropping oversize user events. +#[compio::test] +async fn tcp_new_rejects_over_ceiling_user_event_size() { + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new("bad-serf-opt-node")) + .with_advertise_addr(MaybeResolved::Resolved(bind)); + let serf = + SerfOptions::new().with_max_user_event_size(SerfOptions::DEFAULT_USER_EVENT_SIZE_LIMIT + 1); + let res = + Serf::new::, SocketAddrResolver, FirstAddrResolver, _, _>( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + serf, + gossip_rng().expect("seed gossip rng"), + None, + #[cfg(encryption)] + std::rc::Rc::new(VoidKeyringDelegate), + ) + .await; + match res { + Err(SerfError::InvalidOption(_)) => {} + Err(other) => panic!("expected InvalidOption, got {other:?}"), + Ok(_) => panic!("an over-ceiling max_user_event_size must be rejected at construction"), + } +} + /// A zero `cmd_fairness_budget` (direct builder) starves the command drain under /// an inbound flood, so `Serf::new` rejects it at construction rather than /// spawning a driver whose commands could never make progress. diff --git a/serf-embassy/src/error/mod.rs b/serf-embassy/src/error/mod.rs index 1b9166a4..5a0d57ce 100644 --- a/serf-embassy/src/error/mod.rs +++ b/serf-embassy/src/error/mod.rs @@ -4,7 +4,7 @@ use alloc::boxed::Box; use core::{fmt, time::Duration}; -use serf_embedded::{JoinFailed, SerfError}; +use serf_embedded::{InvalidOptions, JoinFailed, SerfError}; /// Why constructing a [`Serf`](crate::Serf) node failed. /// @@ -68,6 +68,12 @@ pub enum InitError { /// [`Serf::new_with_rng`](crate::Serf::new_with_rng) to supply your own RNGs /// and avoid the platform entropy draw entirely. Entropy, + /// The serf-level [`SerfOptions`](crate::SerfOptions) failed + /// [`validate`](crate::SerfOptions::validate): `max_user_event_size` exceeds + /// the configured `user_event_size_limit` ceiling, or a coalescing quiescent + /// period is not strictly less than its coalesce period. Carries the typed + /// cause. + InvalidSerfOptions(InvalidOptions), } impl InitError { @@ -157,10 +163,17 @@ impl fmt::Display for InitError { InitError::Resolve(e) => write!(f, "advertise address resolution failed: {e}"), InitError::NoAddresses => f.write_str("advertise address resolution returned no addresses"), InitError::Entropy => f.write_str("entropy source failed while seeding the RNGs"), + InitError::InvalidSerfOptions(e) => write!(f, "invalid serf options: {e}"), } } } +impl From for InitError { + fn from(e: serf_embedded::MemberlistInitError) -> Self { + InitError::Engine(e.into()) + } +} + impl From for InitError { fn from(e: serf_embedded::InitError) -> Self { InitError::Engine(e) diff --git a/serf-embassy/src/lib.rs b/serf-embassy/src/lib.rs index f63147ab..ac3a69ba 100644 --- a/serf-embassy/src/lib.rs +++ b/serf-embassy/src/lib.rs @@ -54,7 +54,7 @@ pub use memberlist_proto::{EndpointOptions, Instant, Node, Rng, SeedableRng, Sma // is self-contained: the event set, the resolver result + admission predicates, the // transform config, the await-result join types, and serf's config / error types. pub use serf_embedded::{ - AliveDelegate, DEFAULT_EVENT_BUFFER_CAP, Event, JoinFailed, JoinId, LabelError, + AliveDelegate, DEFAULT_EVENT_BUFFER_CAP, Event, InvalidOptions, JoinFailed, JoinId, LabelError, MAX_RESOLVED_ADDRS_PER_SEED, MaybeOwned, MaybeResolved, MergeDelegate, ReachedSet, ReconnectDelegate, ResolvedAddrs, SerfError, SerfOptions, TransformOptions, socket_addr_is_routable, diff --git a/serf-embassy/src/serf/mod.rs b/serf-embassy/src/serf/mod.rs index 94be1c6c..3678b46e 100644 --- a/serf-embassy/src/serf/mod.rs +++ b/serf-embassy/src/serf/mod.rs @@ -384,6 +384,12 @@ where let embedded_cfg = embedded_options(&cfg); validate_runtime_config(&embedded_cfg, &transform, ep_cfg.gossip_mtu()) .map_err(InitError::from)?; + // Reject a serf-level configuration the engine cannot honor (an over-ceiling + // `max_user_event_size`, or a self-contradictory coalescing pair) at the same + // deterministic preflight, before resolving the advertise address or binding. + serf_opts + .validate() + .map_err(InitError::InvalidSerfOptions)?; // Resolve the advertise address into a single wire `SocketAddr`, then re-type // `ep_cfg` so the rest of construction — and the engine — only sees the resolved diff --git a/serf-embedded/src/engine/mod.rs b/serf-embedded/src/engine/mod.rs index e265c2a9..39b1ba3b 100644 --- a/serf-embedded/src/engine/mod.rs +++ b/serf-embedded/src/engine/mod.rs @@ -61,12 +61,15 @@ use serf_proto::{ }; use memberlist_embedded::{ - GossipIo, InitError, Options, StreamIo, TransformOptions, + GossipIo, InitError as MemberlistInitError, Options, StreamIo, TransformOptions, reliable::{ConnState, Connection, ReliablePlane}, socket_addr_is_routable, validate_runtime_config, }; -use crate::cidr::{CidrFilter, cidr_blocks}; +use crate::{ + cidr::{CidrFilter, cidr_blocks}, + error::InitError, +}; /// The largest the encrypted wrapper can inflate a gossip datagram, or `0` when /// no encryption backend is built in. serf's gossip plane carries only the @@ -545,7 +548,10 @@ where /// # Errors /// /// Returns [`InitError`] instead of panicking when the configuration is - /// invalid: a zero/over-ceiling gossip MTU, a zero port or close timeout, a + /// invalid: serf options that fail + /// [`SerfOptions::validate`](serf_proto::options::Options::validate) (a + /// self-contradictory coalescing pair, an over-ceiling `max_user_event_size`), + /// a zero/over-ceiling gossip MTU, a zero port or close timeout, a /// non-routable or port-mismatched advertise address, a machine-endpoint init /// failure, or (with an encryption backend built in) an unusable keyring. pub fn try_new_at_with_rng( @@ -557,6 +563,13 @@ where gossip_rng: G, serf_rng: SR, ) -> Result { + // Reject an invalid serf configuration up front, in the one funnel every + // constructor (and every wrapping driver) passes through, so no engine can + // be built past the construction-time checks. + serf_opts + .validate() + .map_err(InitError::InvalidSerfOptions)?; + // Validate every advertise-independent config field (port, gossip-MTU // ceiling, close timeout, and the encryption keyring) up front, sharing the // reused preflight so the deterministic checks live in ONE place. @@ -570,13 +583,13 @@ where // rotation ops (`promote`/`remove_secondary`) match on bytes alone, so such a // ring would make every later byte-keyed op ambiguous and let a rotation // promote or remove the wrong cipher's key. Failing fast here — surfaced through - // the existing encryption `InitError` channel, the closest typed construction + // the memberlist half's encryption channel, the closest typed construction // error — beats a latent ambiguous rotation, and establishes the chokepoint // invariant that the live keyring is cross-cipher-collision-free at all times. #[cfg(encryption)] if let Some(keyring) = transform.encryption.keyring() { if keyring_carries_cross_cipher_twin(keyring) { - return Err(EncryptionError::KeyMismatch.into()); + return Err(InitError::Memberlist(EncryptionError::KeyMismatch.into())); } } @@ -587,12 +600,12 @@ where // Reject a non-routable advertise address before the endpoint exists: a node // must advertise an address its peers can route a reply to. if !socket_addr_is_routable(&advertise) { - return Err(InitError::NonRoutableAdvertiseAddr(advertise)); + return Err(MemberlistInitError::NonRoutableAdvertiseAddr(advertise).into()); } // The advertised port must match the single bound port (one port serves both // the gossip and reliable planes; a direct embedded interface has no NAT). if advertise.port() != cfg.port { - return Err(InitError::AdvertisePortMismatch); + return Err(MemberlistInitError::AdvertisePortMismatch.into()); } // Size the inbound-gossip scratch from the configured gossip MTU, keeping the @@ -613,8 +626,9 @@ where // Build the inner memberlist `Endpoint` (the SWIM machine serf sits on) with // the injected gossip RNG. `try_new_at` maps a machine init failure to - // `InitError::Endpoint` and starts its timers from a consistent origin. - let mut ep = Endpoint::try_new_at(ep_cfg, now, gossip_rng).map_err(InitError::Endpoint)?; + // `MemberlistInitError::Endpoint` and starts its timers from a consistent origin. + let mut ep = + Endpoint::try_new_at(ep_cfg, now, gossip_rng).map_err(MemberlistInitError::Endpoint)?; // Install the routable-address admission filter on the raw `Endpoint` BEFORE // it is wrapped: the machine consults it inline for every inbound Alive, so a diff --git a/serf-embedded/src/engine/tests.rs b/serf-embedded/src/engine/tests.rs index 75e8cdab..92c970c3 100644 --- a/serf-embedded/src/engine/tests.rs +++ b/serf-embedded/src/engine/tests.rs @@ -167,11 +167,42 @@ fn non_routable_advertise_is_rejected() { test_rng(), ); assert!( - matches!(result, Err(InitError::NonRoutableAdvertiseAddr(_))), + matches!( + result, + Err(InitError::Memberlist( + crate::MemberlistInitError::NonRoutableAdvertiseAddr(_) + )) + ), "a non-routable advertise address must fail construction" ); } +/// An over-ceiling `max_user_event_size` is rejected in the engine's own +/// construction funnel, so a driver built directly on the engine cannot bypass +/// the serf-options validation the wrapping drivers enforce. +#[test] +fn over_ceiling_user_event_size_is_rejected_at_engine_construction() { + let cfg = Options::new() + .with_port(7946) + .with_close_timeout(Duration::from_secs(10)); + let ep_cfg = EndpointOptions::new(SmolStr::new("test"), node_addr(7946)); + let now = Instant::from_origin(Duration::from_secs(86_400)); + let serf_opts = + SerfOptions::new().with_max_user_event_size(SerfOptions::DEFAULT_USER_EVENT_SIZE_LIMIT + 1); + let result = SerfEngine::::try_new_at( + cfg, + TransformOptions::default(), + ep_cfg, + serf_opts, + now, + test_rng(), + ); + assert!( + matches!(result, Err(InitError::InvalidSerfOptions(_))), + "an over-ceiling max_user_event_size must fail engine construction" + ); +} + /// `start` then a single `pump` advances the machine without panicking and /// returns a wakeup deadline; the single-node engine tracks no remote members. #[test] @@ -2272,7 +2303,7 @@ fn construction_rejects_a_keyring_with_cross_cipher_byte_twins() { // ciphers — an ambiguous ring for the byte-keyed rotation ops. let keyring = Keyring::with_secondaries(aes256(0x11), [chacha(0x11)]); match try_make_encrypted_engine("a", 7946, keyring) { - Err(InitError::Encryption(_)) => {} + Err(InitError::Memberlist(crate::MemberlistInitError::Encryption(_))) => {} Err(other) => { panic!("expected InitError::Encryption for a cross-cipher twin keyring, got {other:?}") } diff --git a/serf-embedded/src/error/mod.rs b/serf-embedded/src/error/mod.rs new file mode 100644 index 00000000..f7f59d69 --- /dev/null +++ b/serf-embedded/src/error/mod.rs @@ -0,0 +1,55 @@ +//! The engine's construction error. +//! +//! [`SerfEngine`](crate::SerfEngine) construction validates two independent +//! configuration surfaces: the shared memberlist runtime configuration +//! (transports, keys, advertise address — rejected as +//! [`memberlist_embedded::InitError`]) and serf's own options (rejected as +//! [`InvalidOptions`](serf_proto::options::InvalidOptions) by +//! [`Options::validate`](serf_proto::options::Options::validate)). This enum +//! carries both, so every constructor — including a driver built directly on the +//! engine — enforces the full validation rather than only the memberlist half. + +#[cfg(test)] +mod tests; + +use serf_proto::options::InvalidOptions; + +/// Construction failure of a [`SerfEngine`](crate::SerfEngine). +#[derive(Debug)] +#[non_exhaustive] +pub enum InitError { + /// The shared memberlist runtime configuration was rejected. + Memberlist(memberlist_embedded::InitError), + /// Serf's own options failed + /// [`Options::validate`](serf_proto::options::Options::validate) — a + /// self-contradictory coalescing pair or an over-ceiling + /// `max_user_event_size`. + InvalidSerfOptions(InvalidOptions), +} + +impl core::fmt::Display for InitError { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::Memberlist(e) => write!(f, "{e}"), + Self::InvalidSerfOptions(e) => write!(f, "invalid serf options: {e}"), + } + } +} + +impl core::error::Error for InitError { + fn source(&self) -> Option<&(dyn core::error::Error + 'static)> { + match self { + // The memberlist half implements `Error` only on feature sets beyond the + // minimal no_std build; its cause is carried through this variant's + // `Display` instead of the source chain. + Self::Memberlist(_) => None, + Self::InvalidSerfOptions(e) => Some(e), + } + } +} + +impl From for InitError { + fn from(e: memberlist_embedded::InitError) -> Self { + Self::Memberlist(e) + } +} diff --git a/serf-embedded/src/error/tests.rs b/serf-embedded/src/error/tests.rs new file mode 100644 index 00000000..f83cf6c2 --- /dev/null +++ b/serf-embedded/src/error/tests.rs @@ -0,0 +1,33 @@ +use serf_proto::options::Options as SerfOptions; + +use super::InitError; + +#[test] +fn display_prefixes_the_serf_options_arm() { + let invalid = SerfOptions::new() + .with_max_user_event_size(SerfOptions::DEFAULT_USER_EVENT_SIZE_LIMIT + 1) + .validate() + .expect_err("an over-ceiling max_user_event_size must fail validation"); + let e = InitError::InvalidSerfOptions(invalid); + let rendered = std::format!("{e}"); + assert!( + rendered.starts_with("invalid serf options: "), + "the serf-options arm names its surface: {rendered}" + ); + assert!( + rendered.contains("max_user_event_size"), + "the inner cause is carried through: {rendered}" + ); +} + +#[test] +fn source_exposes_the_inner_error() { + use core::error::Error as _; + + let invalid = SerfOptions::new() + .with_max_user_event_size(SerfOptions::DEFAULT_USER_EVENT_SIZE_LIMIT + 1) + .validate() + .expect_err("an over-ceiling max_user_event_size must fail validation"); + let e = InitError::InvalidSerfOptions(invalid); + assert!(e.source().is_some(), "the chained cause is preserved"); +} diff --git a/serf-embedded/src/lib.rs b/serf-embedded/src/lib.rs index 6c2330e5..8926f417 100644 --- a/serf-embedded/src/lib.rs +++ b/serf-embedded/src/lib.rs @@ -23,6 +23,9 @@ compile_error!("serf-embedded requires the `std` or `alloc` feature"); mod cidr; pub mod engine; +pub mod error; + +pub use error::InitError; pub use engine::{DEFAULT_EVENT_BUFFER_CAP, JoinFailed, JoinId, ReachedSet, SerfEngine}; @@ -39,7 +42,9 @@ pub use memberlist_embedded::{ // The datagram + pooled-stream I/O seams a driver supplies to `SerfEngine::pump`. GossipIo, GossipMtuTooLarge, - InitError, + // The memberlist half of the engine's construction error, carried inside + // `InitError::Memberlist`. + InitError as MemberlistInitError, // Engine sizing (ports / close timeout / CIDR policy). Options, StreamIo, @@ -80,5 +85,5 @@ pub use memberlist_embedded::{EncryptionOptions, Keyring, SecretKey}; /// serf's own [`Endpoint`](serf_proto::endpoint::Endpoint) configuration, distinct /// from the memberlist-layer engine [`Options`]. -pub use serf_proto::options::Options as SerfOptions; +pub use serf_proto::options::{InvalidOptions, Options as SerfOptions}; pub use serf_proto::{ReconnectDelegate, endpoint::Error as SerfError, event::Event}; diff --git a/serf-proto/src/endpoint/mod.rs b/serf-proto/src/endpoint/mod.rs index f75cd24c..8f2715f3 100644 --- a/serf-proto/src/endpoint/mod.rs +++ b/serf-proto/src/endpoint/mod.rs @@ -4141,7 +4141,10 @@ where } /// Expose the effective broadcast queue depth cap for assertions. - #[cfg(test)] + /// + /// Gated like the unit suite that calls it (`mod tests` is `tcp`-gated), so a + /// quic-only build does not carry an uncalled seam. + #[cfg(all(test, feature = "tcp"))] pub(crate) fn test_queue_max(&self) -> usize { self.queue_max() } diff --git a/serf-proto/src/options/mod.rs b/serf-proto/src/options/mod.rs index 3c92ac73..22dd1187 100644 --- a/serf-proto/src/options/mod.rs +++ b/serf-proto/src/options/mod.rs @@ -64,6 +64,8 @@ pub struct Options { // ── size limits ─────────────────────────────────────────────────────────── /// Maximum `name + payload` byte size for a user event. max_user_event_size: usize, + /// Ceiling on the configured `max_user_event_size`, enforced at construction. + user_event_size_limit: usize, /// Maximum inbound payload size for a query message. query_size_limit: usize, /// Maximum outbound payload size for a query response. @@ -104,6 +106,13 @@ impl Options { pub const DEFAULT_MAX_COALESCED_USER_EVENTS: core::num::NonZeroUsize = core::num::NonZeroUsize::new(1024).unwrap(); + /// Default value of [`user_event_size_limit`](Options::user_event_size_limit): + /// Go serf's fixed 9 KB `UserEventSizeLimit` (the legacy + /// `serf-core/src/serf.rs` `USER_EVENT_SIZE_LIMIT` port), under which a user + /// event fits a single UDP gossip packet with pragmatic IP-fragmentation + /// headroom. + pub const DEFAULT_USER_EVENT_SIZE_LIMIT: usize = 9 * 1024; + /// Returns a new `Options` with all defaults as specified in Go serf /// `options.go` and the legacy `serf-core/src/options.rs` port. pub fn new() -> Self { @@ -124,6 +133,7 @@ impl Options { event_buffer_size: 512, query_buffer_size: 512, max_user_event_size: 512, + user_event_size_limit: Self::DEFAULT_USER_EVENT_SIZE_LIMIT, query_size_limit: 1024, query_response_size_limit: 1024, max_queue_depth: 4096, @@ -235,6 +245,32 @@ impl Options { self.max_user_event_size } + /// Ceiling on the configured [`max_user_event_size`](Options::max_user_event_size), + /// enforced at construction by [`validate`](Options::validate). + /// + /// Defaults to [`DEFAULT_USER_EVENT_SIZE_LIMIT`](Options::DEFAULT_USER_EVENT_SIZE_LIMIT) + /// (9 KiB — Go serf's fixed limit), so a default configuration behaves + /// identically to Go. Raising it is a deliberate double opt-in for larger + /// events, with real operational costs the ceiling cannot remove: + /// + /// - **It is effectively a cluster-wide protocol parameter.** Every node + /// drops inbound user events larger than its *local* `max_user_event_size`, + /// silently, as a flood bound — a node raised above its peers has its + /// events partially and invisibly lost. Keep the pair uniform across the + /// cluster. + /// - **Transport limits still bind.** User events ride the unreliable gossip + /// plane: past the MTU a UDP datagram is IP-fragmented and one lost fragment + /// loses the whole event (gossip never retransmits), QUIC datagrams are + /// capped by the negotiated datagram size, and embedded device MTUs bind far + /// lower. An event the transport cannot frame surfaces as a send error + /// regardless of this ceiling. + /// - **Memory amplification.** Buffered structures (the user coalescer, the + /// event ring, the rebroadcast queue) scale with per-event size × their + /// entry caps. + pub const fn user_event_size_limit(&self) -> usize { + self.user_event_size_limit + } + /// Maximum inbound payload size for a query. pub const fn query_size_limit(&self) -> usize { self.query_size_limit @@ -426,6 +462,22 @@ impl Options { self } + /// Sets `user_event_size_limit` (see + /// [`user_event_size_limit`](Options::user_event_size_limit) for the + /// operational caveats of raising it; keep it uniform across the cluster). + pub fn with_user_event_size_limit(mut self, v: usize) -> Self { + self.user_event_size_limit = v; + self + } + + /// Sets `user_event_size_limit` in place (see + /// [`user_event_size_limit`](Options::user_event_size_limit) for the + /// operational caveats of raising it). + pub fn set_user_event_size_limit(&mut self, v: usize) -> &mut Self { + self.user_event_size_limit = v; + self + } + /// Sets `query_size_limit`. pub fn with_query_size_limit(mut self, v: usize) -> Self { self.query_size_limit = v; @@ -494,20 +546,31 @@ impl Options { self } - /// Validates the coalescing configuration. + /// Validates the configuration. /// - /// When a coalescing pair is enabled (both periods non-zero) the quiescent - /// period must be strictly less than the coalesce period: the quiescent window - /// is the "went quiet" fast-flush, and the coalesce period is the - /// maximum-delay cap. If quiescent `>=` coalesce the quiescent window can - /// never bind — almost always a misconfiguration. This mirrors the semantics - /// documented on the legacy `serf-core/src/options.rs` period fields. + /// Two classes are checked: /// - /// Returns `Ok(())` when coalescing is disabled or the invariant holds for - /// every enabled pair. The Sans-I/O [`Endpoint`](crate::endpoint::Endpoint) - /// construction is infallible and tolerates any configuration (its flush - /// deadline is always `min(coalesce, quiescent)`); a driver that wants to - /// reject a nonsensical configuration up front calls this. + /// - **Coalescing periods.** When a coalescing pair is enabled (both periods + /// non-zero) the quiescent period must be strictly less than the coalesce + /// period: the quiescent window is the "went quiet" fast-flush, and the + /// coalesce period is the maximum-delay cap. If quiescent `>=` coalesce the + /// quiescent window can never bind — almost always a misconfiguration. This + /// mirrors the semantics documented on the legacy `serf-core/src/options.rs` + /// period fields. + /// - **User-event size ceiling.** `max_user_event_size` must not exceed the + /// configured [`user_event_size_limit`](Options::user_event_size_limit) + /// (default 9 KiB — Go serf's fixed limit, under which a user event fits a + /// single UDP gossip packet with headroom), and the ceiling itself must be + /// nonzero. This mirrors the Go serf construction-time check on the + /// configured limit while letting a deliberate configuration raise the + /// ceiling for networks that can carry more. + /// + /// Returns `Ok(())` when every check passes. The Sans-I/O + /// [`Endpoint`](crate::endpoint::Endpoint) construction is infallible and + /// tolerates any configuration (its coalescer flush deadline is always + /// `min(coalesce, quiescent)`, and the per-event size limit is enforced at + /// send time); a driver that wants to reject a nonsensical configuration up + /// front calls this. pub fn validate(&self) -> Result<(), InvalidOptions> { if self.member_coalesce_enabled() && self.quiescent_period >= self.coalesce_period { return Err(InvalidOptions::MemberCoalesce(CoalesceConfig { @@ -521,6 +584,12 @@ impl Options { quiescent_period: self.user_quiescent_period, })); } + if self.user_event_size_limit == 0 || self.max_user_event_size > self.user_event_size_limit { + return Err(InvalidOptions::UserEventSize(UserEventSizeConfig { + max_user_event_size: self.max_user_event_size, + limit: self.user_event_size_limit, + })); + } Ok(()) } } @@ -544,9 +613,32 @@ impl core::fmt::Display for CoalesceConfig { } } -/// Error returned by [`Options::validate`] for a self-contradictory coalescing -/// configuration (an enabled quiescent period not strictly less than its -/// coalesce period). +/// The user-event size configuration that failed [`Options::validate`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct UserEventSizeConfig { + /// The configured `max_user_event_size`. + pub max_user_event_size: usize, + /// The configured ceiling ([`Options::user_event_size_limit`]). + pub limit: usize, +} + +impl core::fmt::Display for UserEventSizeConfig { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + if self.limit == 0 { + write!(f, "user_event_size_limit must be nonzero") + } else { + write!( + f, + "max_user_event_size ({}) must not exceed user_event_size_limit ({})", + self.max_user_event_size, self.limit + ) + } + } +} + +/// Error returned by [`Options::validate`] for a configuration that cannot be +/// honored: a coalescing pair whose quiescent period is not strictly less than +/// its coalesce period, or a `max_user_event_size` above the absolute ceiling. #[derive(Debug, Clone, Copy, PartialEq, Eq, thiserror::Error)] #[non_exhaustive] pub enum InvalidOptions { @@ -558,6 +650,11 @@ pub enum InvalidOptions { /// coalesce period while user coalescing is enabled. #[error("user-event coalescing: {0}")] UserCoalesce(CoalesceConfig), + /// The configured `max_user_event_size` exceeds the configured + /// [`user_event_size_limit`](Options::user_event_size_limit) ceiling, or the + /// ceiling itself is zero. + #[error("user-event size: {0}")] + UserEventSize(UserEventSizeConfig), } impl Default for Options { diff --git a/serf-proto/src/options/tests.rs b/serf-proto/src/options/tests.rs index a5d6f0e0..27d767e9 100644 --- a/serf-proto/src/options/tests.rs +++ b/serf-proto/src/options/tests.rs @@ -217,3 +217,104 @@ fn validate_rejects_quiescent_not_less_than_coalesce() { Err(InvalidOptions::UserCoalesce(_)) )); } + +// ── user-event size ceiling ────────────────────────────────────────────────── + +#[test] +fn default_user_event_size_limit_matches_go_serf() { + // The default ceiling is 9 KiB, matching Go serf's fixed construction-time + // limit, so a default configuration behaves identically to Go. + assert_eq!(Options::DEFAULT_USER_EVENT_SIZE_LIMIT, 9 * 1024); + assert_eq!(Options::DEFAULT_USER_EVENT_SIZE_LIMIT, 9216); + assert_eq!( + Options::new().user_event_size_limit(), + Options::DEFAULT_USER_EVENT_SIZE_LIMIT + ); +} + +#[test] +fn validate_rejects_max_user_event_size_over_ceiling() { + let over = Options::new().with_max_user_event_size(Options::DEFAULT_USER_EVENT_SIZE_LIMIT + 1); + assert!(matches!( + over.validate(), + Err(InvalidOptions::UserEventSize(_)) + )); +} + +#[test] +fn validate_accepts_max_user_event_size_at_or_below_ceiling() { + // Exactly the ceiling is accepted. + assert!( + Options::new() + .with_max_user_event_size(Options::DEFAULT_USER_EVENT_SIZE_LIMIT) + .validate() + .is_ok() + ); + // Below the ceiling is accepted. + assert!( + Options::new() + .with_max_user_event_size(Options::DEFAULT_USER_EVENT_SIZE_LIMIT - 1) + .validate() + .is_ok() + ); + // The default (512) is well below the ceiling. + assert!(Options::new().validate().is_ok()); +} + +#[test] +fn raised_ceiling_admits_a_larger_max_user_event_size() { + // Raising the ceiling is the deliberate double opt-in: both knobs must move + // for a configuration past Go's fixed limit to validate. + let raised = Options::new() + .with_user_event_size_limit(64 * 1024) + .with_max_user_event_size(16 * 1024); + assert!(raised.validate().is_ok()); + + // Turning only the size knob without the ceiling still rejects. + let size_only = Options::new().with_max_user_event_size(16 * 1024); + assert!(matches!( + size_only.validate(), + Err(InvalidOptions::UserEventSize(_)) + )); +} + +#[test] +fn lowered_ceiling_rejects_an_unadjusted_max_user_event_size() { + // A ceiling below the configured (here: default 512) per-event size makes the + // pair incoherent and is rejected. + let lowered = Options::new().with_user_event_size_limit(256); + assert!(matches!( + lowered.validate(), + Err(InvalidOptions::UserEventSize(_)) + )); + // Lowering both knobs coherently validates. + assert!( + Options::new() + .with_user_event_size_limit(256) + .with_max_user_event_size(256) + .validate() + .is_ok() + ); +} + +#[test] +fn zero_ceiling_is_rejected() { + // A zero ceiling can never be satisfied meaningfully; disabling user events + // is expressed through max_user_event_size, not the ceiling. + let zero = Options::new() + .with_user_event_size_limit(0) + .with_max_user_event_size(0); + assert!(matches!( + zero.validate(), + Err(InvalidOptions::UserEventSize(_)) + )); +} + +#[test] +fn user_event_size_limit_builder_pair_round_trips() { + let built = Options::new().with_user_event_size_limit(12 * 1024); + assert_eq!(built.user_event_size_limit(), 12 * 1024); + let mut set = Options::new(); + set.set_user_event_size_limit(10 * 1024); + assert_eq!(set.user_event_size_limit(), 10 * 1024); +} diff --git a/serf-reactor/src/serf/mod.rs b/serf-reactor/src/serf/mod.rs index ae31e920..0fcf263b 100644 --- a/serf-reactor/src/serf/mod.rs +++ b/serf-reactor/src/serf/mod.rs @@ -47,7 +47,7 @@ use crate::{ }, delegate::Delegate, driver::options::RuntimeOptions, - error::{JoinFailed, Result, SerfError}, + error::{InvalidOption, JoinFailed, Result, SerfError}, events::EventStream, resolver::{AdvertiseAddrResolver, Resolver}, shared::Shared, @@ -171,6 +171,13 @@ where // Reject runtime knobs a zero capacity would deterministically break BEFORE // binding any socket or spawning the detached driver. runtime_options.validate().map_err(T::Error::from)?; + // Reject a serf-level configuration a driver cannot honor (an over-ceiling + // `max_user_event_size`, or a self-contradictory coalescing pair) at the same + // early stage, before any socket is bound or the detached driver is spawned. + serf_options + .validate() + .map_err(|e| SerfError::InvalidOption(InvalidOption::new("serf_options", e.to_string()))) + .map_err(T::Error::from)?; // Cache the join deadline before `runtime_options` moves into the bundle. let join_deadline = runtime_options.join_deadline(); // Cache `query_timeout_mult` before `serf_options` moves into the bundle, so diff --git a/serf-reactor/src/serf/tests.rs b/serf-reactor/src/serf/tests.rs index 35254879..c490e925 100644 --- a/serf-reactor/src/serf/tests.rs +++ b/serf-reactor/src/serf/tests.rs @@ -246,6 +246,36 @@ async fn tcp_new_rejects_zero_event_queue_cap() { assert_tcp_new_rejects(RuntimeOptions::new().with_event_queue_cap(0)).await; } +/// An over-ceiling `max_user_event_size` in the serf options is rejected by +/// `Serf::tcp` at construction — before binding a socket or spawning the detached +/// driver — rather than returning `Ok` and later dropping oversize user events. +#[tokio::test] +async fn tcp_new_rejects_over_ceiling_user_event_size() { + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new("bad-serf-opt-node")) + .with_advertise_addr(MaybeResolved::Resolved(bind)); + let serf = + SerfOptions::new().with_max_user_event_size(SerfOptions::DEFAULT_USER_EVENT_SIZE_LIMIT + 1); + let res = Serf::::tcp( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + serf, + None, + #[cfg(encryption)] + std::sync::Arc::new(crate::VoidKeyringDelegate), + ) + .await; + match res { + Err(SerfError::InvalidOption(_)) => {} + Err(other) => panic!("expected InvalidOption, got {other:?}"), + Ok(_) => panic!("an over-ceiling max_user_event_size must be rejected at construction"), + } +} + /// A `Bounded(0)` observation channel sourced from a serde config is rejected. #[cfg(feature = "serde")] #[tokio::test] diff --git a/serf-smoltcp/src/error/mod.rs b/serf-smoltcp/src/error/mod.rs index 81e4b523..904dbaf0 100644 --- a/serf-smoltcp/src/error/mod.rs +++ b/serf-smoltcp/src/error/mod.rs @@ -3,6 +3,7 @@ use core::fmt; use memberlist_proto::EndpointInitError; +use serf_embedded::InvalidOptions; use crate::interface::{HardwareAddress, IpCidr, Medium, Route}; use core::net::SocketAddr; @@ -201,6 +202,12 @@ pub enum InitError { /// immediately — the drain never runs and an in-flight push/pull response is /// truncated. Must be non-zero. ZeroCloseTimeout, + /// The serf-level [`SerfOptions`](crate::SerfOptions) failed + /// [`validate`](crate::SerfOptions::validate): `max_user_event_size` exceeds + /// the configured `user_event_size_limit` ceiling, or a coalescing quiescent + /// period is not strictly less than its coalesce period. Carries the typed + /// cause. + InvalidSerfOptions(InvalidOptions), } /// The configured gossip MTU exceeds the largest plaintext payload whose on-wire @@ -312,6 +319,7 @@ impl fmt::Display for InitError { f.write_str("udp_rx_packets and udp_tx_packets must be non-zero") } InitError::ZeroCloseTimeout => f.write_str("close_timeout must be non-zero"), + InitError::InvalidSerfOptions(e) => write!(f, "invalid serf options: {e}"), } } } @@ -320,16 +328,29 @@ impl InitError { /// Map a [`SerfEngine`](serf_embedded::SerfEngine) construction error into the /// driver's [`InitError`]. /// - /// The driver pre-validates the port, gossip MTU, close timeout, and advertise - /// address before building the engine, so in practice the engine fails only with - /// [`Endpoint`](serf_embedded::InitError::Endpoint) (machine init) or - /// [`Encryption`](serf_embedded::InitError::Encryption) (an unusable keyring). - /// The remaining variants are mapped to their driver equivalents anyway so the - /// conversion is total and stays correct if the driver's pre-checks are ever - /// reordered or relaxed. + /// The driver pre-validates the port, gossip MTU, close timeout, advertise + /// address, and serf options before building the engine, so in practice the + /// engine fails only with + /// [`Endpoint`](serf_embedded::MemberlistInitError::Endpoint) (machine init) or + /// [`Encryption`](serf_embedded::MemberlistInitError::Encryption) (an unusable + /// keyring). The remaining variants are mapped to their driver equivalents + /// anyway so the conversion is total and stays correct if the driver's + /// pre-checks are ever reordered or relaxed. pub(crate) fn from_embedded(e: serf_embedded::InitError) -> Self { - use serf_embedded::InitError as E; match e { + serf_embedded::InitError::InvalidSerfOptions(inner) => InitError::InvalidSerfOptions(inner), + serf_embedded::InitError::Memberlist(m) => Self::from_memberlist(m), + // The engine's own error is `#[non_exhaustive]`; treat an unknown future + // arm like the memberlist wildcard in `from_memberlist`. + _ => InitError::Endpoint(EndpointInitError::AwarenessMultiplierZero), + } + } + + /// Map the memberlist half of the engine's construction error — also the type + /// the shared advertise-independent preflight returns directly. + pub(crate) fn from_memberlist(m: serf_embedded::MemberlistInitError) -> Self { + use serf_embedded::MemberlistInitError as E; + match m { E::NonRoutableAdvertiseAddr(addr) => InitError::NonRoutableAdvertiseAddr(addr), E::AdvertisePortMismatch => InitError::AdvertisePortMismatch, E::ZeroPort => InitError::ZeroPort, @@ -341,9 +362,9 @@ impl InitError { E::Endpoint(inner) => InitError::Endpoint(inner), #[cfg(encryption)] E::Encryption(inner) => InitError::Encryption(inner), - // `serf_embedded::InitError` is `#[non_exhaustive]`, so a wildcard is - // required even though every variant it defines today is handled above and - // this arm is unreachable. A future engine-only failure mode reaching here + // The memberlist half is `#[non_exhaustive]`, so a wildcard is required + // even though every variant it defines today is handled above and this + // arm is unreachable. A future engine-only failure mode reaching here // surfaces as a generic endpoint-init failure and would warrant its own // driver variant when added. _ => InitError::Endpoint(EndpointInitError::AwarenessMultiplierZero), diff --git a/serf-smoltcp/src/error/tests.rs b/serf-smoltcp/src/error/tests.rs index 34f58d28..7a691fda 100644 --- a/serf-smoltcp/src/error/tests.rs +++ b/serf-smoltcp/src/error/tests.rs @@ -56,6 +56,12 @@ fn all_variants() -> Vec { InitError::TcpRxBufferTooLarge, InitError::ZeroUdpPackets, InitError::ZeroCloseTimeout, + InitError::InvalidSerfOptions( + crate::SerfOptions::new() + .with_max_user_event_size(crate::SerfOptions::DEFAULT_USER_EVENT_SIZE_LIMIT + 1) + .validate() + .expect_err("an over-ceiling max_user_event_size is invalid"), + ), ] } @@ -115,36 +121,36 @@ fn from_encryption_error() { // and routes any future (non_exhaustive) variant to a generic endpoint error. #[test] fn from_embedded_maps_each_mode() { - use serf_embedded::InitError as E; + use serf_embedded::MemberlistInitError as E; assert!(matches!( - InitError::from_embedded(E::ZeroPort), + InitError::from_memberlist(E::ZeroPort), InitError::ZeroPort )); assert!(matches!( - InitError::from_embedded(E::AdvertisePortMismatch), + InitError::from_memberlist(E::AdvertisePortMismatch), InitError::AdvertisePortMismatch )); assert!(matches!( - InitError::from_embedded(E::ZeroCloseTimeout), + InitError::from_memberlist(E::ZeroCloseTimeout), InitError::ZeroCloseTimeout )); assert!(matches!( - InitError::from_embedded(E::NonRoutableAdvertiseAddr(sample_socket_addr())), + InitError::from_memberlist(E::NonRoutableAdvertiseAddr(sample_socket_addr())), InitError::NonRoutableAdvertiseAddr(_) )); assert!(matches!( - InitError::from_embedded(E::Endpoint(EndpointInitError::AwarenessMultiplierZero)), + InitError::from_memberlist(E::Endpoint(EndpointInitError::AwarenessMultiplierZero)), InitError::Endpoint(_) )); #[cfg(encryption)] assert!(matches!( - InitError::from_embedded(E::Encryption(memberlist_proto::EncryptionError::AuthFailed)), + InitError::from_memberlist(E::Encryption(memberlist_proto::EncryptionError::AuthFailed)), InitError::Encryption(_) )); // The carried ceiling/value survive the GossipMtuTooLarge remap. - let mapped = InitError::from_embedded(E::GossipMtuTooLarge(serf_embedded::GossipMtuTooLarge { + let mapped = InitError::from_memberlist(E::GossipMtuTooLarge(serf_embedded::GossipMtuTooLarge { gossip_mtu: 99_999, ceiling: 65_467, })); diff --git a/serf-smoltcp/src/lib.rs b/serf-smoltcp/src/lib.rs index cc76b9ed..54b101cf 100644 --- a/serf-smoltcp/src/lib.rs +++ b/serf-smoltcp/src/lib.rs @@ -39,7 +39,7 @@ pub use memberlist_proto::{EndpointOptions, Instant}; // is self-contained: the event set, the resolver result + admission predicates, the // transform config, the await-result join types, and serf's config / error types. pub use serf_embedded::{ - AliveDelegate, DEFAULT_EVENT_BUFFER_CAP, Event, JoinFailed, JoinId, LabelError, + AliveDelegate, DEFAULT_EVENT_BUFFER_CAP, Event, InvalidOptions, JoinFailed, JoinId, LabelError, MAX_RESOLVED_ADDRS_PER_SEED, MaybeOwned, MaybeResolved, MergeDelegate, ReachedSet, ReconnectDelegate, ResolvedAddrs, SerfError, SerfOptions, TransformOptions, socket_addr_is_routable, diff --git a/serf-smoltcp/src/serf/mod.rs b/serf-smoltcp/src/serf/mod.rs index 42eccc5f..b8ff2aa1 100644 --- a/serf-smoltcp/src/serf/mod.rs +++ b/serf-smoltcp/src/serf/mod.rs @@ -295,7 +295,13 @@ where // Advertise-independent config preflight before touching the link layer. let embedded_cfg = embedded_options(&cfg); validate_runtime_config(&embedded_cfg, &transform, ep_cfg.gossip_mtu()) - .map_err(InitError::from_embedded)?; + .map_err(InitError::from_memberlist)?; + // Reject a serf-level configuration the engine cannot honor (an over-ceiling + // `max_user_event_size`, or a self-contradictory coalescing pair) at the same + // deterministic preflight, before drawing entropy or touching the link layer. + serf_opts + .validate() + .map_err(InitError::InvalidSerfOptions)?; // Resolve the advertise address, then re-type `ep_cfg` so the rest of // construction only ever sees the resolved wire `SocketAddr`. @@ -385,7 +391,13 @@ where { let embedded_cfg = embedded_options(&cfg); validate_runtime_config(&embedded_cfg, &transform, ep_cfg.gossip_mtu()) - .map_err(InitError::from_embedded)?; + .map_err(InitError::from_memberlist)?; + // Reject a serf-level configuration the engine cannot honor (an over-ceiling + // `max_user_event_size`, or a self-contradictory coalescing pair) at the same + // deterministic preflight, before drawing entropy or touching the link layer. + serf_opts + .validate() + .map_err(InitError::InvalidSerfOptions)?; let resolved_advertise = resolver .resolve(ep_cfg.advertise_addr_ref()) diff --git a/serf-smoltcp/src/serf/tests.rs b/serf-smoltcp/src/serf/tests.rs index 8ce54441..f2d23652 100644 --- a/serf-smoltcp/src/serf/tests.rs +++ b/serf-smoltcp/src/serf/tests.rs @@ -146,6 +146,56 @@ fn zero_port_is_rejected() { assert!(matches!(err, crate::InitError::ZeroPort)); } +#[test] +fn over_ceiling_user_event_size_is_rejected() { + let mut dev = NullDevice; + // A `max_user_event_size` above the absolute ceiling is rejected at + // construction rather than dropping oversize user events at send time. + let Err(err) = Serf::::try_new( + Options::new(), + ip_iface(1), + TransformOptions::default(), + EndpointOptions::new( + SmolStr::new("a"), + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)), 7946), + ), + SerfOptions::new().with_max_user_event_size(SerfOptions::DEFAULT_USER_EVENT_SIZE_LIMIT + 1), + &SocketAddrResolver, + &mut dev, + now(), + ) else { + panic!("an over-ceiling max_user_event_size must be rejected"); + }; + assert!(matches!(err, crate::InitError::InvalidSerfOptions(_))); +} + +// The explicit-RNG constructor runs the same preflight, so an over-ceiling +// configuration cannot slip in through the production entropy-seeded path. +#[test] +fn over_ceiling_user_event_size_is_rejected_by_with_rng() { + use memberlist_proto::{SeedableRng, SmallRng}; + + let mut dev = NullDevice; + let Err(err) = Serf::::with_rng( + Options::new(), + ip_iface(1), + TransformOptions::default(), + EndpointOptions::new( + SmolStr::new("a"), + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)), 7946), + ), + SerfOptions::new().with_max_user_event_size(SerfOptions::DEFAULT_USER_EVENT_SIZE_LIMIT + 1), + &SocketAddrResolver, + &mut dev, + now(), + SmallRng::seed_from_u64(1), + SmallRng::seed_from_u64(2), + ) else { + panic!("an over-ceiling max_user_event_size must be rejected by with_rng"); + }; + assert!(matches!(err, crate::InitError::InvalidSerfOptions(_))); +} + /// A [`GossipIo`] that replays a fixed list of pre-encoded datagrams once (one per /// `recv`), draining `send`. Feeds a controlled flood of inbound gossip straight to /// the engine's pump. From 2b64cc077f10c297930ff69a0f3dcbb191536398 Mon Sep 17 00:00:00 2001 From: al8n Date: Sat, 11 Jul 2026 23:35:07 +1200 Subject: [PATCH 080/128] feat(reactor): plumb SWIM failure-detection knobs through the transport options MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The TCP/TLS/QUIC reactor transports built their inner memberlist `EndpointOptions` with only the user-broadcast tiers and the optional push/pull interval, so probe / gossip / suspicion timing always ran at the memberlist LAN defaults (~1s probe, ~500ms probe timeout, ~200ms gossip, suspicion_mult 4). A two-node cluster then takes several seconds to move a dead peer to Failed, which is too slow for CI-time failure-detection tests. Add flat passthrough overrides mirroring the `EndpointOptions` names and the existing `push_pull_interval` convention — `with_probe_interval`, `with_probe_timeout`, `with_gossip_interval`, `with_suspicion_mult`, `with_suspicion_max_timeout_mult` — on all three reactor transport options, each applied to the coordinator's `EndpointOptions` in `Transport::run` only when set. Every knob defaults to `None`, so the coordinator keeps its own default and existing behavior is unchanged; tests opt into fast timing. --- serf-reactor/src/quic/mod.rs | 142 +++++++++++++++++++++++++++++++++++ serf-reactor/src/tcp/mod.rs | 142 +++++++++++++++++++++++++++++++++++ serf-reactor/src/tls/mod.rs | 142 +++++++++++++++++++++++++++++++++++ 3 files changed, 426 insertions(+) diff --git a/serf-reactor/src/quic/mod.rs b/serf-reactor/src/quic/mod.rs index 761bebf8..c420647a 100644 --- a/serf-reactor/src/quic/mod.rs +++ b/serf-reactor/src/quic/mod.rs @@ -80,6 +80,21 @@ pub struct QuicTransportOptions> { /// coordinator default; `Some(Duration::ZERO)` disables periodic push/pull /// entirely. See [`with_push_pull_interval`](Self::with_push_pull_interval). push_pull_interval: Option, + /// SWIM probe interval override. `None` keeps the coordinator default. See + /// [`with_probe_interval`](Self::with_probe_interval). + probe_interval: Option, + /// SWIM direct-ping timeout override. `None` keeps the coordinator default. See + /// [`with_probe_timeout`](Self::with_probe_timeout). + probe_timeout: Option, + /// Gossip interval override. `None` keeps the coordinator default. See + /// [`with_gossip_interval`](Self::with_gossip_interval). + gossip_interval: Option, + /// SWIM suspicion multiplier override. `None` keeps the coordinator default. See + /// [`with_suspicion_mult`](Self::with_suspicion_mult). + suspicion_mult: Option, + /// SWIM suspicion max-timeout multiplier override. `None` keeps the coordinator + /// default. See [`with_suspicion_max_timeout_mult`](Self::with_suspicion_max_timeout_mult). + suspicion_max_timeout_mult: Option, /// Gossip-encryption policy. The default (no keyring) leaves the gossip datagrams /// plaintext; attaching a keyring via [`with_encryption`](Self::with_encryption) /// makes the coordinator's `encrypt_gossip`/`decrypt_gossip` AEAD-protect them. @@ -102,6 +117,11 @@ impl QuicTransportOptions { advertise_addr: None, quic_config: None, push_pull_interval: None, + probe_interval: None, + probe_timeout: None, + gossip_interval: None, + suspicion_mult: None, + suspicion_max_timeout_mult: None, #[cfg(encryption)] encryption: EncryptionOptions::new(), } @@ -147,6 +167,66 @@ impl QuicTransportOptions { self } + /// Builder: override the memberlist SWIM probe interval — how often the + /// coordinator probes a random peer for liveness. + /// + /// `None` (the default) keeps the coordinator default (~1s). A shorter interval + /// speeds failure detection at the cost of more probe traffic; it also shortens + /// the suspicion timeout, which scales with the probe interval. + #[must_use] + #[inline] + pub const fn with_probe_interval(mut self, interval: Duration) -> Self { + self.probe_interval = Some(interval); + self + } + + /// Builder: override the memberlist SWIM direct-ping timeout — how long the + /// coordinator waits for a probe ack before escalating to indirect probes. + /// + /// `None` (the default) keeps the coordinator default (~500ms). It must + /// comfortably exceed the real network round-trip, or a live peer whose ack is + /// merely slow is falsely suspected. + #[must_use] + #[inline] + pub const fn with_probe_timeout(mut self, timeout: Duration) -> Self { + self.probe_timeout = Some(timeout); + self + } + + /// Builder: override the memberlist gossip interval — how often the coordinator + /// flushes queued gossip to a random subset of peers. + /// + /// `None` (the default) keeps the coordinator default (~200ms). + #[must_use] + #[inline] + pub const fn with_gossip_interval(mut self, interval: Duration) -> Self { + self.gossip_interval = Some(interval); + self + } + + /// Builder: override the memberlist SWIM suspicion multiplier — how long a + /// suspected peer is held in the Suspect state before being declared Failed. + /// + /// The minimum suspicion timeout is `suspicion_mult * log10(N+1) * probe_interval`. + /// `None` (the default) keeps the coordinator default. + #[must_use] + #[inline] + pub const fn with_suspicion_mult(mut self, mult: u32) -> Self { + self.suspicion_mult = Some(mult); + self + } + + /// Builder: override the memberlist SWIM suspicion max-timeout multiplier — the + /// upper bound on the suspicion timeout as a multiple of the minimum. + /// + /// `None` (the default) keeps the coordinator default. + #[must_use] + #[inline] + pub const fn with_suspicion_max_timeout_mult(mut self, mult: u32) -> Self { + self.suspicion_max_timeout_mult = Some(mult); + self + } + /// Builder: gossip-encryption policy. /// /// The default (no keyring) keeps the gossip datagrams plaintext, so an @@ -191,6 +271,36 @@ impl QuicTransportOptions { self.push_pull_interval } + /// The SWIM probe-interval override, if set. + #[inline] + pub const fn probe_interval(&self) -> Option { + self.probe_interval + } + + /// The SWIM probe-timeout override, if set. + #[inline] + pub const fn probe_timeout(&self) -> Option { + self.probe_timeout + } + + /// The gossip-interval override, if set. + #[inline] + pub const fn gossip_interval(&self) -> Option { + self.gossip_interval + } + + /// The SWIM suspicion-multiplier override, if set. + #[inline] + pub const fn suspicion_mult(&self) -> Option { + self.suspicion_mult + } + + /// The SWIM suspicion max-timeout-multiplier override, if set. + #[inline] + pub const fn suspicion_max_timeout_mult(&self) -> Option { + self.suspicion_max_timeout_mult + } + /// Gossip-encryption policy. #[cfg(encryption)] #[cfg_attr( @@ -230,6 +340,14 @@ where /// [`Transport::run`]. `None` keeps the default; `Some(Duration::ZERO)` disables /// periodic anti-entropy. push_pull_interval: Option, + /// SWIM failure-detection overrides applied to the coordinator's + /// `EndpointOptions` in [`Transport::run`]. Each `None` keeps the coordinator + /// default. + probe_interval: Option, + probe_timeout: Option, + gossip_interval: Option, + suspicion_mult: Option, + suspicion_max_timeout_mult: Option, /// Independent OS-seeded seed for the serf core's RNG, drawn once per node in /// [`Transport::new`] and consumed when [`Transport::run`] builds the endpoint via /// `new_with_rng`. Distinct from the coordinator's gossip RNG so serf's query IDs @@ -331,6 +449,11 @@ where gossip_socket, quic_config, push_pull_interval: options.push_pull_interval, + probe_interval: options.probe_interval, + probe_timeout: options.probe_timeout, + gossip_interval: options.gossip_interval, + suspicion_mult: options.suspicion_mult, + suspicion_max_timeout_mult: options.suspicion_max_timeout_mult, serf_rng, #[cfg(encryption)] encryption: options.encryption, @@ -370,6 +493,25 @@ where if let Some(interval) = self.push_pull_interval { inner_opts = inner_opts.with_push_pull_interval(interval); } + // Caller-supplied SWIM failure-detection overrides: each left unset keeps the + // coordinator's own default. Lowering these speeds up failure detection (probe + // cadence, ack timeout, gossip cadence, and the suspicion timeout that scales + // with the probe interval). + if let Some(v) = self.probe_interval { + inner_opts = inner_opts.with_probe_interval(v); + } + if let Some(v) = self.probe_timeout { + inner_opts = inner_opts.with_probe_timeout(v); + } + if let Some(v) = self.gossip_interval { + inner_opts = inner_opts.with_gossip_interval(v); + } + if let Some(v) = self.suspicion_mult { + inner_opts = inner_opts.with_suspicion_mult(v); + } + if let Some(v) = self.suspicion_max_timeout_mult { + inner_opts = inner_opts.with_suspicion_max_timeout_mult(v); + } // The shared UDP socket also carries raw QUIC packets, whose size is governed by // the quinn `EndpointConfig`'s accepted max UDP payload — which a caller can set // above the serf gossip MTU (quinn's default 1472 already exceeds the 1400 diff --git a/serf-reactor/src/tcp/mod.rs b/serf-reactor/src/tcp/mod.rs index e1f89baf..9aa85575 100644 --- a/serf-reactor/src/tcp/mod.rs +++ b/serf-reactor/src/tcp/mod.rs @@ -57,6 +57,21 @@ pub struct TcpTransportOptions> { /// coordinator default; `Some(Duration::ZERO)` disables periodic push/pull /// entirely. See [`with_push_pull_interval`](Self::with_push_pull_interval). push_pull_interval: Option, + /// SWIM probe interval override. `None` keeps the coordinator default. See + /// [`with_probe_interval`](Self::with_probe_interval). + probe_interval: Option, + /// SWIM direct-ping timeout override. `None` keeps the coordinator default. See + /// [`with_probe_timeout`](Self::with_probe_timeout). + probe_timeout: Option, + /// Gossip interval override. `None` keeps the coordinator default. See + /// [`with_gossip_interval`](Self::with_gossip_interval). + gossip_interval: Option, + /// SWIM suspicion multiplier override. `None` keeps the coordinator default. See + /// [`with_suspicion_mult`](Self::with_suspicion_mult). + suspicion_mult: Option, + /// SWIM suspicion max-timeout multiplier override. `None` keeps the coordinator + /// default. See [`with_suspicion_max_timeout_mult`](Self::with_suspicion_max_timeout_mult). + suspicion_max_timeout_mult: Option, /// Gossip-and-reliable encryption policy. The default (no keyring) leaves both /// planes plaintext; attaching a keyring via [`with_encryption`](Self::with_encryption) /// makes the coordinator's `encrypt_gossip`/`decrypt_gossip` (and the plain-TCP @@ -77,6 +92,11 @@ impl TcpTransportOptions { advertise_addr: None, stream: StreamTransportOptions::new(), push_pull_interval: None, + probe_interval: None, + probe_timeout: None, + gossip_interval: None, + suspicion_mult: None, + suspicion_max_timeout_mult: None, #[cfg(encryption)] encryption: EncryptionOptions::new(), } @@ -119,6 +139,66 @@ impl TcpTransportOptions { self } + /// Builder: override the memberlist SWIM probe interval — how often the + /// coordinator probes a random peer for liveness. + /// + /// `None` (the default) keeps the coordinator default (~1s). A shorter interval + /// speeds failure detection at the cost of more probe traffic; it also shortens + /// the suspicion timeout, which scales with the probe interval. + #[must_use] + #[inline] + pub const fn with_probe_interval(mut self, interval: Duration) -> Self { + self.probe_interval = Some(interval); + self + } + + /// Builder: override the memberlist SWIM direct-ping timeout — how long the + /// coordinator waits for a probe ack before escalating to indirect probes. + /// + /// `None` (the default) keeps the coordinator default (~500ms). It must + /// comfortably exceed the real network round-trip, or a live peer whose ack is + /// merely slow is falsely suspected. + #[must_use] + #[inline] + pub const fn with_probe_timeout(mut self, timeout: Duration) -> Self { + self.probe_timeout = Some(timeout); + self + } + + /// Builder: override the memberlist gossip interval — how often the coordinator + /// flushes queued gossip to a random subset of peers. + /// + /// `None` (the default) keeps the coordinator default (~200ms). + #[must_use] + #[inline] + pub const fn with_gossip_interval(mut self, interval: Duration) -> Self { + self.gossip_interval = Some(interval); + self + } + + /// Builder: override the memberlist SWIM suspicion multiplier — how long a + /// suspected peer is held in the Suspect state before being declared Failed. + /// + /// The minimum suspicion timeout is `suspicion_mult * log10(N+1) * probe_interval`. + /// `None` (the default) keeps the coordinator default. + #[must_use] + #[inline] + pub const fn with_suspicion_mult(mut self, mult: u32) -> Self { + self.suspicion_mult = Some(mult); + self + } + + /// Builder: override the memberlist SWIM suspicion max-timeout multiplier — the + /// upper bound on the suspicion timeout as a multiple of the minimum. + /// + /// `None` (the default) keeps the coordinator default. + #[must_use] + #[inline] + pub const fn with_suspicion_max_timeout_mult(mut self, mult: u32) -> Self { + self.suspicion_max_timeout_mult = Some(mult); + self + } + /// Builder: gossip-and-reliable encryption policy. #[cfg(encryption)] #[cfg_attr( @@ -156,6 +236,36 @@ impl TcpTransportOptions { self.push_pull_interval } + /// The SWIM probe-interval override, if set. + #[inline] + pub const fn probe_interval(&self) -> Option { + self.probe_interval + } + + /// The SWIM probe-timeout override, if set. + #[inline] + pub const fn probe_timeout(&self) -> Option { + self.probe_timeout + } + + /// The gossip-interval override, if set. + #[inline] + pub const fn gossip_interval(&self) -> Option { + self.gossip_interval + } + + /// The SWIM suspicion-multiplier override, if set. + #[inline] + pub const fn suspicion_mult(&self) -> Option { + self.suspicion_mult + } + + /// The SWIM suspicion max-timeout-multiplier override, if set. + #[inline] + pub const fn suspicion_max_timeout_mult(&self) -> Option { + self.suspicion_max_timeout_mult + } + /// Gossip-and-reliable encryption policy. #[cfg(encryption)] #[cfg_attr( @@ -196,6 +306,14 @@ where /// [`Transport::run`]. `None` keeps the default; `Some(Duration::ZERO)` disables /// periodic anti-entropy. push_pull_interval: Option, + /// SWIM failure-detection overrides applied to the coordinator's + /// `EndpointOptions` in [`Transport::run`]. Each `None` keeps the coordinator + /// default. + probe_interval: Option, + probe_timeout: Option, + gossip_interval: Option, + suspicion_mult: Option, + suspicion_max_timeout_mult: Option, /// Independent OS-seeded seed for the serf core's RNG, drawn once per node in /// [`Transport::new`] and consumed when [`Transport::run`] builds the endpoint. serf_rng: StdRng, @@ -317,6 +435,11 @@ where tcp_listener, stream_options: options.stream, push_pull_interval: options.push_pull_interval, + probe_interval: options.probe_interval, + probe_timeout: options.probe_timeout, + gossip_interval: options.gossip_interval, + suspicion_mult: options.suspicion_mult, + suspicion_max_timeout_mult: options.suspicion_max_timeout_mult, serf_rng, #[cfg(encryption)] encryption: options.encryption, @@ -355,6 +478,25 @@ where if let Some(interval) = self.push_pull_interval { inner_opts = inner_opts.with_push_pull_interval(interval); } + // Caller-supplied SWIM failure-detection overrides: each left unset keeps the + // coordinator's own default. Lowering these speeds up failure detection (probe + // cadence, ack timeout, gossip cadence, and the suspicion timeout that scales + // with the probe interval). + if let Some(v) = self.probe_interval { + inner_opts = inner_opts.with_probe_interval(v); + } + if let Some(v) = self.probe_timeout { + inner_opts = inner_opts.with_probe_timeout(v); + } + if let Some(v) = self.gossip_interval { + inner_opts = inner_opts.with_gossip_interval(v); + } + if let Some(v) = self.suspicion_mult { + inner_opts = inner_opts.with_suspicion_mult(v); + } + if let Some(v) = self.suspicion_max_timeout_mult { + inner_opts = inner_opts.with_suspicion_max_timeout_mult(v); + } // Snapshot the reliable push/pull exchange timeout from the SAME options the // coordinator is built from, so the driver reconciles an await-result join's // caller deadline against the exact deadline the coordinator will stamp. diff --git a/serf-reactor/src/tls/mod.rs b/serf-reactor/src/tls/mod.rs index 568129ac..b9f927a9 100644 --- a/serf-reactor/src/tls/mod.rs +++ b/serf-reactor/src/tls/mod.rs @@ -112,6 +112,21 @@ pub struct TlsTransportOptions> { /// coordinator default; `Some(Duration::ZERO)` disables periodic push/pull /// entirely. See [`with_push_pull_interval`](Self::with_push_pull_interval). push_pull_interval: Option, + /// SWIM probe interval override. `None` keeps the coordinator default. See + /// [`with_probe_interval`](Self::with_probe_interval). + probe_interval: Option, + /// SWIM direct-ping timeout override. `None` keeps the coordinator default. See + /// [`with_probe_timeout`](Self::with_probe_timeout). + probe_timeout: Option, + /// Gossip interval override. `None` keeps the coordinator default. See + /// [`with_gossip_interval`](Self::with_gossip_interval). + gossip_interval: Option, + /// SWIM suspicion multiplier override. `None` keeps the coordinator default. See + /// [`with_suspicion_mult`](Self::with_suspicion_mult). + suspicion_mult: Option, + /// SWIM suspicion max-timeout multiplier override. `None` keeps the coordinator + /// default. See [`with_suspicion_max_timeout_mult`](Self::with_suspicion_max_timeout_mult). + suspicion_max_timeout_mult: Option, /// Gossip encryption policy. The default (no keyring) leaves the gossip /// datagrams plaintext; attaching a keyring via /// [`with_encryption`](Self::with_encryption) makes the coordinator's @@ -138,6 +153,11 @@ impl TlsTransportOptions { sni_provider: Box::new(|_addr: &SocketAddr| Some("localhost".to_string())), tls_options: None, push_pull_interval: None, + probe_interval: None, + probe_timeout: None, + gossip_interval: None, + suspicion_mult: None, + suspicion_max_timeout_mult: None, #[cfg(encryption)] encryption: EncryptionOptions::new(), } @@ -201,6 +221,66 @@ impl TlsTransportOptions { self } + /// Builder: override the memberlist SWIM probe interval — how often the + /// coordinator probes a random peer for liveness. + /// + /// `None` (the default) keeps the coordinator default (~1s). A shorter interval + /// speeds failure detection at the cost of more probe traffic; it also shortens + /// the suspicion timeout, which scales with the probe interval. + #[must_use] + #[inline] + pub const fn with_probe_interval(mut self, interval: Duration) -> Self { + self.probe_interval = Some(interval); + self + } + + /// Builder: override the memberlist SWIM direct-ping timeout — how long the + /// coordinator waits for a probe ack before escalating to indirect probes. + /// + /// `None` (the default) keeps the coordinator default (~500ms). It must + /// comfortably exceed the real network round-trip, or a live peer whose ack is + /// merely slow is falsely suspected. + #[must_use] + #[inline] + pub const fn with_probe_timeout(mut self, timeout: Duration) -> Self { + self.probe_timeout = Some(timeout); + self + } + + /// Builder: override the memberlist gossip interval — how often the coordinator + /// flushes queued gossip to a random subset of peers. + /// + /// `None` (the default) keeps the coordinator default (~200ms). + #[must_use] + #[inline] + pub const fn with_gossip_interval(mut self, interval: Duration) -> Self { + self.gossip_interval = Some(interval); + self + } + + /// Builder: override the memberlist SWIM suspicion multiplier — how long a + /// suspected peer is held in the Suspect state before being declared Failed. + /// + /// The minimum suspicion timeout is `suspicion_mult * log10(N+1) * probe_interval`. + /// `None` (the default) keeps the coordinator default. + #[must_use] + #[inline] + pub const fn with_suspicion_mult(mut self, mult: u32) -> Self { + self.suspicion_mult = Some(mult); + self + } + + /// Builder: override the memberlist SWIM suspicion max-timeout multiplier — the + /// upper bound on the suspicion timeout as a multiple of the minimum. + /// + /// `None` (the default) keeps the coordinator default. + #[must_use] + #[inline] + pub const fn with_suspicion_max_timeout_mult(mut self, mult: u32) -> Self { + self.suspicion_max_timeout_mult = Some(mult); + self + } + /// Builder: gossip-encryption policy. /// /// The default (no keyring) keeps the gossip datagrams plaintext, so an @@ -245,6 +325,36 @@ impl TlsTransportOptions { self.push_pull_interval } + /// The SWIM probe-interval override, if set. + #[inline] + pub const fn probe_interval(&self) -> Option { + self.probe_interval + } + + /// The SWIM probe-timeout override, if set. + #[inline] + pub const fn probe_timeout(&self) -> Option { + self.probe_timeout + } + + /// The gossip-interval override, if set. + #[inline] + pub const fn gossip_interval(&self) -> Option { + self.gossip_interval + } + + /// The SWIM suspicion-multiplier override, if set. + #[inline] + pub const fn suspicion_mult(&self) -> Option { + self.suspicion_mult + } + + /// The SWIM suspicion max-timeout-multiplier override, if set. + #[inline] + pub const fn suspicion_max_timeout_mult(&self) -> Option { + self.suspicion_max_timeout_mult + } + /// SNI provider closure. #[inline] pub fn sni_provider(&self) -> &(dyn Fn(&SocketAddr) -> Option + Send + Sync) { @@ -300,6 +410,14 @@ where /// [`Transport::run`]. `None` keeps the default; `Some(Duration::ZERO)` disables /// periodic anti-entropy. push_pull_interval: Option, + /// SWIM failure-detection overrides applied to the coordinator's + /// `EndpointOptions` in [`Transport::run`]. Each `None` keeps the coordinator + /// default. + probe_interval: Option, + probe_timeout: Option, + gossip_interval: Option, + suspicion_mult: Option, + suspicion_max_timeout_mult: Option, /// Independent OS-seeded seed for the serf core's RNG, drawn once per node in /// [`Transport::new`] and consumed when [`Transport::run`] builds the endpoint /// via `new_with_rng`. Distinct from the coordinator's gossip RNG so serf's @@ -432,6 +550,11 @@ where sni_provider: options.sni_provider, tls_options, push_pull_interval: options.push_pull_interval, + probe_interval: options.probe_interval, + probe_timeout: options.probe_timeout, + gossip_interval: options.gossip_interval, + suspicion_mult: options.suspicion_mult, + suspicion_max_timeout_mult: options.suspicion_max_timeout_mult, serf_rng, #[cfg(encryption)] encryption: options.encryption, @@ -470,6 +593,25 @@ where if let Some(interval) = self.push_pull_interval { inner_opts = inner_opts.with_push_pull_interval(interval); } + // Caller-supplied SWIM failure-detection overrides: each left unset keeps the + // coordinator's own default. Lowering these speeds up failure detection (probe + // cadence, ack timeout, gossip cadence, and the suspicion timeout that scales + // with the probe interval). + if let Some(v) = self.probe_interval { + inner_opts = inner_opts.with_probe_interval(v); + } + if let Some(v) = self.probe_timeout { + inner_opts = inner_opts.with_probe_timeout(v); + } + if let Some(v) = self.gossip_interval { + inner_opts = inner_opts.with_gossip_interval(v); + } + if let Some(v) = self.suspicion_mult { + inner_opts = inner_opts.with_suspicion_mult(v); + } + if let Some(v) = self.suspicion_max_timeout_mult { + inner_opts = inner_opts.with_suspicion_max_timeout_mult(v); + } // Snapshot the reliable push/pull exchange timeout from the SAME options the // coordinator is built from, so the driver reconciles an await-result join's // caller deadline against the exact deadline the coordinator will stamp. From 0365fe0000c68bf370a47211c98ca7260ddfeb45 Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 00:06:26 +1200 Subject: [PATCH 081/128] test(reactor): add a reusable multi-node fault-injection cluster fixture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add `tests/cluster` — a runtime-generic fixture the real-node lifecycle tests share across the tokio and smol cells. `Cluster::spawn` brings up N ephemeral loopback nodes with fast SWIM failure-detection timing (via the transport probe/gossip/suspicion overrides) plus fast serf reap/reconnect timing, joins them into a converged star, and attaches a detached per-node event collector draining each node's stream into a separate log. `kill_abrupt` shuts a node down — which discards the teardown's best-effort leave, so peers detect a probe-timeout Failed rather than a graceful Leave — while retaining the slot's id, addr, and log for a later `restart` at the same address. The event log is a separate `Arc`, so it survives the last-handle drop and the ordered member-event assertions (`assert_member_events`, `await_member_event`) read a stable per-node history even after the observed node departs. --- serf-reactor/tests/cluster/mod.rs | 388 ++++++++++++++++++++++++++++++ 1 file changed, 388 insertions(+) create mode 100644 serf-reactor/tests/cluster/mod.rs diff --git a/serf-reactor/tests/cluster/mod.rs b/serf-reactor/tests/cluster/mod.rs new file mode 100644 index 00000000..71f78192 --- /dev/null +++ b/serf-reactor/tests/cluster/mod.rs @@ -0,0 +1,388 @@ +//! Reusable multi-node fault-injection fixture for the reactor's real-node serf +//! driver, shared by the runtime-generic scenario bodies in the sibling test +//! binaries. +//! +//! A [`Cluster`] spins up N ephemeral loopback nodes through the ergonomic +//! `Serf::tcp` constructor with fast SWIM failure-detection timing (the transport +//! probe / gossip / suspicion overrides), so an abruptly-killed peer is detected +//! as Failed in well under a second. Each node runs a detached collector that +//! drains its event stream into a shared log; the log is a separate `Arc`, so it +//! survives an abrupt [`kill_abrupt`](Cluster::kill_abrupt) (which drops the +//! node's last handle) and later assertions read an ordered per-node member-event +//! history. +//! +//! Mirrors the legacy Go-parity cluster helpers — `wait_until_num_nodes` / +//! `test_events` in `legacy/serf-core/src/serf/base/tests.rs` — adapted to the +//! reactor's `Send`/`agnostic` model. + +use core::time::Duration; +use std::{ + net::SocketAddr, + sync::{Arc, Mutex}, +}; + +use agnostic::Runtime; +use futures_util::StreamExt; +use serf_proto::{ + event::{Event, MemberEventKind}, + options::Options as SerfOptions, +}; +use serf_reactor::{ + FirstAddrResolver, MaybeResolved, RuntimeOptions, Serf, SocketAddrResolver, TcpTransportOptions, + VoidDelegate, +}; +use smol_str::SmolStr; + +/// A reactor TCP node handle over the agnostic runtime `R`. +pub type Node = Serf; + +/// Wall-clock ceiling for every fixture poll loop, so a convergence or detection +/// regression surfaces as a bounded timeout rather than a hang. +const POLL_TIMEOUT: Duration = Duration::from_secs(20); +/// Poll granularity for the fixture's await loops. +const POLL_STEP: Duration = Duration::from_millis(20); + +/// Failure-detection and reap timing shared by every node in a fault-injection +/// cluster. +/// +/// The probe / gossip / suspicion knobs tune the memberlist SWIM layer (carried +/// on the transport options); the reap / reconnect knobs tune the serf reaper +/// (carried on the serf `Options`). [`fast`](Self::fast) yields CI-speed values +/// that detect an abrupt kill in sub-second time and reap the failed member +/// shortly after (short reconnect timeout). A test that intends to reconnect a +/// killed node raises the reconnect timeout via +/// [`with_reconnect_timeout`](Self::with_reconnect_timeout) so the failed member +/// is held — not reaped — until it rejoins. +#[derive(Clone)] +pub struct ClusterTiming { + probe_interval: Duration, + probe_timeout: Duration, + gossip_interval: Duration, + suspicion_mult: u32, + reap_interval: Duration, + reconnect_interval: Duration, + reconnect_timeout: Duration, + tombstone_timeout: Duration, +} + +impl ClusterTiming { + /// CI-speed timing: sub-second SWIM failure detection on loopback with the + /// failed member reaped shortly after. The probe timeout stays generous + /// relative to a loopback round-trip so a live peer is not falsely suspected + /// under CI scheduling jitter, while the suspicion timeout (fixed at the minimum + /// for a small cluster: `suspicion_mult * probe_interval`) stays short. + pub fn fast() -> Self { + Self { + probe_interval: Duration::from_millis(100), + probe_timeout: Duration::from_millis(100), + gossip_interval: Duration::from_millis(20), + suspicion_mult: 1, + reap_interval: Duration::from_millis(100), + reconnect_interval: Duration::from_millis(100), + reconnect_timeout: Duration::from_millis(1), + tombstone_timeout: Duration::from_millis(1), + } + } + + /// Override the reconnect timeout — the age at which the reaper removes a Failed + /// member. Raise it well beyond the test's kill-to-restart window to hold a + /// failed peer for reconnection instead of reaping it. + #[must_use] + pub fn with_reconnect_timeout(mut self, v: Duration) -> Self { + self.reconnect_timeout = v; + self + } + + /// The transport options for one node: the fast SWIM overrides plus this node's + /// id and advertise address. + fn transport_opts( + &self, + id: &str, + advertise: SocketAddr, + ) -> TcpTransportOptions { + TcpTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(advertise)) + .with_probe_interval(self.probe_interval) + .with_probe_timeout(self.probe_timeout) + .with_gossip_interval(self.gossip_interval) + .with_suspicion_mult(self.suspicion_mult) + } + + /// The serf `Options` for every node: the fast reap / reconnect timing. + fn serf_opts(&self) -> SerfOptions { + SerfOptions::new() + .with_reap_interval(self.reap_interval) + .with_reconnect_interval(self.reconnect_interval) + .with_reconnect_timeout(self.reconnect_timeout) + .with_tombstone_timeout(self.tombstone_timeout) + } +} + +/// One observed member event: its kind and the member ids it names. +#[derive(Clone)] +struct MemberRec { + kind: MemberEventKind, + ids: Vec, +} + +/// One cluster node: its stable id and advertise address, the live handle (taken +/// while killed), and the event log the collector drains into. +struct NodeSlot +where + R: Runtime, +{ + id: SmolStr, + addr: SocketAddr, + serf: Option>, + log: Arc>>, +} + +/// A live multi-node loopback cluster with per-node member-event logs. +pub struct Cluster +where + R: Runtime, +{ + timing: ClusterTiming, + slots: Vec>, +} + +impl Cluster +where + R: Runtime, +{ + /// Spawn `ids.len()` ephemeral loopback nodes with `timing`, attach a per-node + /// event collector, join every non-seed node to the first (a star), and wait + /// for the whole cluster to converge. + pub async fn spawn(ids: &[&str], timing: ClusterTiming) -> Self { + let mut slots = Vec::with_capacity(ids.len()); + for id in ids { + let serf = build_node::(id, loopback_ephemeral(), &timing) + .await + .expect("spawn serf tcp node"); + let addr = serf.advertise_address(); + let log = Arc::new(Mutex::new(Vec::new())); + // Attach the collector before the handle moves into the slot, so no member + // event can slip past between construction and the first join. + attach_collector::(&serf, log.clone()); + slots.push(NodeSlot { + id: SmolStr::new(*id), + addr, + serf: Some(serf), + log, + }); + } + let cluster = Self { timing, slots }; + let seed = cluster.slots[0].addr; + for i in 1..cluster.slots.len() { + cluster + .node(i) + .join(&SocketAddrResolver, MaybeResolved::Resolved(seed), false) + .await + .expect("join reaches the seed node"); + } + cluster.converge(cluster.slots.len()).await; + cluster + } + + /// The live handle for node `i` (panics if the node is currently killed). + pub fn node(&self, i: usize) -> &Node { + self.slots[i].serf.as_ref().expect("node slot is live") + } + + /// The stable id of node `i`. + pub fn id(&self, i: usize) -> SmolStr { + self.slots[i].id.clone() + } + + /// Abruptly kill node `i`: shut its handle down, which discards the teardown's + /// best-effort leave (the gossip socket drops before the leave datagram can be + /// transmitted), so peers detect a probe-timeout Failed rather than a graceful + /// Leave. The slot's id, addr, and event log are retained for a later restart or + /// assertion; the freed port is released before `shutdown` resolves. + pub async fn kill_abrupt(&mut self, i: usize) { + let serf = self.slots[i].serf.take().expect("node slot is live"); + serf.shutdown().await.expect("node shuts down"); + } + + /// Restart a previously-killed node `i` at the SAME id and advertise address, + /// re-attaching a collector to the slot's existing log. The freed port is + /// rebound with a bounded retry to absorb a transient rebind race. + pub async fn restart(&mut self, i: usize) { + assert!( + self.slots[i].serf.is_none(), + "node {i} must be killed before restart" + ); + let id = self.slots[i].id.clone(); + let addr = self.slots[i].addr; + const REBIND_RETRIES: usize = 25; + let mut attempt = 0usize; + let serf = loop { + match build_node::(id.as_str(), addr, &self.timing).await { + Ok(serf) => break serf, + // Ignoring Err: a transient rebind race (the freed port not yet reusable) + // is retried; only the final attempt's error is fatal. + Err(_) if attempt + 1 < REBIND_RETRIES => { + attempt += 1; + R::sleep(POLL_STEP).await; + } + Err(e) => panic!("restart rebind for {id:?} at {addr} failed: {e}"), + } + }; + attach_collector::(&serf, self.slots[i].log.clone()); + self.slots[i].serf = Some(serf); + } + + /// Poll every live node until each reports exactly `expect` members, or fail on + /// the poll timeout. + pub async fn converge(&self, expect: usize) { + R::timeout(POLL_TIMEOUT, async { + loop { + if self + .slots + .iter() + .filter_map(|s| s.serf.as_ref()) + .all(|n| n.num_members() == expect) + { + break; + } + R::sleep(POLL_STEP).await; + } + }) + .await + .expect("cluster converges to the expected member count"); + } + + /// Poll until node `observer` reports exactly `expect` members. + pub async fn await_num_members(&self, observer: usize, expect: usize) { + R::timeout(POLL_TIMEOUT, async { + loop { + if self.node(observer).num_members() == expect { + break; + } + R::sleep(POLL_STEP).await; + } + }) + .await + .expect("observer reaches the expected member count"); + } + + /// Poll until `observer`'s log records a member event of `kind` naming + /// `subject`. + pub async fn await_member_event(&self, observer: usize, subject: &str, kind: MemberEventKind) { + R::timeout(POLL_TIMEOUT, async { + loop { + if self.member_event_kinds(observer, subject).contains(&kind) { + break; + } + R::sleep(POLL_STEP).await; + } + }) + .await + .expect("observer records the expected member event"); + } + + /// Poll until the ordered member-event kinds `observer` recorded about `subject` + /// are at least as long as `expected`, then assert exact equality. Polling first + /// lets a still-in-flight event land; the final assert then catches a wrong, + /// missing, or extra event. + pub async fn assert_member_events( + &self, + observer: usize, + subject: &str, + expected: &[MemberEventKind], + ) { + // Ignoring Err: a poll timeout here just means fewer events than expected + // arrived; the assert_eq below reports the precise sequence mismatch. + let _ = R::timeout(POLL_TIMEOUT, async { + loop { + if self.member_event_kinds(observer, subject).len() >= expected.len() { + break; + } + R::sleep(POLL_STEP).await; + } + }) + .await; + let actual = self.member_event_kinds(observer, subject); + assert_eq!( + actual, expected, + "member events for {subject:?} observed by node {observer}" + ); + } + + /// Shut down every live node, releasing all bound ports. + pub async fn shutdown_all(&mut self) { + for slot in &mut self.slots { + if let Some(serf) = slot.serf.take() { + serf.shutdown().await.expect("node shuts down"); + } + } + } + + /// The ordered member-event kinds `observer` recorded about `subject`. + fn member_event_kinds(&self, observer: usize, subject: &str) -> Vec { + self.slots[observer] + .log + .lock() + .expect("event log lock") + .iter() + .filter(|rec| rec.ids.iter().any(|id| id.as_str() == subject)) + .map(|rec| rec.kind) + .collect() + } +} + +/// An ephemeral loopback bind address (`127.0.0.1:0`). +fn loopback_ephemeral() -> SocketAddr { + "127.0.0.1:0".parse().expect("loopback addr") +} + +/// Spawn a fixture node at `bind` (an ephemeral `:0` for a fresh node, or a +/// concrete addr for a restart) with `timing`. +async fn build_node( + id: &str, + bind: SocketAddr, + timing: &ClusterTiming, +) -> serf_reactor::Result> +where + R: Runtime, +{ + Serf::::tcp( + timing.transport_opts(id, bind), + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + timing.serf_opts(), + None, + #[cfg(encryption)] + std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), + ) + .await +} + +/// Attach a detached collector that drains `serf`'s event stream into `log`, +/// recording every member event (kind + named ids). The collector holds only the +/// stream, so a later kill (last-handle drop) still tears the node down while the +/// log persists. +fn attach_collector(serf: &Node, log: Arc>>) +where + R: Runtime, +{ + let mut stream = serf.events(); + R::spawn_detach(async move { + while let Some(ev) = stream.next().await { + if let Event::Member(me) = ev { + let ids = me + .members() + .iter() + .map(|m| m.node().id_ref().clone()) + .collect(); + log.lock().expect("event log lock").push(MemberRec { + kind: me.kind(), + ids, + }); + } + } + }); +} From e9d63d1b66a70e6fab3b1787d7762d1f6e58cff9 Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 00:06:36 +1200 Subject: [PATCH 082/128] test(reactor): port serf_events_failed and serf_reconnect over the fault fixture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wire the cluster fixture into the tcp real-node harness and add the first two failure-detection lifecycle ports, each running on both the tokio and smol cells: - serf_events_failed: two nodes join, node B is abruptly killed, and node A observes Join -> Failed -> Reap about B — a Failed (not a Leave), proving the kill discards the graceful-leave datagram, then the reaper removes the failed member under the shortened reconnect timeout. - serf_reconnect: node B is killed, detected Failed, then restarted at the same id and address; node A observes Join -> Failed -> Join as the reconnect loop re-dials B and it rejoins (the reconnect timeout is raised past the kill-to-restart window so the failed member is held, not reaped). Both mirror the Go-parity legacy behaviors under legacy/serf-core/src/serf/base/tests/serf/{event,reconnect}.rs. --- serf-reactor/tests/tcp.rs | 115 +++++++++++++++++++++++++++++++++++++- 1 file changed, 112 insertions(+), 3 deletions(-) diff --git a/serf-reactor/tests/tcp.rs b/serf-reactor/tests/tcp.rs index f24f0d1a..812c9133 100644 --- a/serf-reactor/tests/tcp.rs +++ b/serf-reactor/tests/tcp.rs @@ -23,9 +23,11 @@ use std::net::SocketAddr; use agnostic::Runtime; use bytes::Bytes; use futures_util::{StreamExt, future}; -#[cfg(encryption)] -use serf_proto::event::MemberEventKind; -use serf_proto::{event::Event, members::SerfState, options::Options as SerfOptions}; +use serf_proto::{ + event::{Event, MemberEventKind}, + members::SerfState, + options::Options as SerfOptions, +}; #[cfg(encryption)] use serf_reactor::{EncryptionOptions, Keyring, SecretKey, VoidKeyringDelegate}; use serf_reactor::{ @@ -34,6 +36,10 @@ use serf_reactor::{ }; use smol_str::SmolStr; +/// The reusable multi-node fault-injection fixture, shared by the tokio and smol +/// cells below. +mod cluster; + /// A reactor TCP node handle over the agnostic runtime `R`. type Node = Serf; @@ -471,6 +477,89 @@ where b.shutdown().await.expect("rfn-b shuts down"); } +/// Two nodes on loopback: node A joins node B, then B is abruptly killed. Node A +/// must observe the member-event sequence Join → Failed → Reap about B: a Failed +/// (not a Leave), proving the kill discards the graceful-leave datagram, followed +/// by the reaper removing the failed member under the shortened reconnect timeout. +/// +/// Mirrors Go serf `serf_events_failed` +/// (`legacy/serf-core/src/serf/base/tests/serf/event.rs`), whose `test_events` +/// asserts the exact ordered member-event sequence about the shut-down node. +async fn serf_events_failed() +where + R: Runtime, +{ + let mut cluster = cluster::Cluster::::spawn( + &["events-failed-a", "events-failed-b"], + cluster::ClusterTiming::fast(), + ) + .await; + let subject = cluster.id(1); + + cluster.kill_abrupt(1).await; + // A drops back to a single member once B is detected Failed and then reaped. + cluster.await_num_members(0, 1).await; + + cluster + .assert_member_events( + 0, + subject.as_str(), + &[ + MemberEventKind::Join, + MemberEventKind::Failed, + MemberEventKind::Reap, + ], + ) + .await; + + cluster.shutdown_all().await; +} + +/// Two nodes on loopback: node A joins node B, B is abruptly killed and detected +/// Failed, then B is restarted at the same id and advertise address. Node A must +/// observe the sequence Join → Failed → Join about B — the failed member +/// reconnects rather than being reaped, because the reconnect timeout is raised +/// past the kill-to-restart window while the reconnect loop re-dials B. +/// +/// Mirrors Go serf `serf_reconnect` +/// (`legacy/serf-core/src/serf/base/tests/serf/reconnect.rs`). +async fn serf_reconnect() +where + R: Runtime, +{ + let mut cluster = cluster::Cluster::::spawn( + &["reconnect-a", "reconnect-b"], + cluster::ClusterTiming::fast().with_reconnect_timeout(Duration::from_secs(30)), + ) + .await; + let subject = cluster.id(1); + + cluster.kill_abrupt(1).await; + // Wait for A to detect B's failure before B returns, so the Failed event is + // recorded distinctly from the later rejoin. + cluster + .await_member_event(0, subject.as_str(), MemberEventKind::Failed) + .await; + + cluster.restart(1).await; + // The serf reconnect loop re-dials the restarted B, which rejoins the cluster. + cluster.await_num_members(0, 2).await; + + cluster + .assert_member_events( + 0, + subject.as_str(), + &[ + MemberEventKind::Join, + MemberEventKind::Failed, + MemberEventKind::Join, + ], + ) + .await; + + cluster.shutdown_all().await; +} + /// A deterministic test secret key, selecting whichever AEAD cipher this build /// compiled so the encrypted tests work under either backend. #[cfg(encryption)] @@ -666,6 +755,16 @@ mod tokio_cells { super::remove_failed_node_alias_succeeds::().await; } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_events_failed() { + super::serf_events_failed::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_reconnect() { + super::serf_reconnect::().await; + } + #[cfg(encryption)] #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn two_node_join_converges_encrypted() { @@ -730,6 +829,16 @@ mod smol_cells { SmolRuntime::block_on(super::remove_failed_node_alias_succeeds::()); } + #[test] + fn serf_events_failed_smol() { + SmolRuntime::block_on(super::serf_events_failed::()); + } + + #[test] + fn serf_reconnect_smol() { + SmolRuntime::block_on(super::serf_reconnect::()); + } + #[cfg(encryption)] #[test] fn two_node_join_converges_encrypted_smol() { From 8d380c4dc8b283f24c4c39bbb013aa364be93135 Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 00:19:04 +1200 Subject: [PATCH 083/128] test(reactor): give the cluster fixture's fast timing a CI scheduling margin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit probe_timeout now sits below probe_interval so an unanswered probe keeps an indirect window inside its own cycle, and suspicion_mult 3 raises the small-cluster suspicion floor to ~300ms — a live peer survives a couple hundred milliseconds of executor starvation on an oversubscribed runner without a false Failed, while real-kill detection stays sub-second. --- serf-reactor/tests/cluster/mod.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/serf-reactor/tests/cluster/mod.rs b/serf-reactor/tests/cluster/mod.rs index 71f78192..b2faa193 100644 --- a/serf-reactor/tests/cluster/mod.rs +++ b/serf-reactor/tests/cluster/mod.rs @@ -71,12 +71,19 @@ impl ClusterTiming { /// relative to a loopback round-trip so a live peer is not falsely suspected /// under CI scheduling jitter, while the suspicion timeout (fixed at the minimum /// for a small cluster: `suspicion_mult * probe_interval`) stays short. + /// + /// The probe timeout sits BELOW the probe interval so an unanswered probe still + /// has an indirect/fallback window inside its own cycle, and the suspicion + /// multiplier keeps the small-cluster suspicion floor at ~300 ms — a live peer + /// survives a couple hundred milliseconds of executor starvation on an + /// oversubscribed CI runner without being falsely declared Failed, while + /// detection of a real kill stays comfortably sub-second. pub fn fast() -> Self { Self { probe_interval: Duration::from_millis(100), - probe_timeout: Duration::from_millis(100), + probe_timeout: Duration::from_millis(50), gossip_interval: Duration::from_millis(20), - suspicion_mult: 1, + suspicion_mult: 3, reap_interval: Duration::from_millis(100), reconnect_interval: Duration::from_millis(100), reconnect_timeout: Duration::from_millis(1), From 358a4e07606bb9c543c1d04050d1d33dbedf203e Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 02:04:12 +1200 Subject: [PATCH 084/128] refactor(serf-proto): drop the vestigial leave-broadcast wait machinery The inner memberlist leave() now transports queued user broadcasts atomically with the dead-self fan-out: the rank-0 leave intent rides the farewell compound, user parts ahead of the death notice, so peers process the intent before the membership death and read the departure as intentional. The machine's leave() therefore no longer needs a separate wait for the intent to flush. Remove leave_broadcast_deadline and its whole surface: the poll_timeout fold, the after_inner_timeout clear arm, the arming steps in leave()/force_leave(), the public accessor, and the stream/quic super-machine forwarders. Remove the now-unused broadcast_timeout Option (a breaking change): field, default, getter, builder, and its option tests. leave() keeps its exact order (LTIME_MAX gate, clock commit, state=Leaving before the local intent, rank-0 broadcast, synchronous inner leave). Drop the three deadline unit tests; the rest of the leave-FSM suite is unchanged. --- serf-proto/src/endpoint/mod.rs | 78 ++++++++++----------------- serf-proto/src/endpoint/tests.rs | 56 ------------------- serf-proto/src/options/mod.rs | 14 ----- serf-proto/src/options/tests.rs | 2 - serf-proto/src/quic_endpoint/mod.rs | 12 +---- serf-proto/src/stream_endpoint/mod.rs | 12 +---- 6 files changed, 33 insertions(+), 141 deletions(-) diff --git a/serf-proto/src/endpoint/mod.rs b/serf-proto/src/endpoint/mod.rs index 8f2715f3..060ec886 100644 --- a/serf-proto/src/endpoint/mod.rs +++ b/serf-proto/src/endpoint/mod.rs @@ -707,14 +707,6 @@ where next_reconnect: Option, /// Next deadline at which the broadcast queue depth is checked. next_queue_check: Option, - /// Deadline after which we stop waiting for the leave-intent broadcast to - /// propagate and proceed to call the inner `leave()`. - /// - /// Armed in `leave()` at `now + broadcast_timeout`. The driver may - /// short-circuit by polling `user_broadcast_queue_len()` reaching zero; - /// the machine itself fires unconditionally when the deadline elapses. - /// `None` when not in the middle of a graceful leave. - leave_broadcast_deadline: Option, /// Deadline after which the `Leaving → Left` transition fires. /// /// Armed in the `LeftCluster` sieve arm at `now + leave_propagate_delay`. @@ -952,7 +944,6 @@ where next_reap: Some(first_reap), next_reconnect: Some(first_reconnect), next_queue_check: Some(first_queue_check), - leave_broadcast_deadline: None, leave_complete_deadline: None, event_buffer: EventBuffer::new(event_buf_size), query_buffer: QueryBuffer::new(query_buf_size), @@ -1148,7 +1139,7 @@ where /// The earliest serf-level deadline requiring a `handle_timeout` call. /// /// Returns the minimum of serf's own periodic deadlines (reap, reconnect, - /// queue-check, leave-broadcast, leave-complete, and pending-query closes). + /// queue-check, leave-complete, and pending-query closes). /// The composing super-machine folds in the coordinator's own deadline. /// /// A shut-down machine (lost id-conflict vote) schedules no wakeups: its @@ -1174,7 +1165,6 @@ where self.next_reap, self.next_reconnect, self.next_queue_check, - self.leave_broadcast_deadline, self.leave_complete_deadline, query_min, member_flush, @@ -1298,10 +1288,10 @@ where // A lost id-conflict vote in the query-close pass transitions the machine to // Shutdown. Nothing may follow the terminal Event::Shutdown, so skip the rest - // of this pass's deadline work (received-query prune, leave-complete/broadcast - // clears): it would prune protocol state or transition after the terminal + // of this pass's deadline work (received-query prune, leave-complete + // transition): it would prune protocol state or transition after the terminal // event. `serf_poll_timeout` returns `None` once Shutdown, so the un-cleared - // leave deadlines never respin the driver. + // leave-complete deadline never respins the driver. if self.state.is_shutdown() { return; } @@ -1323,14 +1313,6 @@ where } } - // Leave-broadcast: clear the deadline once it has elapsed so poll_timeout - // does not keep returning it and causing driver spin. - if let Some(dl) = self.leave_broadcast_deadline { - if now >= dl { - self.leave_broadcast_deadline = None; - } - } - // Flush any coalescer whose window has closed, delivering the coalesced batch // via `pending_events`. A member/user event fed earlier this tick (during // `drain_inner` / `fire_reap`) arms a future deadline, so it is NOT flushed @@ -2331,8 +2313,8 @@ where /// /// Go serf's `handle_prune` only removes a `Leaving`/`Left` node from /// `left_members` (relying on the invariant that a node is never in both - /// lists) and sleeps `broadcast_timeout + leave_propagate_delay` for a - /// `Leaving` member before erasing. The Sans-I/O machine cannot sleep, so + /// lists) and sleeps its broadcast timeout plus the leave-propagate delay for + /// a `Leaving` member before erasing. The Sans-I/O machine cannot sleep, so /// the prune is immediate; it also scrubs every index list and the /// recent-intent entry so no stale reference to the forgotten node survives /// in any structure. @@ -2607,14 +2589,17 @@ where /// (`handle_node_leave_intent` for the local id), which marks the local /// node as `Leaving` in the membership store and queues a join-refute /// suppression. - /// 3. Enqueue a leave-intent broadcast on the intent tier (rank 0) so - /// peers learn about the leave. (`FIX`: no `on_finished` callback — - /// flushing is bounded by `broadcast_timeout` deadline instead.) - /// 4. Call inner `leave(now)` to begin the memberlist dead-self fan-out. - /// The inner will eventually emit `Event::LeftCluster` once all dead-self - /// packets are drained via `poll_transmit`. - /// 5. Arm `leave_broadcast_deadline = now + broadcast_timeout`. The - /// driver can short-circuit by watching `user_broadcast_queue_len()`. + /// 3. Enqueue the leave-intent broadcast on the intent tier (rank 0) so peers + /// learn the local node is leaving. + /// 4. Call inner `leave(now)` synchronously. The inner packs the payloads + /// still queued on the user-broadcast tiers — the rank-0 intent just + /// enqueued — into its dead-self fan-out, user parts ahead of the death + /// notice, so every farewell recipient receives the intent ATOMICALLY with + /// the dead-self notice in one datagram and processes the intent first, + /// classifying the departure as intentional rather than a failure. The + /// inner emits `Event::LeftCluster` once all dead-self packets drain via + /// `poll_transmit`. No separate wait for the intent to flush exists: the + /// queued intent departs with the dead-self frame, not on its own schedule. /// /// The `Leaving → Left` transition happens later in `handle_timeout` when /// `leave_complete_deadline` (armed on inner `LeftCluster` + `leave_propagate_delay`) @@ -2680,15 +2665,15 @@ where } // 3. Broadcast the leave intent on the intent tier (rank 0) so peers learn - // the local node is leaving without waiting for anti-entropy. The - // driver bounds the flush via the broadcast deadline below. + // the local node is leaving without waiting for anti-entropy. self.broadcast_leave(t, ltime, local_id, false); - // 4. Arm the broadcast-timeout deadline so the driver always has a finite - // wait; it can short-circuit by watching `user_broadcast_queue_len()`. - self.leave_broadcast_deadline = Some(now + self.opts.broadcast_timeout()); - - // 5. Call inner leave; this queues the dead-self fan-out packets. + // 4. Call inner leave synchronously. It packs the rank-0 intent just queued + // (with any other pending user broadcast) into its dead-self fan-out — + // user parts ahead of the death notice — so every farewell recipient + // receives the intent atomically with the dead-self notice and processes + // it first, reading the departure as intentional. This queues the + // resulting fan-out packets for `poll_transmit`. t.leave(now)?; Ok(()) @@ -2703,6 +2688,11 @@ where /// /// Does not require the local endpoint to be `Alive` (callers may want to /// clean up failed nodes before leaving themselves), but rejects `Shutdown`. + /// + /// A leave intent queued here shortly before a local [`leave`](Self::leave) + /// now rides that leave's dead-self fan-out — the inner `leave()` packs the + /// still-pending user broadcasts into its farewell frames — instead of being + /// dropped by the pre-fan-out queue reset. pub(crate) fn force_leave( &mut self, t: &mut T, @@ -2750,9 +2740,6 @@ where // same forced removal. self.broadcast_leave(t, ltime, id, prune); - // Arm the broadcast deadline so the driver knows how long to wait. - self.leave_broadcast_deadline = Some(now + self.opts.broadcast_timeout()); - Ok(()) } @@ -4115,13 +4102,6 @@ where } } - /// Returns the `leave_broadcast_deadline`, if armed. - /// - /// `None` when not in the middle of a graceful leave or force-leave. - pub const fn leave_broadcast_deadline(&self) -> Option { - self.leave_broadcast_deadline - } - /// Returns the `leave_complete_deadline`, if armed. /// /// `None` until the inner `LeftCluster` event has been received. diff --git a/serf-proto/src/endpoint/tests.rs b/serf-proto/src/endpoint/tests.rs index 89839d55..ee7add74 100644 --- a/serf-proto/src/endpoint/tests.rs +++ b/serf-proto/src/endpoint/tests.rs @@ -609,21 +609,6 @@ fn shutdown_prevents_leaving_to_left_transition() { ); } -#[test] -fn leave_arms_broadcast_deadline() { - let mut e = ep(); - assert!(e.leave_broadcast_deadline().is_none()); - e.leave(memberlist_proto::Instant::ORIGIN).unwrap(); - let dl = e - .leave_broadcast_deadline() - .expect("broadcast deadline must be armed"); - // Default broadcast_timeout is 5s; deadline = ORIGIN + 5s. - assert_eq!( - dl, - memberlist_proto::Instant::ORIGIN + core::time::Duration::from_secs(5) - ); -} - #[test] fn force_leave_from_shutdown_is_rejected() { let mut e = ep(); @@ -716,25 +701,6 @@ fn leave_witnesses_and_advances_member_clock() { ); } -#[test] -fn poll_timeout_includes_leave_deadlines_when_armed() { - let mut e = ep(); - // Initially no serf deadlines; poll_timeout may be None (inner idle) or - // Some from inner's own schedule — just confirm it does not panic. - let _ = e.poll_timeout(); - - e.leave(memberlist_proto::Instant::ORIGIN).unwrap(); - // After leave(), leave_broadcast_deadline is armed. - let timeout = e - .poll_timeout() - .expect("must have a deadline after leave()"); - let expected = memberlist_proto::Instant::ORIGIN + core::time::Duration::from_secs(5); - assert!( - timeout <= expected, - "poll_timeout must be ≤ leave_broadcast_deadline ({expected:?}), got {timeout:?}" - ); -} - // ── Task 1.6: reconnector dial-output + reaper deadlines ───────────────────── // Helper: seed a failed member with an explicit address so we can assert what @@ -3426,28 +3392,6 @@ fn stale_self_leave_does_not_trigger_refute() { ); } -// Bug 5: Leave-broadcast deadline never retired. -#[test] -fn leave_broadcast_deadline_is_cleared_after_expiry() { - let mut e = ep(); - // leave() arms leave_broadcast_deadline = ORIGIN + broadcast_timeout (5s). - e.leave(memberlist_proto::Instant::ORIGIN).unwrap(); - assert!( - e.leave_broadcast_deadline().is_some(), - "leave_broadcast_deadline must be armed after leave()" - ); - - // Tick past the deadline. - let past = memberlist_proto::Instant::ORIGIN + core::time::Duration::from_secs(6); - e.handle_timeout(past); - - // The deadline must be cleared. - assert!( - e.leave_broadcast_deadline().is_none(), - "leave_broadcast_deadline must be cleared after expiry" - ); -} - // ── Regression: 6 correctness fixes ────────────────────────────────────────── // ── Fix 1: load_snapshot must advance event_clock / query_clock ─────────────── diff --git a/serf-proto/src/options/mod.rs b/serf-proto/src/options/mod.rs index 22dd1187..33fd70e5 100644 --- a/serf-proto/src/options/mod.rs +++ b/serf-proto/src/options/mod.rs @@ -31,8 +31,6 @@ pub struct Options { /// How long recent join/leave intents are buffered to handle out-of-order /// delivery before the inner memberlist `NodeJoined`/`NodeLeft` event. recent_intent_timeout: Duration, - /// How long to wait for a broadcast (leave, force-remove) to propagate. - broadcast_timeout: Duration, /// Extra delay after calling inner `leave()` before transitioning to `Left`, /// giving in-flight probes time to observe the leave intent. leave_propagate_delay: Duration, @@ -122,7 +120,6 @@ impl Options { reconnect_timeout: Duration::from_secs(3600 * 24), tombstone_timeout: Duration::from_secs(3600 * 24), recent_intent_timeout: Duration::from_secs(60 * 5), - broadcast_timeout: Duration::from_secs(5), leave_propagate_delay: Duration::from_secs(1), queue_check_interval: Duration::from_secs(30), coalesce_period: Duration::ZERO, @@ -175,11 +172,6 @@ impl Options { self.recent_intent_timeout } - /// Broadcast propagation timeout. - pub const fn broadcast_timeout(&self) -> Duration { - self.broadcast_timeout - } - /// Extra delay before transitioning to `Left` after calling inner `leave()`. pub const fn leave_propagate_delay(&self) -> Duration { self.leave_propagate_delay @@ -363,12 +355,6 @@ impl Options { self } - /// Sets `broadcast_timeout`. - pub fn with_broadcast_timeout(mut self, v: Duration) -> Self { - self.broadcast_timeout = v; - self - } - /// Sets `leave_propagate_delay`. pub fn with_leave_propagate_delay(mut self, v: Duration) -> Self { self.leave_propagate_delay = v; diff --git a/serf-proto/src/options/tests.rs b/serf-proto/src/options/tests.rs index 27d767e9..21b4b4e9 100644 --- a/serf-proto/src/options/tests.rs +++ b/serf-proto/src/options/tests.rs @@ -25,7 +25,6 @@ fn options_default_and_new_are_equal() { assert_eq!(a.reconnect_timeout(), b.reconnect_timeout()); assert_eq!(a.tombstone_timeout(), b.tombstone_timeout()); assert_eq!(a.recent_intent_timeout(), b.recent_intent_timeout()); - assert_eq!(a.broadcast_timeout(), b.broadcast_timeout()); assert_eq!(a.leave_propagate_delay(), b.leave_propagate_delay()); assert_eq!(a.queue_check_interval(), b.queue_check_interval()); assert_eq!(a.coalesce_period(), b.coalesce_period()); @@ -56,7 +55,6 @@ fn options_all_defaults() { assert_eq!(o.reconnect_timeout(), Duration::from_secs(3600 * 24)); assert_eq!(o.tombstone_timeout(), Duration::from_secs(3600 * 24)); assert_eq!(o.recent_intent_timeout(), Duration::from_secs(60 * 5)); - assert_eq!(o.broadcast_timeout(), Duration::from_secs(5)); assert_eq!(o.leave_propagate_delay(), Duration::from_secs(1)); assert_eq!(o.queue_check_interval(), Duration::from_secs(30)); // Coalescing disabled by default diff --git a/serf-proto/src/quic_endpoint/mod.rs b/serf-proto/src/quic_endpoint/mod.rs index 8cd5daf6..8bb7970d 100644 --- a/serf-proto/src/quic_endpoint/mod.rs +++ b/serf-proto/src/quic_endpoint/mod.rs @@ -286,11 +286,8 @@ where } } - /// Number of unsent items in the coordinator's user broadcast queue. - /// - /// The driver may poll this during a graceful leave to detect when the - /// leave-intent broadcast has been flushed without waiting the full - /// `broadcast_timeout`. + /// Number of unsent items in the coordinator's user broadcast queue, summed + /// across every tier. Queue-depth introspection for telemetry and tests. pub fn user_broadcast_queue_len(&self) -> usize { self.transport.endpoint_ref().user_broadcast_queue_len() } @@ -757,11 +754,6 @@ where self.core.respond_key(&mut self.transport, req, resp, now) } - /// Forwards to [`Endpoint::leave_broadcast_deadline`]. - pub const fn leave_broadcast_deadline(&self) -> Option { - self.core.leave_broadcast_deadline() - } - /// Forwards to [`Endpoint::leave_complete_deadline`]. pub const fn leave_complete_deadline(&self) -> Option { self.core.leave_complete_deadline() diff --git a/serf-proto/src/stream_endpoint/mod.rs b/serf-proto/src/stream_endpoint/mod.rs index dbba94ac..a3e965bd 100644 --- a/serf-proto/src/stream_endpoint/mod.rs +++ b/serf-proto/src/stream_endpoint/mod.rs @@ -327,11 +327,8 @@ where } } - /// Number of unsent items in the coordinator's user broadcast queue. - /// - /// The driver may poll this during a graceful leave to detect when the - /// leave-intent broadcast has been flushed without waiting the full - /// `broadcast_timeout`. + /// Number of unsent items in the coordinator's user broadcast queue, summed + /// across every tier. Queue-depth introspection for telemetry and tests. pub fn user_broadcast_queue_len(&self) -> usize { self.transport.endpoint_ref().user_broadcast_queue_len() } @@ -768,11 +765,6 @@ where self.transport.set_encryption_options(encryption) } - /// Forwards to [`Endpoint::leave_broadcast_deadline`]. - pub const fn leave_broadcast_deadline(&self) -> Option { - self.core.leave_broadcast_deadline() - } - /// Forwards to [`Endpoint::leave_complete_deadline`]. pub const fn leave_complete_deadline(&self) -> Option { self.core.leave_complete_deadline() From 1a1ddceca8d08cbd5352953da2ea0e2694439fa1 Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 02:06:44 +1200 Subject: [PATCH 085/128] test(serf-proto): pin the graceful-leave ordering and late-intent heal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two peer-side sequences composing handle_node_leave with handle_node_leave_intent: - graceful_leave_intent_then_dead_yields_exactly_leave: a leave intent processed before the dead-self notice (the order the farewell compound delivers) takes the Alive peer to Leaving silently, then Leaving -> Left emits exactly one Member(Leave); the absence of Failed is the discriminator. - dead_then_late_intent_heals_to_leave: the reversed order — dead-self first (Alive -> Failed, Member(Failed)), then the late intent heals Failed -> Left (Member(Leave)) and moves the peer from failed_members to left_members. --- serf-proto/src/endpoint/serf_parity_tests.rs | 99 ++++++++++++++++++++ 1 file changed, 99 insertions(+) diff --git a/serf-proto/src/endpoint/serf_parity_tests.rs b/serf-proto/src/endpoint/serf_parity_tests.rs index c876c1b5..23bdbafa 100644 --- a/serf-proto/src/endpoint/serf_parity_tests.rs +++ b/serf-proto/src/endpoint/serf_parity_tests.rs @@ -220,6 +220,105 @@ fn handle_node_leave_intent_failed_to_left() { assert!(matches!(ev, Some(Event::Member(ref me)) if me.kind() == MemberEventKind::Leave)); } +// ── graceful-leave ordering + late-intent heal (peer side) ────────────────── + +/// Farewell ordering: a leave intent processed before the dead-self notice — +/// the order the farewell compound delivers, user intent part ahead of the +/// `Dead` part — yields EXACTLY one `Member(Leave)`. The intent takes the Alive +/// peer to `Leaving` silently (no event), then the inner `NodeLeft` completes +/// `Leaving → Left` and emits the single `Member(Leave)`. The absence of a +/// `Failed` is the discriminator between an intentional departure and a crash. +#[test] +fn graceful_leave_intent_then_dead_yields_exactly_leave() { + let mut e = ep(); + // A known Alive peer, seeded silently at a low status_time. + e.test_seed_member(2, MemberStatus::Alive, LamportTime::new(3)); + + // One drain cycle in wire order: the leave intent first (Alive → Leaving, + // silent), then the dead-self notice (Leaving → Left, emits Leave). + let rb = e.test_handle_leave_intent(2, LamportTime::new(7), Instant::ORIGIN); + assert!(rb, "a fresh leave intent for an Alive peer rebroadcasts"); + assert_eq!( + e.test_member_status(2), + Some(MemberStatus::Leaving), + "the leave intent must move the Alive peer to Leaving silently" + ); + e.test_inner_node_left(2, Instant::ORIGIN); + assert_eq!( + e.test_member_status(2), + Some(MemberStatus::Left), + "the dead-self notice must complete Leaving → Left" + ); + assert!( + e.test_in_left_members(2), + "the departed peer lands in left_members" + ); + + // Exactly one member event, and it is Leave: the Alive → Leaving step emits + // nothing, so no Failed and no duplicate Leave appears. + let kinds: Vec = core::iter::from_fn(|| e.poll_event()) + .filter_map(|ev| match ev { + Event::Member(me) => Some(me.kind()), + _ => None, + }) + .collect(); + assert_eq!( + kinds, + vec![MemberEventKind::Leave], + "peer-side output must be exactly one Member(Leave)" + ); +} + +/// Reversed order: the dead-self notice arrives before the leave intent, so the +/// Alive peer first goes `Failed` (emitting `Member(Failed)`), then the late +/// intent heals `Failed → Left` (emitting `Member(Leave)`) and moves the peer +/// from failed_members to left_members. Pins the full second-hand sequence that +/// a farewell whose intent part is processed after a prior suspicion-driven +/// failure produces. +#[test] +fn dead_then_late_intent_heals_to_leave() { + let mut e = ep(); + e.test_seed_member(2, MemberStatus::Alive, LamportTime::new(3)); + + // Dead-self notice first: Alive → Failed, emits Member(Failed). + e.test_inner_node_left(2, Instant::ORIGIN); + assert_eq!(e.test_member_status(2), Some(MemberStatus::Failed)); + assert!( + e.test_in_failed_members(2), + "the failed peer lands in failed_members" + ); + + // Late leave intent heals Failed → Left, emits Member(Leave). + let rb = e.test_handle_leave_intent(2, LamportTime::new(7), Instant::ORIGIN); + assert!(rb, "a fresh leave intent for a Failed peer rebroadcasts"); + assert_eq!( + e.test_member_status(2), + Some(MemberStatus::Left), + "the late intent must heal Failed → Left" + ); + assert!( + !e.test_in_failed_members(2), + "the healed peer leaves failed_members" + ); + assert!( + e.test_in_left_members(2), + "the healed peer joins left_members" + ); + + // The full second-hand sequence: Failed then Leave, in order. + let kinds: Vec = core::iter::from_fn(|| e.poll_event()) + .filter_map(|ev| match ev { + Event::Member(me) => Some(me.kind()), + _ => None, + }) + .collect(); + assert_eq!( + kinds, + vec![MemberEventKind::Failed, MemberEventKind::Leave], + "peer-side output must be Failed then Leave" + ); +} + // ── merge_remote_state (G2 witness, G3 left-first, G4 eventJoinIgnore) ─────── /// Helper: build a `PushPullMessage` body as encoded `Bytes`. From abc6f1b1ac084cfe8cfacaf7794af3c2c5a23e00 Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 02:17:36 +1200 Subject: [PATCH 086/128] test(serf-reactor): e2e graceful leave yields [Join, Leave], never Failed Add serf_events_leave over the cluster fixture (tokio + smol): two nodes, B leaves gracefully, and observer A's member-event log for B is exactly [Join, Leave]. The absence of a Failed discriminates the intentional departure from the abrupt-kill path (serf_events_failed, unchanged, kept as the paired [Join, Failed] oracle). B lands in A's Left tombstone view, and a latency canary asserts the graceful leave().await completes well inside the driver's 5s leave timeout, guarding against a reintroduced flush wait. Fixture additions: leave_graceful (mirrors kill_abrupt's slot bookkeeping, returns the leave() latency), await_left_tombstone, a with_tombstone_timeout knob to hold the Left tombstone so no trailing Reap appears, and a short leave_propagate_delay wired into the fast serf options. --- serf-reactor/tests/cluster/mod.rs | 54 +++++++++++++++++++++++++++++++ serf-reactor/tests/tcp.rs | 53 ++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+) diff --git a/serf-reactor/tests/cluster/mod.rs b/serf-reactor/tests/cluster/mod.rs index b2faa193..32b4a30c 100644 --- a/serf-reactor/tests/cluster/mod.rs +++ b/serf-reactor/tests/cluster/mod.rs @@ -25,6 +25,7 @@ use agnostic::Runtime; use futures_util::StreamExt; use serf_proto::{ event::{Event, MemberEventKind}, + members::MemberStatus, options::Options as SerfOptions, }; use serf_reactor::{ @@ -63,6 +64,7 @@ pub struct ClusterTiming { reconnect_interval: Duration, reconnect_timeout: Duration, tombstone_timeout: Duration, + leave_propagate_delay: Duration, } impl ClusterTiming { @@ -88,6 +90,9 @@ impl ClusterTiming { reconnect_interval: Duration::from_millis(100), reconnect_timeout: Duration::from_millis(1), tombstone_timeout: Duration::from_millis(1), + // Short enough to keep a graceful leave sub-second, long enough to give + // in-flight probes a gossip cycle to observe the leave intent. + leave_propagate_delay: Duration::from_millis(100), } } @@ -100,6 +105,16 @@ impl ClusterTiming { self } + /// Override the tombstone timeout — the age at which the reaper removes a + /// gracefully-Left member. Raise it beyond the test window to HOLD a left peer + /// in the tombstone view instead of reaping it, so a graceful-leave assertion + /// observes `[Join, Leave]` without a trailing `Reap`. + #[must_use] + pub fn with_tombstone_timeout(mut self, v: Duration) -> Self { + self.tombstone_timeout = v; + self + } + /// The transport options for one node: the fast SWIM overrides plus this node's /// id and advertise address. fn transport_opts( @@ -123,6 +138,7 @@ impl ClusterTiming { .with_reconnect_interval(self.reconnect_interval) .with_reconnect_timeout(self.reconnect_timeout) .with_tombstone_timeout(self.tombstone_timeout) + .with_leave_propagate_delay(self.leave_propagate_delay) } } @@ -212,6 +228,24 @@ where serf.shutdown().await.expect("node shuts down"); } + /// Gracefully leave node `i`, then release its slot. Calls `leave()` — which + /// the reactor resolves only once the machine's `LeftCluster` fires, so a + /// successful return proves the graceful-leave chain completed — and returns + /// the wall-clock the `leave().await` took (a latency canary against any + /// reintroduced flush wait). Unlike [`kill_abrupt`](Self::kill_abrupt), the + /// farewell (leave intent packed with the dead-self notice) reaches peers + /// before teardown, so peers observe an intentional Leave rather than a + /// probe-timeout Failed. The slot's id, addr, and event log are retained for + /// later assertions. + pub async fn leave_graceful(&mut self, i: usize) -> Duration { + let serf = self.slots[i].serf.take().expect("node slot is live"); + let start = std::time::Instant::now(); + serf.leave().await.expect("node leaves gracefully"); + let elapsed = start.elapsed(); + serf.shutdown().await.expect("node shuts down"); + elapsed + } + /// Restart a previously-killed node `i` at the SAME id and advertise address, /// re-attaching a collector to the slot's existing log. The freed port is /// rebound with a bounded retry to absorb a transient rebind race. @@ -274,6 +308,26 @@ where .expect("observer reaches the expected member count"); } + /// Poll until `observer`'s membership view holds `subject` as a `Left` + /// tombstone — the graceful-leave end state — or fail on the poll timeout. + pub async fn await_left_tombstone(&self, observer: usize, subject: &str) { + R::timeout(POLL_TIMEOUT, async { + loop { + if self + .node(observer) + .members() + .iter() + .any(|m| m.node().id_ref().as_str() == subject && m.status() == MemberStatus::Left) + { + break; + } + R::sleep(POLL_STEP).await; + } + }) + .await + .expect("observer holds the subject as a Left tombstone"); + } + /// Poll until `observer`'s log records a member event of `kind` naming /// `subject`. pub async fn await_member_event(&self, observer: usize, subject: &str, kind: MemberEventKind) { diff --git a/serf-reactor/tests/tcp.rs b/serf-reactor/tests/tcp.rs index 812c9133..0d9b0bd1 100644 --- a/serf-reactor/tests/tcp.rs +++ b/serf-reactor/tests/tcp.rs @@ -515,6 +515,49 @@ where cluster.shutdown_all().await; } +/// Two nodes on loopback: node A joins node B, then B leaves gracefully. Node A +/// must observe the member-event sequence Join → Leave about B: a Leave (not a +/// Failed), proving the farewell — the leave intent packed with the dead-self +/// notice — reached A before B tore down, and B lands in A's Left tombstone +/// view. The absence of a Failed is the discriminator against the abrupt-kill +/// path (`serf_events_failed`), which the two tests together pin. +async fn serf_events_leave() +where + R: Runtime, +{ + // Raise A's tombstone timeout past the test window so the reaper holds B's + // Left tombstone rather than appending a trailing Reap to the observed + // sequence (the fast profile otherwise reaps a left member sub-second). + let mut cluster = cluster::Cluster::::spawn( + &["events-leave-a", "events-leave-b"], + cluster::ClusterTiming::fast().with_tombstone_timeout(Duration::from_secs(30)), + ) + .await; + let subject = cluster.id(1); + + // The graceful leave().await must complete well inside the driver's 5s leave + // timeout — a canary against anyone reintroducing a broadcast-flush wait. + let elapsed = cluster.leave_graceful(1).await; + assert!( + elapsed < Duration::from_secs(3), + "graceful leave().await took {elapsed:?}, expected well under the 5s driver leave timeout" + ); + + // A observes exactly [Join, Leave] about B — never a Failed. + cluster + .assert_member_events( + 0, + subject.as_str(), + &[MemberEventKind::Join, MemberEventKind::Leave], + ) + .await; + + // B lands in A's Left tombstone view. + cluster.await_left_tombstone(0, subject.as_str()).await; + + cluster.shutdown_all().await; +} + /// Two nodes on loopback: node A joins node B, B is abruptly killed and detected /// Failed, then B is restarted at the same id and advertise address. Node A must /// observe the sequence Join → Failed → Join about B — the failed member @@ -760,6 +803,11 @@ mod tokio_cells { super::serf_events_failed::().await; } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_events_leave() { + super::serf_events_leave::().await; + } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn serf_reconnect() { super::serf_reconnect::().await; @@ -834,6 +882,11 @@ mod smol_cells { SmolRuntime::block_on(super::serf_events_failed::()); } + #[test] + fn serf_events_leave_smol() { + SmolRuntime::block_on(super::serf_events_leave::()); + } + #[test] fn serf_reconnect_smol() { SmolRuntime::block_on(super::serf_reconnect::()); From 570d38310b11eae1338e9d315bec6b9772275820 Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 02:32:05 +1200 Subject: [PATCH 087/128] feat(serf-proto): pass the leave intent as the coordinator's explicit farewell MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The intent previously rode the farewell via the rank-0 broadcast queue, whose drain selects by fewest-transmissions-then-largest — an older, larger queued payload of the same rank could crowd the just-queued intent out of the single-datagram budget, and the inner reset then destroyed it. The Reliable leave now carries an explicit farewell payload the coordinator reserves into every dead-self compound ahead of its ordinary drain, so the intent's ride is guaranteed by construction rather than by drain ordering. An encode failure on a degenerate id degrades to a plain inner leave, matching the pre-farewell tolerance for the same case. --- serf-proto/src/endpoint/mod.rs | 29 +++++++++++++++---------- serf-proto/src/endpoint/reliable/mod.rs | 25 ++++++++++++++++----- 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/serf-proto/src/endpoint/mod.rs b/serf-proto/src/endpoint/mod.rs index 060ec886..0ad413e3 100644 --- a/serf-proto/src/endpoint/mod.rs +++ b/serf-proto/src/endpoint/mod.rs @@ -2664,17 +2664,24 @@ where return Err(Error::LeaveClockExhausted); } - // 3. Broadcast the leave intent on the intent tier (rank 0) so peers learn - // the local node is leaving without waiting for anti-entropy. - self.broadcast_leave(t, ltime, local_id, false); - - // 4. Call inner leave synchronously. It packs the rank-0 intent just queued - // (with any other pending user broadcast) into its dead-self fan-out — - // user parts ahead of the death notice — so every farewell recipient - // receives the intent atomically with the dead-self notice and processes - // it first, reading the departure as intentional. This queues the - // resulting fan-out packets for `poll_transmit`. - t.leave(now)?; + // 3. Encode the leave intent and call the inner leave synchronously, + // passing the intent as the explicit farewell payload. The coordinator + // RESERVES it into every dead-self farewell compound ahead of its + // ordinary queue drain — user parts before the death notice — so every + // farewell recipient receives the intent atomically with the dead-self + // notice and processes it first, reading the departure as intentional, + // regardless of what else is queued (an older, larger queued payload + // cannot crowd the reservation out). This queues the resulting fan-out + // packets for `poll_transmit`. + // + // An encode failure (a degenerate id type — a construction-time concern + // the driver surfaces) degrades to a plain inner leave: peers then read + // the departure as a failure until the late-intent heal cannot help, + // matching the pre-farewell tolerance for the same degenerate case. + let farewell = AnyMessage::::Leave(LeaveMessage::new(ltime, local_id, false)) + .encode() + .ok(); + t.leave(now, farewell)?; Ok(()) } diff --git a/serf-proto/src/endpoint/reliable/mod.rs b/serf-proto/src/endpoint/reliable/mod.rs index f0567bc4..3febb7b3 100644 --- a/serf-proto/src/endpoint/reliable/mod.rs +++ b/serf-proto/src/endpoint/reliable/mod.rs @@ -132,11 +132,18 @@ where /// and `poll_transmit` drain remaining output; no new exchanges may be /// initiated. /// + /// `farewell` is an optional already-encoded user frame the coordinator + /// reserves into every dead-self farewell compound ahead of its ordinary + /// queue drain — serf passes its leave intent here so every farewell + /// recipient processes the intent before the death notice from the same + /// frame, regardless of what else is queued. + /// /// # Errors /// /// Returns [`memberlist_proto::Error`] if the coordinator is already in a /// terminal state. - fn leave(&mut self, now: Instant) -> Result<(), memberlist_proto::Error>; + fn leave(&mut self, now: Instant, farewell: Option) + -> Result<(), memberlist_proto::Error>; /// Attach an application payload to outbound probe Ack messages. /// @@ -216,8 +223,12 @@ where } #[inline] - fn leave(&mut self, now: Instant) -> Result<(), memberlist_proto::Error> { - self.leave(now) + fn leave( + &mut self, + now: Instant, + farewell: Option, + ) -> Result<(), memberlist_proto::Error> { + self.leave_with(now, farewell) } #[cfg(feature = "coordinates")] @@ -293,8 +304,12 @@ where } #[inline] - fn leave(&mut self, now: Instant) -> Result<(), memberlist_proto::Error> { - self.leave(now) + fn leave( + &mut self, + now: Instant, + farewell: Option, + ) -> Result<(), memberlist_proto::Error> { + self.leave_with(now, farewell) } #[cfg(feature = "coordinates")] From c7a292f9d61ceb1ac038dc8c4b14b433b457e5a1 Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 03:16:40 +1200 Subject: [PATCH 088/128] feat(serf-proto): admit the leave farewell before mutating any state leave() encoded its leave intent and handed it to the coordinator as the explicit farewell, but the coordinator silently degrades an over-budget farewell to the bare Dead notice. A node with a long (but legal) id could then report a successful leave while peers, seeing only Dead, classify the departure as Failed. Make the admission explicit and pre-mutation. Add farewell_capacity() to the Reliable trait (both coordinators forward to the inner Endpoint's identity-aware bound). In leave(), compute the prospective stamp, encode the intent from it, and check it against farewell_capacity() BEFORE committing the clock, transitioning to Leaving, or applying the local intent. An over-budget or unencodable intent is refused with a new typed error (LeaveFarewellTooLarge / LeaveFarewellEncode) and the machine stays a consistent Alive; only past the admission does any mutation occur, after which the pre-admitted intent rides the dead-self fan-out. --- serf-proto/src/endpoint/mod.rs | 121 +++++++++++++++------- serf-proto/src/endpoint/reliable/mod.rs | 24 +++++ serf-proto/src/endpoint/reliable/tests.rs | 3 + serf-proto/src/endpoint/tests.rs | 110 +++++++++++++++++++- 4 files changed, 215 insertions(+), 43 deletions(-) diff --git a/serf-proto/src/endpoint/mod.rs b/serf-proto/src/endpoint/mod.rs index 0ad413e3..f2bf7dc0 100644 --- a/serf-proto/src/endpoint/mod.rs +++ b/serf-proto/src/endpoint/mod.rs @@ -509,6 +509,28 @@ pub enum Error { /// lifetime (2^63 membership events). #[error("leave clock exhausted: member clock reached the LTIME_MAX integrity floor")] LeaveClockExhausted, + /// `leave()` refused because the encoded leave intent does not fit the + /// farewell budget the coordinator can guarantee beside this node's own + /// death notice. + /// + /// serf reserves its leave intent into every dead-self farewell compound so + /// each recipient reads the departure as intentional; when the intent would + /// not fit — a long (but legal) node id can leave the death notice alone + /// filling the gossip MTU — the leave is refused BEFORE any mutation rather + /// than silently degrading to a bare `Dead` fan-out that peers would classify + /// as a failure. The endpoint stays a consistent `Alive` member. + /// + /// Carries `(encoded_intent_size, farewell_capacity)`; the capacity is `0` + /// when the death notice alone exhausts the gossip MTU (no farewell can ride). + #[error("leave farewell too large: {0} bytes exceeds farewell capacity of {1}")] + LeaveFarewellTooLarge(usize, usize), + /// `leave()` could not encode its leave intent for the farewell compound. + /// + /// Encoding the intent is a prerequisite for admitting it against the farewell + /// budget, so a codec failure (a degenerate id type) refuses the leave BEFORE + /// any mutation instead of degrading to a bare `Dead` fan-out. + #[error("leave farewell encode error: {0}")] + LeaveFarewellEncode(EncodeError), /// `join()` was called while the local endpoint is not `Alive`. /// /// serf only announces its own join intent from the `Alive` state; a @@ -2579,27 +2601,33 @@ where /// - `Alive` → proceeds with the leave chain below. /// /// Leave chain (decision 5 / oracle `api.go` leave()): - /// 1. Post-increment the member clock to stamp the leave ltime — but only - /// after the [`LTIME_MAX`] integrity-floor gate: if the stamp would land at - /// or above `LTIME_MAX` (a clock driven near the floor), return - /// [`Error::LeaveClockExhausted`] without mutating any state. No invalid - /// intent is emitted and no inner leave starts; the endpoint stays a - /// consistent `Alive` member (degraded-but-safe). - /// 2. Set `state = Leaving`, then handle the local leave intent - /// (`handle_node_leave_intent` for the local id), which marks the local - /// node as `Leaving` in the membership store and queues a join-refute + /// 1. Compute the prospective leave ltime by post-incrementing the member + /// clock, WITHOUT committing it, and gate it on the [`LTIME_MAX`] + /// integrity floor: if the stamp would land at or above `LTIME_MAX` (a + /// clock driven near the floor), return [`Error::LeaveClockExhausted`] + /// without mutating any state — the endpoint stays a consistent `Alive` + /// member (degraded-but-safe). + /// 2. Encode the leave intent from that prospective stamp and admit it against + /// the coordinator's `farewell_capacity` — still BEFORE any mutation. The + /// intent MUST ride the dead-self farewell compound so peers read the + /// departure as intentional; if it cannot fit (a long id can leave the + /// death notice alone filling the gossip MTU) the inner leave would degrade + /// to a bare `Dead` fan-out peers classify as a failure, so refuse with + /// [`Error::LeaveFarewellTooLarge`] (or [`Error::LeaveFarewellEncode`] on a + /// codec failure) and leave the machine untouched. Only past this admission + /// does any mutation occur. + /// 3. Commit the stamp, set `state = Leaving`, then handle the local leave + /// intent (`handle_node_leave_intent` for the local id), which marks the + /// local node as `Leaving` in the membership store and queues a join-refute /// suppression. - /// 3. Enqueue the leave-intent broadcast on the intent tier (rank 0) so peers - /// learn the local node is leaving. - /// 4. Call inner `leave(now)` synchronously. The inner packs the payloads - /// still queued on the user-broadcast tiers — the rank-0 intent just - /// enqueued — into its dead-self fan-out, user parts ahead of the death - /// notice, so every farewell recipient receives the intent ATOMICALLY with - /// the dead-self notice in one datagram and processes the intent first, - /// classifying the departure as intentional rather than a failure. The - /// inner emits `Event::LeftCluster` once all dead-self packets drain via - /// `poll_transmit`. No separate wait for the intent to flush exists: the - /// queued intent departs with the dead-self frame, not on its own schedule. + /// 4. Call inner `leave(now, Some(intent))` synchronously, passing the + /// pre-admitted intent as the explicit farewell. The inner RESERVES it into + /// every dead-self farewell compound ahead of its ordinary queue drain, + /// user parts ahead of the death notice, so every farewell recipient + /// receives the intent ATOMICALLY with the dead-self notice in one datagram + /// and processes it first, classifying the departure as intentional rather + /// than a failure. The inner emits `Event::LeftCluster` once all dead-self + /// packets drain via `poll_transmit`. /// /// The `Leaving → Left` transition happens later in `handle_timeout` when /// `leave_complete_deadline` (armed on inner `LeftCluster` + `leave_propagate_delay`) @@ -2643,10 +2671,32 @@ where if !ltime_is_acceptable(stamp) { return Err(Error::LeaveClockExhausted); } + let ltime = LamportTime(stamp); + + // Admit the farewell BEFORE any mutation. serf reserves its leave intent + // into every dead-self farewell compound so each recipient reads the + // departure as intentional; if the intent cannot ride beside the death + // notice the inner leave silently degrades to a bare `Dead` fan-out and + // peers classify the departure as a failure. Encode the intent from the + // prospective (not-yet-committed) stamp and check it against the largest + // payload the coordinator can guarantee to carry, refusing up front — clock + // unadvanced, still `Alive`, nothing queued — rather than reporting a + // success peers read as a failure. An encode failure (a degenerate id type) + // refuses here for the same reason: the intent is a prerequisite for the + // admission, not an optional extra that may be dropped. + let farewell = AnyMessage::::Leave(LeaveMessage::new(ltime, local_id.clone(), false)) + .encode() + .map_err(Error::LeaveFarewellEncode)?; + match t.farewell_capacity() { + Some(capacity) if farewell.len() <= capacity => {} + Some(capacity) => return Err(Error::LeaveFarewellTooLarge(farewell.len(), capacity)), + // `None`: the death notice alone exhausts the gossip MTU, so no farewell + // can ride beside it. Report a zero capacity per the variant's contract. + None => return Err(Error::LeaveFarewellTooLarge(farewell.len(), 0)), + } - // The stamp is acceptable: commit it and transition to Leaving. + // Admission passed: commit the stamp and transition to Leaving. self.clock = stamp; - let ltime = LamportTime(self.clock); self.mark_local_state_dirty(); // 1. Transition to Leaving BEFORE applying the local intent so the @@ -2664,24 +2714,15 @@ where return Err(Error::LeaveClockExhausted); } - // 3. Encode the leave intent and call the inner leave synchronously, - // passing the intent as the explicit farewell payload. The coordinator - // RESERVES it into every dead-self farewell compound ahead of its - // ordinary queue drain — user parts before the death notice — so every - // farewell recipient receives the intent atomically with the dead-self - // notice and processes it first, reading the departure as intentional, - // regardless of what else is queued (an older, larger queued payload - // cannot crowd the reservation out). This queues the resulting fan-out - // packets for `poll_transmit`. - // - // An encode failure (a degenerate id type — a construction-time concern - // the driver surfaces) degrades to a plain inner leave: peers then read - // the departure as a failure until the late-intent heal cannot help, - // matching the pre-farewell tolerance for the same degenerate case. - let farewell = AnyMessage::::Leave(LeaveMessage::new(ltime, local_id, false)) - .encode() - .ok(); - t.leave(now, farewell)?; + // 3. Call the inner leave synchronously, passing the pre-admitted intent as + // the explicit farewell payload. The coordinator RESERVES it into every + // dead-self farewell compound ahead of its ordinary queue drain — user + // parts before the death notice — so every farewell recipient receives + // the intent atomically with the dead-self notice and processes it first, + // reading the departure as intentional, regardless of what else is queued + // (an older, larger queued payload cannot crowd the reservation out). + // This queues the resulting fan-out packets for `poll_transmit`. + t.leave(now, Some(farewell))?; Ok(()) } diff --git a/serf-proto/src/endpoint/reliable/mod.rs b/serf-proto/src/endpoint/reliable/mod.rs index 3febb7b3..8d55b732 100644 --- a/serf-proto/src/endpoint/reliable/mod.rs +++ b/serf-proto/src/endpoint/reliable/mod.rs @@ -124,6 +124,20 @@ where /// `poll_inner_event`). fn start_push_pull(&mut self, peer: A, kind: PushPullKind, now: Instant) -> StreamId; + /// The largest farewell payload [`leave`](Reliable::leave) can guarantee to + /// deliver beside this node's death notice, or `None` when the death notice + /// alone exhausts the gossip MTU. + /// + /// serf checks this BEFORE mutating any leave state: an intent that cannot + /// ride the farewell compound would degrade [`leave`](Reliable::leave) to a + /// bare `Dead` fan-out that peers classify as a failure, so an over-budget + /// farewell is refused up front. Identity-aware — the bound depends on the + /// encoded size of this node's own id. + /// + /// Reaches the inner membership [`Endpoint`] directly, mirroring the other + /// read-only accessors. + fn farewell_capacity(&self) -> Option; + /// Signal that the local node intends to leave the cluster gracefully. /// /// The coordinator disseminates a Leave message, transitions the inner @@ -222,6 +236,11 @@ where self.start_push_pull(peer, kind, now) } + #[inline] + fn farewell_capacity(&self) -> Option { + self.endpoint_ref().farewell_capacity() + } + #[inline] fn leave( &mut self, @@ -303,6 +322,11 @@ where self.start_push_pull(peer, kind, now) } + #[inline] + fn farewell_capacity(&self) -> Option { + self.endpoint_ref().farewell_capacity() + } + #[inline] fn leave( &mut self, diff --git a/serf-proto/src/endpoint/reliable/tests.rs b/serf-proto/src/endpoint/reliable/tests.rs index fa41051c..bb942e0c 100644 --- a/serf-proto/src/endpoint/reliable/tests.rs +++ b/serf-proto/src/endpoint/reliable/tests.rs @@ -22,6 +22,9 @@ where let _ = t.endpoint_ref().local_state_snapshot_bytes(); let _ = t.endpoint_ref().user_broadcast_queue_len(); + // Read-only farewell admission bound (the pre-leave capacity check). + let _ = t.farewell_capacity(); + // Mutating: queue a broadcast at rank 0 (highest priority). let result = t.queue_user_broadcast_ranked(0, Bytes::from_static(b"hello")); assert!( diff --git a/serf-proto/src/endpoint/tests.rs b/serf-proto/src/endpoint/tests.rs index ee7add74..0abb9478 100644 --- a/serf-proto/src/endpoint/tests.rs +++ b/serf-proto/src/endpoint/tests.rs @@ -8,6 +8,7 @@ use crate::{ #[cfg(feature = "coordinates")] use bytes::Bytes; use memberlist_proto::{EndpointOptions, RawRecords, SeedableRng, SmallRng, streams::LabelOptions}; +use smol_str::SmolStr; /// The plain-TCP record layer the unit-test coordinators run over. type TestTransport = RawRecords; @@ -30,9 +31,12 @@ fn default_endpoint_is_send_and_sync() { /// the identity and the SNI provider is unused (the plain-TCP record layer /// ignores it). A fixed cluster label keeps the handshake well-formed for the /// loopback tests that complete a real exchange. -fn coord( - inner: memberlist_proto::Endpoint, -) -> memberlist_proto::streams::StreamEndpoint { +fn coord( + inner: memberlist_proto::Endpoint, +) -> memberlist_proto::streams::StreamEndpoint +where + I: memberlist_proto::Id, +{ memberlist_proto::streams::StreamEndpoint::new( inner, LabelOptions::new_in(Some(b"serf-test".to_vec()), ()), @@ -474,6 +478,106 @@ fn leave_one_below_watermark_succeeds() { ); } +/// Pre-mutation farewell admission: a `leave()` whose encoded intent cannot ride +/// the dead-self farewell compound is refused with [`Error::LeaveFarewellTooLarge`] +/// BEFORE any mutation. The machine stays a consistent `Alive`: the clock does +/// not advance, self-membership never half-transitions to `Leaving`, no intent +/// is queued, no dead-self fan-out is emitted, and a normal operation still works. +/// +/// The reproduction is the finding's own scenario — a long (but legal) node id. +/// memberlist's identity-aware construction floor sizes the gossip MTU for a +/// probe `Ping` (the local id in BOTH node slots plus two advertise addresses), +/// while the farewell budget only has to hold the Dead-self notice (the id +/// twice, no addresses). At the minimum valid MTU the budget left beside the +/// death notice is a couple of addresses' worth — far below the id-sized leave +/// intent — so the intent cannot ride. +#[test] +fn leave_over_farewell_budget_refuses_pre_mutation() { + let addr: core::net::SocketAddr = "127.0.0.1:7946".parse().unwrap(); + let big_id = SmolStr::new("n".repeat(900)); + + // Read the identity-aware gossip-MTU floor off a deliberately-undersized + // construction, so the test tracks the real encoding overhead rather than a + // brittle constant, then build at exactly that floor. + let required = match memberlist_proto::Endpoint::try_new_at( + EndpointOptions::new(big_id.clone(), addr) + .with_gossip_mtu(memberlist_proto::DEFAULT_GOSSIP_MTU), + memberlist_proto::Instant::ORIGIN, + SmallRng::seed_from_u64(0), + ) { + Err(memberlist_proto::EndpointInitError::GossipMtuTooSmall(b)) => b.bound(), + Err(other) => { + panic!("expected a gossip-MTU floor rejection, got a different init error: {other:?}") + } + Ok(_) => panic!("expected the long-id construction to be rejected by the gossip-MTU floor"), + }; + let inner_opts = EndpointOptions::new(big_id.clone(), addr) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()) + .with_gossip_mtu(required); + let inner = memberlist_proto::Endpoint::new_at( + inner_opts, + memberlist_proto::Instant::ORIGIN, + SmallRng::seed_from_u64(0), + ); + let mut e: StreamEndpoint = + StreamEndpoint::new(coord(inner), Options::new()); + // Drain the construction self-join so the machine starts settled. + let _ = e.poll_event(); + + // Seed one live peer: a successful leave WOULD fan a dead-self notice out to + // it, so the absence of any transmit after the refusal is meaningful. Drain + // whatever the seed produced first, and snapshot the clock and queue depth. + e.test_seed_member( + SmolStr::new("peer"), + MemberStatus::Alive, + LamportTime::new(1), + ); + while e.poll_memberlist_transmit().is_some() {} + let clock_before = e.member_time(); + let queue_before = e.user_broadcast_queue_len(); + + let err = e + .leave(memberlist_proto::Instant::ORIGIN) + .expect_err("an over-budget farewell must refuse the leave"); + assert!( + matches!(err, Error::LeaveFarewellTooLarge(len, cap) if len > cap), + "expected LeaveFarewellTooLarge with intent size exceeding capacity, got {err:?}" + ); + + // Unmutated machine: still Alive, clock unadvanced, self-membership still + // Alive (never a half-Leaving), no intent queued, and no dead-self fan-out. + assert!(e.state().is_alive(), "the refused leave must stay Alive"); + assert_eq!( + e.member_time(), + clock_before, + "the refused leave must not advance the member clock" + ); + assert_eq!( + e.test_member_status(big_id.clone()), + Some(MemberStatus::Alive), + "self-membership must stay Alive, not a half-Leaving" + ); + assert_eq!( + e.user_broadcast_queue_len(), + queue_before, + "the refused leave must queue no leave intent" + ); + assert!( + e.poll_memberlist_transmit().is_none(), + "the refused leave must emit no dead-self fan-out" + ); + + // A normal operation still works — the machine is not wedged in a half-state. + // force_leave targets the peer id (a small intent, no farewell reservation), + // so it proceeds regardless of the local id's farewell budget. + e.force_leave( + SmolStr::new("peer"), + false, + memberlist_proto::Instant::ORIGIN, + ) + .expect("a normal membership operation must still work after the refusal"); +} + #[test] fn double_leave_is_rejected() { let mut e = ep(); From b39b6650882273c5d136d95fffa519120ff064a4 Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 03:43:31 +1200 Subject: [PATCH 089/128] fix(serf-reactor): retain the leave farewell under UDP backpressure The gossip egress pops a transmit, encodes/encrypts, and sends best-effort: a poll_send_to that returns Pending drops the datagram and SWIM re-sends on the next round. The graceful-leave fan-out has no next round, so a dropped farewell leaves peers to classify the intentional departure as a failure while leave() reported success. Scope a retention discipline to the leave drain in both readiness pumps. Once Command::Leave has initiated leave() the pump keeps a VecDeque of the encoded-and-transformed datagrams whose plain-UDP send did not complete and retries them FIFO ahead of fresh transmits, so they drain as the socket becomes writable; a Ready(Err) or an encode/encrypt failure is logged (the leave fan-out cannot mask it). Periodic gossip stays best-effort, the QUIC datagram-mode path stays quinn-managed, and the pop-fence is unchanged. Both teardowns flush the retained queue with one bounded sweep before releasing the socket. The retention bookkeeping is a pure helper with a unit test. --- serf-reactor/src/driver/quic/mod.rs | 92 +++++++++++++---- serf-reactor/src/driver/shared.rs | 127 ++++++++++++++++++++++++ serf-reactor/src/driver/shared/tests.rs | 55 ++++++++++ serf-reactor/src/driver/stream/mod.rs | 68 +++++++++++-- 4 files changed, 316 insertions(+), 26 deletions(-) create mode 100644 serf-reactor/src/driver/shared/tests.rs diff --git a/serf-reactor/src/driver/quic/mod.rs b/serf-reactor/src/driver/quic/mod.rs index 501923ab..2cf8412a 100644 --- a/serf-reactor/src/driver/quic/mod.rs +++ b/serf-reactor/src/driver/quic/mod.rs @@ -83,7 +83,10 @@ use crate::{ delegate::Delegate, driver::{ options::RuntimeOptions, - shared::{ExchangeId, dispatch_event_delegate, observation_payload_bytes}, + shared::{ + ExchangeId, LeaveDrain, dispatch_event_delegate, observation_payload_bytes, poll_send_gossip, + retry_retained_leave, trace_leave_transform_error, + }, }, drop_counter::ReactorDropCounter, error::{JoinFailed, Result, SerfError}, @@ -305,6 +308,13 @@ where pending_joins: Vec, /// The in-flight graceful leave, resolved on `LeftCluster`. pending_leave: Option, + /// Set once `leave()` has been initiated. Switches the gossip egress from + /// best-effort (drop on non-completion) to retention for the leave fan-out, + /// which has no next gossip round to re-send a dropped farewell. + leave_initiated: bool, + /// Leave-farewell datagrams retained after a non-completing plain-UDP send, + /// retried FIFO ahead of fresh transmits and flushed before the socket drops. + leave_drain: LeaveDrain, /// Parked `Shutdown` replies — acked only after the UDP socket drops, so a caller /// resuming from `shutdown().await` can rebind the same address. A `Vec` because /// several callers can race `shutdown()`. @@ -376,6 +386,8 @@ where label, pending_joins: Vec::new(), pending_leave: None, + leave_initiated: false, + leave_drain: LeaveDrain::new(), shutdown_reply: Vec::new(), recv_buf: vec![0u8; buf_len.max(1)], iter_drain_cap: driver_opts.iter_drain_cap().max(1), @@ -509,6 +521,9 @@ where let res: Result<()> = self.endpoint.leave(now).map_err(SerfError::from); match res { Ok(()) if was_alive => { + // The leave mutated and fan-out is queued: switch the gossip egress + // to retention so a backpressured farewell is retried, not dropped. + self.leave_initiated = true; self.pending_leave = Some(PendingLeave { repliers: vec![reply], deadline: now + leave_timeout, @@ -736,6 +751,16 @@ where // shared UDP socket in `Udp` mode. Popping the last transmit is the endpoint's // leave-completion fence (it emits `LeftCluster`), so the leave/shutdown // datagrams reach the wire before that fence fires. + // Once `leave()` has been initiated the fan-out is RETAINED on plain-UDP + // backpressure: retry the retained datagrams FIRST (they drain as the socket + // becomes writable), then pop fresh transmits at the unchanged cadence — a + // non-completing send is retained (not dropped) so peers do not read the + // departure as a failure. Periodic gossip stays best-effort. (Datagram-mode + // sends that ride an established QUIC connection are quinn-managed; only the + // plain-UDP sends below are retained.) + if self.leave_initiated { + retry_retained_leave(&mut self.leave_drain, self.socket.as_ref(), cx); + } let encode_opts = EncodeOptions::new(self.label.clone()); let unreliable = self.endpoint.unreliable_transport(); let mut needs_flush = false; @@ -750,14 +775,24 @@ where let (to, msg) = pkt.into_parts(); match encode_outgoing(&msg, &encode_opts) { Ok(b) => (to, b), - Err(_) => continue, + Err(_) => { + if self.leave_initiated { + trace_leave_transform_error(to); + } + continue; + } } } Transmit::Compound(cmp) => { let (to, msgs) = cmp.into_parts(); match encode_outgoing_compound(&msgs, &encode_opts) { Ok(b) => (to, b), - Err(_) => continue, + Err(_) => { + if self.leave_initiated { + trace_leave_transform_error(to); + } + continue; + } } } }; @@ -767,7 +802,12 @@ where { on_wire = match self.endpoint.encrypt_gossip(&on_wire) { Ok(bytes) => bytes, - Err(_) => continue, + Err(_) => { + if self.leave_initiated { + trace_leave_transform_error(peer); + } + continue; + } }; } // `Bytes` so the datagram-queue path and the UDP fallback can share the @@ -776,11 +816,15 @@ where let on_wire = Bytes::from(on_wire); match unreliable { UnreliableTransport::Udp => { - if let Some(socket) = self.socket.as_ref() { - // Ignoring Poll: gossip is best-effort — a full or errored UDP send - // drops the datagram and SWIM recovers on the next round. - let _ = socket.poll_send_to(cx, &on_wire, peer); - } + // Best-effort for periodic gossip; retained for the leave fan-out. + poll_send_gossip( + &mut self.leave_drain, + self.leave_initiated, + self.socket.as_ref(), + cx, + peer, + &on_wire, + ); } UnreliableTransport::Datagram => { match self @@ -800,18 +844,26 @@ where // immediately over the plain-UDP fallback. DatagramSendStatus::NotReady => { needs_flush = true; - if let Some(socket) = self.socket.as_ref() { - // Ignoring Poll: gossip is best-effort — SWIM recovers next round. - let _ = socket.poll_send_to(cx, &on_wire, peer); - } + poll_send_gossip( + &mut self.leave_drain, + self.leave_initiated, + self.socket.as_ref(), + cx, + peer, + &on_wire, + ); } // TooLarge: the connection is already Established (max_size was Some), so // there is no pending Initial to flush; fall back to plain UDP. DatagramSendStatus::TooLarge => { - if let Some(socket) = self.socket.as_ref() { - // Ignoring Poll: gossip is best-effort — SWIM recovers next round. - let _ = socket.poll_send_to(cx, &on_wire, peer); - } + poll_send_gossip( + &mut self.leave_drain, + self.leave_initiated, + self.socket.as_ref(), + cx, + peer, + &on_wire, + ); } } } @@ -1206,6 +1258,12 @@ where if let Some(pl) = this.pending_leave.take() { pl.resolve_all(|| Err(SerfError::Shutdown)); } + // Flush any retained leave-farewell datagrams before releasing the socket: + // the quiescence loop above retains a backpressured graceful-leave fan-out + // rather than dropping it, and it must reach the wire before the socket + // drops. A single bounded sweep (stops at the first backpressure) exhausts + // it without an unbounded wait — the bind must be released for a rebind. + retry_retained_leave(&mut this.leave_drain, this.socket.as_ref(), cx); // Release the bound port BEFORE acking: dropping the agnostic UDP socket // closes its FD synchronously, so a caller resuming from `shutdown().await` // can immediately rebind the same address. diff --git a/serf-reactor/src/driver/shared.rs b/serf-reactor/src/driver/shared.rs index a78e9e18..97dd7956 100644 --- a/serf-reactor/src/driver/shared.rs +++ b/serf-reactor/src/driver/shared.rs @@ -8,6 +8,13 @@ //! recv-loops the gossip socket to kernel-empty and fires `handle_timeout` inline //! — there is no completion-backend past-due drain to build. +#[cfg(any(feature = "tcp", feature = "quic"))] +use agnostic::net::UdpSocket; +#[cfg(any(feature = "tcp", feature = "quic"))] +use core::task::{Context, Poll}; +#[cfg(any(feature = "tcp", feature = "quic"))] +use std::{collections::VecDeque, io, net::SocketAddr, vec::Vec}; + /// Coordinator-allocated handle for one in-flight reliable exchange. /// /// Shared by the stream driver and the per-bridge task so they agree on the same @@ -62,3 +69,123 @@ pub(crate) async fn dispatch_event_delegate( _ => {} } } + +// ── leave-drain retention ───────────────────────────────────────────────────── +// +// Periodic gossip is best-effort: a readiness-based UDP send that returns +// `Pending` (kernel buffer full) or `Ready(Err)` drops the datagram, and SWIM +// re-sends on the next round. The graceful-leave fan-out has NO next round — a +// dropped farewell leaves peers to classify the intentional departure as a +// failure while `leave()` reported success — so once `leave()` has been +// initiated the pump RETAINS the fan-out datagrams whose send did not complete +// and retries them until the socket accepts them (or teardown exhausts them). + +/// Encoded-and-transformed leave-farewell gossip datagrams retained after a +/// non-completing UDP send, keyed by their destination and retried FIFO. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) type LeaveDrain = VecDeque<(SocketAddr, Vec)>; + +/// Surface a leave-farewell datagram the driver could not deliver. Unlike +/// best-effort periodic gossip the drop is logged (the send error is +/// deterministic, config/IO-class, and the fan-out has no next round to mask +/// it). A no-op without the `tracing` feature. +#[cfg(any(feature = "tcp", feature = "quic"))] +fn trace_leave_send_error(_peer: SocketAddr, _err: &io::Error) { + #[cfg(feature = "tracing")] + tracing::debug!(peer = %_peer, error = %_err, "serf leave farewell datagram send failed"); +} + +/// Surface a leave-farewell datagram dropped because it could not be encoded or +/// encrypted — a deterministic config-class failure, unlike a transient +/// best-effort gossip drop. A no-op without the `tracing` feature. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) fn trace_leave_transform_error(_peer: SocketAddr) { + #[cfg(feature = "tracing")] + tracing::debug!(peer = %_peer, "serf leave farewell datagram could not be encoded or encrypted"); +} + +/// Record the outcome of one readiness-based leave-farewell datagram send into +/// `retained`: +/// - `Ready(Ok)` — it left the socket; nothing to retain. +/// - `Ready(Err)` — attempted; logged and NOT retained (a deterministic error +/// the next round cannot mask). +/// - `Pending` — the socket is backpressured; `(peer, datagram)` is retained for +/// the next writable wake (the datagram is copied only on this path). +/// +/// Returns `true` when the datagram was retained, so a caller draining fresh +/// transmits learns the socket is backpressured. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) fn retain_leave_datagram( + retained: &mut LeaveDrain, + peer: SocketAddr, + datagram: &[u8], + outcome: Poll>, +) -> bool { + match outcome { + Poll::Ready(Ok(_)) => false, + Poll::Ready(Err(err)) => { + trace_leave_send_error(peer, &err); + false + } + Poll::Pending => { + retained.push_back((peer, datagram.to_vec())); + true + } + } +} + +/// Send one already-transformed gossip datagram over the plain-UDP socket. +/// +/// Periodic gossip (`leave_initiated == false`) is best-effort: a non-completing +/// send drops the datagram. Once `leave()` has been initiated the leave fan-out +/// is retained on backpressure via [`retain_leave_datagram`]. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) fn poll_send_gossip( + retained: &mut LeaveDrain, + leave_initiated: bool, + socket: Option<&S>, + cx: &mut Context<'_>, + peer: SocketAddr, + on_wire: &[u8], +) where + S: UdpSocket, +{ + let Some(socket) = socket else { + return; + }; + let outcome = socket.poll_send_to(cx, on_wire, peer); + if leave_initiated { + retain_leave_datagram(retained, peer, on_wire, outcome); + } +} + +/// Retry the retained leave-farewell datagrams FIRST (oldest to newest), sending +/// each over `socket` until one is backpressured. A `Ready(Err)` is logged and +/// counts as attempted; a `Pending` keeps that datagram (and every later one) +/// retained and stops the pass. Bounded: a single front-to-back sweep with no +/// re-enqueue of a just-sent datagram, so it cannot loop. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) fn retry_retained_leave( + retained: &mut LeaveDrain, + socket: Option<&S>, + cx: &mut Context<'_>, +) where + S: UdpSocket, +{ + let Some(socket) = socket else { + return; + }; + while let Some((peer, datagram)) = retained.pop_front() { + match socket.poll_send_to(cx, &datagram, peer) { + Poll::Ready(Ok(_)) => {} + Poll::Ready(Err(err)) => trace_leave_send_error(peer, &err), + Poll::Pending => { + retained.push_front((peer, datagram)); + break; + } + } + } +} + +#[cfg(all(test, any(feature = "tcp", feature = "quic")))] +mod tests; diff --git a/serf-reactor/src/driver/shared/tests.rs b/serf-reactor/src/driver/shared/tests.rs new file mode 100644 index 00000000..6ff182be --- /dev/null +++ b/serf-reactor/src/driver/shared/tests.rs @@ -0,0 +1,55 @@ +use super::*; + +/// The leave-drain retention bookkeeping: `Ready(Ok)` sends (retaining nothing), +/// `Pending` retains FIFO at the back (returning `true`), and `Ready(Err)` is +/// attempted — logged, retaining nothing. The datagram bytes are copied only on +/// the retain path. +#[test] +fn retain_leave_datagram_records_only_pending() { + let peer: SocketAddr = "127.0.0.1:7946".parse().unwrap(); + let mut retained: LeaveDrain = VecDeque::new(); + + // Ready(Ok): sent, nothing retained. + assert!(!retain_leave_datagram( + &mut retained, + peer, + b"alpha", + Poll::Ready(Ok(5)) + )); + assert!(retained.is_empty(), "a completed send retains nothing"); + + // Pending: retained at the back, returns true (socket backpressured). + assert!(retain_leave_datagram( + &mut retained, + peer, + b"beta", + Poll::Pending + )); + assert_eq!(retained.len(), 1); + assert_eq!(retained.back().unwrap(), &(peer, b"beta".to_vec())); + + // A second Pending appends behind the first (FIFO order preserved). + let peer2: SocketAddr = "127.0.0.1:7947".parse().unwrap(); + assert!(retain_leave_datagram( + &mut retained, + peer2, + b"gamma", + Poll::Pending + )); + assert_eq!(retained.len(), 2); + assert_eq!(retained.front().unwrap(), &(peer, b"beta".to_vec())); + assert_eq!(retained.back().unwrap(), &(peer2, b"gamma".to_vec())); + + // Ready(Err): attempted (logged), not retained; the queue is unchanged. + assert!(!retain_leave_datagram( + &mut retained, + peer, + b"delta", + Poll::Ready(Err(io::Error::other("send failed"))), + )); + assert_eq!( + retained.len(), + 2, + "an errored send is attempted, not retained" + ); +} diff --git a/serf-reactor/src/driver/stream/mod.rs b/serf-reactor/src/driver/stream/mod.rs index 837aa50c..86bea64e 100644 --- a/serf-reactor/src/driver/stream/mod.rs +++ b/serf-reactor/src/driver/stream/mod.rs @@ -65,7 +65,10 @@ use crate::{ delegate::Delegate, driver::{ options::{RuntimeOptions, StreamTransportOptions}, - shared::{ExchangeId, dispatch_event_delegate, observation_payload_bytes}, + shared::{ + ExchangeId, LeaveDrain, dispatch_event_delegate, observation_payload_bytes, poll_send_gossip, + retry_retained_leave, trace_leave_transform_error, + }, }, drop_counter::ReactorDropCounter, error::{JoinFailed, Result, SerfError}, @@ -347,6 +350,13 @@ where pending_joins: Vec, /// The in-flight graceful leave, resolved on `LeftCluster`. pending_leave: Option, + /// Set once `leave()` has been initiated. Switches the gossip egress from + /// best-effort (drop on non-completion) to retention for the leave fan-out, + /// which has no next gossip round to re-send a dropped farewell. + leave_initiated: bool, + /// Leave-farewell datagrams retained after a non-completing UDP send, retried + /// FIFO ahead of fresh transmits and flushed before the socket drops. + leave_drain: LeaveDrain, /// Parked `Shutdown` replies — acked only after the bind sockets drop, so a /// caller resuming from `shutdown().await` can rebind the same address. A `Vec` /// because several callers can race `shutdown()`. @@ -484,6 +494,8 @@ where label, pending_joins: Vec::new(), pending_leave: None, + leave_initiated: false, + leave_drain: LeaveDrain::new(), shutdown_reply: Vec::new(), bridges: HashMap::new(), accepted_rx, @@ -669,6 +681,9 @@ where let res: Result<()> = self.endpoint.leave(now).map_err(SerfError::from); match res { Ok(()) if was_alive => { + // The leave mutated and fan-out is queued: switch the gossip egress + // to retention so a backpressured farewell is retried, not dropped. + self.leave_initiated = true; self.pending_leave = Some(PendingLeave { repliers: vec![reply], deadline: now + leave_timeout, @@ -1083,6 +1098,15 @@ where // the last transmit is the endpoint's leave-completion fence (it emits // `LeftCluster`), so the leave/shutdown datagrams reach the socket before that // fence fires. + // + // Once `leave()` has been initiated the fan-out is RETAINED on backpressure: + // retry the retained datagrams FIRST (they drain as the socket becomes + // writable), then pop fresh transmits at the unchanged cadence — a + // non-completing send is retained (not dropped) instead of leaving peers to + // read the departure as a failure. Periodic gossip stays best-effort. + if self.leave_initiated { + retry_retained_leave(&mut self.leave_drain, self.socket.as_ref(), cx); + } let encode_opts = EncodeOptions::new(self.label.clone()); let mut sent = 0; while sent < budget { @@ -1095,14 +1119,24 @@ where let (to, msg) = pkt.into_parts(); match encode_outgoing(&msg, &encode_opts) { Ok(b) => (to, b), - Err(_) => continue, + Err(_) => { + if self.leave_initiated { + trace_leave_transform_error(to); + } + continue; + } } } Transmit::Compound(cmp) => { let (to, msgs) = cmp.into_parts(); match encode_outgoing_compound(&msgs, &encode_opts) { Ok(b) => (to, b), - Err(_) => continue, + Err(_) => { + if self.leave_initiated { + trace_leave_transform_error(to); + } + continue; + } } } }; @@ -1112,14 +1146,24 @@ where { on_wire = match self.endpoint.encrypt_gossip(&on_wire) { Ok(bytes) => bytes, - Err(_) => continue, + Err(_) => { + if self.leave_initiated { + trace_leave_transform_error(peer); + } + continue; + } }; } - if let Some(socket) = self.socket.as_ref() { - // Ignoring Poll: gossip is best-effort — a full or errored UDP send drops - // the datagram and SWIM recovers on the next round. - let _ = socket.poll_send_to(cx, &on_wire, peer); - } + // Periodic gossip is best-effort (dropped on non-completion, SWIM re-sends + // next round); the leave fan-out is retained for a retry. + poll_send_gossip( + &mut self.leave_drain, + self.leave_initiated, + self.socket.as_ref(), + cx, + peer, + &on_wire, + ); } worked |= sent > 0; more |= sent == budget; @@ -1487,6 +1531,12 @@ where // scheduled (awaited below). Dropping the gossip socket closes its UDP FD // synchronously. drop(this.accept_shutdown_tx.take()); + // Flush any retained leave-farewell datagrams before releasing the socket. + // The graceful drain runs across polls (retried as the socket becomes + // writable), so a residual queue here is rare; a single bounded sweep + // (stops at the first backpressure) exhausts it without an unbounded wait, + // since the bind must be released for a caller to rebind the address. + retry_retained_leave(&mut this.leave_drain, this.socket.as_ref(), cx); drop(this.socket.take()); } From 3ed76afdd917157156b97fb672f0d00b462a23d5 Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 04:06:51 +1200 Subject: [PATCH 090/128] fix(serf-reactor): leave resolves only after the farewell reaches the socket MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two holes remained in the leave-farewell retention. The raw QUIC egress discarded quinn transmit packets on a backpressured send with a comment claiming quinn retransmits them — false for application DATAGRAM frames, which quinn's loss recovery does not cover — so a datagram-mode farewell could vanish; during a leave those packets now join the same retained queue. And the parked leave resolved on LeftCluster without checking the retained queue, so a caller could get Ok and tear down while farewells sat undelivered; the resolution now waits for the queue to empty (bounded by the caller's existing leave timeout, so Ok means every farewell was accepted by the socket), and the teardown parks on the writable waker under a one-second deadline to drain the residue before releasing the socket instead of dropping it. --- serf-reactor/src/driver/quic/mod.rs | 76 +++++++++++++++++++++++---- serf-reactor/src/driver/shared.rs | 21 ++++++++ serf-reactor/src/driver/stream/mod.rs | 62 ++++++++++++++++++---- serf-reactor/tests/cluster/mod.rs | 7 ++- 4 files changed, 145 insertions(+), 21 deletions(-) diff --git a/serf-reactor/src/driver/quic/mod.rs b/serf-reactor/src/driver/quic/mod.rs index 2cf8412a..3f6f033b 100644 --- a/serf-reactor/src/driver/quic/mod.rs +++ b/serf-reactor/src/driver/quic/mod.rs @@ -84,8 +84,9 @@ use crate::{ driver::{ options::RuntimeOptions, shared::{ - ExchangeId, LeaveDrain, dispatch_event_delegate, observation_payload_bytes, poll_send_gossip, - retry_retained_leave, trace_leave_transform_error, + ExchangeId, LEAVE_DRAIN_TEARDOWN_BOUND, LeaveDrain, dispatch_event_delegate, + observation_payload_bytes, poll_send_gossip, retain_leave_datagram, retry_retained_leave, + trace_leave_drain_residue, trace_leave_transform_error, }, }, drop_counter::ReactorDropCounter, @@ -315,6 +316,12 @@ where /// Leave-farewell datagrams retained after a non-completing plain-UDP send, /// retried FIFO ahead of fresh transmits and flushed before the socket drops. leave_drain: LeaveDrain, + /// `LeftCluster` observed while retained farewells were still queued: the + /// parked leave resolves the moment `leave_drain` empties, so `Ok` from + /// `leave().await` always means the farewell reached the socket. + left_cluster_seen: bool, + /// Deadline bounding the teardown park that drains retained farewells. + leave_drain_deadline: Option, /// Parked `Shutdown` replies — acked only after the UDP socket drops, so a caller /// resuming from `shutdown().await` can rebind the same address. A `Vec` because /// several callers can race `shutdown()`. @@ -388,6 +395,8 @@ where pending_leave: None, leave_initiated: false, leave_drain: LeaveDrain::new(), + left_cluster_seen: false, + leave_drain_deadline: None, shutdown_reply: Vec::new(), recv_buf: vec![0u8; buf_len.max(1)], iter_drain_cap: driver_opts.iter_drain_cap().max(1), @@ -760,6 +769,15 @@ where // plain-UDP sends below are retained.) if self.leave_initiated { retry_retained_leave(&mut self.leave_drain, self.socket.as_ref(), cx); + // A `LeftCluster` observed while these datagrams were still queued + // deferred the parked leave; resolve it now that the socket has accepted + // every retained farewell. + if self.left_cluster_seen && self.leave_drain.is_empty() { + self.left_cluster_seen = false; + if let Some(pl) = self.pending_leave.take() { + pl.resolve_all(|| Ok(())); + } + } } let encode_opts = EncodeOptions::new(self.label.clone()); let unreliable = self.endpoint.unreliable_transport(); @@ -889,8 +907,20 @@ where }; raw_sent += 1; if let Some(socket) = self.socket.as_ref() { - // Ignoring Poll: a dropped QUIC datagram is retransmitted by quinn-proto. - let _ = socket.poll_send_to(cx, &bytes, dest); + let outcome = socket.poll_send_to(cx, &bytes, dest); + if self.leave_initiated { + // The leave fan-out may ride a quinn application DATAGRAM frame, + // which quinn does NOT retransmit — its loss recovery covers the + // reliable streams only. A backpressured send here would lose the + // farewell for good, so it is retained and retried exactly like the + // plain-UDP leave drain. + retain_leave_datagram(&mut self.leave_drain, dest, &bytes, outcome); + } else { + // Ignoring Poll: outside a leave, a dropped handshake/ACK/stream + // packet is recovered by quinn's own loss detection, and a dropped + // datagram-mode gossip frame is re-sent by the next periodic round. + let _ = outcome; + } } } worked |= raw_sent > 0; @@ -1001,10 +1031,19 @@ where matches!(c.outcome(), ExchangeStatus::Succeeded), ); } - if matches!(ev, Event::LeftCluster) - && let Some(pl) = self.pending_leave.take() - { - pl.resolve_all(|| Ok(())); + if matches!(ev, Event::LeftCluster) { + // Resolve the parked leave only once every retained farewell datagram has + // been accepted by the socket: `Ok` from `leave().await` means the leave + // notices reached the transport, not merely that the machine drained its + // fan-out. With retained datagrams still queued, remember the fence and + // resolve when the drain empties (bounded by the caller's leave timeout). + if self.leave_drain.is_empty() { + if let Some(pl) = self.pending_leave.take() { + pl.resolve_all(|| Ok(())); + } + } else { + self.left_cluster_seen = true; + } } // A lost id-conflict vote means the local node MUST stop, exactly as for a // `Command::Shutdown`. Flag shutdown; the pump self-wakes into the teardown @@ -1261,9 +1300,26 @@ where // Flush any retained leave-farewell datagrams before releasing the socket: // the quiescence loop above retains a backpressured graceful-leave fan-out // rather than dropping it, and it must reach the wire before the socket - // drops. A single bounded sweep (stops at the first backpressure) exhausts - // it without an unbounded wait — the bind must be released for a rebind. + // drops. A residue parks the teardown — bounded by a short deadline — + // instead of being dropped: the `Pending` send has the writable waker + // registered while the deadline timer keeps a dead socket from hanging + // shutdown. retry_retained_leave(&mut this.leave_drain, this.socket.as_ref(), cx); + if !this.leave_drain.is_empty() { + let now = Instant::now(); + let deadline = *this + .leave_drain_deadline + .get_or_insert(now + LEAVE_DRAIN_TEARDOWN_BOUND); + if now < deadline { + this.arm_timer(deadline, now); + if let Some(timer) = this.timer.as_mut() + && timer.as_mut().poll(cx).is_pending() + { + return Poll::Pending; + } + } + trace_leave_drain_residue(this.leave_drain.len()); + } // Release the bound port BEFORE acking: dropping the agnostic UDP socket // closes its FD synchronously, so a caller resuming from `shutdown().await` // can immediately rebind the same address. diff --git a/serf-reactor/src/driver/shared.rs b/serf-reactor/src/driver/shared.rs index 97dd7956..abad779c 100644 --- a/serf-reactor/src/driver/shared.rs +++ b/serf-reactor/src/driver/shared.rs @@ -95,6 +95,19 @@ fn trace_leave_send_error(_peer: SocketAddr, _err: &io::Error) { tracing::debug!(peer = %_peer, error = %_err, "serf leave farewell datagram send failed"); } +/// Surface retained leave-farewell datagrams abandoned when the teardown drain +/// deadline won — the socket stayed unwritable, so shutdown proceeds and the +/// affected peers will read the departure as a failure. A no-op without the +/// `tracing` feature. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) fn trace_leave_drain_residue(_count: usize) { + #[cfg(feature = "tracing")] + tracing::debug!( + residual = _count, + "serf leave farewell datagrams abandoned at the teardown drain deadline" + ); +} + /// Surface a leave-farewell datagram dropped because it could not be encoded or /// encrypted — a deterministic config-class failure, unlike a transient /// best-effort gossip drop. A no-op without the `tracing` feature. @@ -114,6 +127,14 @@ pub(crate) fn trace_leave_transform_error(_peer: SocketAddr) { /// /// Returns `true` when the datagram was retained, so a caller draining fresh /// transmits learns the socket is backpressured. +/// Upper bound on how long a pump's teardown parks waiting for retained +/// leave-farewell datagrams to drain before releasing the socket anyway. Keeps +/// shutdown from hanging on a persistently unwritable socket while still giving +/// the farewell a real window to reach the wire. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) const LEAVE_DRAIN_TEARDOWN_BOUND: core::time::Duration = + core::time::Duration::from_secs(1); + #[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) fn retain_leave_datagram( retained: &mut LeaveDrain, diff --git a/serf-reactor/src/driver/stream/mod.rs b/serf-reactor/src/driver/stream/mod.rs index 86bea64e..5a45118f 100644 --- a/serf-reactor/src/driver/stream/mod.rs +++ b/serf-reactor/src/driver/stream/mod.rs @@ -66,8 +66,9 @@ use crate::{ driver::{ options::{RuntimeOptions, StreamTransportOptions}, shared::{ - ExchangeId, LeaveDrain, dispatch_event_delegate, observation_payload_bytes, poll_send_gossip, - retry_retained_leave, trace_leave_transform_error, + ExchangeId, LEAVE_DRAIN_TEARDOWN_BOUND, LeaveDrain, dispatch_event_delegate, + observation_payload_bytes, poll_send_gossip, retry_retained_leave, trace_leave_drain_residue, + trace_leave_transform_error, }, }, drop_counter::ReactorDropCounter, @@ -357,6 +358,12 @@ where /// Leave-farewell datagrams retained after a non-completing UDP send, retried /// FIFO ahead of fresh transmits and flushed before the socket drops. leave_drain: LeaveDrain, + /// `LeftCluster` observed while retained farewells were still queued: the + /// parked leave resolves the moment `leave_drain` empties, so `Ok` from + /// `leave().await` always means the farewell reached the socket. + left_cluster_seen: bool, + /// Deadline bounding the teardown park that drains retained farewells. + leave_drain_deadline: Option, /// Parked `Shutdown` replies — acked only after the bind sockets drop, so a /// caller resuming from `shutdown().await` can rebind the same address. A `Vec` /// because several callers can race `shutdown()`. @@ -496,6 +503,8 @@ where pending_leave: None, leave_initiated: false, leave_drain: LeaveDrain::new(), + left_cluster_seen: false, + leave_drain_deadline: None, shutdown_reply: Vec::new(), bridges: HashMap::new(), accepted_rx, @@ -1106,6 +1115,15 @@ where // read the departure as a failure. Periodic gossip stays best-effort. if self.leave_initiated { retry_retained_leave(&mut self.leave_drain, self.socket.as_ref(), cx); + // A `LeftCluster` observed while these datagrams were still queued + // deferred the parked leave; resolve it now that the socket has accepted + // every retained farewell. + if self.left_cluster_seen && self.leave_drain.is_empty() { + self.left_cluster_seen = false; + if let Some(pl) = self.pending_leave.take() { + pl.resolve_all(|| Ok(())); + } + } } let encode_opts = EncodeOptions::new(self.label.clone()); let mut sent = 0; @@ -1278,10 +1296,19 @@ where matches!(c.outcome(), ExchangeStatus::Succeeded), ); } - if matches!(ev, Event::LeftCluster) - && let Some(pl) = self.pending_leave.take() - { - pl.resolve_all(|| Ok(())); + if matches!(ev, Event::LeftCluster) { + // Resolve the parked leave only once every retained farewell datagram has + // been accepted by the socket: `Ok` from `leave().await` means the leave + // notices reached the transport, not merely that the machine drained its + // fan-out. With retained datagrams still queued, remember the fence and + // resolve when the drain empties (bounded by the caller's leave timeout). + if self.leave_drain.is_empty() { + if let Some(pl) = self.pending_leave.take() { + pl.resolve_all(|| Ok(())); + } + } else { + self.left_cluster_seen = true; + } } // A lost id-conflict vote means the local node MUST stop, exactly as for a // `Command::Shutdown`. Flag shutdown; the pump self-wakes into the teardown @@ -1533,10 +1560,27 @@ where drop(this.accept_shutdown_tx.take()); // Flush any retained leave-farewell datagrams before releasing the socket. // The graceful drain runs across polls (retried as the socket becomes - // writable), so a residual queue here is rare; a single bounded sweep - // (stops at the first backpressure) exhausts it without an unbounded wait, - // since the bind must be released for a caller to rebind the address. + // writable), so a residual queue here is rare. A residue parks the + // teardown — bounded by a short deadline — instead of being dropped: the + // farewell is the peers' only first-hand signal that the departure was + // intentional, and the `Pending` send has the writable waker registered + // while the deadline timer keeps a dead socket from hanging shutdown. retry_retained_leave(&mut this.leave_drain, this.socket.as_ref(), cx); + if !this.leave_drain.is_empty() { + let now = Instant::now(); + let deadline = *this + .leave_drain_deadline + .get_or_insert(now + LEAVE_DRAIN_TEARDOWN_BOUND); + if now < deadline { + this.arm_timer(deadline, now); + if let Some(timer) = this.timer.as_mut() + && timer.as_mut().poll(cx).is_pending() + { + return Poll::Pending; + } + } + trace_leave_drain_residue(this.leave_drain.len()); + } drop(this.socket.take()); } diff --git a/serf-reactor/tests/cluster/mod.rs b/serf-reactor/tests/cluster/mod.rs index 32b4a30c..3899c2e2 100644 --- a/serf-reactor/tests/cluster/mod.rs +++ b/serf-reactor/tests/cluster/mod.rs @@ -235,8 +235,11 @@ where /// reintroduced flush wait). Unlike [`kill_abrupt`](Self::kill_abrupt), the /// farewell (leave intent packed with the dead-self notice) reaches peers /// before teardown, so peers observe an intentional Leave rather than a - /// probe-timeout Failed. The slot's id, addr, and event log are retained for - /// later assertions. + /// probe-timeout Failed. `shutdown()` follows the resolved `leave()` with no + /// intervening delay, deliberately pinning the leave-then-immediate-shutdown + /// ordering: a resolved leave means every farewell datagram was accepted by + /// the socket, so an immediate teardown cannot discard one. The slot's id, + /// addr, and event log are retained for later assertions. pub async fn leave_graceful(&mut self, i: usize) -> Duration { let serf = self.slots[i].serf.take().expect("node slot is live"); let start = std::time::Instant::now(); From 040e6a2985deec3245038bf776424c03b6b9c973 Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 10:59:29 +1200 Subject: [PATCH 091/128] fix(serf-reactor): account farewell delivery failures and drain re-entrantly at teardown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A leave-farewell send that returned Ready(Err) was popped and logged, so the drain emptied and the parked leave resolved Ok even when the datagram never left this host. Classify send errors at the retention layer: per-peer network signals (reset/refused/aborted, host/network-unreachable, addr-not-available — the network's answer to a delivered attempt, and ICMP-reflected on some platforms) stay log-only, while local socket failures set a per-pump leave_send_failed flag. Encode/encrypt failures during the leave fan-out set the same flag — that peer's farewell is deterministically lost. Both resolve sites in both pumps now map the flag onto the caller's result via a shared leave_outcome helper and the new SerfError::LeaveFarewellUndelivered, so Ok from leave().await always means the fan-out was handed to the transport. The stream pump's teardown farewell drain lived inside the one-shot freeze block: its first Pending return skipped the block on re-poll, leaving the socket undropped and the residue undrained while shutdown acked anyway. The drain is now its own re-entrant phase guarded by socket-presence — any teardown wake retries the retained datagrams, parks bounded by the drain deadline, and only then releases the socket. The QUIC teardown re-runs its whole idempotent shutdown branch, so it only needed the failure flag threaded. --- serf-reactor/src/driver/quic/mod.rs | 42 +++++++++++-- serf-reactor/src/driver/shared.rs | 75 ++++++++++++++++++---- serf-reactor/src/driver/shared/tests.rs | 82 ++++++++++++++++++++++++- serf-reactor/src/driver/stream/mod.rs | 54 +++++++++++----- serf-reactor/src/error/mod.rs | 11 ++++ serf-reactor/src/error/tests.rs | 1 + serf-reactor/src/serf/mod.rs | 8 ++- 7 files changed, 236 insertions(+), 37 deletions(-) diff --git a/serf-reactor/src/driver/quic/mod.rs b/serf-reactor/src/driver/quic/mod.rs index 3f6f033b..7a1c940a 100644 --- a/serf-reactor/src/driver/quic/mod.rs +++ b/serf-reactor/src/driver/quic/mod.rs @@ -84,7 +84,7 @@ use crate::{ driver::{ options::RuntimeOptions, shared::{ - ExchangeId, LEAVE_DRAIN_TEARDOWN_BOUND, LeaveDrain, dispatch_event_delegate, + ExchangeId, LEAVE_DRAIN_TEARDOWN_BOUND, LeaveDrain, dispatch_event_delegate, leave_outcome, observation_payload_bytes, poll_send_gossip, retain_leave_datagram, retry_retained_leave, trace_leave_drain_residue, trace_leave_transform_error, }, @@ -322,6 +322,11 @@ where left_cluster_seen: bool, /// Deadline bounding the teardown park that drains retained farewells. leave_drain_deadline: Option, + /// A leave-farewell send failed on the LOCAL socket (not a per-peer network + /// signal): the parked leave resolves + /// [`LeaveFarewellUndelivered`](SerfError::LeaveFarewellUndelivered) instead + /// of a false `Ok`. + leave_send_failed: bool, /// Parked `Shutdown` replies — acked only after the UDP socket drops, so a caller /// resuming from `shutdown().await` can rebind the same address. A `Vec` because /// several callers can race `shutdown()`. @@ -397,6 +402,7 @@ where leave_drain: LeaveDrain::new(), left_cluster_seen: false, leave_drain_deadline: None, + leave_send_failed: false, shutdown_reply: Vec::new(), recv_buf: vec![0u8; buf_len.max(1)], iter_drain_cap: driver_opts.iter_drain_cap().max(1), @@ -768,14 +774,20 @@ where // sends that ride an established QUIC connection are quinn-managed; only the // plain-UDP sends below are retained.) if self.leave_initiated { - retry_retained_leave(&mut self.leave_drain, self.socket.as_ref(), cx); + retry_retained_leave( + &mut self.leave_drain, + self.socket.as_ref(), + cx, + &mut self.leave_send_failed, + ); // A `LeftCluster` observed while these datagrams were still queued // deferred the parked leave; resolve it now that the socket has accepted // every retained farewell. if self.left_cluster_seen && self.leave_drain.is_empty() { self.left_cluster_seen = false; if let Some(pl) = self.pending_leave.take() { - pl.resolve_all(|| Ok(())); + let failed = self.leave_send_failed; + pl.resolve_all(|| leave_outcome(failed)); } } } @@ -796,6 +808,7 @@ where Err(_) => { if self.leave_initiated { trace_leave_transform_error(to); + self.leave_send_failed = true; } continue; } @@ -808,6 +821,7 @@ where Err(_) => { if self.leave_initiated { trace_leave_transform_error(to); + self.leave_send_failed = true; } continue; } @@ -823,6 +837,7 @@ where Err(_) => { if self.leave_initiated { trace_leave_transform_error(peer); + self.leave_send_failed = true; } continue; } @@ -842,6 +857,7 @@ where cx, peer, &on_wire, + &mut self.leave_send_failed, ); } UnreliableTransport::Datagram => { @@ -869,6 +885,7 @@ where cx, peer, &on_wire, + &mut self.leave_send_failed, ); } // TooLarge: the connection is already Established (max_size was Some), so @@ -881,6 +898,7 @@ where cx, peer, &on_wire, + &mut self.leave_send_failed, ); } } @@ -914,7 +932,13 @@ where // reliable streams only. A backpressured send here would lose the // farewell for good, so it is retained and retried exactly like the // plain-UDP leave drain. - retain_leave_datagram(&mut self.leave_drain, dest, &bytes, outcome); + retain_leave_datagram( + &mut self.leave_drain, + dest, + &bytes, + outcome, + &mut self.leave_send_failed, + ); } else { // Ignoring Poll: outside a leave, a dropped handshake/ACK/stream // packet is recovered by quinn's own loss detection, and a dropped @@ -1039,7 +1063,8 @@ where // resolve when the drain empties (bounded by the caller's leave timeout). if self.leave_drain.is_empty() { if let Some(pl) = self.pending_leave.take() { - pl.resolve_all(|| Ok(())); + let failed = self.leave_send_failed; + pl.resolve_all(|| leave_outcome(failed)); } } else { self.left_cluster_seen = true; @@ -1304,7 +1329,12 @@ where // instead of being dropped: the `Pending` send has the writable waker // registered while the deadline timer keeps a dead socket from hanging // shutdown. - retry_retained_leave(&mut this.leave_drain, this.socket.as_ref(), cx); + retry_retained_leave( + &mut this.leave_drain, + this.socket.as_ref(), + cx, + &mut this.leave_send_failed, + ); if !this.leave_drain.is_empty() { let now = Instant::now(); let deadline = *this diff --git a/serf-reactor/src/driver/shared.rs b/serf-reactor/src/driver/shared.rs index abad779c..b863fb89 100644 --- a/serf-reactor/src/driver/shared.rs +++ b/serf-reactor/src/driver/shared.rs @@ -117,16 +117,44 @@ pub(crate) fn trace_leave_transform_error(_peer: SocketAddr) { tracing::debug!(peer = %_peer, "serf leave farewell datagram could not be encoded or encrypted"); } -/// Record the outcome of one readiness-based leave-farewell datagram send into -/// `retained`: -/// - `Ready(Ok)` — it left the socket; nothing to retain. -/// - `Ready(Err)` — attempted; logged and NOT retained (a deterministic error -/// the next round cannot mask). -/// - `Pending` — the socket is backpressured; `(peer, datagram)` is retained for -/// the next writable wake (the datagram is copied only on this path). +/// The final outcome of one graceful leave: `Ok` when every farewell datagram +/// reached the socket, [`LeaveFarewellUndelivered`] when a local send or +/// transform failure lost part of the fan-out. /// -/// Returns `true` when the datagram was retained, so a caller draining fresh -/// transmits learns the socket is backpressured. +/// [`LeaveFarewellUndelivered`]: crate::error::SerfError::LeaveFarewellUndelivered +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) fn leave_outcome(send_failed: bool) -> crate::error::Result<()> { + if send_failed { + Err(crate::error::SerfError::LeaveFarewellUndelivered) + } else { + Ok(()) + } +} + +/// Whether a leave-farewell send error is a LOCAL socket failure (the leave +/// contract is broken — the datagram never left this host) rather than a +/// per-peer network signal. +/// +/// Per-peer signals — a reset/refusal reflected for a peer that is itself gone, +/// or an unreachable route — are the network's answer to a delivered attempt: +/// the reference implementation logs them and proceeds, and failing the whole +/// leave because one peer already died would be wrong (and flaky on platforms +/// that reflect ICMP errors into UDP sends). Everything else — a closed or +/// invalid socket, a broken pipe, an out-of-memory send path — means the +/// farewell could not be handed off at all. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) fn farewell_send_failure_is_local(err: &io::Error) -> bool { + !matches!( + err.kind(), + io::ErrorKind::ConnectionReset + | io::ErrorKind::ConnectionRefused + | io::ErrorKind::ConnectionAborted + | io::ErrorKind::HostUnreachable + | io::ErrorKind::NetworkUnreachable + | io::ErrorKind::AddrNotAvailable + ) +} + /// Upper bound on how long a pump's teardown parks waiting for retained /// leave-farewell datagrams to drain before releasing the socket anyway. Keeps /// shutdown from hanging on a persistently unwritable socket while still giving @@ -135,17 +163,32 @@ pub(crate) fn trace_leave_transform_error(_peer: SocketAddr) { pub(crate) const LEAVE_DRAIN_TEARDOWN_BOUND: core::time::Duration = core::time::Duration::from_secs(1); +/// Record the outcome of one readiness-based leave-farewell datagram send into +/// `retained`: +/// - `Ready(Ok)` — it left the socket; nothing to retain. +/// - `Ready(Err)` — attempted; logged and NOT retained (retrying a socket that +/// errors is futile). A LOCAL failure additionally sets `send_failed`, so the +/// parked leave resolves with an error instead of a false success. +/// - `Pending` — the socket is backpressured; `(peer, datagram)` is retained for +/// the next writable wake (the datagram is copied only on this path). +/// +/// Returns `true` when the datagram was retained, so a caller draining fresh +/// transmits learns the socket is backpressured. #[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) fn retain_leave_datagram( retained: &mut LeaveDrain, peer: SocketAddr, datagram: &[u8], outcome: Poll>, + send_failed: &mut bool, ) -> bool { match outcome { Poll::Ready(Ok(_)) => false, Poll::Ready(Err(err)) => { trace_leave_send_error(peer, &err); + if farewell_send_failure_is_local(&err) { + *send_failed = true; + } false } Poll::Pending => { @@ -168,6 +211,7 @@ pub(crate) fn poll_send_gossip( cx: &mut Context<'_>, peer: SocketAddr, on_wire: &[u8], + send_failed: &mut bool, ) where S: UdpSocket, { @@ -176,13 +220,14 @@ pub(crate) fn poll_send_gossip( }; let outcome = socket.poll_send_to(cx, on_wire, peer); if leave_initiated { - retain_leave_datagram(retained, peer, on_wire, outcome); + retain_leave_datagram(retained, peer, on_wire, outcome, send_failed); } } /// Retry the retained leave-farewell datagrams FIRST (oldest to newest), sending /// each over `socket` until one is backpressured. A `Ready(Err)` is logged and -/// counts as attempted; a `Pending` keeps that datagram (and every later one) +/// counts as attempted — a LOCAL failure sets `send_failed` so the parked leave +/// resolves with an error; a `Pending` keeps that datagram (and every later one) /// retained and stops the pass. Bounded: a single front-to-back sweep with no /// re-enqueue of a just-sent datagram, so it cannot loop. #[cfg(any(feature = "tcp", feature = "quic"))] @@ -190,6 +235,7 @@ pub(crate) fn retry_retained_leave( retained: &mut LeaveDrain, socket: Option<&S>, cx: &mut Context<'_>, + send_failed: &mut bool, ) where S: UdpSocket, { @@ -199,7 +245,12 @@ pub(crate) fn retry_retained_leave( while let Some((peer, datagram)) = retained.pop_front() { match socket.poll_send_to(cx, &datagram, peer) { Poll::Ready(Ok(_)) => {} - Poll::Ready(Err(err)) => trace_leave_send_error(peer, &err), + Poll::Ready(Err(err)) => { + trace_leave_send_error(peer, &err); + if farewell_send_failure_is_local(&err) { + *send_failed = true; + } + } Poll::Pending => { retained.push_front((peer, datagram)); break; diff --git a/serf-reactor/src/driver/shared/tests.rs b/serf-reactor/src/driver/shared/tests.rs index 6ff182be..4a798e13 100644 --- a/serf-reactor/src/driver/shared/tests.rs +++ b/serf-reactor/src/driver/shared/tests.rs @@ -8,25 +8,30 @@ use super::*; fn retain_leave_datagram_records_only_pending() { let peer: SocketAddr = "127.0.0.1:7946".parse().unwrap(); let mut retained: LeaveDrain = VecDeque::new(); + let mut send_failed = false; // Ready(Ok): sent, nothing retained. assert!(!retain_leave_datagram( &mut retained, peer, b"alpha", - Poll::Ready(Ok(5)) + Poll::Ready(Ok(5)), + &mut send_failed, )); assert!(retained.is_empty(), "a completed send retains nothing"); + assert!(!send_failed, "a completed send is not a failure"); // Pending: retained at the back, returns true (socket backpressured). assert!(retain_leave_datagram( &mut retained, peer, b"beta", - Poll::Pending + Poll::Pending, + &mut send_failed, )); assert_eq!(retained.len(), 1); assert_eq!(retained.back().unwrap(), &(peer, b"beta".to_vec())); + assert!(!send_failed, "backpressure is retention, not failure"); // A second Pending appends behind the first (FIFO order preserved). let peer2: SocketAddr = "127.0.0.1:7947".parse().unwrap(); @@ -34,7 +39,8 @@ fn retain_leave_datagram_records_only_pending() { &mut retained, peer2, b"gamma", - Poll::Pending + Poll::Pending, + &mut send_failed, )); assert_eq!(retained.len(), 2); assert_eq!(retained.front().unwrap(), &(peer, b"beta".to_vec())); @@ -46,6 +52,7 @@ fn retain_leave_datagram_records_only_pending() { peer, b"delta", Poll::Ready(Err(io::Error::other("send failed"))), + &mut send_failed, )); assert_eq!( retained.len(), @@ -53,3 +60,72 @@ fn retain_leave_datagram_records_only_pending() { "an errored send is attempted, not retained" ); } + +/// Error accounting: a LOCAL send failure (broken socket — the farewell never +/// left this host) sets `send_failed` so the parked leave resolves with an +/// error, while a per-peer network signal (a reset or unreachable reflected +/// for a peer that is itself gone) is log-only — it neither retains nor fails +/// the leave. +#[test] +fn retain_leave_datagram_flags_only_local_failures() { + let peer: SocketAddr = "127.0.0.1:7946".parse().unwrap(); + let mut retained: LeaveDrain = VecDeque::new(); + + // Per-peer network signals: attempted, not retained, leave still succeeds. + for kind in [ + io::ErrorKind::ConnectionReset, + io::ErrorKind::ConnectionRefused, + io::ErrorKind::ConnectionAborted, + io::ErrorKind::HostUnreachable, + io::ErrorKind::NetworkUnreachable, + io::ErrorKind::AddrNotAvailable, + ] { + let mut send_failed = false; + assert!(!retain_leave_datagram( + &mut retained, + peer, + b"alpha", + Poll::Ready(Err(io::Error::from(kind))), + &mut send_failed, + )); + assert!(retained.is_empty()); + assert!( + !send_failed, + "a per-peer network signal ({kind:?}) must not fail the leave" + ); + } + + // Local socket failures: attempted, not retained, and the leave FAILS. + for kind in [ + io::ErrorKind::NotConnected, + io::ErrorKind::BrokenPipe, + io::ErrorKind::InvalidInput, + io::ErrorKind::Other, + ] { + let mut send_failed = false; + assert!(!retain_leave_datagram( + &mut retained, + peer, + b"alpha", + Poll::Ready(Err(io::Error::from(kind))), + &mut send_failed, + )); + assert!(retained.is_empty()); + assert!( + send_failed, + "a local socket failure ({kind:?}) must fail the leave" + ); + } +} + +/// The leave outcome maps the accumulated failure flag onto the caller-facing +/// `leave().await` result: no failure resolves `Ok`, a local send/transform +/// failure resolves `LeaveFarewellUndelivered`. +#[test] +fn leave_outcome_maps_the_failure_flag() { + assert!(leave_outcome(false).is_ok()); + assert!(matches!( + leave_outcome(true), + Err(crate::error::SerfError::LeaveFarewellUndelivered) + )); +} diff --git a/serf-reactor/src/driver/stream/mod.rs b/serf-reactor/src/driver/stream/mod.rs index 5a45118f..14cd67b7 100644 --- a/serf-reactor/src/driver/stream/mod.rs +++ b/serf-reactor/src/driver/stream/mod.rs @@ -66,7 +66,7 @@ use crate::{ driver::{ options::{RuntimeOptions, StreamTransportOptions}, shared::{ - ExchangeId, LEAVE_DRAIN_TEARDOWN_BOUND, LeaveDrain, dispatch_event_delegate, + ExchangeId, LEAVE_DRAIN_TEARDOWN_BOUND, LeaveDrain, dispatch_event_delegate, leave_outcome, observation_payload_bytes, poll_send_gossip, retry_retained_leave, trace_leave_drain_residue, trace_leave_transform_error, }, @@ -364,6 +364,11 @@ where left_cluster_seen: bool, /// Deadline bounding the teardown park that drains retained farewells. leave_drain_deadline: Option, + /// A leave-farewell send failed on the LOCAL socket (not a per-peer network + /// signal): the parked leave resolves + /// [`LeaveFarewellUndelivered`](SerfError::LeaveFarewellUndelivered) instead + /// of a false `Ok`. + leave_send_failed: bool, /// Parked `Shutdown` replies — acked only after the bind sockets drop, so a /// caller resuming from `shutdown().await` can rebind the same address. A `Vec` /// because several callers can race `shutdown()`. @@ -505,6 +510,7 @@ where leave_drain: LeaveDrain::new(), left_cluster_seen: false, leave_drain_deadline: None, + leave_send_failed: false, shutdown_reply: Vec::new(), bridges: HashMap::new(), accepted_rx, @@ -1114,14 +1120,20 @@ where // non-completing send is retained (not dropped) instead of leaving peers to // read the departure as a failure. Periodic gossip stays best-effort. if self.leave_initiated { - retry_retained_leave(&mut self.leave_drain, self.socket.as_ref(), cx); + retry_retained_leave( + &mut self.leave_drain, + self.socket.as_ref(), + cx, + &mut self.leave_send_failed, + ); // A `LeftCluster` observed while these datagrams were still queued // deferred the parked leave; resolve it now that the socket has accepted // every retained farewell. if self.left_cluster_seen && self.leave_drain.is_empty() { self.left_cluster_seen = false; if let Some(pl) = self.pending_leave.take() { - pl.resolve_all(|| Ok(())); + let failed = self.leave_send_failed; + pl.resolve_all(|| leave_outcome(failed)); } } } @@ -1140,6 +1152,7 @@ where Err(_) => { if self.leave_initiated { trace_leave_transform_error(to); + self.leave_send_failed = true; } continue; } @@ -1152,6 +1165,7 @@ where Err(_) => { if self.leave_initiated { trace_leave_transform_error(to); + self.leave_send_failed = true; } continue; } @@ -1167,6 +1181,7 @@ where Err(_) => { if self.leave_initiated { trace_leave_transform_error(peer); + self.leave_send_failed = true; } continue; } @@ -1181,6 +1196,7 @@ where cx, peer, &on_wire, + &mut self.leave_send_failed, ); } worked |= sent > 0; @@ -1304,7 +1320,8 @@ where // resolve when the drain empties (bounded by the caller's leave timeout). if self.leave_drain.is_empty() { if let Some(pl) = self.pending_leave.take() { - pl.resolve_all(|| Ok(())); + let failed = self.leave_send_failed; + pl.resolve_all(|| leave_outcome(failed)); } } else { self.left_cluster_seen = true; @@ -1555,17 +1572,26 @@ where this.inbound_tx = None; // Dropping `accept_shutdown_tx` cancels the accept task's pending // `accept()`; its `listener` local is released when the task is next - // scheduled (awaited below). Dropping the gossip socket closes its UDP FD - // synchronously. + // scheduled (awaited below). drop(this.accept_shutdown_tx.take()); - // Flush any retained leave-farewell datagrams before releasing the socket. - // The graceful drain runs across polls (retried as the socket becomes - // writable), so a residual queue here is rare. A residue parks the - // teardown — bounded by a short deadline — instead of being dropped: the - // farewell is the peers' only first-hand signal that the departure was - // intentional, and the `Pending` send has the writable waker registered - // while the deadline timer keeps a dead socket from hanging shutdown. - retry_retained_leave(&mut this.leave_drain, this.socket.as_ref(), cx); + } + + // FAREWELL DRAIN (re-entrant), then release the gossip socket (once — + // `socket.is_some()` is the phase guard). Retained leave-farewell + // datagrams are retried while the socket is still held; a residue parks + // the teardown — bounded by a short deadline — instead of being dropped, + // because the farewell is the peers' only first-hand signal that the + // departure was intentional. The `Pending` send registers the writable + // waker and the deadline timer bounds a persistently dead socket; ANY + // teardown wake re-enters this phase until the queue empties or the + // deadline wins. Dropping the socket then closes its UDP FD synchronously. + if this.socket.is_some() { + retry_retained_leave( + &mut this.leave_drain, + this.socket.as_ref(), + cx, + &mut this.leave_send_failed, + ); if !this.leave_drain.is_empty() { let now = Instant::now(); let deadline = *this diff --git a/serf-reactor/src/error/mod.rs b/serf-reactor/src/error/mod.rs index 2f832d08..c777ec86 100644 --- a/serf-reactor/src/error/mod.rs +++ b/serf-reactor/src/error/mod.rs @@ -140,6 +140,17 @@ pub enum SerfError { #[error("leave did not complete within the configured leave timeout")] LeaveTimeout, + /// A graceful [`leave`](crate::Serf::leave) completed locally but the local + /// socket failed while sending the departure fan-out, so at least one peer + /// was never handed the farewell and will classify the departure as a + /// failure. Per-peer network signals (a reset or unreachable reflected for a + /// peer that is itself gone) do NOT raise this — only a local send failure + /// does. + #[error( + "the local socket failed while sending the leave fan-out; peers may classify the departure as a failure" + )] + LeaveFarewellUndelivered, + /// The driver task has shut down and is no longer accepting commands. #[error("driver shut down")] Shutdown, diff --git a/serf-reactor/src/error/tests.rs b/serf-reactor/src/error/tests.rs index 2ee94e47..940208e4 100644 --- a/serf-reactor/src/error/tests.rs +++ b/serf-reactor/src/error/tests.rs @@ -52,6 +52,7 @@ fn every_variant_displays_and_debugs() { SerfError::Entropy(io::Error::other("entropy")), SerfError::Resolve(io::Error::other("dns")), SerfError::LeaveTimeout, + SerfError::LeaveFarewellUndelivered, SerfError::Shutdown, SerfError::NotRunning, SerfError::JoinAllFailed(JoinFailed::new(3, 0)), diff --git a/serf-reactor/src/serf/mod.rs b/serf-reactor/src/serf/mod.rs index 0fcf263b..204346c2 100644 --- a/serf-reactor/src/serf/mod.rs +++ b/serf-reactor/src/serf/mod.rs @@ -798,8 +798,12 @@ impl Serf { } } - /// Gracefully leave the cluster. Resolves once peers have been notified or the - /// configured leave timeout elapses. + /// Gracefully leave the cluster. Resolves `Ok` once the departure fan-out has + /// been handed to the transport (peers have been notified), or an error when + /// the configured leave timeout elapses + /// ([`LeaveTimeout`](SerfError::LeaveTimeout)) or the local socket failed + /// while sending the fan-out + /// ([`LeaveFarewellUndelivered`](SerfError::LeaveFarewellUndelivered)). pub async fn leave(&self) -> Result<()> { let (tx, rx) = oneshot::channel(); self.send(Command::Leave(LeaveCmd { reply: tx }))?; From 8b4b6da6f08d9ec4b35817e59524ce75c39000d9 Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 11:34:40 +1200 Subject: [PATCH 092/128] fix(serf-reactor): egress the queued farewell at teardown and scope raw-QUIC retention MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A leave dispatched in the same command batch as a shutdown queued its dead-self fan-out inside the endpoint, but the stream teardown only retried the already-retained datagrams before dropping the socket — the fan-out was then popped into a socket-less send and silently discarded while shutdown acked cleanly. The teardown now drains the endpoint surfaces to quiescence while the gossip socket still exists, and the parked leave resolves at the teardown itself once nothing is retained: the machine's LeftCluster is propagate-delay-fenced behind a handle_timeout a tearing-down pump never runs, so delivery — surfaces quiescent, retention empty — is the resolution signal, mapped through leave_outcome; a residue past the drain deadline still resolves Err(Shutdown). Same delivery-gated resolution in the QUIC teardown, replacing its unconditional early Err(Shutdown). The teardown's implicit best-effort leave is removed from both pumps: with the teardown now reliably egressing queued transmits it upgraded every abrupt shutdown into a peer-visible graceful Leave, breaking failure detection (a shutdown without an explicit leave is abrupt by design, mirroring the reference implementation's Shutdown; an explicit leave already queued its fan-out at dispatch). Raw-QUIC egress retention is scoped to Datagram mode: in Udp mode no raw quinn packet carries the farewell, so retaining handshake/ACK/stream packets could delay the leave or raise a false LeaveFarewellUndelivered from unrelated traffic. In Datagram mode retain-all stands — quinn-proto gives no per-packet frame provenance, datagram frames are never retransmitted, and a local failure of the shared socket is correlated with farewell loss. New e2e: leave() racing shutdown() (same command batch) still yields Join → Leave at the peer and Ok from leave().await, on both runtimes. --- serf-reactor/src/driver/quic/mod.rs | 57 +++++++++++++++++---------- serf-reactor/src/driver/stream/mod.rs | 38 ++++++++++++++---- serf-reactor/tests/cluster/mod.rs | 25 +++++++++--- serf-reactor/tests/tcp.rs | 46 +++++++++++++++++++++ 4 files changed, 133 insertions(+), 33 deletions(-) diff --git a/serf-reactor/src/driver/quic/mod.rs b/serf-reactor/src/driver/quic/mod.rs index 7a1c940a..6348d45c 100644 --- a/serf-reactor/src/driver/quic/mod.rs +++ b/serf-reactor/src/driver/quic/mod.rs @@ -926,12 +926,15 @@ where raw_sent += 1; if let Some(socket) = self.socket.as_ref() { let outcome = socket.poll_send_to(cx, &bytes, dest); - if self.leave_initiated { - // The leave fan-out may ride a quinn application DATAGRAM frame, - // which quinn does NOT retransmit — its loss recovery covers the - // reliable streams only. A backpressured send here would lose the - // farewell for good, so it is retained and retried exactly like the - // plain-UDP leave drain. + if self.leave_initiated && matches!(unreliable, UnreliableTransport::Datagram) { + // In `Datagram` mode the leave fan-out rides quinn application + // DATAGRAM frames, which quinn does NOT retransmit — its loss + // recovery covers the reliable streams only — and quinn-proto gives + // no per-packet frame provenance, so every raw packet during the + // leave is retained and error-accounted like the plain-UDP leave + // drain (a local failure of the shared socket is correlated with + // farewell loss; retaining an unrelated handshake/ACK packet merely + // delays the drain-empty fence, bounded by the writable wake). retain_leave_datagram( &mut self.leave_drain, dest, @@ -940,9 +943,11 @@ where &mut self.leave_send_failed, ); } else { - // Ignoring Poll: outside a leave, a dropped handshake/ACK/stream - // packet is recovered by quinn's own loss detection, and a dropped - // datagram-mode gossip frame is re-sent by the next periodic round. + // Ignoring Poll: in `Udp` mode the farewell rides only the plain-UDP + // gossip path above — no raw QUIC packet carries it — and outside a + // leave a dropped handshake/ACK/stream packet is recovered by + // quinn's own loss detection while a dropped datagram-mode gossip + // frame is re-sent by the next periodic round. let _ = outcome; } } @@ -1277,14 +1282,15 @@ where progress = true; } - // Shutdown: best-effort leave, flush to quiescence (the leave's `Dead`-self - // notices must reach the wire before the socket drops), fail every parked - // waiter and queued command, release the bound port, then ack. The completion - // latch promises the bind address is free, not that every QUIC connection has - // closed. + // Shutdown: flush to quiescence (an explicit leave's `Dead`-self notices + // must reach the wire before the socket drops), fail every parked waiter + // and queued command, release the bound port, then ack. No implicit leave: + // a shutdown without an explicit `leave()` is abrupt by design (mirroring + // the reference implementation's Shutdown), so peers detect the departure + // as a failure; an explicit leave already queued its fan-out when its + // command was dispatched. The completion latch promises the bind address is + // free, not that every QUIC connection has closed. if this.shared.is_shutdown() { - // Ignoring Err: best-effort leave during shutdown. - let _ = this.endpoint.leave(Instant::now()); // Drain endpoint surfaces to quiescence before reaping: a single // `drain_surfaces` pass is egress-capped, so a large batch of already-queued // `ExchangeCompleted` events would be partially skipped, leaving contacted @@ -1319,9 +1325,6 @@ where ))); } } - if let Some(pl) = this.pending_leave.take() { - pl.resolve_all(|| Err(SerfError::Shutdown)); - } // Flush any retained leave-farewell datagrams before releasing the socket: // the quiescence loop above retains a backpressured graceful-leave fan-out // rather than dropping it, and it must reach the wire before the socket @@ -1335,7 +1338,18 @@ where cx, &mut this.leave_send_failed, ); - if !this.leave_drain.is_empty() { + if this.leave_drain.is_empty() { + // A leave racing this shutdown resolves HERE, on delivery: the + // machine's `LeftCluster` is propagate-delay-fenced behind a + // `handle_timeout` a tearing-down pump never runs, and the fan-out has + // verifiably been handed to the transport (surfaces quiescent, nothing + // retained). A residue instead resolves `Err(Shutdown)` below — the + // farewell did not fully leave this host. + if let Some(pl) = this.pending_leave.take() { + let failed = this.leave_send_failed; + pl.resolve_all(|| leave_outcome(failed)); + } + } else { let now = Instant::now(); let deadline = *this .leave_drain_deadline @@ -1349,6 +1363,9 @@ where } } trace_leave_drain_residue(this.leave_drain.len()); + if let Some(pl) = this.pending_leave.take() { + pl.resolve_all(|| Err(SerfError::Shutdown)); + } } // Release the bound port BEFORE acking: dropping the agnostic UDP socket // closes its FD synchronously, so a caller resuming from `shutdown().await` diff --git a/serf-reactor/src/driver/stream/mod.rs b/serf-reactor/src/driver/stream/mod.rs index 14cd67b7..d095ff32 100644 --- a/serf-reactor/src/driver/stream/mod.rs +++ b/serf-reactor/src/driver/stream/mod.rs @@ -1559,11 +1559,12 @@ where // stream FD has closed. if this.shared.is_shutdown() { if this.accept_shutdown_tx.is_some() { - // FREEZE (one-time). Best-effort leave, freeze every live bridge, drop the - // template inbound sender so the channel can reach all-senders-gone, and - // release the bind sockets. - // Ignoring Err: best-effort leave during shutdown. - let _ = this.endpoint.leave(Instant::now()); + // FREEZE (one-time). Freeze every live bridge, drop the template inbound + // sender so the channel can reach all-senders-gone, and release the bind + // sockets. No implicit leave: a shutdown without an explicit `leave()` is + // abrupt by design (mirroring the reference implementation's Shutdown), + // so peers detect the departure as a failure; an explicit leave already + // queued its fan-out when its command was dispatched. for (_, handle) in this.bridges.drain() { // Ignoring Err: the bridge may have already exited (cancel receiver // gone); the freeze is best-effort. @@ -1577,8 +1578,12 @@ where } // FAREWELL DRAIN (re-entrant), then release the gossip socket (once — - // `socket.is_some()` is the phase guard). Retained leave-farewell - // datagrams are retried while the socket is still held; a residue parks + // `socket.is_some()` is the phase guard). The endpoint surfaces are first + // drained to quiescence WHILE the socket still exists: a leave dispatched + // in the same command batch as the shutdown (or the best-effort one in + // the freeze) queued its dead-self fan-out inside the endpoint, and a + // gossip transmit popped after the socket drops is silently discarded. + // Retained leave-farewell datagrams are then retried; a residue parks // the teardown — bounded by a short deadline — instead of being dropped, // because the farewell is the peers' only first-hand signal that the // departure was intentional. The `Pending` send registers the writable @@ -1586,13 +1591,30 @@ where // teardown wake re-enters this phase until the queue empties or the // deadline wins. Dropping the socket then closes its UDP FD synchronously. if this.socket.is_some() { + loop { + let (_, drain_more, _) = this.drain_surfaces(cx); + if !drain_more { + break; + } + } retry_retained_leave( &mut this.leave_drain, this.socket.as_ref(), cx, &mut this.leave_send_failed, ); - if !this.leave_drain.is_empty() { + if this.leave_drain.is_empty() { + // A leave racing this shutdown resolves HERE, on delivery: the + // machine's `LeftCluster` is propagate-delay-fenced behind a + // `handle_timeout` a tearing-down pump never runs, and the fan-out + // has verifiably been handed to the transport (surfaces quiescent, + // nothing retained). A residue instead falls through to the reap's + // `Err(Shutdown)` — the farewell did not fully leave this host. + if let Some(pl) = this.pending_leave.take() { + let failed = this.leave_send_failed; + pl.resolve_all(|| leave_outcome(failed)); + } + } else { let now = Instant::now(); let deadline = *this .leave_drain_deadline diff --git a/serf-reactor/tests/cluster/mod.rs b/serf-reactor/tests/cluster/mod.rs index 3899c2e2..ea3406e1 100644 --- a/serf-reactor/tests/cluster/mod.rs +++ b/serf-reactor/tests/cluster/mod.rs @@ -218,11 +218,12 @@ where self.slots[i].id.clone() } - /// Abruptly kill node `i`: shut its handle down, which discards the teardown's - /// best-effort leave (the gossip socket drops before the leave datagram can be - /// transmitted), so peers detect a probe-timeout Failed rather than a graceful - /// Leave. The slot's id, addr, and event log are retained for a later restart or - /// assertion; the freed port is released before `shutdown` resolves. + /// Abruptly kill node `i`: shut its handle down without a leave. A shutdown + /// sends no farewell (abrupt by design, mirroring the reference + /// implementation's Shutdown), so peers detect a probe-timeout Failed rather + /// than a graceful Leave. The slot's id, addr, and event log are retained for + /// a later restart or assertion; the freed port is released before `shutdown` + /// resolves. pub async fn kill_abrupt(&mut self, i: usize) { let serf = self.slots[i].serf.take().expect("node slot is live"); serf.shutdown().await.expect("node shuts down"); @@ -249,6 +250,20 @@ where elapsed } + /// Gracefully leave node `i` with a shutdown racing the leave: both commands + /// are issued concurrently, so they typically land in the same driver command + /// batch and the teardown itself must egress the still-queued farewell before + /// releasing the gossip socket. Unlike + /// [`leave_graceful`](Self::leave_graceful) there is no resolved-leave fence + /// ahead of the shutdown — the leave must still resolve `Ok`, proving the + /// farewell reached the transport under the race. + pub async fn leave_with_racing_shutdown(&mut self, i: usize) { + let serf = self.slots[i].serf.take().expect("node slot is live"); + let (leave, shutdown) = futures_util::future::join(serf.leave(), serf.shutdown()).await; + leave.expect("node leaves gracefully despite the racing shutdown"); + shutdown.expect("node shuts down"); + } + /// Restart a previously-killed node `i` at the SAME id and advertise address, /// re-attaching a collector to the slot's existing log. The freed port is /// rebound with a bounded retry to absorb a transient rebind race. diff --git a/serf-reactor/tests/tcp.rs b/serf-reactor/tests/tcp.rs index 0d9b0bd1..7c45cb2b 100644 --- a/serf-reactor/tests/tcp.rs +++ b/serf-reactor/tests/tcp.rs @@ -558,6 +558,42 @@ where cluster.shutdown_all().await; } +/// Two nodes on loopback: node A joins node B, then B's `leave()` and +/// `shutdown()` race — issued concurrently, they typically land in the same +/// driver command batch, so the teardown itself owns egressing the queued +/// farewell before it releases the gossip socket. Node A must still observe +/// Join → Leave (never a Failed) and B's leave must still resolve `Ok`, +/// pinning that an immediate shutdown cannot discard the departure fan-out +/// while it is still queued inside the endpoint. +async fn serf_events_leave_with_racing_shutdown() +where + R: Runtime, +{ + // Hold B's Left tombstone past the test window, as in `serf_events_leave`. + let mut cluster = cluster::Cluster::::spawn( + &["leave-race-a", "leave-race-b"], + cluster::ClusterTiming::fast().with_tombstone_timeout(Duration::from_secs(30)), + ) + .await; + let subject = cluster.id(1); + + cluster.leave_with_racing_shutdown(1).await; + + // A observes exactly [Join, Leave] about B — never a Failed. + cluster + .assert_member_events( + 0, + subject.as_str(), + &[MemberEventKind::Join, MemberEventKind::Leave], + ) + .await; + + // B lands in A's Left tombstone view. + cluster.await_left_tombstone(0, subject.as_str()).await; + + cluster.shutdown_all().await; +} + /// Two nodes on loopback: node A joins node B, B is abruptly killed and detected /// Failed, then B is restarted at the same id and advertise address. Node A must /// observe the sequence Join → Failed → Join about B — the failed member @@ -808,6 +844,11 @@ mod tokio_cells { super::serf_events_leave::().await; } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_events_leave_with_racing_shutdown() { + super::serf_events_leave_with_racing_shutdown::().await; + } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn serf_reconnect() { super::serf_reconnect::().await; @@ -887,6 +928,11 @@ mod smol_cells { SmolRuntime::block_on(super::serf_events_leave::()); } + #[test] + fn serf_events_leave_with_racing_shutdown_smol() { + SmolRuntime::block_on(super::serf_events_leave_with_racing_shutdown::()); + } + #[test] fn serf_reconnect_smol() { SmolRuntime::block_on(super::serf_reconnect::()); From ab68be88db85e652680690d602347d5851d2f437 Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 11:47:55 +1200 Subject: [PATCH 093/128] fix(serf-reactor): route the Datagram-mode leave fan-out over plain UDP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DatagramSendStatus::Queued only proves the farewell entered quinn's internal DATAGRAM queue; congestion control and pacing gate when (and whether) a raw packet actually carries it out, so an empty retention queue was no proof of socket handoff — the parked leave could resolve Ok, and a racing teardown could drop the socket, while the farewell still sat inside quinn. quinn also never retransmits DATAGRAM frames, so that loss is permanent. Once leave() is initiated the gossip egress now sends the fan-out over the plain-UDP path in BOTH unreliable modes: the plain send gives exact socket-handoff semantics (completed, retained on backpressure, or error-classified into the leave outcome), and peers demux plain gossip datagrams in every mode — the NotReady/TooLarge fallbacks already rely on that. No raw QUIC packet ever carries the farewell now, so the raw egress loop reverts to pure best-effort, removing the Datagram-mode retain-all imprecision entirely. New e2e in the QUIC suite (Datagram mode, both runtimes): leave() racing shutdown() must reach the peer as a graceful Leave — never a Failed — and resolve Ok. --- serf-reactor/src/driver/quic/mod.rs | 75 +++++++++++++++-------------- serf-reactor/tests/quic.rs | 71 ++++++++++++++++++++++++++- 2 files changed, 110 insertions(+), 36 deletions(-) diff --git a/serf-reactor/src/driver/quic/mod.rs b/serf-reactor/src/driver/quic/mod.rs index 6348d45c..a416fc1b 100644 --- a/serf-reactor/src/driver/quic/mod.rs +++ b/serf-reactor/src/driver/quic/mod.rs @@ -85,8 +85,8 @@ use crate::{ options::RuntimeOptions, shared::{ ExchangeId, LEAVE_DRAIN_TEARDOWN_BOUND, LeaveDrain, dispatch_event_delegate, leave_outcome, - observation_payload_bytes, poll_send_gossip, retain_leave_datagram, retry_retained_leave, - trace_leave_drain_residue, trace_leave_transform_error, + observation_payload_bytes, poll_send_gossip, retry_retained_leave, trace_leave_drain_residue, + trace_leave_transform_error, }, }, drop_counter::ReactorDropCounter, @@ -311,7 +311,10 @@ where pending_leave: Option, /// Set once `leave()` has been initiated. Switches the gossip egress from /// best-effort (drop on non-completion) to retention for the leave fan-out, - /// which has no next gossip round to re-send a dropped farewell. + /// which has no next gossip round to re-send a dropped farewell — and, in + /// `Datagram` mode, reroutes it onto the plain-UDP path, since a frame + /// queued into quinn is emitted only when congestion control allows and so + /// carries no socket-handoff signal. leave_initiated: bool, /// Leave-farewell datagrams retained after a non-completing plain-UDP send, /// retried FIFO ahead of fresh transmits and flushed before the socket drops. @@ -766,13 +769,13 @@ where // shared UDP socket in `Udp` mode. Popping the last transmit is the endpoint's // leave-completion fence (it emits `LeftCluster`), so the leave/shutdown // datagrams reach the wire before that fence fires. - // Once `leave()` has been initiated the fan-out is RETAINED on plain-UDP - // backpressure: retry the retained datagrams FIRST (they drain as the socket - // becomes writable), then pop fresh transmits at the unchanged cadence — a - // non-completing send is retained (not dropped) so peers do not read the - // departure as a failure. Periodic gossip stays best-effort. (Datagram-mode - // sends that ride an established QUIC connection are quinn-managed; only the - // plain-UDP sends below are retained.) + // Once `leave()` has been initiated the fan-out rides plain UDP in BOTH + // unreliable modes (the Datagram arm reroutes it — see the match below) and + // is RETAINED on backpressure: retry the retained datagrams FIRST (they + // drain as the socket becomes writable), then pop fresh transmits at the + // unchanged cadence — a non-completing send is retained (not dropped) so + // peers do not read the departure as a failure. Periodic gossip stays + // best-effort. if self.leave_initiated { retry_retained_leave( &mut self.leave_drain, @@ -860,6 +863,26 @@ where &mut self.leave_send_failed, ); } + // Once `leave()` has been initiated the fan-out takes the plain-UDP + // path even in `Datagram` mode: a frame queued into quinn is emitted + // only when congestion control and pacing allow, so an empty retention + // queue would not prove the farewell reached the socket — and the + // fan-out has no retry round to absorb that loss. The plain-UDP send + // gives exact socket-handoff semantics (completed, retained, or + // error-classified), and peers demux plain gossip datagrams in every + // mode — the `NotReady`/`TooLarge` fallbacks below rely on exactly + // that. + UnreliableTransport::Datagram if self.leave_initiated => { + poll_send_gossip( + &mut self.leave_drain, + true, + self.socket.as_ref(), + cx, + peer, + &on_wire, + &mut self.leave_send_failed, + ); + } UnreliableTransport::Datagram => { match self .endpoint @@ -925,31 +948,13 @@ where }; raw_sent += 1; if let Some(socket) = self.socket.as_ref() { - let outcome = socket.poll_send_to(cx, &bytes, dest); - if self.leave_initiated && matches!(unreliable, UnreliableTransport::Datagram) { - // In `Datagram` mode the leave fan-out rides quinn application - // DATAGRAM frames, which quinn does NOT retransmit — its loss - // recovery covers the reliable streams only — and quinn-proto gives - // no per-packet frame provenance, so every raw packet during the - // leave is retained and error-accounted like the plain-UDP leave - // drain (a local failure of the shared socket is correlated with - // farewell loss; retaining an unrelated handshake/ACK packet merely - // delays the drain-empty fence, bounded by the writable wake). - retain_leave_datagram( - &mut self.leave_drain, - dest, - &bytes, - outcome, - &mut self.leave_send_failed, - ); - } else { - // Ignoring Poll: in `Udp` mode the farewell rides only the plain-UDP - // gossip path above — no raw QUIC packet carries it — and outside a - // leave a dropped handshake/ACK/stream packet is recovered by - // quinn's own loss detection while a dropped datagram-mode gossip - // frame is re-sent by the next periodic round. - let _ = outcome; - } + // Ignoring Poll: no raw QUIC packet ever carries the leave fan-out — + // once `leave()` is initiated the gossip egress above routes it over + // plain UDP with socket-handoff retention — so raw egress stays + // best-effort: a dropped handshake/ACK/stream packet is recovered by + // quinn's own loss detection, and a dropped datagram-mode gossip frame + // is re-sent by the next periodic round. + let _ = socket.poll_send_to(cx, &bytes, dest); } } worked |= raw_sent > 0; diff --git a/serf-reactor/tests/quic.rs b/serf-reactor/tests/quic.rs index ba18ed8e..d0ec9135 100644 --- a/serf-reactor/tests/quic.rs +++ b/serf-reactor/tests/quic.rs @@ -29,7 +29,11 @@ use rustls::{ version::TLS13, }; use rustls_pki_types::{CertificateDer, PrivateKeyDer}; -use serf_proto::{event::Event, members::SerfState, options::Options as SerfOptions}; +use serf_proto::{ + event::{Event, MemberEventKind}, + members::SerfState, + options::Options as SerfOptions, +}; #[cfg(encryption)] use serf_reactor::{EncryptionOptions, Keyring, SecretKey, VoidKeyringDelegate}; use serf_reactor::{ @@ -437,6 +441,59 @@ where b.shutdown().await.expect("lv-b shuts down"); } +/// Peer-visible graceful departure in `Datagram` mode with a shutdown racing the +/// leave: B's farewell must reach A even though B tears down immediately — the +/// leave fan-out rides plain UDP with socket-handoff retention rather than +/// quinn's congestion-gated datagram queue, so neither the racing teardown nor +/// the QUIC datagram plane can silently discard it. A must classify B's +/// departure as a Leave (never a Failed) and B's `leave().await` must resolve +/// `Ok`. +async fn leave_with_racing_shutdown_reaches_peer_as_leave() +where + R: Runtime, +{ + let b = spawn_node::("race-b").await; + let a = spawn_node::("race-a").await; + let b_addr = b.advertise_address(); + + let mut a_events = a.events(); + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B over QUIC"); + converge(&a, &b).await; + + let (leave, shutdown) = future::join(b.leave(), b.shutdown()).await; + leave.expect("B leaves gracefully despite the racing shutdown"); + shutdown.expect("B shuts down"); + + let kind = R::timeout(Duration::from_secs(30), async { + loop { + match a_events.next().await { + Some(Event::Member(me)) + if me + .members() + .iter() + .any(|m| m.node().id_ref().as_str() == "race-b") + && matches!(me.kind(), MemberEventKind::Leave | MemberEventKind::Failed) => + { + break Some(me.kind()); + } + Some(_) => {} + None => break None, + } + } + }) + .await + .expect("A observes B's departure within the timeout"); + assert_eq!( + kind, + Some(MemberEventKind::Leave), + "A must classify B's racing-shutdown departure as a graceful Leave, not a Failed" + ); + + a.shutdown().await.expect("race-a shuts down"); +} + /// The QUIC driver binds a single UDP socket and drops it (releasing its FD) before /// acking shutdown, so `shutdown().await` releases the bound port before it /// resolves: a second QUIC node binding the SAME advertise address the instant the @@ -714,6 +771,11 @@ mod tokio_cells { super::leave_emits_left_cluster::().await; } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn leave_with_racing_shutdown_reaches_peer_as_leave() { + super::leave_with_racing_shutdown_reaches_peer_as_leave::().await; + } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn quic_shutdown_releases_bound_address_for_rebind() { super::quic_shutdown_releases_bound_address_for_rebind::().await; @@ -767,6 +829,13 @@ mod smol_cells { SmolRuntime::block_on(super::leave_emits_left_cluster::()); } + #[test] + fn leave_with_racing_shutdown_reaches_peer_as_leave_smol() { + SmolRuntime::block_on(super::leave_with_racing_shutdown_reaches_peer_as_leave::< + SmolRuntime, + >()); + } + #[test] fn quic_shutdown_releases_bound_address_for_rebind_smol() { SmolRuntime::block_on(super::quic_shutdown_releases_bound_address_for_rebind::< From acb8124f7afe66cebe2bd30b50de179cde287df4 Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 11:59:32 +1200 Subject: [PATCH 094/128] fix(serf-reactor): classify EADDRNOTAVAIL as local and honor the leave deadline at teardown MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AddrNotAvailable maps to EADDRNOTAVAIL — the bound source address vanished from this host (interface loss), not a peer's answer to a delivered attempt — so it now sets leave_send_failed like every other local failure instead of letting the leave resolve Ok while the farewell never left. The teardown resolution path bypassed the caller's per-leave deadline: a leave configured with a zero or short timeout racing a shutdown could resolve Ok past its window, and under backpressure could park to the independent one-second drain bound first. Both teardowns now reap the deadline exactly like the normal poll (a zero timeout stays a loud immediate LeaveTimeout), and the residue park is bounded by whichever fires first — the drain bound or a still-parked leave's deadline, whose firing resolves LeaveTimeout promptly while the drain keeps the rest of its window; a Ready timer re-enters the phase so the reap and park recompute. New e2e: a zero-leave-timeout leave racing shutdown resolves LeaveTimeout — never Ok — on both runtimes. --- serf-reactor/src/driver/quic/mod.rs | 23 +++++++++++-- serf-reactor/src/driver/shared.rs | 6 ++-- serf-reactor/src/driver/shared/tests.rs | 4 ++- serf-reactor/src/driver/stream/mod.rs | 23 +++++++++++-- serf-reactor/tests/tcp.rs | 46 +++++++++++++++++++++++++ 5 files changed, 92 insertions(+), 10 deletions(-) diff --git a/serf-reactor/src/driver/quic/mod.rs b/serf-reactor/src/driver/quic/mod.rs index a416fc1b..3fa691fd 100644 --- a/serf-reactor/src/driver/quic/mod.rs +++ b/serf-reactor/src/driver/quic/mod.rs @@ -1343,6 +1343,12 @@ where cx, &mut this.leave_send_failed, ); + // The caller's per-leave deadline governs resolution during teardown + // exactly as in the normal poll's reap: a leave that missed its + // configured window resolves `LeaveTimeout` even mid-shutdown (a zero + // timeout is a loud immediate `LeaveTimeout` by contract). A leave + // still parked past this point has a strictly-future deadline. + this.reap_pending_leave(Instant::now()); if this.leave_drain.is_empty() { // A leave racing this shutdown resolves HERE, on delivery: the // machine's `LeftCluster` is propagate-delay-fenced behind a @@ -1356,16 +1362,27 @@ where } } else { let now = Instant::now(); - let deadline = *this + let drain_deadline = *this .leave_drain_deadline .get_or_insert(now + LEAVE_DRAIN_TEARDOWN_BOUND); - if now < deadline { - this.arm_timer(deadline, now); + if now < drain_deadline { + // Park until whichever fires first: the drain bound, or a + // still-parked leave's deadline — whose firing must resolve + // `LeaveTimeout` promptly, while the drain keeps the rest of its + // window. A `Ready` timer re-enters the phase so the reap and the + // park recompute against the new now. + let park_until = this + .pending_leave + .as_ref() + .map_or(drain_deadline, |pl| pl.deadline.min(drain_deadline)); + this.arm_timer(park_until, now); if let Some(timer) = this.timer.as_mut() && timer.as_mut().poll(cx).is_pending() { return Poll::Pending; } + cx.waker().wake_by_ref(); + return Poll::Pending; } trace_leave_drain_residue(this.leave_drain.len()); if let Some(pl) = this.pending_leave.take() { diff --git a/serf-reactor/src/driver/shared.rs b/serf-reactor/src/driver/shared.rs index b863fb89..ca098f49 100644 --- a/serf-reactor/src/driver/shared.rs +++ b/serf-reactor/src/driver/shared.rs @@ -140,8 +140,9 @@ pub(crate) fn leave_outcome(send_failed: bool) -> crate::error::Result<()> { /// the reference implementation logs them and proceeds, and failing the whole /// leave because one peer already died would be wrong (and flaky on platforms /// that reflect ICMP errors into UDP sends). Everything else — a closed or -/// invalid socket, a broken pipe, an out-of-memory send path — means the -/// farewell could not be handed off at all. +/// invalid socket, a vanished source address (`EADDRNOTAVAIL`, e.g. the bound +/// interface disappeared), a broken pipe, an out-of-memory send path — means +/// the farewell could not be handed off at all. #[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) fn farewell_send_failure_is_local(err: &io::Error) -> bool { !matches!( @@ -151,7 +152,6 @@ pub(crate) fn farewell_send_failure_is_local(err: &io::Error) -> bool { | io::ErrorKind::ConnectionAborted | io::ErrorKind::HostUnreachable | io::ErrorKind::NetworkUnreachable - | io::ErrorKind::AddrNotAvailable ) } diff --git a/serf-reactor/src/driver/shared/tests.rs b/serf-reactor/src/driver/shared/tests.rs index 4a798e13..f8bd69aa 100644 --- a/serf-reactor/src/driver/shared/tests.rs +++ b/serf-reactor/src/driver/shared/tests.rs @@ -78,7 +78,6 @@ fn retain_leave_datagram_flags_only_local_failures() { io::ErrorKind::ConnectionAborted, io::ErrorKind::HostUnreachable, io::ErrorKind::NetworkUnreachable, - io::ErrorKind::AddrNotAvailable, ] { let mut send_failed = false; assert!(!retain_leave_datagram( @@ -96,10 +95,13 @@ fn retain_leave_datagram_flags_only_local_failures() { } // Local socket failures: attempted, not retained, and the leave FAILS. + // `AddrNotAvailable` is local — the bound source address vanished from this + // host — not a peer signal. for kind in [ io::ErrorKind::NotConnected, io::ErrorKind::BrokenPipe, io::ErrorKind::InvalidInput, + io::ErrorKind::AddrNotAvailable, io::ErrorKind::Other, ] { let mut send_failed = false; diff --git a/serf-reactor/src/driver/stream/mod.rs b/serf-reactor/src/driver/stream/mod.rs index d095ff32..4c71e0d9 100644 --- a/serf-reactor/src/driver/stream/mod.rs +++ b/serf-reactor/src/driver/stream/mod.rs @@ -1603,6 +1603,12 @@ where cx, &mut this.leave_send_failed, ); + // The caller's per-leave deadline governs resolution during teardown + // exactly as in the normal poll's reap: a leave that missed its + // configured window resolves `LeaveTimeout` even mid-shutdown (a zero + // timeout is a loud immediate `LeaveTimeout` by contract). A leave + // still parked past this point has a strictly-future deadline. + this.reap_pending_leave(Instant::now()); if this.leave_drain.is_empty() { // A leave racing this shutdown resolves HERE, on delivery: the // machine's `LeftCluster` is propagate-delay-fenced behind a @@ -1616,16 +1622,27 @@ where } } else { let now = Instant::now(); - let deadline = *this + let drain_deadline = *this .leave_drain_deadline .get_or_insert(now + LEAVE_DRAIN_TEARDOWN_BOUND); - if now < deadline { - this.arm_timer(deadline, now); + if now < drain_deadline { + // Park until whichever fires first: the drain bound, or a + // still-parked leave's deadline — whose firing must resolve + // `LeaveTimeout` promptly, while the drain keeps the rest of its + // window. A `Ready` timer re-enters the phase so the reap and the + // park recompute against the new now. + let park_until = this + .pending_leave + .as_ref() + .map_or(drain_deadline, |pl| pl.deadline.min(drain_deadline)); + this.arm_timer(park_until, now); if let Some(timer) = this.timer.as_mut() && timer.as_mut().poll(cx).is_pending() { return Poll::Pending; } + cx.waker().wake_by_ref(); + return Poll::Pending; } trace_leave_drain_residue(this.leave_drain.len()); } diff --git a/serf-reactor/tests/tcp.rs b/serf-reactor/tests/tcp.rs index 7c45cb2b..98274188 100644 --- a/serf-reactor/tests/tcp.rs +++ b/serf-reactor/tests/tcp.rs @@ -594,6 +594,40 @@ where cluster.shutdown_all().await; } +/// A leave configured with a zero timeout racing a shutdown resolves +/// `Err(LeaveTimeout)` — never `Ok` — even though the teardown still delivers +/// the fan-out: the caller's per-leave deadline keeps governing resolution +/// during teardown, and a zero timeout is a loud immediate `LeaveTimeout` by +/// contract. +async fn leave_with_zero_timeout_racing_shutdown_times_out() +where + R: Runtime, +{ + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let node = Serf::::tcp( + TcpTransportOptions::::new() + .with_local_id(SmolStr::new("zero-leave")) + .with_advertise_addr(MaybeResolved::Resolved(bind)), + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new().with_leave_timeout(Duration::ZERO), + SerfOptions::new(), + None, + #[cfg(encryption)] + std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), + ) + .await + .expect("spawn zero-leave-timeout node"); + + let (leave, shutdown) = future::join(node.leave(), node.shutdown()).await; + assert!( + matches!(leave, Err(serf_reactor::SerfError::LeaveTimeout)), + "a zero leave timeout must resolve LeaveTimeout even when a shutdown races it, got {leave:?}" + ); + shutdown.expect("node shuts down"); +} + /// Two nodes on loopback: node A joins node B, B is abruptly killed and detected /// Failed, then B is restarted at the same id and advertise address. Node A must /// observe the sequence Join → Failed → Join about B — the failed member @@ -849,6 +883,11 @@ mod tokio_cells { super::serf_events_leave_with_racing_shutdown::().await; } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn leave_with_zero_timeout_racing_shutdown_times_out() { + super::leave_with_zero_timeout_racing_shutdown_times_out::().await; + } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn serf_reconnect() { super::serf_reconnect::().await; @@ -933,6 +972,13 @@ mod smol_cells { SmolRuntime::block_on(super::serf_events_leave_with_racing_shutdown::()); } + #[test] + fn leave_with_zero_timeout_racing_shutdown_times_out_smol() { + SmolRuntime::block_on(super::leave_with_zero_timeout_racing_shutdown_times_out::< + SmolRuntime, + >()); + } + #[test] fn serf_reconnect_smol() { SmolRuntime::block_on(super::serf_reconnect::()); From 45b3065e03056902bc147250ecae261453b4f557 Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 12:11:56 +1200 Subject: [PATCH 095/128] fix(serf-reactor): disambiguate ICMP-class farewell send errors by bounded retry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An errored sendto never accepted the current datagram, and on a shared unconnected UDP socket the kernel may surface an asynchronous error left by an EARLIER packet to a DIFFERENT peer — so treating an allowlisted error kind as this destination's own answer both misattributed stale errors and silently dropped the current farewell (a healthy peer could then classify the departure as a failure while leave() resolved Ok). The retention entry now carries an absorbed-error count. An ICMP-class error (ConnectionReset/ConnectionRefused — what a gone peer's ICMP answer surfaces on every platform, and what Windows reflects after a send to a closed port) re-retains the datagram at the back of the queue for a bounded retry: the retry drains a possibly-stale error slot and re-hands the datagram to the socket, and an error persisting to the limit is credibly this destination's own answer — logged and dropped without failing the leave, as the reference implementation does. Every other kind is now a local delivery failure: ConnectionAborted is a local software abort on Windows, the unreachables usually report the local routing table's answer, and the rest (closed/invalid socket, vanished source address, broken pipe) never left this host — all set leave_send_failed. The retry sweep pops at most the queue's initial length per pass, so a re-retained datagram is re-attempted on the next pump wake, never the same pass; the outcome fold is factored into pure functions covered by exhaustive kind-by-count unit tests. --- serf-reactor/src/driver/shared.rs | 217 ++++++++++++++++++------ serf-reactor/src/driver/shared/tests.rs | 213 ++++++++++++++++++----- 2 files changed, 341 insertions(+), 89 deletions(-) diff --git a/serf-reactor/src/driver/shared.rs b/serf-reactor/src/driver/shared.rs index ca098f49..52b49962 100644 --- a/serf-reactor/src/driver/shared.rs +++ b/serf-reactor/src/driver/shared.rs @@ -79,11 +79,41 @@ pub(crate) async fn dispatch_event_delegate( // failure while `leave()` reported success — so once `leave()` has been // initiated the pump RETAINS the fan-out datagrams whose send did not complete // and retries them until the socket accepts them (or teardown exhausts them). +// +// An errored send never accepted the CURRENT datagram, and on a shared +// unconnected UDP socket the kernel may surface an asynchronous error left by +// an EARLIER packet to a DIFFERENT peer — so an ICMP-reflection-class error +// (reset/refused) is disambiguated by bounded retry: the retry both drains the +// stale error slot and re-hands this datagram to the socket, and an error that +// persists across the retries is credibly this destination's own answer (a +// peer that is itself gone), which the reference implementation logs and +// proceeds past. Every other error kind — aborts (a local software abort on +// Windows), unreachables (usually the LOCAL routing table's answer), a closed +// or invalid socket — is a local delivery failure that fails the leave. + +/// One retained leave-farewell gossip datagram: its destination, the +/// encoded-and-transformed bytes, and how many ICMP-class send errors it has +/// absorbed (bounded by [`FAREWELL_ICMP_ERROR_LIMIT`]). +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) struct LeaveDatagram { + peer: SocketAddr, + bytes: Vec, + icmp_errors: u8, +} -/// Encoded-and-transformed leave-farewell gossip datagrams retained after a -/// non-completing UDP send, keyed by their destination and retried FIFO. +/// Leave-farewell datagrams retained after a non-completing UDP send, retried +/// FIFO. #[cfg(any(feature = "tcp", feature = "quic"))] -pub(crate) type LeaveDrain = VecDeque<(SocketAddr, Vec)>; +pub(crate) type LeaveDrain = VecDeque; + +/// Total ICMP-class (`ConnectionReset` / `ConnectionRefused`) send errors one +/// farewell datagram absorbs before it is dropped as answered-by-the-network. +/// The first errors are ambiguous (a stale asynchronous error from an earlier +/// packet to a different peer may occupy the socket's error slot), so the +/// datagram is retried; at the limit the answer is attributed to this +/// destination itself. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) const FAREWELL_ICMP_ERROR_LIMIT: u8 = 3; /// Surface a leave-farewell datagram the driver could not deliver. Unlike /// best-effort periodic gossip the drop is logged (the send error is @@ -108,6 +138,20 @@ pub(crate) fn trace_leave_drain_residue(_count: usize) { ); } +/// Surface a leave-farewell datagram dropped after absorbing +/// [`FAREWELL_ICMP_ERROR_LIMIT`] ICMP-class send errors: the network's answer +/// is attributed to this destination (a peer that is itself gone), and the +/// reference implementation logs such peers and proceeds. A no-op without the +/// `tracing` feature. +#[cfg(any(feature = "tcp", feature = "quic"))] +fn trace_leave_peer_answered(_peer: SocketAddr) { + #[cfg(feature = "tracing")] + tracing::debug!( + peer = %_peer, + "serf leave farewell dropped after repeated ICMP-class send errors; peer presumed gone" + ); +} + /// Surface a leave-farewell datagram dropped because it could not be encoded or /// encrypted — a deterministic config-class failure, unlike a transient /// best-effort gossip drop. A no-op without the `tracing` feature. @@ -131,28 +175,52 @@ pub(crate) fn leave_outcome(send_failed: bool) -> crate::error::Result<()> { } } -/// Whether a leave-farewell send error is a LOCAL socket failure (the leave -/// contract is broken — the datagram never left this host) rather than a -/// per-peer network signal. +/// The disposition of one errored leave-farewell send. +#[cfg(any(feature = "tcp", feature = "quic"))] +#[derive(Debug, PartialEq, Eq)] +enum ErroredFarewell { + /// Ambiguous ICMP-class error under the retry limit: retain the datagram + /// (with the bumped absorb count) and re-attempt it — the retry drains a + /// possibly-stale asynchronous error slot and re-hands the datagram to the + /// socket. + Retry(u8), + /// ICMP-class errors persisted to [`FAREWELL_ICMP_ERROR_LIMIT`]: the answer + /// is credibly this destination's own (the peer is itself gone). Logged and + /// dropped without failing the leave, as the reference implementation does. + PeerAnswered, + /// Any other error kind: a LOCAL delivery failure — the farewell never left + /// this host — so the leave resolves + /// [`LeaveFarewellUndelivered`](crate::error::SerfError::LeaveFarewellUndelivered). + LocalFailure, +} + +/// Classify one errored farewell send, given how many ICMP-class errors this +/// datagram has already absorbed. /// -/// Per-peer signals — a reset/refusal reflected for a peer that is itself gone, -/// or an unreachable route — are the network's answer to a delivered attempt: -/// the reference implementation logs them and proceeds, and failing the whole -/// leave because one peer already died would be wrong (and flaky on platforms -/// that reflect ICMP errors into UDP sends). Everything else — a closed or -/// invalid socket, a vanished source address (`EADDRNOTAVAIL`, e.g. the bound -/// interface disappeared), a broken pipe, an out-of-memory send path — means -/// the farewell could not be handed off at all. +/// Only `ConnectionReset` / `ConnectionRefused` are the ambiguous +/// ICMP-reflection class: they are what a gone peer's ICMP answer surfaces on +/// every platform (and what Windows reflects routinely after a send to a +/// closed port), and on a shared unconnected socket they may equally be a +/// stale answer to an EARLIER packet for a different peer — hence bounded +/// retry rather than trusting either reading. `ConnectionAborted` is a local +/// software abort on Windows, the unreachables usually report the LOCAL +/// routing table's answer, and everything else (a closed or invalid socket, a +/// vanished source address, a broken pipe) is unambiguously local. #[cfg(any(feature = "tcp", feature = "quic"))] -pub(crate) fn farewell_send_failure_is_local(err: &io::Error) -> bool { - !matches!( +fn classify_errored_farewell(icmp_errors: u8, err: &io::Error) -> ErroredFarewell { + if matches!( err.kind(), - io::ErrorKind::ConnectionReset - | io::ErrorKind::ConnectionRefused - | io::ErrorKind::ConnectionAborted - | io::ErrorKind::HostUnreachable - | io::ErrorKind::NetworkUnreachable - ) + io::ErrorKind::ConnectionReset | io::ErrorKind::ConnectionRefused + ) { + let absorbed = icmp_errors.saturating_add(1); + if absorbed < FAREWELL_ICMP_ERROR_LIMIT { + ErroredFarewell::Retry(absorbed) + } else { + ErroredFarewell::PeerAnswered + } + } else { + ErroredFarewell::LocalFailure + } } /// Upper bound on how long a pump's teardown parks waiting for retained @@ -166,14 +234,16 @@ pub(crate) const LEAVE_DRAIN_TEARDOWN_BOUND: core::time::Duration = /// Record the outcome of one readiness-based leave-farewell datagram send into /// `retained`: /// - `Ready(Ok)` — it left the socket; nothing to retain. -/// - `Ready(Err)` — attempted; logged and NOT retained (retrying a socket that -/// errors is futile). A LOCAL failure additionally sets `send_failed`, so the -/// parked leave resolves with an error instead of a false success. -/// - `Pending` — the socket is backpressured; `(peer, datagram)` is retained for -/// the next writable wake (the datagram is copied only on this path). +/// - `Ready(Err)` — the socket did NOT accept this datagram. An ICMP-class +/// error retains it for a bounded retry (the error slot may hold a stale +/// answer to an earlier packet for a different peer); a LOCAL failure sets +/// `send_failed`, so the parked leave resolves with an error instead of a +/// false success. +/// - `Pending` — the socket is backpressured; the datagram is retained for the +/// next writable wake. /// -/// Returns `true` when the datagram was retained, so a caller draining fresh -/// transmits learns the socket is backpressured. +/// The datagram bytes are copied only on the retain paths. Returns `true` when +/// the datagram was retained. #[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) fn retain_leave_datagram( retained: &mut LeaveDrain, @@ -186,13 +256,31 @@ pub(crate) fn retain_leave_datagram( Poll::Ready(Ok(_)) => false, Poll::Ready(Err(err)) => { trace_leave_send_error(peer, &err); - if farewell_send_failure_is_local(&err) { - *send_failed = true; + match classify_errored_farewell(0, &err) { + ErroredFarewell::Retry(absorbed) => { + retained.push_back(LeaveDatagram { + peer, + bytes: datagram.to_vec(), + icmp_errors: absorbed, + }); + true + } + ErroredFarewell::PeerAnswered => { + trace_leave_peer_answered(peer); + false + } + ErroredFarewell::LocalFailure => { + *send_failed = true; + false + } } - false } Poll::Pending => { - retained.push_back((peer, datagram.to_vec())); + retained.push_back(LeaveDatagram { + peer, + bytes: datagram.to_vec(), + icmp_errors: 0, + }); true } } @@ -224,12 +312,14 @@ pub(crate) fn poll_send_gossip( } } -/// Retry the retained leave-farewell datagrams FIRST (oldest to newest), sending -/// each over `socket` until one is backpressured. A `Ready(Err)` is logged and -/// counts as attempted — a LOCAL failure sets `send_failed` so the parked leave -/// resolves with an error; a `Pending` keeps that datagram (and every later one) -/// retained and stops the pass. Bounded: a single front-to-back sweep with no -/// re-enqueue of a just-sent datagram, so it cannot loop. +/// Retry the retained leave-farewell datagrams FIRST (oldest to newest), +/// sending each over `socket` until one is backpressured. An ICMP-class +/// `Ready(Err)` re-retains the datagram at the BACK of the queue (bounded by +/// its absorb count) so the rest of the queue drains ahead of the re-attempt; a +/// LOCAL failure sets `send_failed` so the parked leave resolves with an error; +/// a `Pending` keeps that datagram (and every later one) retained and stops the +/// pass. Bounded: the pass pops at most the queue's initial length, so a +/// re-retained datagram is re-attempted on the NEXT pass, never this one. #[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) fn retry_retained_leave( retained: &mut LeaveDrain, @@ -242,19 +332,46 @@ pub(crate) fn retry_retained_leave( let Some(socket) = socket else { return; }; - while let Some((peer, datagram)) = retained.pop_front() { - match socket.poll_send_to(cx, &datagram, peer) { - Poll::Ready(Ok(_)) => {} - Poll::Ready(Err(err)) => { - trace_leave_send_error(peer, &err); - if farewell_send_failure_is_local(&err) { - *send_failed = true; + let mut budget = retained.len(); + while budget > 0 { + budget -= 1; + let Some(d) = retained.pop_front() else { + break; + }; + let outcome = socket.poll_send_to(cx, &d.bytes, d.peer); + if settle_retried_farewell(retained, d, outcome, send_failed) { + break; + } + } +} + +/// Fold one retried farewell's send outcome back into the retention state. +/// Returns `true` when the sweep must stop (the socket is backpressured; the +/// datagram went back to the FRONT so FIFO order is preserved). +#[cfg(any(feature = "tcp", feature = "quic"))] +fn settle_retried_farewell( + retained: &mut LeaveDrain, + mut d: LeaveDatagram, + outcome: Poll>, + send_failed: &mut bool, +) -> bool { + match outcome { + Poll::Ready(Ok(_)) => false, + Poll::Ready(Err(err)) => { + trace_leave_send_error(d.peer, &err); + match classify_errored_farewell(d.icmp_errors, &err) { + ErroredFarewell::Retry(absorbed) => { + d.icmp_errors = absorbed; + retained.push_back(d); } + ErroredFarewell::PeerAnswered => trace_leave_peer_answered(d.peer), + ErroredFarewell::LocalFailure => *send_failed = true, } - Poll::Pending => { - retained.push_front((peer, datagram)); - break; - } + false + } + Poll::Pending => { + retained.push_front(d); + true } } } diff --git a/serf-reactor/src/driver/shared/tests.rs b/serf-reactor/src/driver/shared/tests.rs index f8bd69aa..35d40805 100644 --- a/serf-reactor/src/driver/shared/tests.rs +++ b/serf-reactor/src/driver/shared/tests.rs @@ -1,11 +1,18 @@ use super::*; -/// The leave-drain retention bookkeeping: `Ready(Ok)` sends (retaining nothing), -/// `Pending` retains FIFO at the back (returning `true`), and `Ready(Err)` is -/// attempted — logged, retaining nothing. The datagram bytes are copied only on -/// the retain path. +fn entry(peer: SocketAddr, bytes: &[u8], icmp_errors: u8) -> LeaveDatagram { + LeaveDatagram { + peer, + bytes: bytes.to_vec(), + icmp_errors, + } +} + +/// The leave-drain retention bookkeeping for fresh sends: `Ready(Ok)` retains +/// nothing, `Pending` retains FIFO at the back (returning `true`), and the +/// datagram bytes are copied only on the retain paths. #[test] -fn retain_leave_datagram_records_only_pending() { +fn retain_leave_datagram_retains_pending_fifo() { let peer: SocketAddr = "127.0.0.1:7946".parse().unwrap(); let mut retained: LeaveDrain = VecDeque::new(); let mut send_failed = false; @@ -30,7 +37,8 @@ fn retain_leave_datagram_records_only_pending() { &mut send_failed, )); assert_eq!(retained.len(), 1); - assert_eq!(retained.back().unwrap(), &(peer, b"beta".to_vec())); + assert_eq!(retained.back().unwrap().bytes, b"beta".to_vec()); + assert_eq!(retained.back().unwrap().icmp_errors, 0); assert!(!send_failed, "backpressure is retention, not failure"); // A second Pending appends behind the first (FIFO order preserved). @@ -43,67 +51,62 @@ fn retain_leave_datagram_records_only_pending() { &mut send_failed, )); assert_eq!(retained.len(), 2); - assert_eq!(retained.front().unwrap(), &(peer, b"beta".to_vec())); - assert_eq!(retained.back().unwrap(), &(peer2, b"gamma".to_vec())); - - // Ready(Err): attempted (logged), not retained; the queue is unchanged. - assert!(!retain_leave_datagram( - &mut retained, - peer, - b"delta", - Poll::Ready(Err(io::Error::other("send failed"))), - &mut send_failed, - )); - assert_eq!( - retained.len(), - 2, - "an errored send is attempted, not retained" - ); + assert_eq!(retained.front().unwrap().peer, peer); + assert_eq!(retained.back().unwrap().peer, peer2); } -/// Error accounting: a LOCAL send failure (broken socket — the farewell never -/// left this host) sets `send_failed` so the parked leave resolves with an -/// error, while a per-peer network signal (a reset or unreachable reflected -/// for a peer that is itself gone) is log-only — it neither retains nor fails -/// the leave. +/// Error accounting for fresh sends: an ICMP-class error (`ConnectionReset` / +/// `ConnectionRefused`) is ambiguous on a shared unconnected socket — it may +/// be a stale asynchronous answer to an earlier packet for a DIFFERENT peer, +/// and either way the current datagram was not accepted — so the datagram is +/// RETAINED for a bounded retry without failing the leave. Every other kind is +/// a local delivery failure: `send_failed` is set and nothing is retained. #[test] -fn retain_leave_datagram_flags_only_local_failures() { +fn retain_leave_datagram_retries_icmp_and_flags_local_failures() { let peer: SocketAddr = "127.0.0.1:7946".parse().unwrap(); - let mut retained: LeaveDrain = VecDeque::new(); - // Per-peer network signals: attempted, not retained, leave still succeeds. + // ICMP-class: retained with one absorbed error, leave not failed. for kind in [ io::ErrorKind::ConnectionReset, io::ErrorKind::ConnectionRefused, - io::ErrorKind::ConnectionAborted, - io::ErrorKind::HostUnreachable, - io::ErrorKind::NetworkUnreachable, ] { + let mut retained: LeaveDrain = VecDeque::new(); let mut send_failed = false; - assert!(!retain_leave_datagram( + assert!(retain_leave_datagram( &mut retained, peer, b"alpha", Poll::Ready(Err(io::Error::from(kind))), &mut send_failed, )); - assert!(retained.is_empty()); + assert_eq!( + retained.len(), + 1, + "an ICMP-class error must retain ({kind:?})" + ); + assert_eq!(retained.front().unwrap().icmp_errors, 1); assert!( !send_failed, - "a per-peer network signal ({kind:?}) must not fail the leave" + "an ICMP-class error ({kind:?}) must not fail the leave outright" ); } // Local socket failures: attempted, not retained, and the leave FAILS. - // `AddrNotAvailable` is local — the bound source address vanished from this - // host — not a peer signal. + // `ConnectionAborted` is a local software abort on Windows; the + // unreachables usually report the LOCAL routing table's answer; and + // `AddrNotAvailable` means the bound source address vanished from this + // host. for kind in [ + io::ErrorKind::ConnectionAborted, + io::ErrorKind::HostUnreachable, + io::ErrorKind::NetworkUnreachable, + io::ErrorKind::AddrNotAvailable, io::ErrorKind::NotConnected, io::ErrorKind::BrokenPipe, io::ErrorKind::InvalidInput, - io::ErrorKind::AddrNotAvailable, io::ErrorKind::Other, ] { + let mut retained: LeaveDrain = VecDeque::new(); let mut send_failed = false; assert!(!retain_leave_datagram( &mut retained, @@ -120,6 +123,138 @@ fn retain_leave_datagram_flags_only_local_failures() { } } +/// The errored-farewell classifier: ICMP-class errors retry with a bumped +/// absorb count until [`FAREWELL_ICMP_ERROR_LIMIT`], where the answer is +/// attributed to the destination itself; every other kind is local regardless +/// of the count. +#[test] +fn classify_errored_farewell_bounds_icmp_retries() { + for kind in [ + io::ErrorKind::ConnectionReset, + io::ErrorKind::ConnectionRefused, + ] { + let err = io::Error::from(kind); + assert_eq!( + classify_errored_farewell(0, &err), + ErroredFarewell::Retry(1) + ); + assert_eq!( + classify_errored_farewell(1, &err), + ErroredFarewell::Retry(2) + ); + assert_eq!( + classify_errored_farewell(FAREWELL_ICMP_ERROR_LIMIT - 1, &err), + ErroredFarewell::PeerAnswered + ); + assert_eq!( + classify_errored_farewell(u8::MAX, &err), + ErroredFarewell::PeerAnswered, + "the absorb count saturates rather than wrapping" + ); + } + for kind in [ + io::ErrorKind::ConnectionAborted, + io::ErrorKind::HostUnreachable, + io::ErrorKind::NetworkUnreachable, + io::ErrorKind::AddrNotAvailable, + io::ErrorKind::BrokenPipe, + io::ErrorKind::Other, + ] { + let err = io::Error::from(kind); + assert_eq!( + classify_errored_farewell(0, &err), + ErroredFarewell::LocalFailure, + "{kind:?} must be local on the first error" + ); + assert_eq!( + classify_errored_farewell(u8::MAX, &err), + ErroredFarewell::LocalFailure + ); + } +} + +/// The retry sweep's outcome fold: a delivered datagram leaves the queue; an +/// ICMP-class error under the limit re-retains it at the BACK (so the rest of +/// the queue drains ahead of the re-attempt) with the bumped count; at the +/// limit it is dropped as answered without failing the leave; a local error +/// sets `send_failed`; and `Pending` puts it back at the FRONT and stops the +/// sweep (FIFO preserved). +#[test] +fn settle_retried_farewell_dispositions() { + let peer_a: SocketAddr = "127.0.0.1:7946".parse().unwrap(); + let peer_b: SocketAddr = "127.0.0.1:7947".parse().unwrap(); + + // Delivered: not re-added, sweep continues. + let mut retained: LeaveDrain = VecDeque::new(); + let mut send_failed = false; + assert!(!settle_retried_farewell( + &mut retained, + entry(peer_a, b"alpha", 1), + Poll::Ready(Ok(5)), + &mut send_failed, + )); + assert!(retained.is_empty()); + assert!(!send_failed); + + // ICMP-class under the limit: re-retained at the BACK with a bumped count — + // the stale-error disambiguation retry — leaving the queue's head (another + // peer's farewell) to drain first. + let mut retained: LeaveDrain = VecDeque::from([entry(peer_b, b"beta", 0)]); + let mut send_failed = false; + assert!(!settle_retried_farewell( + &mut retained, + entry(peer_a, b"alpha", 1), + Poll::Ready(Err(io::Error::from(io::ErrorKind::ConnectionRefused))), + &mut send_failed, + )); + assert_eq!(retained.len(), 2); + assert_eq!(retained.front().unwrap().peer, peer_b); + assert_eq!(retained.back().unwrap().peer, peer_a); + assert_eq!(retained.back().unwrap().icmp_errors, 2); + assert!(!send_failed); + + // ICMP-class at the limit: dropped as answered-by-the-network, leave intact. + let mut retained: LeaveDrain = VecDeque::new(); + let mut send_failed = false; + assert!(!settle_retried_farewell( + &mut retained, + entry(peer_a, b"alpha", FAREWELL_ICMP_ERROR_LIMIT - 1), + Poll::Ready(Err(io::Error::from(io::ErrorKind::ConnectionReset))), + &mut send_failed, + )); + assert!( + retained.is_empty(), + "an answered peer's farewell is dropped" + ); + assert!(!send_failed, "a gone peer must not fail the whole leave"); + + // Local failure: dropped AND the leave fails. + let mut retained: LeaveDrain = VecDeque::new(); + let mut send_failed = false; + assert!(!settle_retried_farewell( + &mut retained, + entry(peer_a, b"alpha", 0), + Poll::Ready(Err(io::Error::from(io::ErrorKind::BrokenPipe))), + &mut send_failed, + )); + assert!(retained.is_empty()); + assert!(send_failed); + + // Pending: back at the FRONT (count preserved), sweep stops. + let mut retained: LeaveDrain = VecDeque::from([entry(peer_b, b"beta", 0)]); + let mut send_failed = false; + assert!(settle_retried_farewell( + &mut retained, + entry(peer_a, b"alpha", 2), + Poll::Pending, + &mut send_failed, + )); + assert_eq!(retained.len(), 2); + assert_eq!(retained.front().unwrap().peer, peer_a); + assert_eq!(retained.front().unwrap().icmp_errors, 2); + assert!(!send_failed); +} + /// The leave outcome maps the accumulated failure flag onto the caller-facing /// `leave().await` result: no failure resolves `Ok`, a local send/transform /// failure resolves `LeaveFarewellUndelivered`. From d39da2711705aac2ae329f1c73f016b8692eff67 Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 12:22:10 +1200 Subject: [PATCH 096/128] fix(serf-reactor): retry retained farewells exactly once per pump wake MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The retained-datagram retry ran inside the gossip egress, which repeats within drain_surfaces' fixed point, and the teardown invoked it again after its quiescence drain — so a same-batch leave+shutdown could charge a fresh errored send plus two same-poll retries against one datagram, exhausting its whole ICMP allowance (three absorbs) within a single wake against one stale asynchronous error. A healthy destination's farewell was then dropped as answered-by-the-network and the emptied drain let leave() resolve Ok. The retry is hoisted to a single site at the top of each pump's poll, before any fresh sends: a retained datagram now absorbs at most one ICMP-class error per wake, so the bounded retry samples the socket's error slot across distinct wakes — which is the whole disambiguation. Fresh sends only retain; a datagram retained this poll is re-attempted on the next wake (the leave and teardown deadline timers, and the pump's idle wake, are armed while any farewell is outstanding). Both egress prologues and the teardown lost their retry calls; each pump now has exactly one. --- serf-reactor/src/driver/quic/mod.rs | 78 ++++++++++++---------- serf-reactor/src/driver/stream/mod.rs | 94 +++++++++++++++------------ 2 files changed, 93 insertions(+), 79 deletions(-) diff --git a/serf-reactor/src/driver/quic/mod.rs b/serf-reactor/src/driver/quic/mod.rs index 3fa691fd..cd441c0a 100644 --- a/serf-reactor/src/driver/quic/mod.rs +++ b/serf-reactor/src/driver/quic/mod.rs @@ -771,29 +771,12 @@ where // datagrams reach the wire before that fence fires. // Once `leave()` has been initiated the fan-out rides plain UDP in BOTH // unreliable modes (the Datagram arm reroutes it — see the match below) and - // is RETAINED on backpressure: retry the retained datagrams FIRST (they - // drain as the socket becomes writable), then pop fresh transmits at the - // unchanged cadence — a non-completing send is retained (not dropped) so - // peers do not read the departure as a failure. Periodic gossip stays - // best-effort. - if self.leave_initiated { - retry_retained_leave( - &mut self.leave_drain, - self.socket.as_ref(), - cx, - &mut self.leave_send_failed, - ); - // A `LeftCluster` observed while these datagrams were still queued - // deferred the parked leave; resolve it now that the socket has accepted - // every retained farewell. - if self.left_cluster_seen && self.leave_drain.is_empty() { - self.left_cluster_seen = false; - if let Some(pl) = self.pending_leave.take() { - let failed = self.leave_send_failed; - pl.resolve_all(|| leave_outcome(failed)); - } - } - } + // is RETAINED on non-completion instead of dropped. The retained-datagram + // RETRY does NOT run here: this pass repeats inside `drain_surfaces`' + // fixed point, and a retained farewell must absorb at most one ICMP-class + // error per pump wake for the bounded retry to sample the socket's error + // slot across DISTINCT wakes — the single retry site is the top of `poll`. + // Periodic gossip stays best-effort. let encode_opts = EncodeOptions::new(self.label.clone()); let unreliable = self.endpoint.unreliable_transport(); let mut needs_flush = false; @@ -1287,6 +1270,34 @@ where progress = true; } + // Retained leave-farewell retry: exactly ONCE per poll, before any fresh + // sends (normal egress or teardown drain). A retained datagram must absorb + // at most one ICMP-class error per pump wake, so the bounded retry samples + // the socket's error slot across DISTINCT wakes — running it inside + // `drain_surfaces`' fixed point (or again at teardown) could exhaust the + // whole allowance within a single wake against one stale asynchronous + // error. A datagram retained by THIS poll's fresh sends waits for the next + // wake (the leave/teardown deadline timers are always armed while any + // farewell is outstanding). + if this.leave_initiated { + retry_retained_leave( + &mut this.leave_drain, + this.socket.as_ref(), + cx, + &mut this.leave_send_failed, + ); + // A `LeftCluster` observed while these datagrams were still queued + // deferred the parked leave; resolve it now that the socket has accepted + // every retained farewell. + if this.left_cluster_seen && this.leave_drain.is_empty() { + this.left_cluster_seen = false; + if let Some(pl) = this.pending_leave.take() { + let failed = this.leave_send_failed; + pl.resolve_all(|| leave_outcome(failed)); + } + } + } + // Shutdown: flush to quiescence (an explicit leave's `Dead`-self notices // must reach the wire before the socket drops), fail every parked waiter // and queued command, release the bound port, then ack. No implicit leave: @@ -1330,19 +1341,14 @@ where ))); } } - // Flush any retained leave-farewell datagrams before releasing the socket: - // the quiescence loop above retains a backpressured graceful-leave fan-out - // rather than dropping it, and it must reach the wire before the socket - // drops. A residue parks the teardown — bounded by a short deadline — - // instead of being dropped: the `Pending` send has the writable waker - // registered while the deadline timer keeps a dead socket from hanging - // shutdown. - retry_retained_leave( - &mut this.leave_drain, - this.socket.as_ref(), - cx, - &mut this.leave_send_failed, - ); + // Retained leave-farewell datagrams must reach the wire before the + // socket drops. They are retried by the per-poll retry at the top of + // `poll` (never here — a second same-poll retry could exhaust a + // datagram's whole ICMP allowance in one wake); a residue parks the + // teardown — bounded by a short deadline — instead of being dropped: the + // `Pending` send has the writable waker registered while the deadline + // timer keeps a dead socket from hanging shutdown. + // // The caller's per-leave deadline governs resolution during teardown // exactly as in the normal poll's reap: a leave that missed its // configured window resolves `LeaveTimeout` even mid-shutdown (a zero diff --git a/serf-reactor/src/driver/stream/mod.rs b/serf-reactor/src/driver/stream/mod.rs index 4c71e0d9..6864f8a5 100644 --- a/serf-reactor/src/driver/stream/mod.rs +++ b/serf-reactor/src/driver/stream/mod.rs @@ -1114,29 +1114,13 @@ where // `LeftCluster`), so the leave/shutdown datagrams reach the socket before that // fence fires. // - // Once `leave()` has been initiated the fan-out is RETAINED on backpressure: - // retry the retained datagrams FIRST (they drain as the socket becomes - // writable), then pop fresh transmits at the unchanged cadence — a - // non-completing send is retained (not dropped) instead of leaving peers to - // read the departure as a failure. Periodic gossip stays best-effort. - if self.leave_initiated { - retry_retained_leave( - &mut self.leave_drain, - self.socket.as_ref(), - cx, - &mut self.leave_send_failed, - ); - // A `LeftCluster` observed while these datagrams were still queued - // deferred the parked leave; resolve it now that the socket has accepted - // every retained farewell. - if self.left_cluster_seen && self.leave_drain.is_empty() { - self.left_cluster_seen = false; - if let Some(pl) = self.pending_leave.take() { - let failed = self.leave_send_failed; - pl.resolve_all(|| leave_outcome(failed)); - } - } - } + // Once `leave()` has been initiated the fan-out is RETAINED on + // non-completion instead of dropped. The retained-datagram RETRY does NOT + // run here: this pass repeats inside `drain_surfaces`' fixed point, and a + // retained farewell must absorb at most one ICMP-class error per pump + // wake for the bounded retry to sample the socket's error slot across + // DISTINCT wakes — the single retry site is the top of `poll`. Periodic + // gossip stays best-effort. let encode_opts = EncodeOptions::new(self.label.clone()); let mut sent = 0; while sent < budget { @@ -1549,10 +1533,38 @@ where progress = true; } - // Shutdown, in ordered phases: (1) FREEZE — best-effort leave, cancel every - // bridge's reads, drop the template inbound sender, and release the bind - // sockets; (2) DRAIN the bridge inbound channel to all-senders-gone, folding - // every already-read completion into its join's contacted set; (3) REAP the + // Retained leave-farewell retry: exactly ONCE per poll, before any fresh + // sends (normal egress or teardown drain). A retained datagram must absorb + // at most one ICMP-class error per pump wake, so the bounded retry samples + // the socket's error slot across DISTINCT wakes — running it inside + // `drain_surfaces`' fixed point (or again at teardown) could exhaust the + // whole allowance within a single wake against one stale asynchronous + // error. A datagram retained by THIS poll's fresh sends waits for the next + // wake (the leave/teardown deadline timers are always armed while any + // farewell is outstanding). + if this.leave_initiated { + retry_retained_leave( + &mut this.leave_drain, + this.socket.as_ref(), + cx, + &mut this.leave_send_failed, + ); + // A `LeftCluster` observed while these datagrams were still queued + // deferred the parked leave; resolve it now that the socket has accepted + // every retained farewell. + if this.left_cluster_seen && this.leave_drain.is_empty() { + this.left_cluster_seen = false; + if let Some(pl) = this.pending_leave.take() { + let failed = this.leave_send_failed; + pl.resolve_all(|| leave_outcome(failed)); + } + } + } + + // Shutdown, in ordered phases: (1) FREEZE — cancel every bridge's reads, + // drop the template inbound sender, and release the bind sockets; (2) DRAIN + // the bridge inbound channel to all-senders-gone, folding every + // already-read completion into its join's contacted set; (3) REAP the // parked joins/leave and fail the queued commands; (4) await ONLY the accept // task's exit before acking. The completion latch promises the bind address // is free (the UDP gossip socket + the TCP listener), not that every connected @@ -1580,16 +1592,18 @@ where // FAREWELL DRAIN (re-entrant), then release the gossip socket (once — // `socket.is_some()` is the phase guard). The endpoint surfaces are first // drained to quiescence WHILE the socket still exists: a leave dispatched - // in the same command batch as the shutdown (or the best-effort one in - // the freeze) queued its dead-self fan-out inside the endpoint, and a - // gossip transmit popped after the socket drops is silently discarded. - // Retained leave-farewell datagrams are then retried; a residue parks - // the teardown — bounded by a short deadline — instead of being dropped, - // because the farewell is the peers' only first-hand signal that the - // departure was intentional. The `Pending` send registers the writable - // waker and the deadline timer bounds a persistently dead socket; ANY - // teardown wake re-enters this phase until the queue empties or the - // deadline wins. Dropping the socket then closes its UDP FD synchronously. + // in the same command batch as the shutdown queued its dead-self fan-out + // inside the endpoint, and a gossip transmit popped after the socket + // drops is silently discarded. Retained datagrams are retried by the + // per-poll retry at the top of `poll` (never here — a second same-poll + // retry could exhaust a datagram's whole ICMP allowance in one wake); a + // residue parks the teardown — bounded by a short deadline — instead of + // being dropped, because the farewell is the peers' only first-hand + // signal that the departure was intentional. The `Pending` send + // registers the writable waker and the deadline timer bounds a + // persistently dead socket; ANY teardown wake re-enters this phase until + // the queue empties or the deadline wins. Dropping the socket then + // closes its UDP FD synchronously. if this.socket.is_some() { loop { let (_, drain_more, _) = this.drain_surfaces(cx); @@ -1597,12 +1611,6 @@ where break; } } - retry_retained_leave( - &mut this.leave_drain, - this.socket.as_ref(), - cx, - &mut this.leave_send_failed, - ); // The caller's per-leave deadline governs resolution during teardown // exactly as in the normal poll's reap: a leave that missed its // configured window resolves `LeaveTimeout` even mid-shutdown (a zero From 7cca4e5163ed55cdc19820f30f2d6edb13c783a3 Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 12:39:55 +1200 Subject: [PATCH 097/128] fix(serf-reactor): epoch-gate the farewell retry so self-wakes cannot burn the ICMP allowance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Limiting the retry to once per poll was not enough: a loaded pump re-wakes itself (capped ingress, machine-surface backlog set the more flag), and a back-to-back re-poll is not a temporally distinct sample of the socket's error slot — three self-wake polls could still burn a datagram's whole three-error allowance against one stale asynchronous error and falsely drop a healthy peer's farewell as answered. The accounting now carries a retry epoch: any ICMP-class absorb arms retry_after = now + FAREWELL_ICMP_RETRY_INTERVAL (100ms), and the single hoisted retry runs only once the epoch has elapsed, however many polls occur in between. The epoch instant folds into every wake source that must observe it — the normal idle-arm timer target and the teardown residue park (now min(drain bound, leave deadline, retry epoch)) — so the eligible wake arrives exactly when the epoch elapses; backpressure-retained datagrams stay writable-waker-driven (Pending never arms the epoch, so an empty error history retries immediately). The pump-level regression drives a real driver: with the epoch in the future, eight consecutive polls leave a seeded retained farewell untouched; the first poll after the epoch elapses hands it to the socket. The pure-fn suite gains arming assertions (ICMP absorbs arm the epoch; Ok, Pending, and local failures never do). --- serf-reactor/src/driver/quic/mod.rs | 89 ++++++++++++------- serf-reactor/src/driver/shared.rs | 91 ++++++++++++++++--- serf-reactor/src/driver/shared/tests.rs | 111 ++++++++++++++++-------- serf-reactor/src/driver/stream/mod.rs | 82 +++++++++++------ serf-reactor/src/driver/stream/tests.rs | 49 +++++++++++ 5 files changed, 312 insertions(+), 110 deletions(-) diff --git a/serf-reactor/src/driver/quic/mod.rs b/serf-reactor/src/driver/quic/mod.rs index cd441c0a..7a5873bc 100644 --- a/serf-reactor/src/driver/quic/mod.rs +++ b/serf-reactor/src/driver/quic/mod.rs @@ -84,9 +84,9 @@ use crate::{ driver::{ options::RuntimeOptions, shared::{ - ExchangeId, LEAVE_DRAIN_TEARDOWN_BOUND, LeaveDrain, dispatch_event_delegate, leave_outcome, - observation_payload_bytes, poll_send_gossip, retry_retained_leave, trace_leave_drain_residue, - trace_leave_transform_error, + ExchangeId, FarewellAccounting, LEAVE_DRAIN_TEARDOWN_BOUND, LeaveDrain, + dispatch_event_delegate, leave_outcome, observation_payload_bytes, poll_send_gossip, + retry_retained_leave, trace_leave_drain_residue, trace_leave_transform_error, }, }, drop_counter::ReactorDropCounter, @@ -325,11 +325,9 @@ where left_cluster_seen: bool, /// Deadline bounding the teardown park that drains retained farewells. leave_drain_deadline: Option, - /// A leave-farewell send failed on the LOCAL socket (not a per-peer network - /// signal): the parked leave resolves - /// [`LeaveFarewellUndelivered`](SerfError::LeaveFarewellUndelivered) instead - /// of a false `Ok`. - leave_send_failed: bool, + /// Delivery accounting for the leave fan-out: the local-failure flag and the + /// epoch gate spacing ICMP-class retry passes across distinct instants. + farewell: FarewellAccounting, /// Parked `Shutdown` replies — acked only after the UDP socket drops, so a caller /// resuming from `shutdown().await` can rebind the same address. A `Vec` because /// several callers can race `shutdown()`. @@ -405,7 +403,7 @@ where leave_drain: LeaveDrain::new(), left_cluster_seen: false, leave_drain_deadline: None, - leave_send_failed: false, + farewell: FarewellAccounting::new(), shutdown_reply: Vec::new(), recv_buf: vec![0u8; buf_len.max(1)], iter_drain_cap: driver_opts.iter_drain_cap().max(1), @@ -794,7 +792,7 @@ where Err(_) => { if self.leave_initiated { trace_leave_transform_error(to); - self.leave_send_failed = true; + self.farewell.send_failed = true; } continue; } @@ -807,7 +805,7 @@ where Err(_) => { if self.leave_initiated { trace_leave_transform_error(to); - self.leave_send_failed = true; + self.farewell.send_failed = true; } continue; } @@ -823,7 +821,7 @@ where Err(_) => { if self.leave_initiated { trace_leave_transform_error(peer); - self.leave_send_failed = true; + self.farewell.send_failed = true; } continue; } @@ -843,7 +841,8 @@ where cx, peer, &on_wire, - &mut self.leave_send_failed, + now, + &mut self.farewell, ); } // Once `leave()` has been initiated the fan-out takes the plain-UDP @@ -863,7 +862,8 @@ where cx, peer, &on_wire, - &mut self.leave_send_failed, + now, + &mut self.farewell, ); } UnreliableTransport::Datagram => { @@ -891,7 +891,8 @@ where cx, peer, &on_wire, - &mut self.leave_send_failed, + now, + &mut self.farewell, ); } // TooLarge: the connection is already Established (max_size was Some), so @@ -904,7 +905,8 @@ where cx, peer, &on_wire, - &mut self.leave_send_failed, + now, + &mut self.farewell, ); } } @@ -1056,7 +1058,7 @@ where // resolve when the drain empties (bounded by the caller's leave timeout). if self.leave_drain.is_empty() { if let Some(pl) = self.pending_leave.take() { - let failed = self.leave_send_failed; + let failed = self.farewell.send_failed; pl.resolve_all(|| leave_outcome(failed)); } } else { @@ -1160,6 +1162,18 @@ where self.pending_leave.as_ref().map(|pl| pl.deadline) } + /// Earliest instant a retained farewell's epoch-gated retry becomes + /// eligible, folded into the per-poll timer target; `None` when nothing is + /// retained (a backpressure-retained datagram is woken by the socket's + /// writable waker instead). + fn farewell_retry_deadline(&self) -> Option { + if self.leave_drain.is_empty() { + None + } else { + self.farewell.retry_after + } + } + /// Publish a fresh [`SerfSnapshot`] of the endpoint's observable membership. /// Skips the publish when the local node is not yet present in the membership /// store (the local `NodeJoined` sieve has not fired), so `SerfSnapshot::new` @@ -1280,19 +1294,27 @@ where // wake (the leave/teardown deadline timers are always armed while any // farewell is outstanding). if this.leave_initiated { - retry_retained_leave( - &mut this.leave_drain, - this.socket.as_ref(), - cx, - &mut this.leave_send_failed, - ); + // Epoch gate: after an ICMP-class absorb, the next retry pass waits for + // `retry_after` — a self-wake re-poll is NOT a temporally distinct + // sample of the socket's error slot. The gate instant folds into the + // pump's timer targets, so the eligible wake arrives on time. + if this.farewell.retry_after.is_none_or(|t| now >= t) { + this.farewell.retry_after = None; + retry_retained_leave( + &mut this.leave_drain, + this.socket.as_ref(), + cx, + now, + &mut this.farewell, + ); + } // A `LeftCluster` observed while these datagrams were still queued // deferred the parked leave; resolve it now that the socket has accepted // every retained farewell. if this.left_cluster_seen && this.leave_drain.is_empty() { this.left_cluster_seen = false; if let Some(pl) = this.pending_leave.take() { - let failed = this.leave_send_failed; + let failed = this.farewell.send_failed; pl.resolve_all(|| leave_outcome(failed)); } } @@ -1363,7 +1385,7 @@ where // retained). A residue instead resolves `Err(Shutdown)` below — the // farewell did not fully leave this host. if let Some(pl) = this.pending_leave.take() { - let failed = this.leave_send_failed; + let failed = this.farewell.send_failed; pl.resolve_all(|| leave_outcome(failed)); } } else { @@ -1372,15 +1394,19 @@ where .leave_drain_deadline .get_or_insert(now + LEAVE_DRAIN_TEARDOWN_BOUND); if now < drain_deadline { - // Park until whichever fires first: the drain bound, or a - // still-parked leave's deadline — whose firing must resolve + // Park until whichever fires first: the drain bound, a + // still-parked leave's deadline (whose firing must resolve // `LeaveTimeout` promptly, while the drain keeps the rest of its - // window. A `Ready` timer re-enters the phase so the reap and the - // park recompute against the new now. - let park_until = this + // window), or the ICMP retry epoch (so an epoch-gated retry runs + // as soon as it becomes eligible). A `Ready` timer re-enters the + // phase so the reap and the park recompute against the new now. + let mut park_until = this .pending_leave .as_ref() .map_or(drain_deadline, |pl| pl.deadline.min(drain_deadline)); + if let Some(epoch) = this.farewell.retry_after { + park_until = park_until.min(epoch); + } this.arm_timer(park_until, now); if let Some(timer) = this.timer.as_mut() && timer.as_mut().poll(cx).is_pending() @@ -1553,6 +1579,9 @@ where if recv_errored { target = target.min(now + RECV_ERROR_BACKOFF); } + if let Some(epoch) = this.farewell_retry_deadline() { + target = target.min(epoch); + } if this.arm_and_poll_timer(target, now, cx) { more = true; } diff --git a/serf-reactor/src/driver/shared.rs b/serf-reactor/src/driver/shared.rs index 52b49962..4bd617a5 100644 --- a/serf-reactor/src/driver/shared.rs +++ b/serf-reactor/src/driver/shared.rs @@ -13,6 +13,8 @@ use agnostic::net::UdpSocket; #[cfg(any(feature = "tcp", feature = "quic"))] use core::task::{Context, Poll}; #[cfg(any(feature = "tcp", feature = "quic"))] +use memberlist_proto::Instant; +#[cfg(any(feature = "tcp", feature = "quic"))] use std::{collections::VecDeque, io, net::SocketAddr, vec::Vec}; /// Coordinator-allocated handle for one in-flight reliable exchange. @@ -101,6 +103,58 @@ pub(crate) struct LeaveDatagram { icmp_errors: u8, } +#[cfg(all(test, any(feature = "tcp", feature = "quic")))] +impl LeaveDatagram { + /// Test seam: build a retained entry directly (production entries are built + /// only by the retention helpers in this module). + // Test-only: consumed by the tokio-gated pump tests, so a smol-only test + // build sees no caller. + #[allow(dead_code)] + pub(crate) fn for_tests(peer: SocketAddr, bytes: Vec, icmp_errors: u8) -> Self { + Self { + peer, + bytes, + icmp_errors, + } + } +} + +/// Per-pump accounting for the leave-farewell fan-out's delivery. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) struct FarewellAccounting { + /// A LOCAL send/transform failure occurred — the farewell (or part of it) + /// never left this host — so the parked leave resolves + /// [`LeaveFarewellUndelivered`](crate::error::SerfError::LeaveFarewellUndelivered) + /// instead of a false `Ok`. + pub(crate) send_failed: bool, + /// Earliest instant the next retained-farewell retry pass may run. Armed + /// whenever a datagram absorbs an ICMP-class error: consecutive absorbs must + /// sample the socket's error slot at temporally DISTINCT instants, so + /// back-to-back self-wake polls (a loaded pump re-waking itself) cannot burn + /// the bounded allowance against one stale asynchronous error. Folded into + /// the pump's timer targets so the wake arrives when the epoch elapses. + pub(crate) retry_after: Option, +} + +#[cfg(any(feature = "tcp", feature = "quic"))] +impl FarewellAccounting { + pub(crate) const fn new() -> Self { + Self { + send_failed: false, + retry_after: None, + } + } +} + +/// Minimum wall-clock spacing between retained-farewell retry passes once a +/// datagram has absorbed an ICMP-class error. Short enough that the full +/// [`FAREWELL_ICMP_ERROR_LIMIT`] allowance fits comfortably inside the +/// teardown drain bound and any realistic leave timeout; long enough that each +/// retry is a temporally distinct sample of the socket's error slot. +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) const FAREWELL_ICMP_RETRY_INTERVAL: core::time::Duration = + core::time::Duration::from_millis(100); + /// Leave-farewell datagrams retained after a non-completing UDP send, retried /// FIFO. #[cfg(any(feature = "tcp", feature = "quic"))] @@ -235,10 +289,10 @@ pub(crate) const LEAVE_DRAIN_TEARDOWN_BOUND: core::time::Duration = /// `retained`: /// - `Ready(Ok)` — it left the socket; nothing to retain. /// - `Ready(Err)` — the socket did NOT accept this datagram. An ICMP-class -/// error retains it for a bounded retry (the error slot may hold a stale -/// answer to an earlier packet for a different peer); a LOCAL failure sets -/// `send_failed`, so the parked leave resolves with an error instead of a -/// false success. +/// error retains it for an epoch-gated retry (the error slot may hold a +/// stale answer to an earlier packet for a different peer) and arms +/// `acct.retry_after`; a LOCAL failure sets `acct.send_failed`, so the +/// parked leave resolves with an error instead of a false success. /// - `Pending` — the socket is backpressured; the datagram is retained for the /// next writable wake. /// @@ -250,7 +304,8 @@ pub(crate) fn retain_leave_datagram( peer: SocketAddr, datagram: &[u8], outcome: Poll>, - send_failed: &mut bool, + now: Instant, + acct: &mut FarewellAccounting, ) -> bool { match outcome { Poll::Ready(Ok(_)) => false, @@ -258,6 +313,7 @@ pub(crate) fn retain_leave_datagram( trace_leave_send_error(peer, &err); match classify_errored_farewell(0, &err) { ErroredFarewell::Retry(absorbed) => { + acct.retry_after = Some(now + FAREWELL_ICMP_RETRY_INTERVAL); retained.push_back(LeaveDatagram { peer, bytes: datagram.to_vec(), @@ -270,7 +326,7 @@ pub(crate) fn retain_leave_datagram( false } ErroredFarewell::LocalFailure => { - *send_failed = true; + acct.send_failed = true; false } } @@ -292,6 +348,7 @@ pub(crate) fn retain_leave_datagram( /// send drops the datagram. Once `leave()` has been initiated the leave fan-out /// is retained on backpressure via [`retain_leave_datagram`]. #[cfg(any(feature = "tcp", feature = "quic"))] +#[allow(clippy::too_many_arguments)] pub(crate) fn poll_send_gossip( retained: &mut LeaveDrain, leave_initiated: bool, @@ -299,7 +356,8 @@ pub(crate) fn poll_send_gossip( cx: &mut Context<'_>, peer: SocketAddr, on_wire: &[u8], - send_failed: &mut bool, + now: Instant, + acct: &mut FarewellAccounting, ) where S: UdpSocket, { @@ -308,7 +366,7 @@ pub(crate) fn poll_send_gossip( }; let outcome = socket.poll_send_to(cx, on_wire, peer); if leave_initiated { - retain_leave_datagram(retained, peer, on_wire, outcome, send_failed); + retain_leave_datagram(retained, peer, on_wire, outcome, now, acct); } } @@ -325,7 +383,8 @@ pub(crate) fn retry_retained_leave( retained: &mut LeaveDrain, socket: Option<&S>, cx: &mut Context<'_>, - send_failed: &mut bool, + now: Instant, + acct: &mut FarewellAccounting, ) where S: UdpSocket, { @@ -339,21 +398,24 @@ pub(crate) fn retry_retained_leave( break; }; let outcome = socket.poll_send_to(cx, &d.bytes, d.peer); - if settle_retried_farewell(retained, d, outcome, send_failed) { + if settle_retried_farewell(retained, d, outcome, now, acct) { break; } } } /// Fold one retried farewell's send outcome back into the retention state. -/// Returns `true` when the sweep must stop (the socket is backpressured; the -/// datagram went back to the FRONT so FIFO order is preserved). +/// An ICMP-class re-retention arms `acct.retry_after`, epoch-gating the next +/// retry pass. Returns `true` when the sweep must stop (the socket is +/// backpressured; the datagram went back to the FRONT so FIFO order is +/// preserved). #[cfg(any(feature = "tcp", feature = "quic"))] fn settle_retried_farewell( retained: &mut LeaveDrain, mut d: LeaveDatagram, outcome: Poll>, - send_failed: &mut bool, + now: Instant, + acct: &mut FarewellAccounting, ) -> bool { match outcome { Poll::Ready(Ok(_)) => false, @@ -361,11 +423,12 @@ fn settle_retried_farewell( trace_leave_send_error(d.peer, &err); match classify_errored_farewell(d.icmp_errors, &err) { ErroredFarewell::Retry(absorbed) => { + acct.retry_after = Some(now + FAREWELL_ICMP_RETRY_INTERVAL); d.icmp_errors = absorbed; retained.push_back(d); } ErroredFarewell::PeerAnswered => trace_leave_peer_answered(d.peer), - ErroredFarewell::LocalFailure => *send_failed = true, + ErroredFarewell::LocalFailure => acct.send_failed = true, } false } diff --git a/serf-reactor/src/driver/shared/tests.rs b/serf-reactor/src/driver/shared/tests.rs index 35d40805..73e22174 100644 --- a/serf-reactor/src/driver/shared/tests.rs +++ b/serf-reactor/src/driver/shared/tests.rs @@ -9,13 +9,15 @@ fn entry(peer: SocketAddr, bytes: &[u8], icmp_errors: u8) -> LeaveDatagram { } /// The leave-drain retention bookkeeping for fresh sends: `Ready(Ok)` retains -/// nothing, `Pending` retains FIFO at the back (returning `true`), and the -/// datagram bytes are copied only on the retain paths. +/// nothing, `Pending` retains FIFO at the back (returning `true`) without +/// arming the retry epoch, and the datagram bytes are copied only on the +/// retain paths. #[test] fn retain_leave_datagram_retains_pending_fifo() { let peer: SocketAddr = "127.0.0.1:7946".parse().unwrap(); + let now = Instant::now(); let mut retained: LeaveDrain = VecDeque::new(); - let mut send_failed = false; + let mut acct = FarewellAccounting::new(); // Ready(Ok): sent, nothing retained. assert!(!retain_leave_datagram( @@ -23,23 +25,31 @@ fn retain_leave_datagram_retains_pending_fifo() { peer, b"alpha", Poll::Ready(Ok(5)), - &mut send_failed, + now, + &mut acct, )); assert!(retained.is_empty(), "a completed send retains nothing"); - assert!(!send_failed, "a completed send is not a failure"); + assert!(!acct.send_failed, "a completed send is not a failure"); + assert_eq!(acct.retry_after, None); - // Pending: retained at the back, returns true (socket backpressured). + // Pending: retained at the back, returns true (socket backpressured); the + // retry epoch stays unarmed — a writable wake may retry immediately. assert!(retain_leave_datagram( &mut retained, peer, b"beta", Poll::Pending, - &mut send_failed, + now, + &mut acct, )); assert_eq!(retained.len(), 1); assert_eq!(retained.back().unwrap().bytes, b"beta".to_vec()); assert_eq!(retained.back().unwrap().icmp_errors, 0); - assert!(!send_failed, "backpressure is retention, not failure"); + assert!(!acct.send_failed, "backpressure is retention, not failure"); + assert_eq!( + acct.retry_after, None, + "backpressure does not arm the epoch" + ); // A second Pending appends behind the first (FIFO order preserved). let peer2: SocketAddr = "127.0.0.1:7947".parse().unwrap(); @@ -48,7 +58,8 @@ fn retain_leave_datagram_retains_pending_fifo() { peer2, b"gamma", Poll::Pending, - &mut send_failed, + now, + &mut acct, )); assert_eq!(retained.len(), 2); assert_eq!(retained.front().unwrap().peer, peer); @@ -59,25 +70,28 @@ fn retain_leave_datagram_retains_pending_fifo() { /// `ConnectionRefused`) is ambiguous on a shared unconnected socket — it may /// be a stale asynchronous answer to an earlier packet for a DIFFERENT peer, /// and either way the current datagram was not accepted — so the datagram is -/// RETAINED for a bounded retry without failing the leave. Every other kind is -/// a local delivery failure: `send_failed` is set and nothing is retained. +/// RETAINED for an epoch-gated retry (arming `retry_after`) without failing +/// the leave. Every other kind is a local delivery failure: `send_failed` is +/// set and nothing is retained. #[test] fn retain_leave_datagram_retries_icmp_and_flags_local_failures() { let peer: SocketAddr = "127.0.0.1:7946".parse().unwrap(); + let now = Instant::now(); - // ICMP-class: retained with one absorbed error, leave not failed. + // ICMP-class: retained with one absorbed error, epoch armed, leave intact. for kind in [ io::ErrorKind::ConnectionReset, io::ErrorKind::ConnectionRefused, ] { let mut retained: LeaveDrain = VecDeque::new(); - let mut send_failed = false; + let mut acct = FarewellAccounting::new(); assert!(retain_leave_datagram( &mut retained, peer, b"alpha", Poll::Ready(Err(io::Error::from(kind))), - &mut send_failed, + now, + &mut acct, )); assert_eq!( retained.len(), @@ -86,9 +100,14 @@ fn retain_leave_datagram_retries_icmp_and_flags_local_failures() { ); assert_eq!(retained.front().unwrap().icmp_errors, 1); assert!( - !send_failed, + !acct.send_failed, "an ICMP-class error ({kind:?}) must not fail the leave outright" ); + assert_eq!( + acct.retry_after, + Some(now + FAREWELL_ICMP_RETRY_INTERVAL), + "an ICMP-class absorb must arm the retry epoch" + ); } // Local socket failures: attempted, not retained, and the leave FAILS. @@ -107,19 +126,24 @@ fn retain_leave_datagram_retries_icmp_and_flags_local_failures() { io::ErrorKind::Other, ] { let mut retained: LeaveDrain = VecDeque::new(); - let mut send_failed = false; + let mut acct = FarewellAccounting::new(); assert!(!retain_leave_datagram( &mut retained, peer, b"alpha", Poll::Ready(Err(io::Error::from(kind))), - &mut send_failed, + now, + &mut acct, )); assert!(retained.is_empty()); assert!( - send_failed, + acct.send_failed, "a local socket failure ({kind:?}) must fail the leave" ); + assert_eq!( + acct.retry_after, None, + "a local failure does not arm the retry epoch" + ); } } @@ -175,84 +199,95 @@ fn classify_errored_farewell_bounds_icmp_retries() { /// The retry sweep's outcome fold: a delivered datagram leaves the queue; an /// ICMP-class error under the limit re-retains it at the BACK (so the rest of -/// the queue drains ahead of the re-attempt) with the bumped count; at the -/// limit it is dropped as answered without failing the leave; a local error -/// sets `send_failed`; and `Pending` puts it back at the FRONT and stops the -/// sweep (FIFO preserved). +/// the queue drains ahead of the re-attempt) with the bumped count and arms +/// the retry epoch; at the limit it is dropped as answered without failing +/// the leave; a local error sets `send_failed`; and `Pending` puts it back at +/// the FRONT and stops the sweep (FIFO preserved). #[test] fn settle_retried_farewell_dispositions() { let peer_a: SocketAddr = "127.0.0.1:7946".parse().unwrap(); let peer_b: SocketAddr = "127.0.0.1:7947".parse().unwrap(); + let now = Instant::now(); // Delivered: not re-added, sweep continues. let mut retained: LeaveDrain = VecDeque::new(); - let mut send_failed = false; + let mut acct = FarewellAccounting::new(); assert!(!settle_retried_farewell( &mut retained, entry(peer_a, b"alpha", 1), Poll::Ready(Ok(5)), - &mut send_failed, + now, + &mut acct, )); assert!(retained.is_empty()); - assert!(!send_failed); + assert!(!acct.send_failed); + assert_eq!(acct.retry_after, None); // ICMP-class under the limit: re-retained at the BACK with a bumped count — // the stale-error disambiguation retry — leaving the queue's head (another - // peer's farewell) to drain first. + // peer's farewell) to drain first, and arming the retry epoch. let mut retained: LeaveDrain = VecDeque::from([entry(peer_b, b"beta", 0)]); - let mut send_failed = false; + let mut acct = FarewellAccounting::new(); assert!(!settle_retried_farewell( &mut retained, entry(peer_a, b"alpha", 1), Poll::Ready(Err(io::Error::from(io::ErrorKind::ConnectionRefused))), - &mut send_failed, + now, + &mut acct, )); assert_eq!(retained.len(), 2); assert_eq!(retained.front().unwrap().peer, peer_b); assert_eq!(retained.back().unwrap().peer, peer_a); assert_eq!(retained.back().unwrap().icmp_errors, 2); - assert!(!send_failed); + assert!(!acct.send_failed); + assert_eq!(acct.retry_after, Some(now + FAREWELL_ICMP_RETRY_INTERVAL)); // ICMP-class at the limit: dropped as answered-by-the-network, leave intact. let mut retained: LeaveDrain = VecDeque::new(); - let mut send_failed = false; + let mut acct = FarewellAccounting::new(); assert!(!settle_retried_farewell( &mut retained, entry(peer_a, b"alpha", FAREWELL_ICMP_ERROR_LIMIT - 1), Poll::Ready(Err(io::Error::from(io::ErrorKind::ConnectionReset))), - &mut send_failed, + now, + &mut acct, )); assert!( retained.is_empty(), "an answered peer's farewell is dropped" ); - assert!(!send_failed, "a gone peer must not fail the whole leave"); + assert!( + !acct.send_failed, + "a gone peer must not fail the whole leave" + ); // Local failure: dropped AND the leave fails. let mut retained: LeaveDrain = VecDeque::new(); - let mut send_failed = false; + let mut acct = FarewellAccounting::new(); assert!(!settle_retried_farewell( &mut retained, entry(peer_a, b"alpha", 0), Poll::Ready(Err(io::Error::from(io::ErrorKind::BrokenPipe))), - &mut send_failed, + now, + &mut acct, )); assert!(retained.is_empty()); - assert!(send_failed); + assert!(acct.send_failed); // Pending: back at the FRONT (count preserved), sweep stops. let mut retained: LeaveDrain = VecDeque::from([entry(peer_b, b"beta", 0)]); - let mut send_failed = false; + let mut acct = FarewellAccounting::new(); assert!(settle_retried_farewell( &mut retained, entry(peer_a, b"alpha", 2), Poll::Pending, - &mut send_failed, + now, + &mut acct, )); assert_eq!(retained.len(), 2); assert_eq!(retained.front().unwrap().peer, peer_a); assert_eq!(retained.front().unwrap().icmp_errors, 2); - assert!(!send_failed); + assert!(!acct.send_failed); } /// The leave outcome maps the accumulated failure flag onto the caller-facing diff --git a/serf-reactor/src/driver/stream/mod.rs b/serf-reactor/src/driver/stream/mod.rs index 6864f8a5..ccc1e3c4 100644 --- a/serf-reactor/src/driver/stream/mod.rs +++ b/serf-reactor/src/driver/stream/mod.rs @@ -66,9 +66,9 @@ use crate::{ driver::{ options::{RuntimeOptions, StreamTransportOptions}, shared::{ - ExchangeId, LEAVE_DRAIN_TEARDOWN_BOUND, LeaveDrain, dispatch_event_delegate, leave_outcome, - observation_payload_bytes, poll_send_gossip, retry_retained_leave, trace_leave_drain_residue, - trace_leave_transform_error, + ExchangeId, FarewellAccounting, LEAVE_DRAIN_TEARDOWN_BOUND, LeaveDrain, + dispatch_event_delegate, leave_outcome, observation_payload_bytes, poll_send_gossip, + retry_retained_leave, trace_leave_drain_residue, trace_leave_transform_error, }, }, drop_counter::ReactorDropCounter, @@ -364,11 +364,9 @@ where left_cluster_seen: bool, /// Deadline bounding the teardown park that drains retained farewells. leave_drain_deadline: Option, - /// A leave-farewell send failed on the LOCAL socket (not a per-peer network - /// signal): the parked leave resolves - /// [`LeaveFarewellUndelivered`](SerfError::LeaveFarewellUndelivered) instead - /// of a false `Ok`. - leave_send_failed: bool, + /// Delivery accounting for the leave fan-out: the local-failure flag and the + /// epoch gate spacing ICMP-class retry passes across distinct instants. + farewell: FarewellAccounting, /// Parked `Shutdown` replies — acked only after the bind sockets drop, so a /// caller resuming from `shutdown().await` can rebind the same address. A `Vec` /// because several callers can race `shutdown()`. @@ -510,7 +508,7 @@ where leave_drain: LeaveDrain::new(), left_cluster_seen: false, leave_drain_deadline: None, - leave_send_failed: false, + farewell: FarewellAccounting::new(), shutdown_reply: Vec::new(), bridges: HashMap::new(), accepted_rx, @@ -1136,7 +1134,7 @@ where Err(_) => { if self.leave_initiated { trace_leave_transform_error(to); - self.leave_send_failed = true; + self.farewell.send_failed = true; } continue; } @@ -1149,7 +1147,7 @@ where Err(_) => { if self.leave_initiated { trace_leave_transform_error(to); - self.leave_send_failed = true; + self.farewell.send_failed = true; } continue; } @@ -1165,7 +1163,7 @@ where Err(_) => { if self.leave_initiated { trace_leave_transform_error(peer); - self.leave_send_failed = true; + self.farewell.send_failed = true; } continue; } @@ -1180,7 +1178,8 @@ where cx, peer, &on_wire, - &mut self.leave_send_failed, + now, + &mut self.farewell, ); } worked |= sent > 0; @@ -1304,7 +1303,7 @@ where // resolve when the drain empties (bounded by the caller's leave timeout). if self.leave_drain.is_empty() { if let Some(pl) = self.pending_leave.take() { - let failed = self.leave_send_failed; + let failed = self.farewell.send_failed; pl.resolve_all(|| leave_outcome(failed)); } } else { @@ -1407,6 +1406,18 @@ where self.pending_leave.as_ref().map(|pl| pl.deadline) } + /// Earliest instant a retained farewell's epoch-gated retry becomes + /// eligible, folded into the per-poll timer target; `None` when nothing is + /// retained (a backpressure-retained datagram is woken by the socket's + /// writable waker instead). + fn farewell_retry_deadline(&self) -> Option { + if self.leave_drain.is_empty() { + None + } else { + self.farewell.retry_after + } + } + /// The GREATEST join/leave deadline already due (`<= now`), or `None` when none /// is. Mirrors the min-deadline folds ([`Self::min_pending_join_deadline`] /// filters to still-unreplied joins) but takes the max, so the reap watermark @@ -1543,19 +1554,27 @@ where // wake (the leave/teardown deadline timers are always armed while any // farewell is outstanding). if this.leave_initiated { - retry_retained_leave( - &mut this.leave_drain, - this.socket.as_ref(), - cx, - &mut this.leave_send_failed, - ); + // Epoch gate: after an ICMP-class absorb, the next retry pass waits for + // `retry_after` — a self-wake re-poll is NOT a temporally distinct + // sample of the socket's error slot. The gate instant folds into the + // pump's timer targets, so the eligible wake arrives on time. + if this.farewell.retry_after.is_none_or(|t| now >= t) { + this.farewell.retry_after = None; + retry_retained_leave( + &mut this.leave_drain, + this.socket.as_ref(), + cx, + now, + &mut this.farewell, + ); + } // A `LeftCluster` observed while these datagrams were still queued // deferred the parked leave; resolve it now that the socket has accepted // every retained farewell. if this.left_cluster_seen && this.leave_drain.is_empty() { this.left_cluster_seen = false; if let Some(pl) = this.pending_leave.take() { - let failed = this.leave_send_failed; + let failed = this.farewell.send_failed; pl.resolve_all(|| leave_outcome(failed)); } } @@ -1625,7 +1644,7 @@ where // nothing retained). A residue instead falls through to the reap's // `Err(Shutdown)` — the farewell did not fully leave this host. if let Some(pl) = this.pending_leave.take() { - let failed = this.leave_send_failed; + let failed = this.farewell.send_failed; pl.resolve_all(|| leave_outcome(failed)); } } else { @@ -1634,15 +1653,19 @@ where .leave_drain_deadline .get_or_insert(now + LEAVE_DRAIN_TEARDOWN_BOUND); if now < drain_deadline { - // Park until whichever fires first: the drain bound, or a - // still-parked leave's deadline — whose firing must resolve + // Park until whichever fires first: the drain bound, a + // still-parked leave's deadline (whose firing must resolve // `LeaveTimeout` promptly, while the drain keeps the rest of its - // window. A `Ready` timer re-enters the phase so the reap and the - // park recompute against the new now. - let park_until = this + // window), or the ICMP retry epoch (so an epoch-gated retry runs + // as soon as it becomes eligible). A `Ready` timer re-enters the + // phase so the reap and the park recompute against the new now. + let mut park_until = this .pending_leave .as_ref() .map_or(drain_deadline, |pl| pl.deadline.min(drain_deadline)); + if let Some(epoch) = this.farewell.retry_after { + park_until = park_until.min(epoch); + } this.arm_timer(park_until, now); if let Some(timer) = this.timer.as_mut() && timer.as_mut().poll(cx).is_pending() @@ -1960,7 +1983,10 @@ where // Idle: nothing due. Arm + poll the sleep for the next deadline; NO self-wake // (return `Pending` — the armed sleep, a bridge `wake_driver`, or a socket // readiness re-polls us). - let target = reap_deadline.map_or(endpoint_deadline, |d| d.min(endpoint_deadline)); + let mut target = reap_deadline.map_or(endpoint_deadline, |d| d.min(endpoint_deadline)); + if let Some(epoch) = this.farewell_retry_deadline() { + target = target.min(epoch); + } this.arm_timer(target, now); if let Some(timer) = this.timer.as_mut() && timer.as_mut().poll(cx).is_ready() diff --git a/serf-reactor/src/driver/stream/tests.rs b/serf-reactor/src/driver/stream/tests.rs index 2374a450..02e5b266 100644 --- a/serf-reactor/src/driver/stream/tests.rs +++ b/serf-reactor/src/driver/stream/tests.rs @@ -1258,3 +1258,52 @@ async fn join_deadline_above_stream_timeout_does_not_reap_premature_failure() { "the join reached its seed once the clamped deadline widened the watermark: {reached:?}" ); } + +/// The retained-farewell retry is epoch-gated: while `retry_after` lies in the +/// future, ANY number of pump polls — including back-to-back self-wake +/// re-polls — leaves a retained datagram untouched (a re-poll is NOT a +/// temporally distinct sample of the socket's error slot, so it must not burn +/// the bounded ICMP allowance); once the epoch elapses, the next poll's single +/// hoisted retry sends it and the drain empties. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn farewell_retry_waits_for_its_epoch_across_polls() { + let (mut driver, _obs_rx, _shared) = build_driver(8, 8).await; + // Address the retained farewell to the driver's own bound socket, so an + // ELIGIBLE retry deterministically completes `Ready(Ok)` on loopback. + let dest = driver + .socket + .as_ref() + .expect("gossip socket held") + .local_addr() + .expect("gossip socket local addr"); + driver.leave_initiated = true; + driver + .leave_drain + .push_back(crate::driver::shared::LeaveDatagram::for_tests( + dest, + b"farewell".to_vec(), + 1, + )); + driver.farewell.retry_after = Some(Instant::now() + Duration::from_secs(3600)); + + for i in 0..8 { + let _ = poll_once(&mut driver); + assert_eq!( + driver.leave_drain.len(), + 1, + "poll {i}: the epoch gate must hold the retained farewell across re-polls" + ); + } + + // The epoch elapses: the very next poll's single retry pass sends it. + driver.farewell.retry_after = Some(Instant::now()); + let _ = poll_once(&mut driver); + assert!( + driver.leave_drain.is_empty(), + "an eligible retry must hand the retained farewell to the socket" + ); + assert!( + !driver.farewell.send_failed, + "a completed loopback send must not fail the leave" + ); +} From 0b179bbf19c10e7c4321ae415f4570d34bc09093 Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 13:21:24 +1200 Subject: [PATCH 098/128] feat(serf): surface Vivaldi coordinates on the handle and seed the ack payload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The machine computes coordinates but no handle exposed them: the snapshot now carries the local coordinate (populated by both reactor pumps on every republish, read lock-free via Serf::coordinate()), and a new driver command answers Serf::cached_coordinate(id) from the machine's per-peer cache in every lifecycle state. Wiring the accessors surfaced a bootstrap deadlock in the machine itself: the probe-ack coordinate payload was installed only by a successful coordinate update, but no update can succeed until some ack carries a payload — so no coordinate ever moved off the origin and no peer cache ever filled. The reference implementation re-encodes the current coordinate on every ack; the stored-payload model now seeds the origin coordinate once at the first drain (one-shot, at the drain chokepoint every driver path funnels through) and stays refreshed by every subsequent update. New coverage: a machine test pins the first-drain seed, and a two-node e2e pins that both accessors go Some within the probe cadence on both runtimes. --- serf-driver/src/snapshot/mod.rs | 29 ++++++++++++++++ serf-proto/src/endpoint/mod.rs | 36 +++++++++++++++++++ serf-proto/src/endpoint/tests.rs | 18 ++++++++++ serf-proto/src/stream_endpoint/mod.rs | 6 ++++ serf-reactor/src/command/mod.rs | 20 +++++++++++ serf-reactor/src/driver/quic/mod.rs | 15 ++++++++ serf-reactor/src/driver/stream/mod.rs | 15 ++++++++ serf-reactor/src/serf/mod.rs | 32 +++++++++++++++++ serf-reactor/tests/tcp.rs | 50 +++++++++++++++++++++++++++ 9 files changed, 221 insertions(+) diff --git a/serf-driver/src/snapshot/mod.rs b/serf-driver/src/snapshot/mod.rs index 938008c3..02ab1d4f 100644 --- a/serf-driver/src/snapshot/mod.rs +++ b/serf-driver/src/snapshot/mod.rs @@ -32,6 +32,11 @@ pub struct SerfSnapshot { query_clock: LamportTime, alive_count: usize, member_count: usize, + /// The local node's Vivaldi coordinate at the instant of the snapshot, set by + /// the publishing driver via [`with_coordinate`](Self::with_coordinate); + /// `None` when coordinates are disabled or no coordinate exists yet. + #[cfg(feature = "coordinates")] + coordinate: Option, } impl SerfSnapshot { @@ -80,9 +85,33 @@ impl SerfSnapshot { query_clock, alive_count, member_count, + #[cfg(feature = "coordinates")] + coordinate: None, } } + /// Attach the local node's Vivaldi coordinate to the snapshot (builder form, + /// called by the publishing driver after [`new`](Self::new)). + #[cfg(feature = "coordinates")] + #[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] + #[must_use] + pub fn with_coordinate(mut self, coordinate: Option) -> Self { + self.coordinate = coordinate; + self + } + + /// The local node's Vivaldi coordinate at the instant of the snapshot. + /// + /// `None` when coordinates are disabled + /// (`Options::with_disable_coordinates(true)`), when the publishing driver + /// does not forward them, or when no coordinate exists yet. + #[cfg(feature = "coordinates")] + #[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] + #[must_use] + pub const fn coordinate(&self) -> Option<&serf_proto::typed::Coordinate> { + self.coordinate.as_ref() + } + /// All known members (full [`Member`], carrying tags and status) — alive, leaving, left, /// and failed within the reap window. #[must_use] diff --git a/serf-proto/src/endpoint/mod.rs b/serf-proto/src/endpoint/mod.rs index f2bf7dc0..5c6c790c 100644 --- a/serf-proto/src/endpoint/mod.rs +++ b/serf-proto/src/endpoint/mod.rs @@ -820,6 +820,10 @@ where /// Removed when a node is reaped from membership (G13). #[cfg(feature = "coordinates")] coord_cache: crate::FxHashMap, + /// Whether the INITIAL coordinate ack payload has been installed on the + /// inner endpoint (see `seed_coordinate_ack`). + #[cfg(feature = "coordinates")] + coord_ack_seeded: bool, /// The `now` instant threaded into the most recent poll/handle call. /// /// The inner `poll_event` loop (drain_inner) fires synchronously from @@ -986,6 +990,8 @@ where #[cfg(feature = "coordinates")] coord_client, #[cfg(feature = "coordinates")] + coord_ack_seeded: false, + #[cfg(feature = "coordinates")] coord_cache: crate::FxHashMap::default(), #[cfg(test)] last_dial_addr: None, @@ -1391,6 +1397,8 @@ where where T: Reliable, { + #[cfg(feature = "coordinates")] + self.seed_coordinate_ack(t); while let Some(ev) = t.poll_inner_event() { self.on_inner_event(t, ev); } @@ -1399,6 +1407,34 @@ where } } + /// One-shot: install the INITIAL coordinate ack payload so a peer's very + /// first probe round-trip already observes a coordinate. Without the seed + /// the coordinate plane deadlocks at bootstrap: `handle_ping_completed` + /// refreshes the ack payload only after a successful update, but no update + /// can succeed until some ack carries a payload. The reference + /// implementation avoids this by re-encoding the current coordinate on + /// every ack; the stored-payload model needs the origin coordinate seeded + /// once, then stays refreshed by every subsequent update. + #[cfg(feature = "coordinates")] + fn seed_coordinate_ack(&mut self, t: &mut T) + where + T: Reliable, + { + if self.coord_ack_seeded { + return; + } + match self.coord_client.as_ref() { + Some(cc) => { + let payload = coord_ack_payload(cc.get_coordinate()); + if t.set_ack_payload(payload).is_ok() { + self.coord_ack_seeded = true; + } + } + // Coordinates disabled at construction: nothing will ever be seeded. + None => self.coord_ack_seeded = true, + } + } + /// Dispatch a single inner `memberlist_proto::Event` to the matching serf /// handler. /// diff --git a/serf-proto/src/endpoint/tests.rs b/serf-proto/src/endpoint/tests.rs index 0abb9478..ab8daa22 100644 --- a/serf-proto/src/endpoint/tests.rs +++ b/serf-proto/src/endpoint/tests.rs @@ -3227,6 +3227,24 @@ fn make_coord_payload(coord: &crate::typed::Coordinate) -> Bytes { Bytes::from(buf) } +/// A fresh coordinates-enabled endpoint installs the ORIGIN coordinate as its +/// probe-ack payload on its first drain, so a peer's very first probe +/// round-trip already observes a coordinate. Without the seed the coordinate +/// plane deadlocks at bootstrap: updates refresh the payload, but no update +/// can ever succeed until some ack carries one. +#[cfg(feature = "coordinates")] +#[test] +fn coordinate_ack_payload_is_seeded_on_first_drain() { + let mut e = ep_with_coords(); + e.handle_timeout(memberlist_proto::Instant::ORIGIN); + let payload = e.test_ack_payload(); + assert!( + !payload.is_empty(), + "the first drain must install the initial coordinate ack payload" + ); + assert_eq!(payload[0], 1u8, "PING_VERSION leads the seeded ack payload"); +} + #[cfg(feature = "coordinates")] #[test] fn ping_completed_updates_local_coordinate_and_caches_remote() { diff --git a/serf-proto/src/stream_endpoint/mod.rs b/serf-proto/src/stream_endpoint/mod.rs index a3e965bd..fede179e 100644 --- a/serf-proto/src/stream_endpoint/mod.rs +++ b/serf-proto/src/stream_endpoint/mod.rs @@ -873,6 +873,12 @@ where .test_handle_leave_intent(&mut self.transport, id, ltime, now) } + /// The inner memberlist endpoint's stored probe-ack payload (test read). + #[cfg(all(test, feature = "coordinates"))] + pub(crate) fn test_ack_payload(&self) -> bytes::Bytes { + self.transport.endpoint_ref().ack_payload_bytes() + } + /// Forwards to [`Endpoint::test_inner_node_joined`]. #[cfg(test)] pub(crate) fn test_inner_node_joined(&mut self, id: I, now: Instant) diff --git a/serf-reactor/src/command/mod.rs b/serf-reactor/src/command/mod.rs index 22f34523..468259c7 100644 --- a/serf-reactor/src/command/mod.rs +++ b/serf-reactor/src/command/mod.rs @@ -273,6 +273,19 @@ pub(crate) struct ListKeysCmd { pub(crate) reply: Sender>, } +/// Payload for [`Command::CachedCoordinate`]. +/// +/// Requires the `coordinates` feature. +#[cfg(feature = "coordinates")] +pub(crate) struct CachedCoordinateCmd { + /// The peer whose most-recently-observed coordinate is requested. + pub(crate) id: I, + /// One-shot reply channel delivering the peer's cached coordinate, `None` + /// when coordinates are disabled or no RTT sample has arrived from the peer. + #[cfg(any(feature = "tcp", feature = "quic"))] + pub(crate) reply: Sender>>, +} + /// Payload for [`Command::Shutdown`]. pub(crate) struct ShutdownCmd { /// One-shot reply channel for the shutdown acknowledgement. @@ -360,6 +373,13 @@ pub(crate) enum Command { )] ListKeys(ListKeysCmd), + /// Read a peer's most-recently-observed Vivaldi coordinate. + /// + /// Requires the `coordinates` feature. + #[cfg(feature = "coordinates")] + #[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] + CachedCoordinate(CachedCoordinateCmd), + /// Signal the driver task to shut down gracefully. Shutdown(ShutdownCmd), } diff --git a/serf-reactor/src/driver/quic/mod.rs b/serf-reactor/src/driver/quic/mod.rs index 7a5873bc..b2194c82 100644 --- a/serf-reactor/src/driver/quic/mod.rs +++ b/serf-reactor/src/driver/quic/mod.rs @@ -70,6 +70,8 @@ use serf_proto::{ }; use smallvec::SmallVec; +#[cfg(feature = "coordinates")] +use crate::command::CachedCoordinateCmd; #[cfg(encryption)] use crate::command::{KeyCmd, ListKeysCmd}; #[cfg(encryption)] @@ -674,6 +676,13 @@ where // Ignoring Err: caller dropped the reply receiver. let _ = reply.send(res); } + #[cfg(feature = "coordinates")] + Command::CachedCoordinate(CachedCoordinateCmd { id, reply }) => { + // A read-only probe of the coordinate cache: answerable in every + // lifecycle state (post-leave introspection stays valid). + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(Ok(self.endpoint.cached_coordinate(&id))); + } Command::Shutdown(ShutdownCmd { reply }) => { // Do NOT ack inline: the UDP socket is still bound. Flag shutdown and park // the reply; the teardown branch acks every parked caller only AFTER it @@ -1192,6 +1201,8 @@ where LamportTime::from(self.endpoint.event_time()), LamportTime::from(self.endpoint.query_time()), ); + #[cfg(feature = "coordinates")] + let snap = snap.with_coordinate(self.endpoint.get_coordinate()); self.shared.publish(snap); } @@ -1255,6 +1266,10 @@ where Command::ListKeys(ListKeysCmd { reply, .. }) => { let _ = reply.send(Err(SerfError::Shutdown)); } + #[cfg(feature = "coordinates")] + Command::CachedCoordinate(CachedCoordinateCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } Command::Shutdown(ShutdownCmd { reply }) => { let _ = reply.send(Err(SerfError::Shutdown)); } diff --git a/serf-reactor/src/driver/stream/mod.rs b/serf-reactor/src/driver/stream/mod.rs index ccc1e3c4..d92e71b9 100644 --- a/serf-reactor/src/driver/stream/mod.rs +++ b/serf-reactor/src/driver/stream/mod.rs @@ -52,6 +52,8 @@ use serf_proto::{ }; use smallvec::SmallVec; +#[cfg(feature = "coordinates")] +use crate::command::CachedCoordinateCmd; #[cfg(encryption)] use crate::command::{KeyCmd, ListKeysCmd}; #[cfg(encryption)] @@ -831,6 +833,13 @@ where // Ignoring Err: caller dropped the reply receiver. let _ = reply.send(res); } + #[cfg(feature = "coordinates")] + Command::CachedCoordinate(CachedCoordinateCmd { id, reply }) => { + // A read-only probe of the coordinate cache: answerable in every + // lifecycle state (post-leave introspection stays valid). + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(Ok(self.endpoint.cached_coordinate(&id))); + } Command::Shutdown(ShutdownCmd { reply }) => { // Do NOT ack inline: the gossip socket and TCP listener are still bound. // Flag shutdown and park the reply; the teardown branch acks every parked @@ -1467,6 +1476,8 @@ where LamportTime::from(self.endpoint.event_time()), LamportTime::from(self.endpoint.query_time()), ); + #[cfg(feature = "coordinates")] + let snap = snap.with_coordinate(self.endpoint.get_coordinate()); self.shared.publish(snap); } @@ -1513,6 +1524,10 @@ where Command::ListKeys(ListKeysCmd { reply, .. }) => { let _ = reply.send(Err(SerfError::Shutdown)); } + #[cfg(feature = "coordinates")] + Command::CachedCoordinate(CachedCoordinateCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } Command::Shutdown(ShutdownCmd { reply }) => { let _ = reply.send(Err(SerfError::Shutdown)); } diff --git a/serf-reactor/src/serf/mod.rs b/serf-reactor/src/serf/mod.rs index 204346c2..ea8dc2ea 100644 --- a/serf-reactor/src/serf/mod.rs +++ b/serf-reactor/src/serf/mod.rs @@ -29,6 +29,8 @@ use serf_proto::{ use smallvec::SmallVec; use smol_str::SmolStr; +#[cfg(feature = "coordinates")] +use crate::command::CachedCoordinateCmd; #[cfg(encryption)] use crate::command::{KeyCmd, ListKeysCmd}; #[cfg(encryption)] @@ -676,6 +678,36 @@ impl Serf { self.shared.datagrams_sent() } + /// The local node's current Vivaldi network coordinate, read lock-free from + /// the latest published snapshot. + /// + /// `None` when coordinates are disabled + /// (`Options::with_disable_coordinates(true)`) or before the driver's first + /// snapshot publish. Coordinates converge as probe round-trips accumulate; + /// estimate inter-node RTT by comparing two nodes' coordinates. + #[cfg(feature = "coordinates")] + #[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] + #[must_use] + pub fn coordinate(&self) -> Option { + self.shared.load_snapshot().coordinate().cloned() + } + + /// The most-recently-observed Vivaldi coordinate of the peer `id`, updated on + /// each successful probe round-trip from that peer. + /// + /// Resolves `None` when coordinates are disabled or no RTT sample has been + /// received from `id` yet. + #[cfg(feature = "coordinates")] + #[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] + pub async fn cached_coordinate(&self, id: I) -> Result> { + let (tx, rx) = oneshot::channel(); + self.send(Command::CachedCoordinate(CachedCoordinateCmd { + id, + reply: tx, + }))?; + await_reply(rx).await + } + /// Send `cmd` to the driver, failing fast if the node has shut down. fn send(&self, cmd: Command) -> Result<()> { if self.shared.is_shutdown() { diff --git a/serf-reactor/tests/tcp.rs b/serf-reactor/tests/tcp.rs index 98274188..1b6d952d 100644 --- a/serf-reactor/tests/tcp.rs +++ b/serf-reactor/tests/tcp.rs @@ -594,6 +594,44 @@ where cluster.shutdown_all().await; } +/// After a two-node join, probe round-trips feed the Vivaldi coordinate +/// client on both nodes: the local coordinate surfaces through the published +/// snapshot (`coordinate()`), and the peer's coordinate surfaces through the +/// driver round-trip (`cached_coordinate(id)`). Both accessors must go `Some` +/// within the probe cadence — the discriminator that the driver actually +/// forwards coordinates rather than merely compiling the feature. +#[cfg(feature = "coordinates")] +async fn coordinates_surface_on_the_handle() +where + R: Runtime, +{ + let mut cluster = + cluster::Cluster::::spawn(&["coord-a", "coord-b"], cluster::ClusterTiming::fast()).await; + let b_id = cluster.id(1); + + // Probe RTTs accumulate at the fast profile's 100ms cadence; both surfaces + // must appear well inside the fixture's poll ceiling. + let deadline = std::time::Instant::now() + Duration::from_secs(20); + loop { + let local = cluster.node(0).coordinate(); + let cached = cluster + .node(0) + .cached_coordinate(b_id.clone()) + .await + .expect("cached_coordinate round-trips through the driver"); + if local.is_some() && cached.is_some() { + break; + } + assert!( + std::time::Instant::now() < deadline, + "coordinates must surface on the handle: local={local:?} cached={cached:?}" + ); + R::sleep(Duration::from_millis(50)).await; + } + + cluster.shutdown_all().await; +} + /// A leave configured with a zero timeout racing a shutdown resolves /// `Err(LeaveTimeout)` — never `Ok` — even though the teardown still delivers /// the fan-out: the caller's per-leave deadline keeps governing resolution @@ -883,6 +921,12 @@ mod tokio_cells { super::serf_events_leave_with_racing_shutdown::().await; } + #[cfg(feature = "coordinates")] + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn coordinates_surface_on_the_handle() { + super::coordinates_surface_on_the_handle::().await; + } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn leave_with_zero_timeout_racing_shutdown_times_out() { super::leave_with_zero_timeout_racing_shutdown_times_out::().await; @@ -972,6 +1016,12 @@ mod smol_cells { SmolRuntime::block_on(super::serf_events_leave_with_racing_shutdown::()); } + #[cfg(feature = "coordinates")] + #[test] + fn coordinates_surface_on_the_handle_smol() { + SmolRuntime::block_on(super::coordinates_surface_on_the_handle::()); + } + #[test] fn leave_with_zero_timeout_racing_shutdown_times_out_smol() { SmolRuntime::block_on(super::leave_with_zero_timeout_racing_shutdown_times_out::< From 552e24f93a17381be4be0d8cee8e5ec34933a8c0 Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 13:36:36 +1200 Subject: [PATCH 099/128] feat(serf): per-key-op relay factor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Key-management queries always issued with relay_factor 0 — there was no per-op relay-redundancy knob. The machine's four key ops and the shared internal_query now carry an explicit relay factor (the conflict query keeps 0), both composed endpoints forward it, and both async handles gain install_key_with / use_key_with / remove_key_with / list_keys_with taking the factor; the plain forms delegate with 0, byte-identical to before. The KeyRequest event token now exposes relay_factor(), so an answering driver observes the issuer's requested redundancy — a machine test pins that the issued query's factor reaches the loopback-surfaced responder token. --- serf-compio/src/command/mod.rs | 8 +++++ serf-compio/src/driver/quic/mod.rs | 25 ++++++++++--- serf-compio/src/driver/stream/mod.rs | 25 ++++++++++--- serf-compio/src/serf/mod.rs | 52 +++++++++++++++++++++++++++ serf-proto/src/endpoint/mod.rs | 22 ++++++++++-- serf-proto/src/endpoint/tests.rs | 48 ++++++++++++++++++++++--- serf-proto/src/event/mod.rs | 6 ++++ serf-proto/src/quic_endpoint/mod.rs | 19 +++++++--- serf-proto/src/stream_endpoint/mod.rs | 19 +++++++--- serf-reactor/src/command/mod.rs | 8 +++++ serf-reactor/src/driver/quic/mod.rs | 29 ++++++++++++--- serf-reactor/src/driver/stream/mod.rs | 29 ++++++++++++--- serf-reactor/src/serf/mod.rs | 52 +++++++++++++++++++++++++++ 13 files changed, 304 insertions(+), 38 deletions(-) diff --git a/serf-compio/src/command/mod.rs b/serf-compio/src/command/mod.rs index 86acdccd..ca048fa3 100644 --- a/serf-compio/src/command/mod.rs +++ b/serf-compio/src/command/mod.rs @@ -252,6 +252,10 @@ pub(crate) struct SetTagsCmd { pub(crate) struct KeyCmd { /// The secret key to install, promote, or remove. pub(crate) key: SecretKey, + /// Relay redundancy for the responses: each node relays its answer through + /// this many random intermediaries (`0` = direct-only). + #[cfg(any(feature = "tcp", feature = "quic"))] + pub(crate) relay_factor: u8, /// Wall-clock instant passed to the machine's key-op call. #[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) now: Instant, @@ -265,6 +269,10 @@ pub(crate) struct KeyCmd { /// Requires the `aes-gcm` or `chacha20-poly1305` feature. #[cfg(encryption)] pub(crate) struct ListKeysCmd { + /// Relay redundancy for the responses: each node relays its answer through + /// this many random intermediaries (`0` = direct-only). + #[cfg(any(feature = "tcp", feature = "quic"))] + pub(crate) relay_factor: u8, /// Wall-clock instant passed to `Endpoint::list_keys`. #[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) now: Instant, diff --git a/serf-compio/src/driver/quic/mod.rs b/serf-compio/src/driver/quic/mod.rs index 9dca5919..76cc25c0 100644 --- a/serf-compio/src/driver/quic/mod.rs +++ b/serf-compio/src/driver/quic/mod.rs @@ -909,11 +909,14 @@ async fn dispatch_command( #[cfg(encryption)] Command::InstallKey(KeyCmd { key, + relay_factor, now: at, reply, }) => { let res = if running { - endpoint.install_key(key, at).map_err(SerfError::from) + endpoint + .install_key(key, relay_factor, at) + .map_err(SerfError::from) } else { Err(SerfError::NotRunning) }; @@ -923,11 +926,14 @@ async fn dispatch_command( #[cfg(encryption)] Command::UseKey(KeyCmd { key, + relay_factor, now: at, reply, }) => { let res = if running { - endpoint.use_key(key, at).map_err(SerfError::from) + endpoint + .use_key(key, relay_factor, at) + .map_err(SerfError::from) } else { Err(SerfError::NotRunning) }; @@ -937,11 +943,14 @@ async fn dispatch_command( #[cfg(encryption)] Command::RemoveKey(KeyCmd { key, + relay_factor, now: at, reply, }) => { let res = if running { - endpoint.remove_key(key, at).map_err(SerfError::from) + endpoint + .remove_key(key, relay_factor, at) + .map_err(SerfError::from) } else { Err(SerfError::NotRunning) }; @@ -949,9 +958,15 @@ async fn dispatch_command( let _ = reply.send(res); } #[cfg(encryption)] - Command::ListKeys(ListKeysCmd { now: at, reply }) => { + Command::ListKeys(ListKeysCmd { + relay_factor, + now: at, + reply, + }) => { let res = if running { - endpoint.list_keys(at).map_err(SerfError::from) + endpoint + .list_keys(relay_factor, at) + .map_err(SerfError::from) } else { Err(SerfError::NotRunning) }; diff --git a/serf-compio/src/driver/stream/mod.rs b/serf-compio/src/driver/stream/mod.rs index dae385c2..37226b3f 100644 --- a/serf-compio/src/driver/stream/mod.rs +++ b/serf-compio/src/driver/stream/mod.rs @@ -1210,11 +1210,14 @@ async fn dispatch_command( #[cfg(encryption)] Command::InstallKey(KeyCmd { key, + relay_factor, now: at, reply, }) => { let res = if running { - endpoint.install_key(key, at).map_err(SerfError::from) + endpoint + .install_key(key, relay_factor, at) + .map_err(SerfError::from) } else { Err(SerfError::NotRunning) }; @@ -1224,11 +1227,14 @@ async fn dispatch_command( #[cfg(encryption)] Command::UseKey(KeyCmd { key, + relay_factor, now: at, reply, }) => { let res = if running { - endpoint.use_key(key, at).map_err(SerfError::from) + endpoint + .use_key(key, relay_factor, at) + .map_err(SerfError::from) } else { Err(SerfError::NotRunning) }; @@ -1238,11 +1244,14 @@ async fn dispatch_command( #[cfg(encryption)] Command::RemoveKey(KeyCmd { key, + relay_factor, now: at, reply, }) => { let res = if running { - endpoint.remove_key(key, at).map_err(SerfError::from) + endpoint + .remove_key(key, relay_factor, at) + .map_err(SerfError::from) } else { Err(SerfError::NotRunning) }; @@ -1250,9 +1259,15 @@ async fn dispatch_command( let _ = reply.send(res); } #[cfg(encryption)] - Command::ListKeys(ListKeysCmd { now: at, reply }) => { + Command::ListKeys(ListKeysCmd { + relay_factor, + now: at, + reply, + }) => { let res = if running { - endpoint.list_keys(at).map_err(SerfError::from) + endpoint + .list_keys(relay_factor, at) + .map_err(SerfError::from) } else { Err(SerfError::NotRunning) }; diff --git a/serf-compio/src/serf/mod.rs b/serf-compio/src/serf/mod.rs index cf11fd54..8c13d66f 100644 --- a/serf-compio/src/serf/mod.rs +++ b/serf-compio/src/serf/mod.rs @@ -661,9 +661,22 @@ where doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] pub async fn install_key(&self, key: SecretKey) -> Result { + self.install_key_with(key, 0).await + } + + /// As [`install_key`](Self::install_key), with the responses relayed through + /// `relay_factor` random intermediary nodes for delivery redundancy (`0` = + /// direct-only, the plain form's behavior). + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub async fn install_key_with(&self, key: SecretKey, relay_factor: u8) -> Result { let (tx, rx) = oneshot::channel(); self.send(Command::InstallKey(KeyCmd { key, + relay_factor, now: Instant::now(), reply: tx, }))?; @@ -677,9 +690,22 @@ where doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] pub async fn use_key(&self, key: SecretKey) -> Result { + self.use_key_with(key, 0).await + } + + /// As [`use_key`](Self::use_key), with the responses relayed through + /// `relay_factor` random intermediary nodes for delivery redundancy (`0` = + /// direct-only, the plain form's behavior). + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub async fn use_key_with(&self, key: SecretKey, relay_factor: u8) -> Result { let (tx, rx) = oneshot::channel(); self.send(Command::UseKey(KeyCmd { key, + relay_factor, now: Instant::now(), reply: tx, }))?; @@ -693,9 +719,22 @@ where doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] pub async fn remove_key(&self, key: SecretKey) -> Result { + self.remove_key_with(key, 0).await + } + + /// As [`remove_key`](Self::remove_key), with the responses relayed through + /// `relay_factor` random intermediary nodes for delivery redundancy (`0` = + /// direct-only, the plain form's behavior). + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub async fn remove_key_with(&self, key: SecretKey, relay_factor: u8) -> Result { let (tx, rx) = oneshot::channel(); self.send(Command::RemoveKey(KeyCmd { key, + relay_factor, now: Instant::now(), reply: tx, }))?; @@ -709,8 +748,21 @@ where doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] pub async fn list_keys(&self) -> Result { + self.list_keys_with(0).await + } + + /// As [`list_keys`](Self::list_keys), with the responses relayed through + /// `relay_factor` random intermediary nodes for delivery redundancy (`0` = + /// direct-only, the plain form's behavior). + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub async fn list_keys_with(&self, relay_factor: u8) -> Result { let (tx, rx) = oneshot::channel(); self.send(Command::ListKeys(ListKeysCmd { + relay_factor, now: Instant::now(), reply: tx, }))?; diff --git a/serf-proto/src/endpoint/mod.rs b/serf-proto/src/endpoint/mod.rs index 5c6c790c..393804ac 100644 --- a/serf-proto/src/endpoint/mod.rs +++ b/serf-proto/src/endpoint/mod.rs @@ -4992,6 +4992,7 @@ where name: SmolStr, payload: Bytes, purpose: QueryPurpose, + relay_factor: u8, now: Instant, ) -> Result where @@ -5023,7 +5024,7 @@ where ), filters: vec![], flags: QueryFlag::empty(), - relay_factor: 0, + relay_factor, timeout, name, payload, @@ -5111,6 +5112,7 @@ where SmolStr::new("_serf_conflict"), payload, QueryPurpose::Conflict, + 0, now, ); } @@ -5365,7 +5367,9 @@ where self.pending_events.push_back(Event::KeyResponse(kr)); } - /// Issue a cluster-wide `install_key` query for `key`. + /// Issue a cluster-wide `install_key` query for `key`. `relay_factor` + /// relays each response through that many random intermediary nodes for + /// delivery redundancy (`0` = direct-only, the default posture). /// /// Requires the `aes-gcm` or `chacha20-poly1305` feature. #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] @@ -5377,6 +5381,7 @@ where &mut self, t: &mut T, key: memberlist_proto::SecretKey, + relay_factor: u8, now: Instant, ) -> Result where @@ -5390,6 +5395,7 @@ where SmolStr::new("_serf_install_key"), payload, QueryPurpose::Key, + relay_factor, now, ) } @@ -5406,6 +5412,7 @@ where &mut self, t: &mut T, key: memberlist_proto::SecretKey, + relay_factor: u8, now: Instant, ) -> Result where @@ -5419,6 +5426,7 @@ where SmolStr::new("_serf_use_key"), payload, QueryPurpose::Key, + relay_factor, now, ) } @@ -5435,6 +5443,7 @@ where &mut self, t: &mut T, key: memberlist_proto::SecretKey, + relay_factor: u8, now: Instant, ) -> Result where @@ -5448,6 +5457,7 @@ where SmolStr::new("_serf_remove_key"), payload, QueryPurpose::Key, + relay_factor, now, ) } @@ -5460,7 +5470,12 @@ where docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] - pub(crate) fn list_keys(&mut self, t: &mut T, now: Instant) -> Result + pub(crate) fn list_keys( + &mut self, + t: &mut T, + relay_factor: u8, + now: Instant, + ) -> Result where T: Reliable, I: Clone + Data, @@ -5472,6 +5487,7 @@ where SmolStr::new("_serf_list_keys"), payload, QueryPurpose::Key, + relay_factor, now, ) } diff --git a/serf-proto/src/endpoint/tests.rs b/serf-proto/src/endpoint/tests.rs index ab8daa22..c5cf43c7 100644 --- a/serf-proto/src/endpoint/tests.rs +++ b/serf-proto/src/endpoint/tests.rs @@ -2769,19 +2769,19 @@ fn shutdown_refuses_key_management() { // Every issuance funnels through internal_query → ensure_not_shutdown. assert!( - matches!(e.list_keys(now), Err(Error::Shutdown)), + matches!(e.list_keys(0, now), Err(Error::Shutdown)), "list_keys must be refused after shutdown" ); assert!( - matches!(e.install_key(key, now), Err(Error::Shutdown)), + matches!(e.install_key(key, 0, now), Err(Error::Shutdown)), "install_key must be refused after shutdown" ); assert!( - matches!(e.use_key(key, now), Err(Error::Shutdown)), + matches!(e.use_key(key, 0, now), Err(Error::Shutdown)), "use_key must be refused after shutdown" ); assert!( - matches!(e.remove_key(key, now), Err(Error::Shutdown)), + matches!(e.remove_key(key, 0, now), Err(Error::Shutdown)), "remove_key must be refused after shutdown" ); @@ -6246,7 +6246,7 @@ fn local_key_op_self_applies_when_inbound_cap_is_full() { // Issue a local install_key: the initiating node MUST process its own query // and surface Event::KeyRequest even though the inbound cap is full. let now = memberlist_proto::Instant::ORIGIN; - e.install_key(test_key, now) + e.install_key(test_key, 0, now) .expect("install_key must succeed"); let ev = e @@ -6266,6 +6266,44 @@ fn local_key_op_self_applies_when_inbound_cap_is_full() { } } +/// The per-op `relay_factor` rides the issued key query end-to-end: the +/// initiating node loopback-processes its own `_serf_install_key`, and the +/// surfaced [`Event::KeyRequest`] token carries the issuer's relay factor — +/// which the responding driver hands to `respond_key`, driving the relay +/// fan-out on every answering node. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn key_op_relay_factor_rides_the_issued_query() { + use crate::event::Event; + use memberlist_proto::SecretKey; + + #[cfg(feature = "aes-gcm")] + let test_key = SecretKey::Aes128([7u8; 16]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let test_key = SecretKey::ChaCha20Poly1305([7u8; 32]); + + let mut e = ep(); + while e.poll_event().is_some() {} + + e.install_key(test_key, 2, memberlist_proto::Instant::ORIGIN) + .expect("install_key with a relay factor must issue"); + + let ev = e + .poll_event() + .expect("the initiating node loopback-processes its own key query"); + match ev { + Event::KeyRequest(req) => assert_eq!( + req.relay_factor(), + 2, + "the issued query's relay factor must reach the responder token" + ), + other => panic!( + "expected Event::KeyRequest, got {:?}", + core::mem::discriminant(&other) + ), + } +} + // ── Key op-shape check ──────────────────────────────────────────────────────── /// `_serf_install_key`, `_serf_use_key`, `_serf_remove_key` with `key = None` diff --git a/serf-proto/src/event/mod.rs b/serf-proto/src/event/mod.rs index 6a65fd21..f38149e0 100644 --- a/serf-proto/src/event/mod.rs +++ b/serf-proto/src/event/mod.rs @@ -383,6 +383,12 @@ impl KeyRequest { &self.from } + /// Relay redundancy requested by the issuer: the answering node relays its + /// response through this many random intermediary nodes (`0` = direct-only). + pub const fn relay_factor(&self) -> u8 { + self.relay_factor + } + /// Opaque query id (forwarded to `Endpoint::respond_key`). pub const fn id(&self) -> u32 { self.id diff --git a/serf-proto/src/quic_endpoint/mod.rs b/serf-proto/src/quic_endpoint/mod.rs index 8bb7970d..f96f6e8d 100644 --- a/serf-proto/src/quic_endpoint/mod.rs +++ b/serf-proto/src/quic_endpoint/mod.rs @@ -551,12 +551,15 @@ where pub fn use_key( &mut self, key: memberlist_proto::SecretKey, + relay_factor: u8, now: Instant, ) -> Result where I: Clone + Data, { - self.core.use_key(&mut self.transport, key, now) + self + .core + .use_key(&mut self.transport, key, relay_factor, now) } /// Issue a cluster-wide `remove_key` query to remove `key` from all nodes. @@ -570,12 +573,15 @@ where pub fn remove_key( &mut self, key: memberlist_proto::SecretKey, + relay_factor: u8, now: Instant, ) -> Result where I: Clone + Data, { - self.core.remove_key(&mut self.transport, key, now) + self + .core + .remove_key(&mut self.transport, key, relay_factor, now) } /// Issue a cluster-wide `list_keys` query to enumerate installed keys. @@ -586,11 +592,11 @@ where docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] - pub fn list_keys(&mut self, now: Instant) -> Result + pub fn list_keys(&mut self, relay_factor: u8, now: Instant) -> Result where I: Clone + Data, { - self.core.list_keys(&mut self.transport, now) + self.core.list_keys(&mut self.transport, relay_factor, now) } /// Install a new encryption key into the local keyring. @@ -604,12 +610,15 @@ where pub fn install_key( &mut self, key: memberlist_proto::SecretKey, + relay_factor: u8, now: Instant, ) -> Result where I: Clone + Data, { - self.core.install_key(&mut self.transport, key, now) + self + .core + .install_key(&mut self.transport, key, relay_factor, now) } /// Forwards to [`Endpoint::state`]. diff --git a/serf-proto/src/stream_endpoint/mod.rs b/serf-proto/src/stream_endpoint/mod.rs index fede179e..c65b6a7f 100644 --- a/serf-proto/src/stream_endpoint/mod.rs +++ b/serf-proto/src/stream_endpoint/mod.rs @@ -509,13 +509,16 @@ where pub fn use_key( &mut self, key: memberlist_proto::SecretKey, + relay_factor: u8, now: Instant, ) -> Result where I: Clone + Data, A: Clone + Data, { - self.core.use_key(&mut self.transport, key, now) + self + .core + .use_key(&mut self.transport, key, relay_factor, now) } /// Issue a cluster-wide `remove_key` query to remove `key` from all nodes. @@ -529,13 +532,16 @@ where pub fn remove_key( &mut self, key: memberlist_proto::SecretKey, + relay_factor: u8, now: Instant, ) -> Result where I: Clone + Data, A: Clone + Data, { - self.core.remove_key(&mut self.transport, key, now) + self + .core + .remove_key(&mut self.transport, key, relay_factor, now) } /// Issue a cluster-wide `list_keys` query to enumerate installed keys. @@ -546,12 +552,12 @@ where docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] - pub fn list_keys(&mut self, now: Instant) -> Result + pub fn list_keys(&mut self, relay_factor: u8, now: Instant) -> Result where I: Clone + Data, A: Clone + Data, { - self.core.list_keys(&mut self.transport, now) + self.core.list_keys(&mut self.transport, relay_factor, now) } /// Forwards to [`Endpoint::state`]. @@ -1298,13 +1304,16 @@ where pub fn install_key( &mut self, key: memberlist_proto::SecretKey, + relay_factor: u8, now: Instant, ) -> Result where I: Clone + Data, A: Clone + Data, { - self.core.install_key(&mut self.transport, key, now) + self + .core + .install_key(&mut self.transport, key, relay_factor, now) } /// Forwards to [`Endpoint::test_register_conflict_query`]. diff --git a/serf-reactor/src/command/mod.rs b/serf-reactor/src/command/mod.rs index 468259c7..9f332b74 100644 --- a/serf-reactor/src/command/mod.rs +++ b/serf-reactor/src/command/mod.rs @@ -252,6 +252,10 @@ pub(crate) struct SetTagsCmd { pub(crate) struct KeyCmd { /// The secret key to install, promote, or remove. pub(crate) key: SecretKey, + /// Relay redundancy for the responses: each node relays its answer through + /// this many random intermediaries (`0` = direct-only). + #[cfg(any(feature = "tcp", feature = "quic"))] + pub(crate) relay_factor: u8, /// Wall-clock instant passed to the machine's key-op call. #[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) now: Instant, @@ -265,6 +269,10 @@ pub(crate) struct KeyCmd { /// Requires the `aes-gcm` or `chacha20-poly1305` feature. #[cfg(encryption)] pub(crate) struct ListKeysCmd { + /// Relay redundancy for the responses: each node relays its answer through + /// this many random intermediaries (`0` = direct-only). + #[cfg(any(feature = "tcp", feature = "quic"))] + pub(crate) relay_factor: u8, /// Wall-clock instant passed to `Endpoint::list_keys`. #[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) now: Instant, diff --git a/serf-reactor/src/driver/quic/mod.rs b/serf-reactor/src/driver/quic/mod.rs index b2194c82..79a50411 100644 --- a/serf-reactor/src/driver/quic/mod.rs +++ b/serf-reactor/src/driver/quic/mod.rs @@ -627,11 +627,15 @@ where #[cfg(encryption)] Command::InstallKey(KeyCmd { key, + relay_factor, now: at, reply, }) => { let res = if running { - self.endpoint.install_key(key, at).map_err(SerfError::from) + self + .endpoint + .install_key(key, relay_factor, at) + .map_err(SerfError::from) } else { Err(SerfError::NotRunning) }; @@ -641,11 +645,15 @@ where #[cfg(encryption)] Command::UseKey(KeyCmd { key, + relay_factor, now: at, reply, }) => { let res = if running { - self.endpoint.use_key(key, at).map_err(SerfError::from) + self + .endpoint + .use_key(key, relay_factor, at) + .map_err(SerfError::from) } else { Err(SerfError::NotRunning) }; @@ -655,11 +663,15 @@ where #[cfg(encryption)] Command::RemoveKey(KeyCmd { key, + relay_factor, now: at, reply, }) => { let res = if running { - self.endpoint.remove_key(key, at).map_err(SerfError::from) + self + .endpoint + .remove_key(key, relay_factor, at) + .map_err(SerfError::from) } else { Err(SerfError::NotRunning) }; @@ -667,9 +679,16 @@ where let _ = reply.send(res); } #[cfg(encryption)] - Command::ListKeys(ListKeysCmd { now: at, reply }) => { + Command::ListKeys(ListKeysCmd { + relay_factor, + now: at, + reply, + }) => { let res = if running { - self.endpoint.list_keys(at).map_err(SerfError::from) + self + .endpoint + .list_keys(relay_factor, at) + .map_err(SerfError::from) } else { Err(SerfError::NotRunning) }; diff --git a/serf-reactor/src/driver/stream/mod.rs b/serf-reactor/src/driver/stream/mod.rs index d92e71b9..4b3fef0f 100644 --- a/serf-reactor/src/driver/stream/mod.rs +++ b/serf-reactor/src/driver/stream/mod.rs @@ -784,11 +784,15 @@ where #[cfg(encryption)] Command::InstallKey(KeyCmd { key, + relay_factor, now: at, reply, }) => { let res = if running { - self.endpoint.install_key(key, at).map_err(SerfError::from) + self + .endpoint + .install_key(key, relay_factor, at) + .map_err(SerfError::from) } else { Err(SerfError::NotRunning) }; @@ -798,11 +802,15 @@ where #[cfg(encryption)] Command::UseKey(KeyCmd { key, + relay_factor, now: at, reply, }) => { let res = if running { - self.endpoint.use_key(key, at).map_err(SerfError::from) + self + .endpoint + .use_key(key, relay_factor, at) + .map_err(SerfError::from) } else { Err(SerfError::NotRunning) }; @@ -812,11 +820,15 @@ where #[cfg(encryption)] Command::RemoveKey(KeyCmd { key, + relay_factor, now: at, reply, }) => { let res = if running { - self.endpoint.remove_key(key, at).map_err(SerfError::from) + self + .endpoint + .remove_key(key, relay_factor, at) + .map_err(SerfError::from) } else { Err(SerfError::NotRunning) }; @@ -824,9 +836,16 @@ where let _ = reply.send(res); } #[cfg(encryption)] - Command::ListKeys(ListKeysCmd { now: at, reply }) => { + Command::ListKeys(ListKeysCmd { + relay_factor, + now: at, + reply, + }) => { let res = if running { - self.endpoint.list_keys(at).map_err(SerfError::from) + self + .endpoint + .list_keys(relay_factor, at) + .map_err(SerfError::from) } else { Err(SerfError::NotRunning) }; diff --git a/serf-reactor/src/serf/mod.rs b/serf-reactor/src/serf/mod.rs index ea8dc2ea..8f09dde7 100644 --- a/serf-reactor/src/serf/mod.rs +++ b/serf-reactor/src/serf/mod.rs @@ -930,9 +930,22 @@ impl Serf { doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] pub async fn install_key(&self, key: SecretKey) -> Result { + self.install_key_with(key, 0).await + } + + /// As [`install_key`](Self::install_key), with the responses relayed through + /// `relay_factor` random intermediary nodes for delivery redundancy (`0` = + /// direct-only, the plain form's behavior). + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub async fn install_key_with(&self, key: SecretKey, relay_factor: u8) -> Result { let (tx, rx) = oneshot::channel(); self.send(Command::InstallKey(KeyCmd { key, + relay_factor, now: Instant::now(), reply: tx, }))?; @@ -946,9 +959,22 @@ impl Serf { doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] pub async fn use_key(&self, key: SecretKey) -> Result { + self.use_key_with(key, 0).await + } + + /// As [`use_key`](Self::use_key), with the responses relayed through + /// `relay_factor` random intermediary nodes for delivery redundancy (`0` = + /// direct-only, the plain form's behavior). + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub async fn use_key_with(&self, key: SecretKey, relay_factor: u8) -> Result { let (tx, rx) = oneshot::channel(); self.send(Command::UseKey(KeyCmd { key, + relay_factor, now: Instant::now(), reply: tx, }))?; @@ -962,9 +988,22 @@ impl Serf { doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] pub async fn remove_key(&self, key: SecretKey) -> Result { + self.remove_key_with(key, 0).await + } + + /// As [`remove_key`](Self::remove_key), with the responses relayed through + /// `relay_factor` random intermediary nodes for delivery redundancy (`0` = + /// direct-only, the plain form's behavior). + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub async fn remove_key_with(&self, key: SecretKey, relay_factor: u8) -> Result { let (tx, rx) = oneshot::channel(); self.send(Command::RemoveKey(KeyCmd { key, + relay_factor, now: Instant::now(), reply: tx, }))?; @@ -978,8 +1017,21 @@ impl Serf { doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] pub async fn list_keys(&self) -> Result { + self.list_keys_with(0).await + } + + /// As [`list_keys`](Self::list_keys), with the responses relayed through + /// `relay_factor` random intermediary nodes for delivery redundancy (`0` = + /// direct-only, the plain form's behavior). + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub async fn list_keys_with(&self, relay_factor: u8) -> Result { let (tx, rx) = oneshot::channel(); self.send(Command::ListKeys(ListKeysCmd { + relay_factor, now: Instant::now(), reply: tx, }))?; From ab465df66f51e149441e805120fd73f6ccff8544 Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 13:42:37 +1200 Subject: [PATCH 100/128] feat(serf): operator stats aggregate and encryption visibility on the handle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The snapshot now carries the live endpoint readings a driver attaches at publish time — the node-awareness health score, the gossip broadcast queue depth (total across the intent/event/query tiers), the encryption flag, and (under coordinates) the coordinate-reset counter — through additive observability accessors on the machine layers (health_score and coordinate_resets forwarders on both composed endpoints). SerfStats assembles the aggregate from one snapshot: member/failed/left counts, the health score, the three Lamport clocks, the queue depth, and the encryption flag. The handle reads all of it lock-free: stats(), encryption_enabled() (the keyring's presence is fixed at construction — rotation replaces its contents, never the ring itself), and health_score(). Coverage: a snapshot unit pins the assembly (counts by status, ops-stats builder, zero-posture defaults); the converge e2e asserts the healthy two-node aggregate and the plaintext posture; the key-rotation e2e asserts an encrypted node reports its keyring. --- serf-driver/src/lib.rs | 2 +- serf-driver/src/snapshot/mod.rs | 189 ++++++++++++++++++++++++++ serf-driver/src/snapshot/tests.rs | 41 ++++++ serf-proto/src/endpoint/mod.rs | 9 ++ serf-proto/src/quic_endpoint/mod.rs | 14 ++ serf-proto/src/stream_endpoint/mod.rs | 14 ++ serf-reactor/src/driver/quic/mod.rs | 13 +- serf-reactor/src/driver/stream/mod.rs | 13 +- serf-reactor/src/serf/mod.rs | 26 ++++ serf-reactor/tests/key_rotation.rs | 7 + serf-reactor/tests/tcp.rs | 10 ++ 11 files changed, 335 insertions(+), 3 deletions(-) diff --git a/serf-driver/src/lib.rs b/serf-driver/src/lib.rs index b1efa0e9..b420deab 100644 --- a/serf-driver/src/lib.rs +++ b/serf-driver/src/lib.rs @@ -33,4 +33,4 @@ pub use keyring::{KeyApplyOutcome, apply_key_request, keyring_carries_cross_ciph pub use observation::observation_payload_bytes; #[cfg(any(feature = "tcp", feature = "quic"))] #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] -pub use snapshot::SerfSnapshot; +pub use snapshot::{SerfSnapshot, SerfStats}; diff --git a/serf-driver/src/snapshot/mod.rs b/serf-driver/src/snapshot/mod.rs index 02ab1d4f..ea8deb38 100644 --- a/serf-driver/src/snapshot/mod.rs +++ b/serf-driver/src/snapshot/mod.rs @@ -37,6 +37,22 @@ pub struct SerfSnapshot { /// `None` when coordinates are disabled or no coordinate exists yet. #[cfg(feature = "coordinates")] coordinate: Option, + /// The node-awareness health score at the instant of the snapshot (`0` = + /// healthy; higher stretches the failure-detection timeouts). Set by the + /// publishing driver via [`with_ops_stats`](Self::with_ops_stats). + health_score: usize, + /// Depth of the gossip broadcast queue (the total across the intent, event, + /// and query tiers) at the instant of the snapshot. Set by the publishing + /// driver via [`with_ops_stats`](Self::with_ops_stats). + broadcast_queue_depth: usize, + /// Whether a gossip/reliable encryption keyring is configured on the node. + /// Set by the publishing driver via [`with_ops_stats`](Self::with_ops_stats). + encrypted: bool, + /// The number of times the local Vivaldi coordinate was reset after + /// degenerating; `None` when coordinates are disabled. Set by the publishing + /// driver via [`with_coordinate_resets`](Self::with_coordinate_resets). + #[cfg(feature = "coordinates")] + coordinate_resets: Option, } impl SerfSnapshot { @@ -87,6 +103,92 @@ impl SerfSnapshot { member_count, #[cfg(feature = "coordinates")] coordinate: None, + health_score: 0, + broadcast_queue_depth: 0, + encrypted: false, + #[cfg(feature = "coordinates")] + coordinate_resets: None, + } + } + + /// Attach the operator statistics a driver reads live from its endpoint + /// (builder form, called by the publishing driver after [`new`](Self::new)). + #[must_use] + pub fn with_ops_stats( + mut self, + health_score: usize, + broadcast_queue_depth: usize, + encrypted: bool, + ) -> Self { + self.health_score = health_score; + self.broadcast_queue_depth = broadcast_queue_depth; + self.encrypted = encrypted; + self + } + + /// Attach the coordinate-reset counter (builder form). + #[cfg(feature = "coordinates")] + #[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] + #[must_use] + pub fn with_coordinate_resets(mut self, resets: Option) -> Self { + self.coordinate_resets = resets; + self + } + + /// The node-awareness health score at the instant of the snapshot (`0` = + /// healthy; higher stretches the failure-detection timeouts). + #[must_use] + pub const fn health_score(&self) -> usize { + self.health_score + } + + /// Depth of the gossip broadcast queue (the total across the intent, event, + /// and query tiers) at the instant of the snapshot. + #[must_use] + pub const fn broadcast_queue_depth(&self) -> usize { + self.broadcast_queue_depth + } + + /// Whether a gossip/reliable encryption keyring is configured on the node. + #[must_use] + pub const fn encrypted(&self) -> bool { + self.encrypted + } + + /// The number of times the local Vivaldi coordinate was reset after + /// degenerating; `None` when coordinates are disabled. + #[cfg(feature = "coordinates")] + #[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] + #[must_use] + pub const fn coordinate_resets(&self) -> Option { + self.coordinate_resets + } + + /// Assemble the aggregate operator statistics from this snapshot. + #[must_use] + pub fn stats(&self) -> SerfStats { + let failed = self + .members + .iter() + .filter(|m| m.status() == MemberStatus::Failed) + .count(); + let left = self + .members + .iter() + .filter(|m| m.status() == MemberStatus::Left) + .count(); + SerfStats { + members: self.member_count, + failed, + left, + health_score: self.health_score, + member_clock: self.member_clock, + event_clock: self.event_clock, + query_clock: self.query_clock, + broadcast_queue_depth: self.broadcast_queue_depth, + encrypted: self.encrypted, + #[cfg(feature = "coordinates")] + coordinate_resets: self.coordinate_resets, } } @@ -228,3 +330,90 @@ impl SerfSnapshot { #[cfg(test)] mod tests; + +/// Aggregate operator statistics assembled from one [`SerfSnapshot`]. +/// +/// The counts and clocks come from the snapshot's member view; the health +/// score, broadcast queue depth, and encryption flag are the live endpoint +/// readings the publishing driver attached at the same instant. +#[derive(Debug, Clone)] +pub struct SerfStats { + members: usize, + failed: usize, + left: usize, + health_score: usize, + member_clock: LamportTime, + event_clock: LamportTime, + query_clock: LamportTime, + broadcast_queue_depth: usize, + encrypted: bool, + #[cfg(feature = "coordinates")] + coordinate_resets: Option, +} + +impl SerfStats { + /// The count of all known members (alive + leaving + left + failed). + #[must_use] + pub const fn members(&self) -> usize { + self.members + } + + /// The count of members currently in the failed state. + #[must_use] + pub const fn failed(&self) -> usize { + self.failed + } + + /// The count of members currently in the gracefully-left state. + #[must_use] + pub const fn left(&self) -> usize { + self.left + } + + /// The node-awareness health score (`0` = healthy; higher stretches the + /// failure-detection timeouts). + #[must_use] + pub const fn health_score(&self) -> usize { + self.health_score + } + + /// The member Lamport clock. + #[must_use] + pub const fn member_clock(&self) -> LamportTime { + self.member_clock + } + + /// The event Lamport clock. + #[must_use] + pub const fn event_clock(&self) -> LamportTime { + self.event_clock + } + + /// The query Lamport clock. + #[must_use] + pub const fn query_clock(&self) -> LamportTime { + self.query_clock + } + + /// Depth of the gossip broadcast queue (the total across the intent, event, + /// and query tiers). + #[must_use] + pub const fn broadcast_queue_depth(&self) -> usize { + self.broadcast_queue_depth + } + + /// Whether a gossip/reliable encryption keyring is configured on the node. + #[must_use] + pub const fn encrypted(&self) -> bool { + self.encrypted + } + + /// The number of times the local Vivaldi coordinate was reset after + /// degenerating; `None` when coordinates are disabled. + #[cfg(feature = "coordinates")] + #[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] + #[must_use] + pub const fn coordinate_resets(&self) -> Option { + self.coordinate_resets + } +} diff --git a/serf-driver/src/snapshot/tests.rs b/serf-driver/src/snapshot/tests.rs index 212c41be..96029cc1 100644 --- a/serf-driver/src/snapshot/tests.rs +++ b/serf-driver/src/snapshot/tests.rs @@ -215,3 +215,44 @@ fn new_panics_when_local_id_absent() { LamportTime::ZERO, ); } + +/// `stats()` assembles the aggregate from the member view plus the +/// driver-attached live readings: failed/left counts come from the statuses, +/// and the ops-stats builder carries health score, queue depth, and the +/// encryption flag; the defaults are the zero posture. +#[test] +fn stats_assembles_counts_and_ops_readings() { + let members = vec![ + make_member(1, "127.0.0.1:7946", MemberStatus::Alive), + make_member(2, "127.0.0.1:7947", MemberStatus::Failed), + make_member(3, "127.0.0.1:7948", MemberStatus::Failed), + make_member(4, "127.0.0.1:7949", MemberStatus::Left), + make_member(5, "127.0.0.1:7950", MemberStatus::Leaving), + ]; + let snap = SerfSnapshot::new( + members, + &1u32, + SerfState::Alive, + LamportTime::new(7), + LamportTime::new(8), + LamportTime::new(9), + ); + + // Defaults before the driver attaches its live readings. + let zero = snap.stats(); + assert_eq!(zero.health_score(), 0); + assert_eq!(zero.broadcast_queue_depth(), 0); + assert!(!zero.encrypted()); + + let snap = snap.with_ops_stats(2, 5, true); + let stats = snap.stats(); + assert_eq!(stats.members(), 5); + assert_eq!(stats.failed(), 2); + assert_eq!(stats.left(), 1, "Leaving is not Left"); + assert_eq!(stats.health_score(), 2); + assert_eq!(stats.broadcast_queue_depth(), 5); + assert!(stats.encrypted()); + assert_eq!(stats.member_clock(), LamportTime::new(7)); + assert_eq!(stats.event_clock(), LamportTime::new(8)); + assert_eq!(stats.query_clock(), LamportTime::new(9)); +} diff --git a/serf-proto/src/endpoint/mod.rs b/serf-proto/src/endpoint/mod.rs index 393804ac..a0967239 100644 --- a/serf-proto/src/endpoint/mod.rs +++ b/serf-proto/src/endpoint/mod.rs @@ -5944,6 +5944,15 @@ where pub fn cached_coordinate(&self, node: &I) -> Option { self.coord_cache.get(node).cloned() } + + /// The number of times the local Vivaldi coordinate has been reset after + /// degenerating (a defense the coordinate client applies when its model + /// stops converging). `None` when coordinates are disabled. + #[cfg(feature = "coordinates")] + #[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] + pub fn coordinate_resets(&self) -> Option { + self.coord_client.as_ref().map(|cc| cc.stats().resets()) + } } pub(crate) mod reliable; diff --git a/serf-proto/src/quic_endpoint/mod.rs b/serf-proto/src/quic_endpoint/mod.rs index f96f6e8d..11556198 100644 --- a/serf-proto/src/quic_endpoint/mod.rs +++ b/serf-proto/src/quic_endpoint/mod.rs @@ -292,6 +292,20 @@ where self.transport.endpoint_ref().user_broadcast_queue_len() } + /// The inner memberlist node-awareness health score: `0` = healthy, higher + /// values stretch the failure-detection timeouts (the node believes itself + /// degraded). + pub fn health_score(&self) -> usize { + self.transport.endpoint_ref().health_score() + } + + /// Forwards to [`Endpoint::coordinate_resets`]. + #[cfg(feature = "coordinates")] + #[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] + pub fn coordinate_resets(&self) -> Option { + self.core.coordinate_resets() + } + // ── driver-owned transport surface (additive forwarders) ──────────────────── // // These reach the memberlist QUIC coordinator's already-public driver methods. diff --git a/serf-proto/src/stream_endpoint/mod.rs b/serf-proto/src/stream_endpoint/mod.rs index c65b6a7f..1ec7e31d 100644 --- a/serf-proto/src/stream_endpoint/mod.rs +++ b/serf-proto/src/stream_endpoint/mod.rs @@ -333,6 +333,20 @@ where self.transport.endpoint_ref().user_broadcast_queue_len() } + /// The inner memberlist node-awareness health score: `0` = healthy, higher + /// values stretch the failure-detection timeouts (the node believes itself + /// degraded). + pub fn health_score(&self) -> usize { + self.transport.endpoint_ref().health_score() + } + + /// Forwards to [`Endpoint::coordinate_resets`]. + #[cfg(feature = "coordinates")] + #[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] + pub fn coordinate_resets(&self) -> Option { + self.core.coordinate_resets() + } + // ── driver-owned transport surface (additive forwarders) ──────────────────── // // These reach the memberlist coordinator's already-public driver methods. The diff --git a/serf-reactor/src/driver/quic/mod.rs b/serf-reactor/src/driver/quic/mod.rs index 79a50411..540c8e9e 100644 --- a/serf-reactor/src/driver/quic/mod.rs +++ b/serf-reactor/src/driver/quic/mod.rs @@ -1221,7 +1221,18 @@ where LamportTime::from(self.endpoint.query_time()), ); #[cfg(feature = "coordinates")] - let snap = snap.with_coordinate(self.endpoint.get_coordinate()); + let snap = snap + .with_coordinate(self.endpoint.get_coordinate()) + .with_coordinate_resets(self.endpoint.coordinate_resets()); + #[cfg(encryption)] + let encrypted = self.endpoint.encryption_options().keyring().is_some(); + #[cfg(not(encryption))] + let encrypted = false; + let snap = snap.with_ops_stats( + self.endpoint.health_score(), + self.endpoint.user_broadcast_queue_len(), + encrypted, + ); self.shared.publish(snap); } diff --git a/serf-reactor/src/driver/stream/mod.rs b/serf-reactor/src/driver/stream/mod.rs index 4b3fef0f..156ed1c0 100644 --- a/serf-reactor/src/driver/stream/mod.rs +++ b/serf-reactor/src/driver/stream/mod.rs @@ -1496,7 +1496,18 @@ where LamportTime::from(self.endpoint.query_time()), ); #[cfg(feature = "coordinates")] - let snap = snap.with_coordinate(self.endpoint.get_coordinate()); + let snap = snap + .with_coordinate(self.endpoint.get_coordinate()) + .with_coordinate_resets(self.endpoint.coordinate_resets()); + #[cfg(encryption)] + let encrypted = self.endpoint.encryption_options().keyring().is_some(); + #[cfg(not(encryption))] + let encrypted = false; + let snap = snap.with_ops_stats( + self.endpoint.health_score(), + self.endpoint.user_broadcast_queue_len(), + encrypted, + ); self.shared.publish(snap); } diff --git a/serf-reactor/src/serf/mod.rs b/serf-reactor/src/serf/mod.rs index 8f09dde7..b79cc641 100644 --- a/serf-reactor/src/serf/mod.rs +++ b/serf-reactor/src/serf/mod.rs @@ -678,6 +678,32 @@ impl Serf { self.shared.datagrams_sent() } + /// Aggregate operator statistics — member/failed/left counts, the health + /// score, the three Lamport clocks, the broadcast queue depth, and the + /// encryption flag — assembled lock-free from the latest published snapshot. + #[must_use] + pub fn stats(&self) -> serf_driver::SerfStats { + self.shared.load_snapshot().stats() + } + + /// Whether a gossip/reliable encryption keyring is configured on this node, + /// read lock-free from the latest published snapshot. The keyring's presence + /// is fixed at construction (key rotation replaces its contents, never adds + /// or removes the ring itself), so this answer is stable for the node's + /// lifetime once the driver's first snapshot lands. + #[must_use] + pub fn encryption_enabled(&self) -> bool { + self.shared.load_snapshot().encrypted() + } + + /// The node-awareness health score, read lock-free from the latest published + /// snapshot: `0` = healthy; higher values stretch the failure-detection + /// timeouts (the node believes itself degraded). + #[must_use] + pub fn health_score(&self) -> usize { + self.shared.load_snapshot().health_score() + } + /// The local node's current Vivaldi network coordinate, read lock-free from /// the latest published snapshot. /// diff --git a/serf-reactor/tests/key_rotation.rs b/serf-reactor/tests/key_rotation.rs index faa9f0e7..13e2039f 100644 --- a/serf-reactor/tests/key_rotation.rs +++ b/serf-reactor/tests/key_rotation.rs @@ -182,6 +182,13 @@ where .expect("join reaches node B over the encrypted reliable plane"); converge(&a, &b).await; + // An encrypted node reports its keyring through the operator surface. + assert!( + a.encryption_enabled(), + "a node constructed with a keyring must report encryption enabled" + ); + assert!(a.stats().encrypted()); + // Subscribe before issuing any key op so no KeyResponse races the subscription. let mut a_events = a.events(); let mut b_events = b.events(); diff --git a/serf-reactor/tests/tcp.rs b/serf-reactor/tests/tcp.rs index 1b6d952d..dd7bfc25 100644 --- a/serf-reactor/tests/tcp.rs +++ b/serf-reactor/tests/tcp.rs @@ -127,6 +127,16 @@ where assert_eq!(a.num_members(), 2, "A sees the 2-member cluster"); assert_eq!(b.num_members(), 2, "B sees the 2-member cluster"); + // The operator aggregate reflects the converged view and the live endpoint + // readings: full member count, nothing failed or left, a healthy awareness + // score, and no keyring on these plaintext nodes. + let stats = a.stats(); + assert_eq!(stats.members(), 2); + assert_eq!(stats.failed(), 0); + assert_eq!(stats.left(), 0); + assert_eq!(stats.health_score(), 0, "a healthy node scores 0"); + assert!(!a.encryption_enabled(), "no keyring is configured"); + a.shutdown().await.expect("conv-a shuts down"); b.shutdown().await.expect("conv-b shuts down"); } From c7d0296a2860b72a1b4a789df88f03b38bb2a9bb Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 13:49:53 +1200 Subject: [PATCH 101/128] =?UTF-8?q?feat(serf-reactor):=20FileKeyringDelega?= =?UTF-8?q?te=20=E2=80=94=20turnkey=20file=20persistence=20for=20key=20rot?= =?UTF-8?q?ations?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The legacy keyring-file option diverged into the app-owned KeyringDelegate observer; this supplies the turnkey impl so file-backed persistence is one constructor argument again. Rotations write one lowercase-hex line per key (primary first), each decoding to the self-describing algorithm-tag + raw key bytes wire form — two ciphers sharing a key length stay distinguishable — via a sibling temp file and an atomic rename so a crash mid-write never truncates the previous ring. load() rebuilds the ring at construction (None on first boot; malformed contents are typed parse errors, never panics). keyring_updated has no error channel, so a failed write surfaces through tracing while the wire keeps the rotated ring. serf-proto's tag+bytes SecretKey codec pair (already used by the key message bridge) is publicized for the format; transient key-material buffers are zeroized. Unit suite: rotation round-trip, first-boot None, malformed-file taxonomy, atomic replacement. --- serf-proto/src/bridge/mod.rs | 4 +- serf-proto/src/lib.rs | 6 + serf-reactor/Cargo.toml | 9 +- serf-reactor/src/delegate/keyring_file/mod.rs | 165 ++++++++++++++++++ .../src/delegate/keyring_file/tests.rs | 100 +++++++++++ serf-reactor/src/delegate/mod.rs | 8 + serf-reactor/src/lib.rs | 6 + 7 files changed, 295 insertions(+), 3 deletions(-) create mode 100644 serf-reactor/src/delegate/keyring_file/mod.rs create mode 100644 serf-reactor/src/delegate/keyring_file/tests.rs diff --git a/serf-proto/src/bridge/mod.rs b/serf-proto/src/bridge/mod.rs index 18f90abf..41b3d412 100644 --- a/serf-proto/src/bridge/mod.rs +++ b/serf-proto/src/bridge/mod.rs @@ -655,7 +655,7 @@ where /// The transient plaintext buffer is zeroed before it is freed so the raw key /// material does not linger in heap memory after this call returns. #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] -fn secret_key_to_bytes(key: &SecretKey) -> Bytes { +pub fn secret_key_to_bytes(key: &SecretKey) -> Bytes { use zeroize::Zeroize as _; let raw = key.as_bytes(); let mut buf = Vec::with_capacity(1 + raw.len()); @@ -671,7 +671,7 @@ fn secret_key_to_bytes(key: &SecretKey) -> Bytes { /// Returns [`BridgeError::InvalidValue`] when the tag is unknown to this build /// or the byte count does not match the algorithm's expected key length. #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] -fn secret_key_from_bytes(buf: &Bytes) -> Result { +pub fn secret_key_from_bytes(buf: &Bytes) -> Result { if buf.is_empty() { return Err(BridgeError::InvalidValue( "key bytes must carry at least the algorithm tag".into(), diff --git a/serf-proto/src/lib.rs b/serf-proto/src/lib.rs index a18a08be..c505946d 100644 --- a/serf-proto/src/lib.rs +++ b/serf-proto/src/lib.rs @@ -28,6 +28,12 @@ compile_error!("serf-proto requires the `std` or `alloc` feature"); pub(crate) use any::{AnyMessage, EncodeError}; #[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) use bridge::BridgeError; +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) +)] +pub use bridge::{BridgeError as SecretKeyCodecError, secret_key_from_bytes, secret_key_to_bytes}; #[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) use framing::{FrameError, MessageType}; #[cfg(any(feature = "tcp", feature = "quic"))] diff --git a/serf-reactor/Cargo.toml b/serf-reactor/Cargo.toml index 84a7d3d3..b348f601 100644 --- a/serf-reactor/Cargo.toml +++ b/serf-reactor/Cargo.toml @@ -24,11 +24,17 @@ quic-rustls-ring = ["quic", "serf-proto/quic-rustls-ring", "serf-driver/quic-rus coordinates = ["serf-proto/coordinates", "serf-driver/coordinates"] # Encryption backends — forward to serf-proto (key-management messages) and to # memberlist-proto (the coordinator's gossip-encryption keyring + encrypt/decrypt). -aes-gcm = ["serf-proto/aes-gcm", "serf-driver/aes-gcm", "memberlist-proto/aes-gcm"] +aes-gcm = [ + "serf-proto/aes-gcm", + "serf-driver/aes-gcm", + "memberlist-proto/aes-gcm", + "dep:zeroize", +] chacha20-poly1305 = [ "serf-proto/chacha20-poly1305", "serf-driver/chacha20-poly1305", "memberlist-proto/chacha20-poly1305", + "dep:zeroize", ] # Regex-backed tag-filter matching. tag-regex = ["serf-proto/tag-regex", "serf-driver/tag-regex"] @@ -58,6 +64,7 @@ smol_str.workspace = true smallvec.workspace = true bytes = { workspace = true } thiserror.workspace = true +zeroize = { version = "1", optional = true, default-features = false } futures-channel.workspace = true futures-util = { workspace = true, default-features = false, features = [ "std", diff --git a/serf-reactor/src/delegate/keyring_file/mod.rs b/serf-reactor/src/delegate/keyring_file/mod.rs new file mode 100644 index 00000000..f114feb3 --- /dev/null +++ b/serf-reactor/src/delegate/keyring_file/mod.rs @@ -0,0 +1,165 @@ +//! [`FileKeyringDelegate`] — turnkey file persistence for keyring rotations. + +use std::{ + io, + path::{Path, PathBuf}, +}; + +use memberlist_proto::{Keyring, SecretKey}; +use serf_proto::{secret_key_from_bytes, secret_key_to_bytes}; + +use super::KeyringDelegate; + +/// A [`KeyringDelegate`] that persists every keyring rotation to a file, plus +/// a [`load`](Self::load) to rebuild the ring at construction — the turnkey +/// replacement for the legacy keyring-file option, with the persistence +/// app-owned like any other delegate. +/// +/// FORMAT: one lowercase-hex line per key, the PRIMARY key first, each line +/// decoding to `[algorithm_tag][raw_key_bytes]`. The leading tag byte keeps +/// the encoding self-describing, so two ciphers sharing a key length (AES-256 +/// and ChaCha20-Poly1305 are both 32 bytes) stay distinguishable. The file +/// holds raw key material by design — protect it with filesystem permissions +/// exactly as the reference implementation's keyring file required. +/// +/// Rotations are written via a sibling temp file and an atomic rename, so a +/// crash mid-write never truncates the previous ring. `keyring_updated` has no +/// error channel; a failed write is surfaced through `tracing` (a no-op +/// without the `tracing` feature) and the wire keeps the rotated ring +/// regardless — the file is the durable copy, not the live one. +pub struct FileKeyringDelegate { + path: PathBuf, +} + +impl FileKeyringDelegate { + /// A delegate persisting to `path`. + pub fn new(path: impl Into) -> Self { + Self { path: path.into() } + } + + /// The persistence path. + #[must_use] + pub fn path(&self) -> &Path { + &self.path + } + + /// Load a keyring previously persisted by this delegate. + /// + /// Returns `Ok(None)` when the file does not exist (first boot). The first + /// line is the primary key; the rest are secondaries in decrypt-trial order. + /// + /// # Errors + /// + /// [`KeyringFileError::Io`] on a read failure other than not-found; + /// [`KeyringFileError::Parse`] on a malformed line (bad hex, an unknown + /// algorithm tag, a key length not matching its tag, or an empty file). + pub fn load(&self) -> Result, KeyringFileError> { + let raw = match std::fs::read_to_string(&self.path) { + Ok(s) => s, + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(KeyringFileError::Io(e)), + }; + let mut keys = raw + .lines() + .map(str::trim) + .filter(|l| !l.is_empty()) + .map(parse_key_line); + let primary = keys + .next() + .transpose()? + .ok_or_else(|| KeyringFileError::Parse("the keyring file holds no keys".into()))?; + let secondaries = keys.collect::, _>>()?; + Ok(Some(Keyring::with_secondaries(primary, secondaries))) + } + + /// Serialize `keyring` into the file format (primary first, one hex line per + /// key) and write it via temp-file + atomic rename. + fn persist(&self, keyring: &Keyring) -> Result<(), KeyringFileError> { + use zeroize::Zeroize as _; + let mut out = String::new(); + push_key_line(&mut out, keyring.primary_ref()); + for key in keyring.secondaries() { + push_key_line(&mut out, key); + } + let tmp = self.path.with_extension("tmp"); + let res = std::fs::write(&tmp, out.as_bytes()) + .and_then(|()| std::fs::rename(&tmp, &self.path)) + .map_err(KeyringFileError::Io); + out.zeroize(); + res + } +} + +impl KeyringDelegate for FileKeyringDelegate { + fn keyring_updated(&self, keyring: &Keyring) { + if let Err(_err) = self.persist(keyring) { + #[cfg(feature = "tracing")] + tracing::warn!( + path = %self.path.display(), + error = %_err, + "serf keyring rotation could not be persisted; the wire keeps the rotated ring" + ); + } + } +} + +/// Errors from [`FileKeyringDelegate::load`]. +#[derive(Debug, thiserror::Error)] +pub enum KeyringFileError { + /// Reading or writing the keyring file failed. + #[error(transparent)] + Io(#[from] io::Error), + /// The file's contents are not a valid keyring serialization. + #[error("malformed keyring file: {0}")] + Parse(String), +} + +/// Append one key as a lowercase-hex `[tag][bytes]` line. The transient +/// tagged buffer is zeroed before it is freed. +fn push_key_line(out: &mut String, key: &SecretKey) { + use core::fmt::Write as _; + let tagged = secret_key_to_bytes(key); + for b in tagged.iter() { + // Ignoring Err: writing hex digits into a String cannot fail. + let _ = write!(out, "{b:02x}"); + } + out.push('\n'); +} + +/// Parse one lowercase-hex `[tag][bytes]` line into a [`SecretKey`]. +fn parse_key_line(line: &str) -> Result { + use zeroize::Zeroize as _; + if line.len() % 2 != 0 { + return Err(KeyringFileError::Parse("odd-length hex key line".into())); + } + let mut buf = Vec::with_capacity(line.len() / 2); + for pair in line.as_bytes().chunks_exact(2) { + let hi = hex_val(pair[0]); + let lo = hex_val(pair[1]); + match (hi, lo) { + (Some(h), Some(l)) => buf.push((h << 4) | l), + _ => { + buf.zeroize(); + return Err(KeyringFileError::Parse( + "non-hex character in key line".into(), + )); + } + } + } + let bytes = bytes::Bytes::copy_from_slice(&buf); + buf.zeroize(); + secret_key_from_bytes(&bytes).map_err(|e| KeyringFileError::Parse(e.to_string())) +} + +/// The value of one lowercase/uppercase hex digit. +fn hex_val(c: u8) -> Option { + match c { + b'0'..=b'9' => Some(c - b'0'), + b'a'..=b'f' => Some(c - b'a' + 10), + b'A'..=b'F' => Some(c - b'A' + 10), + _ => None, + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-reactor/src/delegate/keyring_file/tests.rs b/serf-reactor/src/delegate/keyring_file/tests.rs new file mode 100644 index 00000000..75e7db27 --- /dev/null +++ b/serf-reactor/src/delegate/keyring_file/tests.rs @@ -0,0 +1,100 @@ +use super::*; + +fn tmp_path(name: &str) -> PathBuf { + let mut p = std::env::temp_dir(); + p.push(format!("serf-keyring-file-{name}-{}", std::process::id())); + p +} + +/// A persisted rotation round-trips: `keyring_updated` writes the ring +/// (primary first), `load` rebuilds it with the same primary and secondaries. +#[test] +fn rotation_round_trips_through_the_file() { + let path = tmp_path("roundtrip"); + let delegate = FileKeyringDelegate::new(&path); + + #[cfg(feature = "aes-gcm")] + let (primary, secondary) = (SecretKey::Aes128([1u8; 16]), SecretKey::Aes256([2u8; 32])); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let (primary, secondary) = ( + SecretKey::ChaCha20Poly1305([1u8; 32]), + SecretKey::ChaCha20Poly1305([2u8; 32]), + ); + + let ring = Keyring::with_secondaries(primary, [secondary]); + delegate.keyring_updated(&ring); + + let loaded = delegate + .load() + .expect("load parses the persisted file") + .expect("the file exists after a rotation"); + assert_eq!(loaded.primary_ref(), ring.primary_ref()); + assert_eq!(loaded.secondaries(), ring.secondaries()); + + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} + +/// First boot: a missing file loads as `None`, not an error. +#[test] +fn missing_file_loads_as_none() { + let delegate = FileKeyringDelegate::new(tmp_path("missing")); + assert!( + delegate + .load() + .expect("not-found is not an error") + .is_none() + ); +} + +/// Malformed contents are a parse error, never a panic: odd-length hex, +/// non-hex characters, an unknown algorithm tag, and an empty file. +#[test] +fn malformed_files_are_parse_errors() { + for (name, contents) in [ + ("odd", "abc\n"), + ("nonhex", "zz\n"), + ("badtag", "ff00112233445566778899aabbccddeeff\n"), + ("empty", "\n"), + ] { + let path = tmp_path(name); + std::fs::write(&path, contents).expect("write test file"); + let delegate = FileKeyringDelegate::new(&path); + assert!( + matches!(delegate.load(), Err(KeyringFileError::Parse(_))), + "{name} must be a parse error" + ); + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); + } +} + +/// A second rotation atomically replaces the file: the newest ring wins and +/// no temp-file residue remains. +#[test] +fn a_second_rotation_replaces_the_first() { + let path = tmp_path("replace"); + let delegate = FileKeyringDelegate::new(&path); + + #[cfg(feature = "aes-gcm")] + let (first, second) = (SecretKey::Aes128([3u8; 16]), SecretKey::Aes128([4u8; 16])); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let (first, second) = ( + SecretKey::ChaCha20Poly1305([3u8; 32]), + SecretKey::ChaCha20Poly1305([4u8; 32]), + ); + + delegate.keyring_updated(&Keyring::new(first)); + delegate.keyring_updated(&Keyring::new(second)); + + let loaded = delegate.load().expect("parses").expect("exists"); + assert_eq!(loaded.primary_ref(), &second); + assert!(loaded.secondaries().is_empty()); + assert!( + !path.with_extension("tmp").exists(), + "the atomic rename must consume the temp file" + ); + + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} diff --git a/serf-reactor/src/delegate/mod.rs b/serf-reactor/src/delegate/mod.rs index 9da01f88..b5fb385a 100644 --- a/serf-reactor/src/delegate/mod.rs +++ b/serf-reactor/src/delegate/mod.rs @@ -13,10 +13,18 @@ //! block), and `MergeDelegate` is an async admission veto supplied at //! construction rather than an observation hook. +#[cfg(encryption)] +mod keyring_file; mod void; pub use void::{NoopMergeDelegate, VoidDelegate}; +#[cfg(encryption)] +#[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) +)] +pub use keyring_file::{FileKeyringDelegate, KeyringFileError}; #[cfg(encryption)] pub use void::VoidKeyringDelegate; diff --git a/serf-reactor/src/lib.rs b/serf-reactor/src/lib.rs index 8ca54e2f..23c8a4e5 100644 --- a/serf-reactor/src/lib.rs +++ b/serf-reactor/src/lib.rs @@ -157,6 +157,12 @@ pub use delegate::{ VoidDelegate, }; +#[cfg(encryption)] +#[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) +)] +pub use delegate::{FileKeyringDelegate, KeyringFileError}; #[cfg(encryption)] #[cfg_attr( docsrs, From 455537f3130f0dc3ef1e81c37ad55df2949a6960 Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 14:06:35 +1200 Subject: [PATCH 102/128] feat(serf): wire the join-merge veto through to the machine predicate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reactor exported an async MergeDelegate that nothing ever invoked — by the time any event surfaces, the inner machine has already applied the push/pull merge, so a driver-side async gate can never veto anything. The machine's own synchronous MergeDelegate predicate is the real admission point: it runs inline for EVERY push/pull merge (a join and a periodic anti-entropy refresh alike, deliberately tighter than the reference implementation's join-only gate), and a vetoed peer set is never applied. Both composed serf endpoints now forward set_merge_delegate to the inner machine, every handle constructor accepts an optional boxed predicate (installed by each transport's run body exactly like the reconnect delegate), and the inert async trait and its noop impl are retired in favor of re-exporting the machine trait — an application needing async policy resolves it ahead of time and answers from the resolved state. An e2e pins the wiring: a recording delegate on the joined node observes at least one notify_merge carrying the joining peer's state. (Consultation is the stable assertion — a merge-only veto is transient by design, since a rejected peer can still be admitted moments later through gossip Alives, as in the reference implementation.) --- serf-proto/src/quic_endpoint/mod.rs | 13 ++++ serf-proto/src/stream_endpoint/mod.rs | 13 ++++ serf-reactor/src/delegate/mod.rs | 45 ++++--------- serf-reactor/src/delegate/tests.rs | 26 +++++--- serf-reactor/src/delegate/void.rs | 22 +------ serf-reactor/src/lib.rs | 3 +- serf-reactor/src/quic/mod.rs | 4 ++ serf-reactor/src/serf/mod.rs | 14 +++++ serf-reactor/src/serf/tests.rs | 4 ++ serf-reactor/src/tcp/mod.rs | 4 ++ serf-reactor/src/tls/mod.rs | 4 ++ serf-reactor/src/transport/runtime.rs | 7 +++ serf-reactor/tests/cluster/mod.rs | 1 + serf-reactor/tests/key_rotation.rs | 1 + serf-reactor/tests/quic.rs | 3 + serf-reactor/tests/tcp.rs | 91 +++++++++++++++++++++++++++ serf-reactor/tests/tls.rs | 1 + 17 files changed, 194 insertions(+), 62 deletions(-) diff --git a/serf-proto/src/quic_endpoint/mod.rs b/serf-proto/src/quic_endpoint/mod.rs index 11556198..ec611490 100644 --- a/serf-proto/src/quic_endpoint/mod.rs +++ b/serf-proto/src/quic_endpoint/mod.rs @@ -299,6 +299,19 @@ where self.transport.endpoint_ref().health_score() } + /// Install a custom join-merge predicate on the inner memberlist machine. + /// + /// The machine consults it inline for EVERY push/pull merge — a join and a + /// periodic anti-entropy refresh alike — before applying the remote member + /// state; returning `false` cancels the merge, so a vetoed peer set is never + /// admitted. + pub fn set_merge_delegate( + &mut self, + delegate: impl memberlist_proto::delegate::MergeDelegate, + ) { + self.transport.set_merge_delegate(delegate); + } + /// Forwards to [`Endpoint::coordinate_resets`]. #[cfg(feature = "coordinates")] #[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] diff --git a/serf-proto/src/stream_endpoint/mod.rs b/serf-proto/src/stream_endpoint/mod.rs index 1ec7e31d..9a3720fc 100644 --- a/serf-proto/src/stream_endpoint/mod.rs +++ b/serf-proto/src/stream_endpoint/mod.rs @@ -340,6 +340,19 @@ where self.transport.endpoint_ref().health_score() } + /// Install a custom join-merge predicate on the inner memberlist machine. + /// + /// The machine consults it inline for EVERY push/pull merge — a join and a + /// periodic anti-entropy refresh alike — before applying the remote member + /// state; returning `false` cancels the merge, so a vetoed peer set is never + /// admitted. + pub fn set_merge_delegate( + &mut self, + delegate: impl memberlist_proto::delegate::MergeDelegate, + ) { + self.transport.set_merge_delegate(delegate); + } + /// Forwards to [`Endpoint::coordinate_resets`]. #[cfg(feature = "coordinates")] #[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] diff --git a/serf-reactor/src/delegate/mod.rs b/serf-reactor/src/delegate/mod.rs index b5fb385a..c39e8684 100644 --- a/serf-reactor/src/delegate/mod.rs +++ b/serf-reactor/src/delegate/mod.rs @@ -3,21 +3,21 @@ //! //! Composes four observation sub-traits (`MemberDelegate` / `UserEventDelegate` //! / `QueryDelegate` / `KeyringDelegate`) and a join-admission veto -//! (`MergeDelegate`). Every observation hook returns a `Send` future +//! (`MergeDelegate`, the machine's sync predicate). Every observation hook returns a `Send` future //! (`-> impl Future + Send`, not `async fn`) so the observation //! task can run on a multi-threaded agnostic runtime; the delegate as a whole is //! `Send + Sync + 'static` and is held behind an `Arc`. //! //! `KeyringDelegate` and `MergeDelegate` are separate from the observation //! `Delegate` composite: `KeyringDelegate` is sync (keyring ops must not -//! block), and `MergeDelegate` is an async admission veto supplied at +//! block), and `MergeDelegate` is the machine's inline admission veto supplied at //! construction rather than an observation hook. #[cfg(encryption)] mod keyring_file; mod void; -pub use void::{NoopMergeDelegate, VoidDelegate}; +pub use void::VoidDelegate; #[cfg(encryption)] #[cfg_attr( @@ -207,40 +207,21 @@ pub trait KeyringDelegate: Send + Sync + 'static { } } -/// Async veto hook invoked by the driver on the join path before accepting -/// remote member state from a push-pull exchange. +/// The join-merge veto predicate, re-exported from the machine. /// -/// `Ok(())` permits the merge; `Err(Self::Error)` cancels it. The driver wraps -/// the concrete error into [`SerfError`](crate::SerfError) before forwarding it -/// to the join caller. -/// -/// The hook is **async and driver-side** deliberately: the application may need -/// to consult an ACL service or other async resource before deciding whether to -/// accept a batch of remote peers. A synchronous (Sans-I/O) filter would -/// preclude that. `notify_merge` returns a `Send` future so the driver can drive -/// it on a multi-threaded runtime; the delegate is `Send + Sync + 'static`. +/// Supplied at construction (the `merge_delegate` argument) and installed into +/// the memberlist machine, which consults it INLINE for every push/pull merge +/// — a join and a periodic anti-entropy refresh alike — before applying the +/// remote member state. Returning `false` cancels the merge, so a vetoed peer +/// set is never admitted. The predicate is synchronous by design: it runs +/// inside the machine's drain, so an application needing async I/O (an ACL +/// service, say) resolves its policy ahead of time and answers from that +/// resolved state here. /// /// Requires a stream or QUIC transport feature (`tcp` or `quic`). #[cfg(any(feature = "tcp", feature = "quic"))] #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] -pub trait MergeDelegate: Send + Sync + 'static { - /// The veto/error type this delegate reports when a merge is cancelled. - type Error; - - /// Called before the driver accepts inbound push-pull peer state. - /// - /// `peers` is the slice of remote [`Member`]s the cluster is about to merge. - /// Return `Ok(())` to proceed, or `Err(e)` to cancel the merge. - /// - /// The default implementation always permits the merge. - fn notify_merge( - &self, - peers: &[Arc>], - ) -> impl Future> + Send + '_ { - let _ = peers; // Unused in the default permit-all impl; an overriding delegate inspects it. - async { Ok(()) } - } -} +pub use memberlist_proto::delegate::MergeDelegate; #[cfg(test)] mod tests; diff --git a/serf-reactor/src/delegate/tests.rs b/serf-reactor/src/delegate/tests.rs index 2a9a6e27..48c83d29 100644 --- a/serf-reactor/src/delegate/tests.rs +++ b/serf-reactor/src/delegate/tests.rs @@ -15,23 +15,35 @@ fn void_delegate_satisfies_observation_composite() { assert_delegate(&v); } -/// `NoopMergeDelegate` satisfies `MergeDelegate` with `Error = Infallible` — a -/// type-level check; no I/O needed. +/// A boxed machine merge delegate threads through the constructor slot — a +/// type-level check that the re-exported trait and its Box blanket compose. #[test] -fn noop_merge_delegate_satisfies_trait() { +fn boxed_merge_delegate_satisfies_trait() { + struct AcceptAll; + impl MergeDelegate for AcceptAll { + fn notify_merge( + &self, + _peers: memberlist_proto::MaybeOwned< + '_, + [memberlist_proto::typed::NodeState], + >, + ) -> bool { + true + } + } fn assert_merge(_: &T) where - T: MergeDelegate, + T: MergeDelegate, { } - assert_merge(&NoopMergeDelegate); + let boxed: Box> = Box::new(AcceptAll); + assert_merge(&boxed); } /// The reactor delegate surface is `Send + Sync + 'static`: the driver holds it /// behind an `Arc` shared across the runtime's worker threads. #[test] -fn void_delegate_and_noop_merge_are_send_sync() { +fn void_delegate_is_send_sync() { fn assert_send_sync() {} assert_send_sync::>(); - assert_send_sync::(); } diff --git a/serf-reactor/src/delegate/void.rs b/serf-reactor/src/delegate/void.rs index 0be2ab44..39864b1e 100644 --- a/serf-reactor/src/delegate/void.rs +++ b/serf-reactor/src/delegate/void.rs @@ -5,7 +5,7 @@ use core::marker::PhantomData; #[cfg(any(feature = "tcp", feature = "quic"))] -use super::{Delegate, MemberDelegate, MergeDelegate, QueryDelegate, UserEventDelegate}; +use super::{Delegate, MemberDelegate, QueryDelegate, UserEventDelegate}; #[cfg(encryption)] use super::KeyringDelegate; @@ -79,26 +79,6 @@ where type Address = A; } -/// A merge delegate that always permits merges. -/// -/// The default delegate for drivers that do not need join admission control. -/// Its associated error type is [`core::convert::Infallible`], reflecting that -/// `notify_merge` can never fail. -/// -/// Requires a stream or QUIC transport feature (`tcp` or `quic`). -#[cfg(any(feature = "tcp", feature = "quic"))] -#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] -pub struct NoopMergeDelegate; - -#[cfg(any(feature = "tcp", feature = "quic"))] -impl MergeDelegate for NoopMergeDelegate -where - I: Send + Sync + 'static, - A: Send + Sync + 'static, -{ - type Error = core::convert::Infallible; -} - /// A keyring delegate that persists nothing. /// /// The default for nodes that do not need to observe key rotations. The driver diff --git a/serf-reactor/src/lib.rs b/serf-reactor/src/lib.rs index 23c8a4e5..bff3f39c 100644 --- a/serf-reactor/src/lib.rs +++ b/serf-reactor/src/lib.rs @@ -153,8 +153,7 @@ pub use tls::{SniProvider, TlsOptions, TlsTransport, TlsTransportOptions}; #[cfg(any(feature = "tcp", feature = "quic"))] #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] pub use delegate::{ - Delegate, MemberDelegate, MergeDelegate, NoopMergeDelegate, QueryDelegate, UserEventDelegate, - VoidDelegate, + Delegate, MemberDelegate, MergeDelegate, QueryDelegate, UserEventDelegate, VoidDelegate, }; #[cfg(encryption)] diff --git a/serf-reactor/src/quic/mod.rs b/serf-reactor/src/quic/mod.rs index c420647a..d3ea0023 100644 --- a/serf-reactor/src/quic/mod.rs +++ b/serf-reactor/src/quic/mod.rs @@ -545,6 +545,10 @@ where runtime.member_drop, ) .with_reconnect_delegate(runtime.reconnect_delegate); + let mut endpoint = endpoint; + if let Some(md) = runtime.merge_delegate { + endpoint.set_merge_delegate(md); + } let driver = crate::driver::quic::spawn_quic_driver::( endpoint, diff --git a/serf-reactor/src/serf/mod.rs b/serf-reactor/src/serf/mod.rs index b79cc641..b4f30e70 100644 --- a/serf-reactor/src/serf/mod.rs +++ b/serf-reactor/src/serf/mod.rs @@ -160,6 +160,7 @@ where serf_options: SerfOptions, gossip_rng: G, reconnect_delegate: Option>>, + merge_delegate: Option>>, #[cfg(encryption)] keyring: Arc, ) -> core::result::Result where @@ -212,6 +213,7 @@ where user_drop_writer, member_drop_writer, reconnect_delegate, + merge_delegate, #[cfg(encryption)] keyring, ); @@ -271,6 +273,7 @@ where runtime_options: RuntimeOptions, serf_options: SerfOptions, reconnect_delegate: Option>>, + merge_delegate: Option>>, #[cfg(encryption)] keyring: Arc, ) -> Result where @@ -287,6 +290,7 @@ where serf_options, crate::gossip_rng()?, reconnect_delegate, + merge_delegate, #[cfg(encryption)] keyring, ) @@ -305,6 +309,7 @@ where serf_options: SerfOptions, gossip_rng: G, reconnect_delegate: Option>>, + merge_delegate: Option>>, #[cfg(encryption)] keyring: Arc, ) -> Result where @@ -322,6 +327,7 @@ where serf_options, gossip_rng, reconnect_delegate, + merge_delegate, #[cfg(encryption)] keyring, ) @@ -373,6 +379,7 @@ where runtime_options: RuntimeOptions, serf_options: SerfOptions, reconnect_delegate: Option>>, + merge_delegate: Option>>, #[cfg(encryption)] keyring: Arc, ) -> Result where @@ -389,6 +396,7 @@ where serf_options, crate::gossip_rng()?, reconnect_delegate, + merge_delegate, #[cfg(encryption)] keyring, ) @@ -407,6 +415,7 @@ where serf_options: SerfOptions, gossip_rng: G, reconnect_delegate: Option>>, + merge_delegate: Option>>, #[cfg(encryption)] keyring: Arc, ) -> Result where @@ -424,6 +433,7 @@ where serf_options, gossip_rng, reconnect_delegate, + merge_delegate, #[cfg(encryption)] keyring, ) @@ -474,6 +484,7 @@ where runtime_options: RuntimeOptions, serf_options: SerfOptions, reconnect_delegate: Option>>, + merge_delegate: Option>>, #[cfg(encryption)] keyring: Arc, ) -> Result where @@ -490,6 +501,7 @@ where serf_options, crate::gossip_rng()?, reconnect_delegate, + merge_delegate, #[cfg(encryption)] keyring, ) @@ -508,6 +520,7 @@ where serf_options: SerfOptions, gossip_rng: G, reconnect_delegate: Option>>, + merge_delegate: Option>>, #[cfg(encryption)] keyring: Arc, ) -> Result where @@ -525,6 +538,7 @@ where serf_options, gossip_rng, reconnect_delegate, + merge_delegate, #[cfg(encryption)] keyring, ) diff --git a/serf-reactor/src/serf/tests.rs b/serf-reactor/src/serf/tests.rs index c490e925..5e2bfdd2 100644 --- a/serf-reactor/src/serf/tests.rs +++ b/serf-reactor/src/serf/tests.rs @@ -60,6 +60,7 @@ async fn try_spawn_node_at(id: &str, bind: SocketAddr) -> Result No RuntimeOptions::new(), serf_options, None, + None, #[cfg(encryption)] std::sync::Arc::new(crate::VoidKeyringDelegate), ) @@ -222,6 +224,7 @@ async fn assert_tcp_new_rejects(runtime: RuntimeOptions) { runtime, SerfOptions::new(), None, + None, #[cfg(encryption)] std::sync::Arc::new(crate::VoidKeyringDelegate), ) @@ -265,6 +268,7 @@ async fn tcp_new_rejects_over_ceiling_user_event_size() { RuntimeOptions::new(), serf, None, + None, #[cfg(encryption)] std::sync::Arc::new(crate::VoidKeyringDelegate), ) diff --git a/serf-reactor/src/tcp/mod.rs b/serf-reactor/src/tcp/mod.rs index 9aa85575..25f10d0e 100644 --- a/serf-reactor/src/tcp/mod.rs +++ b/serf-reactor/src/tcp/mod.rs @@ -535,6 +535,10 @@ where runtime.member_drop, ) .with_reconnect_delegate(runtime.reconnect_delegate); + let mut endpoint = endpoint; + if let Some(md) = runtime.merge_delegate { + endpoint.set_merge_delegate(md); + } let driver = crate::driver::stream::spawn_stream_driver::( endpoint, diff --git a/serf-reactor/src/tls/mod.rs b/serf-reactor/src/tls/mod.rs index b9f927a9..a27f62c8 100644 --- a/serf-reactor/src/tls/mod.rs +++ b/serf-reactor/src/tls/mod.rs @@ -658,6 +658,10 @@ where runtime.member_drop, ) .with_reconnect_delegate(runtime.reconnect_delegate); + let mut endpoint = endpoint; + if let Some(md) = runtime.merge_delegate { + endpoint.set_merge_delegate(md); + } let driver = crate::driver::stream::spawn_stream_driver::, D, G, StdRng>( diff --git a/serf-reactor/src/transport/runtime.rs b/serf-reactor/src/transport/runtime.rs index 91b1bd5a..5d3a98bf 100644 --- a/serf-reactor/src/transport/runtime.rs +++ b/serf-reactor/src/transport/runtime.rs @@ -44,6 +44,11 @@ pub struct TransportRuntime { /// installed into the endpoint by `T::run` before the endpoint moves into the /// detached pump. `None` keeps the flat configured reap timeouts. pub(crate) reconnect_delegate: Option>>, + /// Optional join-merge veto predicate, installed into the machine by `T::run` + /// before the endpoint moves into the detached pump. The machine consults it + /// inline for EVERY push/pull merge; `None` admits every peer set. + pub(crate) merge_delegate: + Option>>, /// The driver's keyring delegate, applied to inbound key-management requests. /// Present only under an encryption backend. #[cfg(encryption)] @@ -64,6 +69,7 @@ impl TransportRuntime { user_drop: ReactorDropCounter, member_drop: ReactorDropCounter, reconnect_delegate: Option>>, + merge_delegate: Option>>, #[cfg(encryption)] keyring: Arc, ) -> Self { Self { @@ -75,6 +81,7 @@ impl TransportRuntime { user_drop, member_drop, reconnect_delegate, + merge_delegate, #[cfg(encryption)] keyring, } diff --git a/serf-reactor/tests/cluster/mod.rs b/serf-reactor/tests/cluster/mod.rs index ea3406e1..14e02949 100644 --- a/serf-reactor/tests/cluster/mod.rs +++ b/serf-reactor/tests/cluster/mod.rs @@ -434,6 +434,7 @@ where RuntimeOptions::new(), timing.serf_opts(), None, + None, #[cfg(encryption)] std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), ) diff --git a/serf-reactor/tests/key_rotation.rs b/serf-reactor/tests/key_rotation.rs index 13e2039f..d2a45a68 100644 --- a/serf-reactor/tests/key_rotation.rs +++ b/serf-reactor/tests/key_rotation.rs @@ -109,6 +109,7 @@ where RuntimeOptions::new(), SerfOptions::new(), None, + None, keyring, ) .await diff --git a/serf-reactor/tests/quic.rs b/serf-reactor/tests/quic.rs index d0ec9135..8608f983 100644 --- a/serf-reactor/tests/quic.rs +++ b/serf-reactor/tests/quic.rs @@ -163,6 +163,7 @@ where RuntimeOptions::new(), SerfOptions::new(), None, + None, #[cfg(encryption)] std::sync::Arc::new(VoidKeyringDelegate), ) @@ -518,6 +519,7 @@ where RuntimeOptions::new(), SerfOptions::new(), None, + None, #[cfg(encryption)] std::sync::Arc::new(VoidKeyringDelegate), ) @@ -574,6 +576,7 @@ where RuntimeOptions::new(), SerfOptions::new(), None, + None, std::sync::Arc::new(VoidKeyringDelegate), ) .await diff --git a/serf-reactor/tests/tcp.rs b/serf-reactor/tests/tcp.rs index dd7bfc25..3c1be804 100644 --- a/serf-reactor/tests/tcp.rs +++ b/serf-reactor/tests/tcp.rs @@ -82,6 +82,7 @@ where RuntimeOptions::new(), SerfOptions::new(), None, + None, #[cfg(encryption)] std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), ) @@ -334,6 +335,7 @@ where Some(Box::new(ReapImmediately { target: SmolStr::new("rd-b"), })), + None, #[cfg(encryption)] std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), ) @@ -642,6 +644,83 @@ where cluster.shutdown_all().await; } +/// The constructor-supplied merge delegate is the predicate the machine +/// consults: with a recording accept-all delegate installed on B, A's join +/// push-pull drives at least one `notify_merge` on B carrying A's node state. +/// (A veto here gates only the push/pull application — a rejected peer can +/// still be admitted moments later through gossip Alives, exactly as in the +/// reference implementation, so the stable assertion is consultation, not +/// permanent exclusion.) +async fn merge_delegate_is_consulted_on_join() +where + R: Runtime, +{ + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct RecordingMerge { + hits: std::sync::Arc, + saw_peer: std::sync::Arc, + } + impl serf_reactor::MergeDelegate for RecordingMerge { + fn notify_merge( + &self, + peers: memberlist_proto::MaybeOwned< + '_, + [memberlist_proto::typed::NodeState], + >, + ) -> bool { + self.hits.fetch_add(1, Ordering::Relaxed); + if peers.iter().any(|p| p.id_ref().as_str() == "merge-a") { + self.saw_peer.fetch_add(1, Ordering::Relaxed); + } + true + } + } + + let hits = std::sync::Arc::new(AtomicUsize::new(0)); + let saw_peer = std::sync::Arc::new(AtomicUsize::new(0)); + + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let b = Serf::::tcp( + TcpTransportOptions::::new() + .with_local_id(SmolStr::new("merge-b")) + .with_advertise_addr(MaybeResolved::Resolved(bind)), + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + None, + Some(Box::new(RecordingMerge { + hits: hits.clone(), + saw_peer: saw_peer.clone(), + })), + #[cfg(encryption)] + std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), + ) + .await + .expect("spawn merge-b"); + let a = spawn_node::("merge-a").await; + let b_addr = b.advertise_address(); + + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + converge(&a, &b).await; + + assert!( + hits.load(Ordering::Relaxed) > 0, + "the machine must consult the constructor-supplied merge delegate on the join push/pull" + ); + assert!( + saw_peer.load(Ordering::Relaxed) > 0, + "the consulted peer set must carry the joining node's state" + ); + + a.shutdown().await.expect("merge-a shuts down"); + b.shutdown().await.expect("merge-b shuts down"); +} + /// A leave configured with a zero timeout racing a shutdown resolves /// `Err(LeaveTimeout)` — never `Ok` — even though the teardown still delivers /// the fan-out: the caller's per-leave deadline keeps governing resolution @@ -662,6 +741,7 @@ where RuntimeOptions::new().with_leave_timeout(Duration::ZERO), SerfOptions::new(), None, + None, #[cfg(encryption)] std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), ) @@ -752,6 +832,7 @@ where RuntimeOptions::new(), SerfOptions::new(), None, + None, std::sync::Arc::new(VoidKeyringDelegate), ) .await @@ -937,6 +1018,11 @@ mod tokio_cells { super::coordinates_surface_on_the_handle::().await; } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn merge_delegate_is_consulted_on_join() { + super::merge_delegate_is_consulted_on_join::().await; + } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn leave_with_zero_timeout_racing_shutdown_times_out() { super::leave_with_zero_timeout_racing_shutdown_times_out::().await; @@ -1032,6 +1118,11 @@ mod smol_cells { SmolRuntime::block_on(super::coordinates_surface_on_the_handle::()); } + #[test] + fn merge_delegate_is_consulted_on_join_smol() { + SmolRuntime::block_on(super::merge_delegate_is_consulted_on_join::()); + } + #[test] fn leave_with_zero_timeout_racing_shutdown_times_out_smol() { SmolRuntime::block_on(super::leave_with_zero_timeout_racing_shutdown_times_out::< diff --git a/serf-reactor/tests/tls.rs b/serf-reactor/tests/tls.rs index 3d709301..cbf61bcc 100644 --- a/serf-reactor/tests/tls.rs +++ b/serf-reactor/tests/tls.rs @@ -133,6 +133,7 @@ where RuntimeOptions::new(), SerfOptions::new(), None, + None, #[cfg(encryption)] std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), ) From 2b6905f24a3de783733546974c9315eba84c10be Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 14:24:24 +1200 Subject: [PATCH 103/128] feat(serf-reactor): snapshot persistence and rejoin-after-restart MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The machine has owned the snapshot record format and the replay fold since the port, but no driver ever touched a file — snapshot_path was a silent no-op and rejoin_after_leave was inert. The driver now owns the file end to end: a SnapshotOptions constructor argument names the path (and compaction threshold), the Serf constructor reads and decodes it so a corrupt record fails construction loudly (SerfError::SnapshotOpen; a truncated tail from a crash mid-append is tolerated and repaired to a whole-record boundary), the transport body replays the records — honoring rejoin_after_leave at the Leave marker — and hands the result to Endpoint::load_snapshot, which recovers the clock floors and re-dials the recovered peers through the normal push/pull machinery. The pump appends at its event chokepoint: Alive/NotAlive per surfaced member event, the advancing clock floors, and the Leave marker on LeftCluster; appends are buffered and flushed per batch, and past the threshold the file is compacted to the live alive-set + clocks via a sibling temp file and an atomic rename. Coverage: five snapshotter units (reopen replay, the leave gate both ways, torn-tail tolerance + repair, corrupt-middle refusal, compaction identity) and two e2e scenarios on both runtimes — an abruptly-killed node reboots from its snapshot and rejoins with NO join call, and a cleanly-left node stays solo on the default posture but recovers its membership when rejoin_after_leave opts in. --- serf-reactor/src/driver/mod.rs | 2 + serf-reactor/src/driver/options/mod.rs | 45 ++++ serf-reactor/src/driver/quic/mod.rs | 40 +++ serf-reactor/src/driver/quic/tests.rs | 1 + serf-reactor/src/driver/snapshotter/mod.rs | 251 +++++++++++++++++++ serf-reactor/src/driver/snapshotter/tests.rs | 178 +++++++++++++ serf-reactor/src/driver/stream/mod.rs | 40 +++ serf-reactor/src/driver/stream/tests.rs | 1 + serf-reactor/src/error/mod.rs | 8 + serf-reactor/src/lib.rs | 8 +- serf-reactor/src/quic/mod.rs | 17 +- serf-reactor/src/serf/mod.rs | 26 ++ serf-reactor/src/serf/tests.rs | 4 + serf-reactor/src/tcp/mod.rs | 17 +- serf-reactor/src/tls/mod.rs | 17 +- serf-reactor/src/transport/runtime.rs | 6 + serf-reactor/tests/cluster/mod.rs | 1 + serf-reactor/tests/key_rotation.rs | 1 + serf-reactor/tests/quic.rs | 3 + serf-reactor/tests/tcp.rs | 147 +++++++++++ serf-reactor/tests/tls.rs | 1 + 21 files changed, 809 insertions(+), 5 deletions(-) create mode 100644 serf-reactor/src/driver/snapshotter/mod.rs create mode 100644 serf-reactor/src/driver/snapshotter/tests.rs diff --git a/serf-reactor/src/driver/mod.rs b/serf-reactor/src/driver/mod.rs index b9215a78..4bb77442 100644 --- a/serf-reactor/src/driver/mod.rs +++ b/serf-reactor/src/driver/mod.rs @@ -6,6 +6,8 @@ //! `StreamEndpoint` / `QuicEndpoint` and pump it as a quinn-style `Future::poll`. pub(crate) mod options; +#[cfg(any(feature = "tcp", feature = "quic"))] +pub(crate) mod snapshotter; #[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) mod shared; diff --git a/serf-reactor/src/driver/options/mod.rs b/serf-reactor/src/driver/options/mod.rs index 3b2d291c..3235ad17 100644 --- a/serf-reactor/src/driver/options/mod.rs +++ b/serf-reactor/src/driver/options/mod.rs @@ -602,5 +602,50 @@ impl Default for StreamTransportOptions { } } +/// Default snapshot compaction threshold: the append file is rewritten to the +/// live state once it grows past this many bytes. +pub const DEFAULT_SNAPSHOT_COMPACT_THRESHOLD: u64 = 128 * 1024; + +/// Snapshot persistence wiring: where the driver appends membership records +/// and when it compacts the file. Supplied as a constructor argument (`None` +/// disables persistence); the replay-vs-fresh-start decision after a clean +/// leave is the serf option `rejoin_after_leave`. +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct SnapshotOptions { + path: std::path::PathBuf, + compact_threshold: u64, +} + +impl SnapshotOptions { + /// Persist to `path`, compacting at the default threshold. + pub fn new(path: impl Into) -> Self { + Self { + path: path.into(), + compact_threshold: DEFAULT_SNAPSHOT_COMPACT_THRESHOLD, + } + } + + /// The snapshot file path. + #[must_use] + pub fn path(&self) -> &std::path::Path { + &self.path + } + + /// Rewrite the file to the live state once it grows past `bytes` + /// (`0` = never compact). + #[must_use] + pub const fn with_compact_threshold(mut self, bytes: u64) -> Self { + self.compact_threshold = bytes; + self + } + + /// The compaction threshold in bytes (`0` = never compact). + #[must_use] + pub const fn compact_threshold(&self) -> u64 { + self.compact_threshold + } +} + #[cfg(test)] mod tests; diff --git a/serf-reactor/src/driver/quic/mod.rs b/serf-reactor/src/driver/quic/mod.rs index 540c8e9e..47bc120f 100644 --- a/serf-reactor/src/driver/quic/mod.rs +++ b/serf-reactor/src/driver/quic/mod.rs @@ -361,6 +361,9 @@ where /// it here rather than relying on the real socket returning `Pending`. #[cfg(test)] recv_force_pending: bool, + /// Snapshot persistence: appends a record for every surfaced membership + /// change and the advancing clock floors, `None` when persistence is off. + snapshotter: Option>, /// The driver's keyring delegate: applies inbound key-management ops and produces /// the `respond_key` answer. Present only under an encryption backend. #[cfg(encryption)] @@ -387,6 +390,7 @@ where obs_payload_budget: Option, driver_opts: RuntimeOptions, label: Option, + snapshotter: Option>, #[cfg(encryption)] keyring: Arc, ) -> Self { let buf_len = recv_buf_len_for(endpoint.gossip_mtu(), quic_max_udp_payload); @@ -418,6 +422,7 @@ where recv_errors_remaining: 0, #[cfg(test)] recv_force_pending: false, + snapshotter, #[cfg(encryption)] keyring, } @@ -1062,6 +1067,39 @@ where /// on `LeftCluster`, begin teardown on a lost id-conflict `Shutdown`, and answer /// an inbound key-management request. fn account_event(&mut self, ev: &Event) { + // Snapshot persistence: append the surfaced membership change, the + // advancing clock floors, and the clean-leave marker, then flush (and + // compact past the threshold) — a change is durable once this poll + // returns. Restart replay + `load_snapshot` recovers the state. + if let Some(snap) = self.snapshotter.as_mut() { + match ev { + Event::Member(me) => { + use serf_proto::event::MemberEventKind as MK; + let alive = matches!(me.kind(), MK::Join | MK::Update); + for m in me.members() { + snap.append_member(alive, m.node()); + } + } + Event::LeftCluster => snap.append_leave(), + _ => {} + } + if matches!(ev, Event::Member(_) | Event::LeftCluster) { + snap.append_clocks( + LamportTime::from(self.endpoint.member_time()), + LamportTime::from(self.endpoint.event_time()), + LamportTime::from(self.endpoint.query_time()), + ); + let endpoint = &self.endpoint; + snap.flush_and_maybe_compact(|| { + endpoint + .members_snapshot() + .iter() + .filter(|m| m.status() == serf_proto::members::MemberStatus::Alive) + .map(|m| m.node().clone()) + .collect() + }); + } + } if let Event::ExchangeCompleted(c) = ev && c.kind() == ExchangeKind::PushPull { @@ -1737,6 +1775,7 @@ pub(crate) fn spawn_quic_driver( delegate: D, driver_opts: RuntimeOptions, label: Option, + snapshotter: Option>, #[cfg(encryption)] keyring: Arc, ) -> QuicDriver where @@ -1780,6 +1819,7 @@ where obs_payload_budget, driver_opts, label, + snapshotter, #[cfg(encryption)] keyring, ) diff --git a/serf-reactor/src/driver/quic/tests.rs b/serf-reactor/src/driver/quic/tests.rs index 788a49ce..416b76cd 100644 --- a/serf-reactor/src/driver/quic/tests.rs +++ b/serf-reactor/src/driver/quic/tests.rs @@ -275,6 +275,7 @@ mod gate { None, RuntimeOptions::new().with_iter_drain_cap(iter_drain_cap), None, + None, #[cfg(encryption)] Arc::new(crate::VoidKeyringDelegate), ); diff --git a/serf-reactor/src/driver/snapshotter/mod.rs b/serf-reactor/src/driver/snapshotter/mod.rs new file mode 100644 index 00000000..da8d3406 --- /dev/null +++ b/serf-reactor/src/driver/snapshotter/mod.rs @@ -0,0 +1,251 @@ +//! Driver-side snapshot persistence: an append-only record file the pump feeds +//! on every surfaced membership change, replayed at construction to recover +//! membership and the Lamport clocks across a restart. +//! +//! The machine owns the record format ([`SnapshotRecord`]) and the replay fold +//! ([`serf_proto::snapshot::ReplayResult`]); this module owns the file. The +//! `Serf` constructor reads and decodes the file (so a corrupt snapshot fails +//! construction loudly), the transport body replays it into the endpoint +//! (`load_snapshot` re-dials the recovered peers through the normal push/pull +//! machinery), and the pump appends records at its event chokepoint. + +use std::{ + fs, + io::{self, Write as _}, + net::SocketAddr, + path::PathBuf, +}; + +use memberlist_proto::Data; +use serf_proto::{LamportTime, snapshot::SnapshotRecord}; + +use super::options::SnapshotOptions; + +/// The snapshot writer paired with the records already on disk, as handed +/// from the constructor (which opens and decodes) to the transport body +/// (which replays and pumps). +pub(crate) type OpenedSnapshot = (Snapshotter, Vec>); + +/// The append-side of the snapshot file, held by the driver pump. +/// +/// Writes are buffered and flushed after every append batch — a surfaced +/// membership change is durable once the pump's poll returns. When the file +/// grows past the compaction threshold, the next append rewrites it to just +/// the current alive set and clock floors (via a sibling temp file and an +/// atomic rename), exactly the state a replay needs. +pub(crate) struct Snapshotter { + path: PathBuf, + file: io::BufWriter, + bytes_written: u64, + compact_threshold: u64, + last_member_clock: LamportTime, + last_event_clock: LamportTime, + last_query_clock: LamportTime, + _id: core::marker::PhantomData, +} + +impl Snapshotter +where + I: Data + Clone + Eq + core::hash::Hash, +{ + /// Open (or create) the snapshot file for appending, returning the writer + /// and the decoded records already on disk. + /// + /// A truncated TAIL (a crash mid-append) is tolerated: decoding stops at a + /// partial trailing record and appends continue after the last whole one. A + /// malformed record BEFORE the tail (an unknown tag or an undecodable node) + /// is a hard error — the file is not trustworthy. + pub(crate) fn open(opts: &SnapshotOptions) -> Result, SnapshotOpenError> { + let path = opts.path().to_path_buf(); + let raw = match fs::read(&path) { + Ok(b) => b, + Err(e) if e.kind() == io::ErrorKind::NotFound => Vec::new(), + Err(e) => return Err(SnapshotOpenError::Io(e)), + }; + + let mut records = Vec::new(); + let mut cursor = 0usize; + while cursor < raw.len() { + match SnapshotRecord::::decode(&raw[cursor..]) { + Ok((rec, consumed)) => { + records.push(rec); + cursor += consumed; + } + Err(serf_proto::snapshot::SnapshotError::Truncated { .. }) => { + // A partial trailing record from a crash mid-append: replay what is + // whole and let the writer continue after it. + break; + } + Err(e) => return Err(SnapshotOpenError::Corrupt(e)), + } + } + + // Truncate any partial tail so new appends start on a whole-record + // boundary; then reopen for appending. + let file = fs::OpenOptions::new() + .create(true) + .truncate(false) + .write(true) + .open(&path) + .map_err(SnapshotOpenError::Io)?; + file.set_len(cursor as u64).map_err(SnapshotOpenError::Io)?; + drop(file); + let file = fs::OpenOptions::new() + .append(true) + .open(&path) + .map_err(SnapshotOpenError::Io)?; + + Ok(( + Self { + path, + file: io::BufWriter::new(file), + bytes_written: cursor as u64, + compact_threshold: opts.compact_threshold(), + last_member_clock: LamportTime::ZERO, + last_event_clock: LamportTime::ZERO, + last_query_clock: LamportTime::ZERO, + _id: core::marker::PhantomData, + }, + records, + )) + } + + /// Append one record, best-effort. An encode or write failure is surfaced + /// through `tracing` (the wire state is authoritative; the file is the + /// durable copy) and the record is dropped. + fn append(&mut self, record: &SnapshotRecord) { + match record.encode() { + Ok(bytes) => { + if let Err(_err) = self.file.write_all(&bytes) { + #[cfg(feature = "tracing")] + tracing::warn!( + path = %self.path.display(), + error = %_err, + "serf snapshot append failed; the record is dropped" + ); + return; + } + self.bytes_written += bytes.len() as u64; + } + Err(_err) => { + #[cfg(feature = "tracing")] + tracing::warn!( + path = %self.path.display(), + error = %_err, + "serf snapshot record could not be encoded; the record is dropped" + ); + } + } + } + + /// Append the membership records for one surfaced member event: `Alive` for + /// a joined or updated member, `NotAlive` for a left, failed, or reaped one. + pub(crate) fn append_member( + &mut self, + alive: bool, + node: &memberlist_proto::Node, + ) { + let record = if alive { + SnapshotRecord::Alive(node.clone()) + } else { + SnapshotRecord::NotAlive(node.clone()) + }; + self.append(&record); + } + + /// Append any clock high-water marks that advanced since the last append. + pub(crate) fn append_clocks( + &mut self, + member: LamportTime, + event: LamportTime, + query: LamportTime, + ) { + if member > self.last_member_clock { + self.append(&SnapshotRecord::Clock(member)); + self.last_member_clock = member; + } + if event > self.last_event_clock { + self.append(&SnapshotRecord::EventClock(event)); + self.last_event_clock = event; + } + if query > self.last_query_clock { + self.append(&SnapshotRecord::QueryClock(query)); + self.last_query_clock = query; + } + } + + /// Append the leave marker: the local node cleanly left the cluster. On the + /// next start, replay clears the recovered state unless + /// `rejoin_after_leave` ignores it. + pub(crate) fn append_leave(&mut self) { + self.append(&SnapshotRecord::Leave); + } + + /// Flush the buffered appends to the OS, and — when the file has grown past + /// the compaction threshold — rewrite it to just `alive` and the clock + /// floors via a sibling temp file and an atomic rename. + pub(crate) fn flush_and_maybe_compact( + &mut self, + alive: impl FnOnce() -> Vec>, + ) { + if let Err(_err) = self.file.flush() { + #[cfg(feature = "tracing")] + tracing::warn!( + path = %self.path.display(), + error = %_err, + "serf snapshot flush failed" + ); + } + if self.compact_threshold == 0 || self.bytes_written < self.compact_threshold { + return; + } + let mut fresh: Vec = Vec::new(); + for rec in [ + SnapshotRecord::::Clock(self.last_member_clock), + SnapshotRecord::EventClock(self.last_event_clock), + SnapshotRecord::QueryClock(self.last_query_clock), + ] { + if let Ok(b) = rec.encode() { + fresh.extend_from_slice(&b); + } + } + for node in alive() { + if let Ok(b) = SnapshotRecord::::Alive(node).encode() { + fresh.extend_from_slice(&b); + } + } + let tmp = self.path.with_extension("compact"); + let replaced = fs::write(&tmp, &fresh) + .and_then(|()| fs::rename(&tmp, &self.path)) + .and_then(|()| fs::OpenOptions::new().append(true).open(&self.path)); + match replaced { + Ok(file) => { + self.file = io::BufWriter::new(file); + self.bytes_written = fresh.len() as u64; + } + Err(_err) => { + #[cfg(feature = "tracing")] + tracing::warn!( + path = %self.path.display(), + error = %_err, + "serf snapshot compaction failed; appends continue on the grown file" + ); + } + } + } +} + +/// Errors surfaced by [`Snapshotter::open`] — construction-time failures, so a +/// node never starts against a snapshot it cannot trust. +#[derive(Debug, thiserror::Error)] +pub enum SnapshotOpenError { + /// Reading or opening the snapshot file failed. + #[error(transparent)] + Io(#[from] io::Error), + /// A record before the tail is malformed — the file is not trustworthy. + #[error("corrupt snapshot record: {0}")] + Corrupt(serf_proto::snapshot::SnapshotError), +} + +#[cfg(test)] +mod tests; diff --git a/serf-reactor/src/driver/snapshotter/tests.rs b/serf-reactor/src/driver/snapshotter/tests.rs new file mode 100644 index 00000000..39e616ed --- /dev/null +++ b/serf-reactor/src/driver/snapshotter/tests.rs @@ -0,0 +1,178 @@ +use std::net::SocketAddr; + +use memberlist_proto::Node; +use serf_proto::snapshot::ReplayResult; +use smol_str::SmolStr; + +use super::*; + +fn opts(name: &str) -> SnapshotOptions { + let mut p = std::env::temp_dir(); + p.push(format!("serf-snapshotter-{name}-{}", std::process::id())); + // Ignoring Err: a leftover file from a previous run is fine to lose. + let _ = fs::remove_file(&p); + SnapshotOptions::new(p) +} + +fn node(id: &str, port: u16) -> Node { + Node::new( + SmolStr::new(id), + format!("127.0.0.1:{port}").parse().unwrap(), + ) +} + +fn cleanup(o: &SnapshotOptions) { + // Ignoring Err: best-effort test-file cleanup. + let _ = fs::remove_file(o.path()); +} + +/// Appended membership and clock records replay across a reopen: the alive +/// set reflects joins minus removals, and the clock floors are the high-water +/// marks. +#[test] +fn appends_replay_across_reopen() { + let o = opts("roundtrip"); + { + let (mut snap, records) = Snapshotter::::open(&o).expect("first open of a fresh path"); + assert!(records.is_empty(), "a fresh path replays to nothing"); + snap.append_member(true, &node("a", 7001)); + snap.append_member(true, &node("b", 7002)); + snap.append_clocks( + LamportTime::new(5), + LamportTime::new(3), + LamportTime::new(2), + ); + snap.append_member(false, &node("b", 7002)); + snap.flush_and_maybe_compact(Vec::new); + } + + let (_snap, records) = Snapshotter::::open(&o).expect("reopen parses"); + let replay = ReplayResult::replay(records, false); + assert_eq!(replay.alive_nodes, vec![node("a", 7001)]); + assert_eq!(replay.last_clock, LamportTime::new(5)); + assert_eq!(replay.last_event_clock, LamportTime::new(3)); + assert_eq!(replay.last_query_clock, LamportTime::new(2)); + cleanup(&o); +} + +/// The clean-leave marker clears the recovered state on replay unless +/// `rejoin_after_leave` ignores it. +#[test] +fn leave_marker_gates_the_replay() { + let o = opts("leave-gate"); + { + let (mut snap, _) = Snapshotter::::open(&o).expect("open"); + snap.append_member(true, &node("a", 7001)); + snap.append_clocks(LamportTime::new(9), LamportTime::ZERO, LamportTime::ZERO); + snap.append_leave(); + snap.flush_and_maybe_compact(Vec::new); + } + + let (_s, records) = Snapshotter::::open(&o).expect("reopen"); + let fresh = ReplayResult::replay(records.clone(), false); + assert!(fresh.alive_nodes.is_empty(), "a clean leave starts fresh"); + assert_eq!(fresh.last_clock, LamportTime::ZERO); + + let rejoin = ReplayResult::replay(records, true); + assert_eq!( + rejoin.alive_nodes, + vec![node("a", 7001)], + "rejoin_after_leave preserves the recovered membership" + ); + assert_eq!(rejoin.last_clock, LamportTime::new(9)); + cleanup(&o); +} + +/// A truncated tail (a crash mid-append) is tolerated: whole records replay, +/// the partial tail is dropped, and new appends land on a whole-record +/// boundary. +#[test] +fn truncated_tail_is_tolerated_and_repaired() { + let o = opts("torn-tail"); + { + let (mut snap, _) = Snapshotter::::open(&o).expect("open"); + snap.append_member(true, &node("a", 7001)); + snap.flush_and_maybe_compact(Vec::new); + } + // Simulate a torn append: a record tag with a length that promises more + // bytes than exist. + { + use std::io::Write as _; + let mut f = fs::OpenOptions::new() + .append(true) + .open(o.path()) + .expect("append to test file"); + f.write_all(&[0x00, 0xff, 0xff]).expect("write torn tail"); + } + + let (mut snap, records) = Snapshotter::::open(&o).expect("torn tail tolerated"); + let replay = ReplayResult::replay(records, false); + assert_eq!(replay.alive_nodes, vec![node("a", 7001)]); + + // Appends continue cleanly on the repaired boundary. + snap.append_member(true, &node("b", 7002)); + snap.flush_and_maybe_compact(Vec::new); + drop(snap); + let (_s, records) = Snapshotter::::open(&o).expect("reopen after repair"); + let replay = ReplayResult::replay(records, false); + assert_eq!(replay.alive_nodes, vec![node("a", 7001), node("b", 7002)]); + cleanup(&o); +} + +/// A malformed record BEFORE the tail is a hard open error, never a silent +/// partial replay. +#[test] +fn corrupt_middle_record_refuses_to_open() { + let o = opts("corrupt"); + { + use std::io::Write as _; + let mut f = fs::File::create(o.path()).expect("create test file"); + // An unknown tag followed by a whole valid record's worth of bytes. + f.write_all(&[0xEE]).expect("write bogus tag"); + let rec = SnapshotRecord::::Comment + .encode() + .expect("encode comment"); + f.write_all(&rec).expect("write trailing record"); + } + assert!( + matches!( + Snapshotter::::open(&o), + Err(SnapshotOpenError::Corrupt(_)) + ), + "an unknown tag before the tail must refuse the open" + ); + cleanup(&o); +} + +/// Past the compaction threshold, the file is rewritten to just the clock +/// floors and the caller-supplied live set — and replays identically. +#[test] +fn compaction_rewrites_to_the_live_state() { + let o = opts("compact").with_compact_threshold(64); + { + let (mut snap, _) = Snapshotter::::open(&o).expect("open"); + // Churn well past 64 bytes: many joins and removals of a transient peer. + for i in 0..32u16 { + snap.append_member(true, &node("transient", 8000 + i)); + snap.append_member(false, &node("transient", 8000 + i)); + } + snap.append_clocks( + LamportTime::new(7), + LamportTime::new(6), + LamportTime::new(5), + ); + snap.flush_and_maybe_compact(|| vec![node("kept", 7001)]); + } + let size = fs::metadata(o.path()).expect("stat").len(); + assert!( + size < 128, + "compaction must shrink the churned file, got {size} bytes" + ); + let (_s, records) = Snapshotter::::open(&o).expect("reopen compacted"); + let replay = ReplayResult::replay(records, false); + assert_eq!(replay.alive_nodes, vec![node("kept", 7001)]); + assert_eq!(replay.last_clock, LamportTime::new(7)); + assert_eq!(replay.last_event_clock, LamportTime::new(6)); + assert_eq!(replay.last_query_clock, LamportTime::new(5)); + cleanup(&o); +} diff --git a/serf-reactor/src/driver/stream/mod.rs b/serf-reactor/src/driver/stream/mod.rs index 156ed1c0..1dd945d2 100644 --- a/serf-reactor/src/driver/stream/mod.rs +++ b/serf-reactor/src/driver/stream/mod.rs @@ -456,6 +456,9 @@ where close_timeout: Duration, dial_timeout: Duration, bridge_recv_buf_len: usize, + /// Snapshot persistence: appends a record for every surfaced membership + /// change and the advancing clock floors, `None` when persistence is off. + snapshotter: Option>, /// The driver's keyring delegate: applies inbound key-management ops and /// produces the `respond_key` answer. Present only under an encryption backend. #[cfg(encryption)] @@ -487,6 +490,7 @@ where stream_opts: StreamTransportOptions, label: Option, stream_timeout: Duration, + snapshotter: Option>, #[cfg(encryption)] keyring: Arc, ) -> Self { let buf_len = endpoint @@ -535,6 +539,7 @@ where close_timeout: stream_opts.close_timeout(), dial_timeout: stream_opts.dial_timeout(), bridge_recv_buf_len: stream_opts.bridge_recv_buf_len(), + snapshotter, #[cfg(encryption)] keyring, } @@ -1307,6 +1312,39 @@ where /// on `LeftCluster`, begin teardown on a lost id-conflict `Shutdown`, and answer /// an inbound key-management request. fn account_event(&mut self, ev: &Event) { + // Snapshot persistence: append the surfaced membership change, the + // advancing clock floors, and the clean-leave marker, then flush (and + // compact past the threshold) — a change is durable once this poll + // returns. Restart replay + `load_snapshot` recovers the state. + if let Some(snap) = self.snapshotter.as_mut() { + match ev { + Event::Member(me) => { + use serf_proto::event::MemberEventKind as MK; + let alive = matches!(me.kind(), MK::Join | MK::Update); + for m in me.members() { + snap.append_member(alive, m.node()); + } + } + Event::LeftCluster => snap.append_leave(), + _ => {} + } + if matches!(ev, Event::Member(_) | Event::LeftCluster) { + snap.append_clocks( + LamportTime::from(self.endpoint.member_time()), + LamportTime::from(self.endpoint.event_time()), + LamportTime::from(self.endpoint.query_time()), + ); + let endpoint = &self.endpoint; + snap.flush_and_maybe_compact(|| { + endpoint + .members_snapshot() + .iter() + .filter(|m| m.status() == serf_proto::members::MemberStatus::Alive) + .map(|m| m.node().clone()) + .collect() + }); + } + } if let Event::ExchangeCompleted(c) = ev && c.kind() == ExchangeKind::PushPull { @@ -2223,6 +2261,7 @@ pub(crate) fn spawn_stream_driver( stream_opts: StreamTransportOptions, label: Option, stream_timeout: Duration, + snapshotter: Option>, #[cfg(encryption)] keyring: Arc, ) -> StreamDriver where @@ -2284,6 +2323,7 @@ where stream_opts, label, stream_timeout, + snapshotter, #[cfg(encryption)] keyring, ) diff --git a/serf-reactor/src/driver/stream/tests.rs b/serf-reactor/src/driver/stream/tests.rs index 02e5b266..77eab963 100644 --- a/serf-reactor/src/driver/stream/tests.rs +++ b/serf-reactor/src/driver/stream/tests.rs @@ -182,6 +182,7 @@ async fn build_driver_with_stream_timeout( StreamTransportOptions::new().with_bridge_inbound_cap(bridge_inbound_cap), None, stream_timeout, + None, #[cfg(encryption)] Arc::new(crate::VoidKeyringDelegate), ); diff --git a/serf-reactor/src/error/mod.rs b/serf-reactor/src/error/mod.rs index c777ec86..a282e570 100644 --- a/serf-reactor/src/error/mod.rs +++ b/serf-reactor/src/error/mod.rs @@ -151,6 +151,14 @@ pub enum SerfError { )] LeaveFarewellUndelivered, + /// The snapshot file supplied at construction could not be read or holds a + /// malformed record — the node refuses to start against state it cannot + /// trust. + #[cfg(any(feature = "tcp", feature = "quic"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] + #[error(transparent)] + SnapshotOpen(#[from] crate::driver::snapshotter::SnapshotOpenError), + /// The driver task has shut down and is no longer accepting commands. #[error("driver shut down")] Shutdown, diff --git a/serf-reactor/src/lib.rs b/serf-reactor/src/lib.rs index bff3f39c..958217c1 100644 --- a/serf-reactor/src/lib.rs +++ b/serf-reactor/src/lib.rs @@ -188,6 +188,10 @@ pub use events::EventStream; pub use driver::options::{ Channel, DEFAULT_BRIDGE_INBOUND_CAP, DEFAULT_BRIDGE_RECV_BUF_LEN, DEFAULT_CLOSE_TIMEOUT, DEFAULT_DIAL_TIMEOUT, DEFAULT_EVENT_QUEUE_CAP, DEFAULT_IDLE_WAKE_INTERVAL, - DEFAULT_ITER_DRAIN_CAP, DEFAULT_LEAVE_TIMEOUT, DEFAULT_OBSERVATION_CHANNEL, ParseChannelError, - RuntimeOptions, StreamTransportOptions, + DEFAULT_ITER_DRAIN_CAP, DEFAULT_LEAVE_TIMEOUT, DEFAULT_OBSERVATION_CHANNEL, + DEFAULT_SNAPSHOT_COMPACT_THRESHOLD, ParseChannelError, RuntimeOptions, SnapshotOptions, + StreamTransportOptions, }; +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub use driver::snapshotter::SnapshotOpenError; diff --git a/serf-reactor/src/quic/mod.rs b/serf-reactor/src/quic/mod.rs index d3ea0023..7df778ac 100644 --- a/serf-reactor/src/quic/mod.rs +++ b/serf-reactor/src/quic/mod.rs @@ -532,6 +532,7 @@ where // Serf's core RNG is seeded from its own OS-drawn entropy (`self.serf_rng`), // independent of the coordinator's gossip RNG, so two nodes never share the // query-ID / relay-selection stream. + let rejoin_after_leave; let endpoint = serf_proto::QuicEndpoint::< Self::Id, G, @@ -539,7 +540,10 @@ where crate::drop_counter::ReactorDropCounter, >::new_with_rng_in( coord, - runtime.serf_options, + { + rejoin_after_leave = runtime.serf_options.rejoin_after_leave(); + runtime.serf_options + }, self.serf_rng, runtime.user_drop, runtime.member_drop, @@ -549,6 +553,16 @@ where if let Some(md) = runtime.merge_delegate { endpoint.set_merge_delegate(md); } + let snapshotter = match runtime.snapshot { + Some((writer, records)) => { + let replay = serf_proto::snapshot::ReplayResult::replay(records, rejoin_after_leave); + // Ignoring Err: load_snapshot refuses only on a machine that already + // lost an id-conflict vote; a freshly built endpoint is Alive. + let _ = endpoint.load_snapshot(replay, memberlist_proto::Instant::now()); + Some(writer) + } + None => None, + }; let driver = crate::driver::quic::spawn_quic_driver::( endpoint, @@ -559,6 +573,7 @@ where runtime.delegate, runtime.driver_options, None, + snapshotter, #[cfg(encryption)] runtime.keyring, ); diff --git a/serf-reactor/src/serf/mod.rs b/serf-reactor/src/serf/mod.rs index b4f30e70..b744311a 100644 --- a/serf-reactor/src/serf/mod.rs +++ b/serf-reactor/src/serf/mod.rs @@ -161,6 +161,7 @@ where gossip_rng: G, reconnect_delegate: Option>>, merge_delegate: Option>>, + snapshot: Option, #[cfg(encryption)] keyring: Arc, ) -> core::result::Result where @@ -187,6 +188,18 @@ where // the handle can compute `default_query_timeout` without a driver round-trip. let query_timeout_mult = serf_options.query_timeout_mult(); + // Open (and decode) the snapshot file BEFORE any socket binds, so a + // corrupt or unreadable file fails construction loudly instead of + // silently starting a node with amnesia. + let snapshot = match snapshot { + Some(opts) => { + let (writer, records) = + crate::driver::snapshotter::Snapshotter::open(&opts).map_err(SerfError::from)?; + Some((writer, records)) + } + None => None, + }; + let transport = T::new(options, resolver, advertise_resolver).await?; let local_id = transport.local_id().clone(); let advertise = *transport.advertise_address(); @@ -214,6 +227,7 @@ where member_drop_writer, reconnect_delegate, merge_delegate, + snapshot, #[cfg(encryption)] keyring, ); @@ -274,6 +288,7 @@ where serf_options: SerfOptions, reconnect_delegate: Option>>, merge_delegate: Option>>, + snapshot: Option, #[cfg(encryption)] keyring: Arc, ) -> Result where @@ -291,6 +306,7 @@ where crate::gossip_rng()?, reconnect_delegate, merge_delegate, + snapshot, #[cfg(encryption)] keyring, ) @@ -310,6 +326,7 @@ where gossip_rng: G, reconnect_delegate: Option>>, merge_delegate: Option>>, + snapshot: Option, #[cfg(encryption)] keyring: Arc, ) -> Result where @@ -328,6 +345,7 @@ where gossip_rng, reconnect_delegate, merge_delegate, + snapshot, #[cfg(encryption)] keyring, ) @@ -380,6 +398,7 @@ where serf_options: SerfOptions, reconnect_delegate: Option>>, merge_delegate: Option>>, + snapshot: Option, #[cfg(encryption)] keyring: Arc, ) -> Result where @@ -397,6 +416,7 @@ where crate::gossip_rng()?, reconnect_delegate, merge_delegate, + snapshot, #[cfg(encryption)] keyring, ) @@ -416,6 +436,7 @@ where gossip_rng: G, reconnect_delegate: Option>>, merge_delegate: Option>>, + snapshot: Option, #[cfg(encryption)] keyring: Arc, ) -> Result where @@ -434,6 +455,7 @@ where gossip_rng, reconnect_delegate, merge_delegate, + snapshot, #[cfg(encryption)] keyring, ) @@ -485,6 +507,7 @@ where serf_options: SerfOptions, reconnect_delegate: Option>>, merge_delegate: Option>>, + snapshot: Option, #[cfg(encryption)] keyring: Arc, ) -> Result where @@ -502,6 +525,7 @@ where crate::gossip_rng()?, reconnect_delegate, merge_delegate, + snapshot, #[cfg(encryption)] keyring, ) @@ -521,6 +545,7 @@ where gossip_rng: G, reconnect_delegate: Option>>, merge_delegate: Option>>, + snapshot: Option, #[cfg(encryption)] keyring: Arc, ) -> Result where @@ -539,6 +564,7 @@ where gossip_rng, reconnect_delegate, merge_delegate, + snapshot, #[cfg(encryption)] keyring, ) diff --git a/serf-reactor/src/serf/tests.rs b/serf-reactor/src/serf/tests.rs index 5e2bfdd2..445f1565 100644 --- a/serf-reactor/src/serf/tests.rs +++ b/serf-reactor/src/serf/tests.rs @@ -61,6 +61,7 @@ async fn try_spawn_node_at(id: &str, bind: SocketAddr) -> Result No serf_options, None, None, + None, #[cfg(encryption)] std::sync::Arc::new(crate::VoidKeyringDelegate), ) @@ -225,6 +227,7 @@ async fn assert_tcp_new_rejects(runtime: RuntimeOptions) { SerfOptions::new(), None, None, + None, #[cfg(encryption)] std::sync::Arc::new(crate::VoidKeyringDelegate), ) @@ -269,6 +272,7 @@ async fn tcp_new_rejects_over_ceiling_user_event_size() { serf, None, None, + None, #[cfg(encryption)] std::sync::Arc::new(crate::VoidKeyringDelegate), ) diff --git a/serf-reactor/src/tcp/mod.rs b/serf-reactor/src/tcp/mod.rs index 25f10d0e..5a86d2f9 100644 --- a/serf-reactor/src/tcp/mod.rs +++ b/serf-reactor/src/tcp/mod.rs @@ -520,6 +520,7 @@ where coord.set_encryption_options(self.encryption); // Serf's core RNG is seeded from its own OS-drawn entropy (`self.serf_rng`), // independent of the coordinator's gossip RNG. + let rejoin_after_leave; let endpoint = serf_proto::StreamEndpoint::< Self::Id, SocketAddr, @@ -529,7 +530,10 @@ where crate::drop_counter::ReactorDropCounter, >::new_with_rng_in( coord, - runtime.serf_options, + { + rejoin_after_leave = runtime.serf_options.rejoin_after_leave(); + runtime.serf_options + }, self.serf_rng, runtime.user_drop, runtime.member_drop, @@ -539,6 +543,16 @@ where if let Some(md) = runtime.merge_delegate { endpoint.set_merge_delegate(md); } + let snapshotter = match runtime.snapshot { + Some((writer, records)) => { + let replay = serf_proto::snapshot::ReplayResult::replay(records, rejoin_after_leave); + // Ignoring Err: load_snapshot refuses only on a machine that already + // lost an id-conflict vote; a freshly built endpoint is Alive. + let _ = endpoint.load_snapshot(replay, memberlist_proto::Instant::now()); + Some(writer) + } + None => None, + }; let driver = crate::driver::stream::spawn_stream_driver::( endpoint, @@ -551,6 +565,7 @@ where self.stream_options, None, stream_timeout, + snapshotter, #[cfg(encryption)] runtime.keyring, ); diff --git a/serf-reactor/src/tls/mod.rs b/serf-reactor/src/tls/mod.rs index a27f62c8..809dfa44 100644 --- a/serf-reactor/src/tls/mod.rs +++ b/serf-reactor/src/tls/mod.rs @@ -643,6 +643,7 @@ where // Serf's core RNG is seeded from its own OS-drawn entropy (`self.serf_rng`), // independent of the coordinator's gossip RNG, so two nodes never share the // query-ID / relay-selection stream. + let rejoin_after_leave; let endpoint = serf_proto::StreamEndpoint::< Self::Id, SocketAddr, @@ -652,7 +653,10 @@ where crate::drop_counter::ReactorDropCounter, >::new_with_rng_in( coord, - runtime.serf_options, + { + rejoin_after_leave = runtime.serf_options.rejoin_after_leave(); + runtime.serf_options + }, self.serf_rng, runtime.user_drop, runtime.member_drop, @@ -662,6 +666,16 @@ where if let Some(md) = runtime.merge_delegate { endpoint.set_merge_delegate(md); } + let snapshotter = match runtime.snapshot { + Some((writer, records)) => { + let replay = serf_proto::snapshot::ReplayResult::replay(records, rejoin_after_leave); + // Ignoring Err: load_snapshot refuses only on a machine that already + // lost an id-conflict vote; a freshly built endpoint is Alive. + let _ = endpoint.load_snapshot(replay, memberlist_proto::Instant::now()); + Some(writer) + } + None => None, + }; let driver = crate::driver::stream::spawn_stream_driver::, D, G, StdRng>( @@ -675,6 +689,7 @@ where self.stream_options, None, stream_timeout, + snapshotter, #[cfg(encryption)] runtime.keyring, ); diff --git a/serf-reactor/src/transport/runtime.rs b/serf-reactor/src/transport/runtime.rs index 5d3a98bf..4b10f293 100644 --- a/serf-reactor/src/transport/runtime.rs +++ b/serf-reactor/src/transport/runtime.rs @@ -49,6 +49,10 @@ pub struct TransportRuntime { /// inline for EVERY push/pull merge; `None` admits every peer set. pub(crate) merge_delegate: Option>>, + /// Snapshot persistence, opened (and its file decoded) by the `Serf` + /// constructor so a corrupt file fails construction: `T::run` replays the + /// records into the endpoint and hands the writer to the pump. + pub(crate) snapshot: Option>, /// The driver's keyring delegate, applied to inbound key-management requests. /// Present only under an encryption backend. #[cfg(encryption)] @@ -70,6 +74,7 @@ impl TransportRuntime { member_drop: ReactorDropCounter, reconnect_delegate: Option>>, merge_delegate: Option>>, + snapshot: Option>, #[cfg(encryption)] keyring: Arc, ) -> Self { Self { @@ -82,6 +87,7 @@ impl TransportRuntime { member_drop, reconnect_delegate, merge_delegate, + snapshot, #[cfg(encryption)] keyring, } diff --git a/serf-reactor/tests/cluster/mod.rs b/serf-reactor/tests/cluster/mod.rs index 14e02949..e9d39088 100644 --- a/serf-reactor/tests/cluster/mod.rs +++ b/serf-reactor/tests/cluster/mod.rs @@ -435,6 +435,7 @@ where timing.serf_opts(), None, None, + None, #[cfg(encryption)] std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), ) diff --git a/serf-reactor/tests/key_rotation.rs b/serf-reactor/tests/key_rotation.rs index d2a45a68..116c9dbe 100644 --- a/serf-reactor/tests/key_rotation.rs +++ b/serf-reactor/tests/key_rotation.rs @@ -110,6 +110,7 @@ where SerfOptions::new(), None, None, + None, keyring, ) .await diff --git a/serf-reactor/tests/quic.rs b/serf-reactor/tests/quic.rs index 8608f983..6a456627 100644 --- a/serf-reactor/tests/quic.rs +++ b/serf-reactor/tests/quic.rs @@ -164,6 +164,7 @@ where SerfOptions::new(), None, None, + None, #[cfg(encryption)] std::sync::Arc::new(VoidKeyringDelegate), ) @@ -520,6 +521,7 @@ where SerfOptions::new(), None, None, + None, #[cfg(encryption)] std::sync::Arc::new(VoidKeyringDelegate), ) @@ -577,6 +579,7 @@ where SerfOptions::new(), None, None, + None, std::sync::Arc::new(VoidKeyringDelegate), ) .await diff --git a/serf-reactor/tests/tcp.rs b/serf-reactor/tests/tcp.rs index 3c1be804..a3103768 100644 --- a/serf-reactor/tests/tcp.rs +++ b/serf-reactor/tests/tcp.rs @@ -83,6 +83,7 @@ where SerfOptions::new(), None, None, + None, #[cfg(encryption)] std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), ) @@ -336,6 +337,7 @@ where target: SmolStr::new("rd-b"), })), None, + None, #[cfg(encryption)] std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), ) @@ -644,6 +646,128 @@ where cluster.shutdown_all().await; } +/// Build a node persisting membership to `snapshot` (fast probe/gossip timing +/// so failure detection inside the scenario window stays sub-second). +async fn spawn_node_with_snapshot( + id: &str, + snapshot: serf_reactor::SnapshotOptions, + rejoin_after_leave: bool, +) -> Node +where + R: Runtime, +{ + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + Serf::::tcp( + TcpTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(bind)), + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new().with_rejoin_after_leave(rejoin_after_leave), + None, + None, + Some(snapshot), + #[cfg(encryption)] + std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), + ) + .await + .expect("spawn snapshot-backed serf tcp node") +} + +/// A unique snapshot path under the system temp dir. +fn snapshot_path(name: &str) -> std::path::PathBuf { + let mut p = std::env::temp_dir(); + p.push(format!("serf-e2e-snap-{name}-{}", std::process::id())); + // Ignoring Err: a leftover file from a previous run is fine to lose. + let _ = std::fs::remove_file(&p); + p +} + +/// Restart-and-rejoin: B persists its membership, is abruptly killed, and a +/// fresh B booted from the SAME snapshot re-dials its recovered peers through +/// the machine's own push/pull machinery — both nodes converge to two members +/// again WITHOUT any explicit join call on the restarted node. +async fn snapshot_restart_rejoins_the_cluster() +where + R: Runtime, +{ + let path = snapshot_path("rejoin"); + let a = spawn_node::("snap-a").await; + let b = + spawn_node_with_snapshot::("snap-b", serf_reactor::SnapshotOptions::new(&path), false).await; + let a_addr = a.advertise_address(); + + b.join(&SocketAddrResolver, MaybeResolved::Resolved(a_addr), false) + .await + .expect("join reaches node A"); + converge(&a, &b).await; + + // Abrupt kill: no leave marker lands in the snapshot. + b.shutdown().await.expect("snap-b shuts down"); + + // A fresh B from the same snapshot auto-rejoins A (no join call). + let b2 = + spawn_node_with_snapshot::("snap-b", serf_reactor::SnapshotOptions::new(&path), false).await; + converge(&a, &b2).await; + assert_eq!( + b2.num_members(), + 2, + "the restarted node recovers its membership from the snapshot" + ); + + a.shutdown().await.expect("snap-a shuts down"); + b2.shutdown().await.expect("snap-b2 shuts down"); + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} + +/// The clean-leave gate: after a graceful leave, a restart with the default +/// `rejoin_after_leave = false` starts fresh (no auto-rejoin), while a restart +/// opting in with `rejoin_after_leave = true` recovers the pre-leave +/// membership and rejoins. +async fn snapshot_leave_gate_controls_rejoin() +where + R: Runtime, +{ + let path = snapshot_path("leave-gate"); + let a = spawn_node::("gate-a").await; + let b = + spawn_node_with_snapshot::("gate-b", serf_reactor::SnapshotOptions::new(&path), false).await; + let a_addr = a.advertise_address(); + + b.join(&SocketAddrResolver, MaybeResolved::Resolved(a_addr), false) + .await + .expect("join reaches node A"); + converge(&a, &b).await; + + // Graceful leave: the Leave marker lands in the snapshot. + b.leave().await.expect("gate-b leaves gracefully"); + b.shutdown().await.expect("gate-b shuts down"); + + // Default posture: the leave clears the recovered state — no auto-rejoin. + let b2 = + spawn_node_with_snapshot::("gate-b", serf_reactor::SnapshotOptions::new(&path), false).await; + R::sleep(Duration::from_millis(1500)).await; + assert_eq!( + b2.num_members(), + 1, + "a cleanly-left node must not auto-rejoin unless opted in" + ); + b2.shutdown().await.expect("gate-b2 shuts down"); + + // Opt-in posture: the Leave marker is ignored and the membership recovers. + let b3 = + spawn_node_with_snapshot::("gate-b", serf_reactor::SnapshotOptions::new(&path), true).await; + converge(&a, &b3).await; + + a.shutdown().await.expect("gate-a shuts down"); + b3.shutdown().await.expect("gate-b3 shuts down"); + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} + /// The constructor-supplied merge delegate is the predicate the machine /// consults: with a recording accept-all delegate installed on B, A's join /// push-pull drives at least one `notify_merge` on B carrying A's node state. @@ -695,6 +819,7 @@ where hits: hits.clone(), saw_peer: saw_peer.clone(), })), + None, #[cfg(encryption)] std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), ) @@ -742,6 +867,7 @@ where SerfOptions::new(), None, None, + None, #[cfg(encryption)] std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), ) @@ -833,6 +959,7 @@ where SerfOptions::new(), None, None, + None, std::sync::Arc::new(VoidKeyringDelegate), ) .await @@ -1018,6 +1145,16 @@ mod tokio_cells { super::coordinates_surface_on_the_handle::().await; } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn snapshot_restart_rejoins_the_cluster() { + super::snapshot_restart_rejoins_the_cluster::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn snapshot_leave_gate_controls_rejoin() { + super::snapshot_leave_gate_controls_rejoin::().await; + } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn merge_delegate_is_consulted_on_join() { super::merge_delegate_is_consulted_on_join::().await; @@ -1118,6 +1255,16 @@ mod smol_cells { SmolRuntime::block_on(super::coordinates_surface_on_the_handle::()); } + #[test] + fn snapshot_restart_rejoins_the_cluster_smol() { + SmolRuntime::block_on(super::snapshot_restart_rejoins_the_cluster::()); + } + + #[test] + fn snapshot_leave_gate_controls_rejoin_smol() { + SmolRuntime::block_on(super::snapshot_leave_gate_controls_rejoin::()); + } + #[test] fn merge_delegate_is_consulted_on_join_smol() { SmolRuntime::block_on(super::merge_delegate_is_consulted_on_join::()); diff --git a/serf-reactor/tests/tls.rs b/serf-reactor/tests/tls.rs index cbf61bcc..ff1d7054 100644 --- a/serf-reactor/tests/tls.rs +++ b/serf-reactor/tests/tls.rs @@ -134,6 +134,7 @@ where SerfOptions::new(), None, None, + None, #[cfg(encryption)] std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), ) From 25579cea6c3ad5c9c9e652ba6a75089dea381ed7 Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 14:26:09 +1200 Subject: [PATCH 104/128] docs(serf-proto): the wire-evolution stance replacing version negotiation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Additive changes ride proto3 field semantics (new optional fields default on old nodes; unknown message tags are consumed and dropped); breaking changes ship as a new cluster generation fenced by the cluster label, cut over blue/green; the delegate surface is compile-time API versioned by semver. No in-band version negotiation exists by design — the stance that replaces the legacy protocol_version and delegate_version knobs. --- serf-proto/src/lib.rs | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/serf-proto/src/lib.rs b/serf-proto/src/lib.rs index c505946d..bf06461d 100644 --- a/serf-proto/src/lib.rs +++ b/serf-proto/src/lib.rs @@ -3,6 +3,40 @@ //! //! Depends on `memberlist-proto` for the `Data`/`DataRef` codec primitives; defines serf's //! own message set and framing on top of them. +//! +//! # Wire evolution +//! +//! The legacy serf carried two negotiation knobs — `protocol_version` and +//! `delegate_version` — so mixed-version clusters could gate features at +//! runtime. This stack deliberately carries neither; the wire forms new↔new +//! clusters only, and there is no per-message version field to dispatch on. +//! What replaces them: +//! +//! - **Additive evolution rides proto3 semantics.** Every message body is a +//! proto3 message, and the typed bridge enforces presence only for the +//! fields the protocol requires. A new optional field decodes as its +//! default on nodes that predate it and is skipped (not erred) by nodes +//! that do not know it, so a feature with a sound default posture for old +//! peers ships with no negotiation and no flag day. Never reuse or +//! renumber a field, change a field's wire type, or make an optional field +//! required — those are breaking changes and take the generation path +//! below. The framing envelope is additive the same way: an unknown +//! message tag is dropped with its body length consumed, so a new message +//! type degrades to a no-op on old nodes rather than a parse failure. +//! +//! - **Breaking changes are a new cluster generation, fenced by the cluster +//! label.** The gossip codec stamps every packet and stream with the +//! configured label and ingress drops anything mismatched. A change that +//! cannot be expressed additively ships as a new deployment under a new +//! label, stood up beside the old one and cut over blue/green: nodes of +//! different generations never exchange state, so incompatible layouts +//! never meet on a socket — the label check does the work version +//! negotiation used to approximate, without mixed-version protocol paths +//! to test and maintain. +//! +//! - **Delegates are a compile-time surface.** The delegate traits are Rust +//! API versioned by the crate's semver; there is nothing to negotiate on +//! the wire. #![cfg_attr(not(feature = "std"), no_std)] #![deny(missing_docs)] #![cfg_attr(docsrs, feature(doc_cfg))] From 5500e780484cf3f6c5c760ad4bac8ec375c187fc Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 15:38:36 +1200 Subject: [PATCH 105/128] fix(serf-reactor): harden the snapshot compaction and keyring persistence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Compaction preserved only the clocks and live alive-set, silently erasing a clean-leave marker: a churned or low-threshold file rewritten after leave() would auto-rejoin on the default posture. The writer now tracks clean-left state (set by the Leave append, cleared by membership activity, recovered from the tail on reopen) and re-emits the marker LAST in the compacted file, which therefore replays exactly like the sequence it replaces under both rejoin postures — pinned by a threshold-1 compaction test both ways, plus the post-leave-activity case. The compacted replacement is now written, synced, and OPENED before the rename, so no fallible operation remains after the swap — a rename that succeeded can no longer pair with a failed reopen leaving the writer on the unlinked inode. (Also fixes a bug the rework surfaced: truncate+append is a rejected OpenOptions combination, so the previous chain silently fell to the grown-file branch.) The keyring file's replacing temp inode is born owner-only (0600 on Unix), so a rotation can never widen a restrictive mode under a permissive umask — pinned by a Unix permissions test. And keyring_updated no longer touches storage inline on the pump: rotations hand off to a dedicated persistence thread through an unbounded channel (rare, small payloads), keeping the callback non-blocking as the delegate contract requires. The merge-veto docs now state the boundary plainly: a push/pull filter, not admission control — a rejected peer can still enter through gossiped Alives, exactly as in the reference implementation. --- serf-reactor/src/delegate/keyring_file/mod.rs | 96 +++++++++++++------ .../src/delegate/keyring_file/tests.rs | 64 ++++++++++++- serf-reactor/src/delegate/mod.rs | 16 +++- serf-reactor/src/driver/snapshotter/mod.rs | 48 +++++++++- serf-reactor/src/driver/snapshotter/tests.rs | 55 +++++++++++ 5 files changed, 237 insertions(+), 42 deletions(-) diff --git a/serf-reactor/src/delegate/keyring_file/mod.rs b/serf-reactor/src/delegate/keyring_file/mod.rs index f114feb3..f51708e2 100644 --- a/serf-reactor/src/delegate/keyring_file/mod.rs +++ b/serf-reactor/src/delegate/keyring_file/mod.rs @@ -3,6 +3,7 @@ use std::{ io, path::{Path, PathBuf}, + sync::mpsc, }; use memberlist_proto::{Keyring, SecretKey}; @@ -22,19 +23,43 @@ use super::KeyringDelegate; /// holds raw key material by design — protect it with filesystem permissions /// exactly as the reference implementation's keyring file required. /// -/// Rotations are written via a sibling temp file and an atomic rename, so a -/// crash mid-write never truncates the previous ring. `keyring_updated` has no -/// error channel; a failed write is surfaced through `tracing` (a no-op -/// without the `tracing` feature) and the wire keeps the rotated ring -/// regardless — the file is the durable copy, not the live one. +/// Rotations are handed to a dedicated persistence thread — `keyring_updated` +/// runs inline on the driver pump, which must never block on storage, so the +/// callback only clones the ring into an unbounded channel (rotations are +/// rare and small) and the worker does the file I/O. Each write goes to a +/// sibling temp file created with owner-only permissions (`0600` on Unix), +/// synced, then atomically renamed over the destination, so a crash mid-write +/// never truncates the previous ring and a restrictive mode on the key file +/// is never widened by a rotation. `keyring_updated` has no error channel; a +/// failed write is surfaced through `tracing` (a no-op without the `tracing` +/// feature) and the wire keeps the rotated ring regardless — the file is the +/// durable copy, not the live one. pub struct FileKeyringDelegate { path: PathBuf, + /// Hand-off to the persistence thread; dropping the delegate drops the + /// sender, and the worker exits after draining what was queued. + worker: mpsc::Sender, } impl FileKeyringDelegate { /// A delegate persisting to `path`. pub fn new(path: impl Into) -> Self { - Self { path: path.into() } + let path: PathBuf = path.into(); + let (worker, jobs) = mpsc::channel::(); + let worker_path = path.clone(); + std::thread::spawn(move || { + while let Ok(ring) = jobs.recv() { + if let Err(_err) = persist(&worker_path, &ring) { + #[cfg(feature = "tracing")] + tracing::warn!( + path = %worker_path.display(), + error = %_err, + "serf keyring rotation could not be persisted; the wire keeps the rotated ring" + ); + } + } + }); + Self { path, worker } } /// The persistence path. @@ -71,35 +96,48 @@ impl FileKeyringDelegate { let secondaries = keys.collect::, _>>()?; Ok(Some(Keyring::with_secondaries(primary, secondaries))) } +} - /// Serialize `keyring` into the file format (primary first, one hex line per - /// key) and write it via temp-file + atomic rename. - fn persist(&self, keyring: &Keyring) -> Result<(), KeyringFileError> { - use zeroize::Zeroize as _; - let mut out = String::new(); - push_key_line(&mut out, keyring.primary_ref()); - for key in keyring.secondaries() { - push_key_line(&mut out, key); - } - let tmp = self.path.with_extension("tmp"); - let res = std::fs::write(&tmp, out.as_bytes()) - .and_then(|()| std::fs::rename(&tmp, &self.path)) - .map_err(KeyringFileError::Io); - out.zeroize(); - res +/// Serialize `keyring` into the file format (primary first, one hex line per +/// key) and write it via an owner-only temp file, sync, and atomic rename. +fn persist(path: &Path, keyring: &Keyring) -> Result<(), KeyringFileError> { + use std::io::Write as _; + use zeroize::Zeroize as _; + let mut out = String::new(); + push_key_line(&mut out, keyring.primary_ref()); + for key in keyring.secondaries() { + push_key_line(&mut out, key); + } + let tmp = path.with_extension("tmp"); + let mut opts = std::fs::OpenOptions::new(); + opts.create(true).truncate(true).write(true); + // The file holds raw key material: the temp inode that will REPLACE the + // destination is born owner-only, so a rotation can never widen a + // restrictive mode on the key file (`fs::write` would inherit the umask). + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + opts.mode(0o600); } + let res = opts + .open(&tmp) + .and_then(|mut file| { + file.write_all(out.as_bytes())?; + file.sync_all() + }) + .and_then(|()| std::fs::rename(&tmp, path)) + .map_err(KeyringFileError::Io); + out.zeroize(); + res } impl KeyringDelegate for FileKeyringDelegate { fn keyring_updated(&self, keyring: &Keyring) { - if let Err(_err) = self.persist(keyring) { - #[cfg(feature = "tracing")] - tracing::warn!( - path = %self.path.display(), - error = %_err, - "serf keyring rotation could not be persisted; the wire keeps the rotated ring" - ); - } + // Non-blocking hand-off: the pump must never wait on storage. Ignoring + // Err: the worker exits only when this sender is dropped, so a send can + // only fail during teardown, where losing the final rotation write is + // acceptable (the wire state is authoritative). + let _ = self.worker.send(keyring.clone()); } } diff --git a/serf-reactor/src/delegate/keyring_file/tests.rs b/serf-reactor/src/delegate/keyring_file/tests.rs index 75e7db27..f4472d36 100644 --- a/serf-reactor/src/delegate/keyring_file/tests.rs +++ b/serf-reactor/src/delegate/keyring_file/tests.rs @@ -1,5 +1,17 @@ use super::*; +/// Bounded wait for the persistence worker to materialize the file state +/// `pred` accepts (rotations hand off to a dedicated thread). +fn await_file(mut probe: impl FnMut() -> Option) -> T { + for _ in 0..200 { + if let Some(v) = probe() { + return v; + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + panic!("the persistence worker did not materialize the expected file state"); +} + fn tmp_path(name: &str) -> PathBuf { let mut p = std::env::temp_dir(); p.push(format!("serf-keyring-file-{name}-{}", std::process::id())); @@ -24,10 +36,7 @@ fn rotation_round_trips_through_the_file() { let ring = Keyring::with_secondaries(primary, [secondary]); delegate.keyring_updated(&ring); - let loaded = delegate - .load() - .expect("load parses the persisted file") - .expect("the file exists after a rotation"); + let loaded = await_file(|| delegate.load().ok().flatten()); assert_eq!(loaded.primary_ref(), ring.primary_ref()); assert_eq!(loaded.secondaries(), ring.secondaries()); @@ -87,7 +96,13 @@ fn a_second_rotation_replaces_the_first() { delegate.keyring_updated(&Keyring::new(first)); delegate.keyring_updated(&Keyring::new(second)); - let loaded = delegate.load().expect("parses").expect("exists"); + let loaded = await_file(|| { + delegate + .load() + .ok() + .flatten() + .filter(|r| r.primary_ref() == &second) + }); assert_eq!(loaded.primary_ref(), &second); assert!(loaded.secondaries().is_empty()); assert!( @@ -98,3 +113,42 @@ fn a_second_rotation_replaces_the_first() { // Ignoring Err: best-effort test-file cleanup. let _ = std::fs::remove_file(&path); } + +/// A rotation never widens the key file's mode: the replacing temp inode is +/// born owner-only, so a `0600` destination stays `0600` (and a fresh file is +/// created `0600`), even under a permissive umask. +#[cfg(unix)] +#[test] +fn rotation_preserves_owner_only_permissions() { + use std::os::unix::fs::PermissionsExt as _; + + let path = tmp_path("perms"); + let delegate = FileKeyringDelegate::new(&path); + + #[cfg(feature = "aes-gcm")] + let (first, second) = (SecretKey::Aes128([5u8; 16]), SecretKey::Aes128([6u8; 16])); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let (first, second) = ( + SecretKey::ChaCha20Poly1305([5u8; 32]), + SecretKey::ChaCha20Poly1305([6u8; 32]), + ); + + delegate.keyring_updated(&Keyring::new(first)); + await_file(|| path.exists().then_some(())); + let mode = std::fs::metadata(&path).expect("stat").permissions().mode() & 0o777; + assert_eq!(mode, 0o600, "a fresh key file is born owner-only"); + + delegate.keyring_updated(&Keyring::new(second)); + await_file(|| { + delegate + .load() + .ok() + .flatten() + .filter(|r| r.primary_ref() == &second) + }); + let mode = std::fs::metadata(&path).expect("stat").permissions().mode() & 0o777; + assert_eq!(mode, 0o600, "a rotation must not widen the key file's mode"); + + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} diff --git a/serf-reactor/src/delegate/mod.rs b/serf-reactor/src/delegate/mod.rs index c39e8684..01b7d09b 100644 --- a/serf-reactor/src/delegate/mod.rs +++ b/serf-reactor/src/delegate/mod.rs @@ -212,11 +212,17 @@ pub trait KeyringDelegate: Send + Sync + 'static { /// Supplied at construction (the `merge_delegate` argument) and installed into /// the memberlist machine, which consults it INLINE for every push/pull merge /// — a join and a periodic anti-entropy refresh alike — before applying the -/// remote member state. Returning `false` cancels the merge, so a vetoed peer -/// set is never admitted. The predicate is synchronous by design: it runs -/// inside the machine's drain, so an application needing async I/O (an ACL -/// service, say) resolves its policy ahead of time and answers from that -/// resolved state here. +/// remote member state. Returning `false` cancels that merge: the vetoed peer +/// set is not applied from the exchange. +/// +/// This is a PUSH/PULL FILTER, not an admission-control boundary: a rejected +/// peer can still enter membership moments later through gossiped Alive +/// messages, exactly as in the reference implementation. Do not rely on it +/// for durable exclusion or as an ACL — it bounds what a single state +/// exchange can bulk-admit, nothing more. The predicate is synchronous by +/// design: it runs inside the machine's drain, so an application needing +/// async I/O (an ACL service, say) resolves its policy ahead of time and +/// answers from that resolved state here. /// /// Requires a stream or QUIC transport feature (`tcp` or `quic`). #[cfg(any(feature = "tcp", feature = "quic"))] diff --git a/serf-reactor/src/driver/snapshotter/mod.rs b/serf-reactor/src/driver/snapshotter/mod.rs index da8d3406..a04d87e2 100644 --- a/serf-reactor/src/driver/snapshotter/mod.rs +++ b/serf-reactor/src/driver/snapshotter/mod.rs @@ -41,6 +41,12 @@ pub(crate) struct Snapshotter { last_member_clock: LamportTime, last_event_clock: LamportTime, last_query_clock: LamportTime, + /// The local node has cleanly left (a `Leave` marker was appended) with no + /// membership activity since. Compaction must re-emit the marker LAST so + /// the rewritten file still replays to a gated fresh start under + /// `rejoin_after_leave = false` while preserving the pre-leave membership + /// for the opt-in posture. + clean_left: bool, _id: core::marker::PhantomData, } @@ -104,6 +110,15 @@ where last_member_clock: LamportTime::ZERO, last_event_clock: LamportTime::ZERO, last_query_clock: LamportTime::ZERO, + clean_left: records + .iter() + .rev() + .find_map(|r| match r { + SnapshotRecord::Leave => Some(true), + SnapshotRecord::Alive(_) | SnapshotRecord::NotAlive(_) => Some(false), + _ => None, + }) + .unwrap_or(false), _id: core::marker::PhantomData, }, records, @@ -150,6 +165,7 @@ where } else { SnapshotRecord::NotAlive(node.clone()) }; + self.clean_left = false; self.append(&record); } @@ -178,6 +194,7 @@ where /// next start, replay clears the recovered state unless /// `rejoin_after_leave` ignores it. pub(crate) fn append_leave(&mut self) { + self.clean_left = true; self.append(&SnapshotRecord::Leave); } @@ -214,10 +231,35 @@ where fresh.extend_from_slice(&b); } } + // A clean leave survives compaction: re-emitted LAST, so the rewritten + // file replays to the gated fresh start under `rejoin_after_leave = + // false` and to the preserved membership under the opt-in posture — + // exactly like the original record sequence it replaces. + if self.clean_left + && let Ok(b) = SnapshotRecord::::Leave.encode() + { + fresh.extend_from_slice(&b); + } + // Write and OPEN the replacement before the rename, so no fallible + // operation remains after the swap: a failure here leaves the grown file + // authoritative and appends continue on it, while a completed rename is + // always paired with a live append handle on the SAME inode. let tmp = self.path.with_extension("compact"); - let replaced = fs::write(&tmp, &fresh) - .and_then(|()| fs::rename(&tmp, &self.path)) - .and_then(|()| fs::OpenOptions::new().append(true).open(&self.path)); + // Plain write mode (append + truncate is a rejected combination): the + // handle's cursor sits at end-of-file after the write below, and this + // writer is the file's only one, so subsequent appends continue from the + // cursor exactly as an append-mode handle would. + let replaced = fs::OpenOptions::new() + .create(true) + .truncate(true) + .write(true) + .open(&tmp) + .and_then(|mut file| { + file.write_all(&fresh)?; + file.sync_all()?; + Ok(file) + }) + .and_then(|file| fs::rename(&tmp, &self.path).map(|()| file)); match replaced { Ok(file) => { self.file = io::BufWriter::new(file); diff --git a/serf-reactor/src/driver/snapshotter/tests.rs b/serf-reactor/src/driver/snapshotter/tests.rs index 39e616ed..191d2e1e 100644 --- a/serf-reactor/src/driver/snapshotter/tests.rs +++ b/serf-reactor/src/driver/snapshotter/tests.rs @@ -176,3 +176,58 @@ fn compaction_rewrites_to_the_live_state() { assert_eq!(replay.last_query_clock, LamportTime::new(5)); cleanup(&o); } + +/// The clean-leave gate survives compaction: with a tiny threshold forcing a +/// rewrite on the very batch that carried the Leave marker, the compacted +/// file still replays to a gated fresh start under the default posture and +/// to the preserved membership under `rejoin_after_leave = true`. +#[test] +fn compaction_preserves_the_clean_leave_gate() { + let o = opts("compact-leave").with_compact_threshold(1); + { + let (mut snap, _) = Snapshotter::::open(&o).expect("open"); + snap.append_member(true, &node("peer", 7001)); + snap.append_clocks(LamportTime::new(4), LamportTime::ZERO, LamportTime::ZERO); + snap.append_leave(); + // Threshold 1: this flush compacts, rewriting the file. + snap.flush_and_maybe_compact(|| vec![node("peer", 7001)]); + } + + let (_s, records) = Snapshotter::::open(&o).expect("reopen compacted"); + let fresh = ReplayResult::replay(records.clone(), false); + assert!( + fresh.alive_nodes.is_empty(), + "the compacted file must still gate a clean leave on the default posture" + ); + assert_eq!(fresh.last_clock, LamportTime::ZERO); + + let rejoin = ReplayResult::replay(records, true); + assert_eq!( + rejoin.alive_nodes, + vec![node("peer", 7001)], + "the opt-in posture must still recover the pre-leave membership" + ); + assert_eq!(rejoin.last_clock, LamportTime::new(4)); + cleanup(&o); +} + +/// Membership activity after a leave clears the clean-left state: the next +/// compaction does not re-emit a stale Leave marker over live members. +#[test] +fn membership_after_a_leave_clears_the_compacted_gate() { + let o = opts("compact-rejoined").with_compact_threshold(1); + { + let (mut snap, _) = Snapshotter::::open(&o).expect("open"); + snap.append_leave(); + snap.append_member(true, &node("peer", 7001)); + snap.flush_and_maybe_compact(|| vec![node("peer", 7001)]); + } + let (_s, records) = Snapshotter::::open(&o).expect("reopen"); + let fresh = ReplayResult::replay(records, false); + assert_eq!( + fresh.alive_nodes, + vec![node("peer", 7001)], + "post-leave membership must survive the default-posture replay" + ); + cleanup(&o); +} From f18617be355e191d70e3face50be06acd2ffaa53 Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 16:52:00 +1200 Subject: [PATCH 106/128] fix(serf-reactor): exclusive keyring temps, persistence-gated key responses, leave-last snapshot order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keyring persistence: each rotation writes through an exclusively-created (create_new), owner-only, OS-entropy-named sibling temp — key bytes can no longer land in a pre-existing inode or behind a planted symlink — and construction sweeps the legacy fixed-name temp and abandoned temps. keyring_updated now returns a persistence acknowledgement: the pumps park a rotated op's key response until it resolves, folding a failure into the response (result = false carrying the error) exactly as the reference implementation folds a keyring-file write error, bounded by the requester's response deadline, with the re-poll cadence folded into the idle-arm timer target. FileKeyringDelegate acknowledges from its persistence worker, and dropping the delegate joins that worker after it drains, so a shutdown cannot discard a rotation the wire already carries. Snapshot: both pumps append the clock floors BEFORE the leave marker, so a clean shutdown ends the file at the Leave record — the terminal shape compaction preserves and replay expects (a trailing clock record would resurrect floors the default no-rejoin posture zeroes). The snapshotter pins original-versus-compacted replay equivalence under both postures and the leave-gate e2e asserts the on-disk tail shape. serf-embedded: the key-management ops pass relay_factor through (the plain forms pin 0, _with variants expose it), restoring the encryption-feature build the widened endpoint signatures had broken. --- serf-embedded/src/engine/mod.rs | 80 ++++++- serf-reactor/src/delegate/keyring_file/mod.rs | 222 ++++++++++++++---- .../src/delegate/keyring_file/tests.rs | 217 +++++++++++++---- serf-reactor/src/delegate/mod.rs | 67 +++++- serf-reactor/src/driver/quic/mod.rs | 148 ++++++++++-- serf-reactor/src/driver/shared.rs | 70 ++++++ serf-reactor/src/driver/snapshotter/tests.rs | 61 +++++ serf-reactor/src/driver/stream/mod.rs | 147 ++++++++++-- serf-reactor/src/lib.rs | 4 +- serf-reactor/tests/key_rotation.rs | 81 ++++++- serf-reactor/tests/tcp.rs | 41 +++- 11 files changed, 984 insertions(+), 154 deletions(-) diff --git a/serf-embedded/src/engine/mod.rs b/serf-embedded/src/engine/mod.rs index 39b1ba3b..6b4c091a 100644 --- a/serf-embedded/src/engine/mod.rs +++ b/serf-embedded/src/engine/mod.rs @@ -1424,44 +1424,114 @@ where } /// Issue a cluster-wide `install_key` query to add `key` to every node's - /// keyring. + /// keyring. Responses return direct-only; see + /// [`install_key_with`](Self::install_key_with) for relayed delivery. #[cfg(encryption)] #[cfg_attr( docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] pub fn install_key(&mut self, key: SecretKey, now: Instant) -> Result { - self.endpoint.install_key(key, now) + self.install_key_with(key, 0, now) + } + + /// As [`install_key`](Self::install_key), with the responses relayed through + /// `relay_factor` random intermediary nodes for delivery redundancy (`0` = + /// direct-only, the plain form's behavior). + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn install_key_with( + &mut self, + key: SecretKey, + relay_factor: u8, + now: Instant, + ) -> Result { + self.endpoint.install_key(key, relay_factor, now) } /// Issue a cluster-wide `use_key` query to promote `key` to primary. + /// Responses return direct-only; see [`use_key_with`](Self::use_key_with) + /// for relayed delivery. #[cfg(encryption)] #[cfg_attr( docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] pub fn use_key(&mut self, key: SecretKey, now: Instant) -> Result { - self.endpoint.use_key(key, now) + self.use_key_with(key, 0, now) + } + + /// As [`use_key`](Self::use_key), with the responses relayed through + /// `relay_factor` random intermediary nodes for delivery redundancy (`0` = + /// direct-only, the plain form's behavior). + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn use_key_with( + &mut self, + key: SecretKey, + relay_factor: u8, + now: Instant, + ) -> Result { + self.endpoint.use_key(key, relay_factor, now) } /// Issue a cluster-wide `remove_key` query to remove `key` from all nodes. + /// Responses return direct-only; see + /// [`remove_key_with`](Self::remove_key_with) for relayed delivery. #[cfg(encryption)] #[cfg_attr( docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] pub fn remove_key(&mut self, key: SecretKey, now: Instant) -> Result { - self.endpoint.remove_key(key, now) + self.remove_key_with(key, 0, now) + } + + /// As [`remove_key`](Self::remove_key), with the responses relayed through + /// `relay_factor` random intermediary nodes for delivery redundancy (`0` = + /// direct-only, the plain form's behavior). + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn remove_key_with( + &mut self, + key: SecretKey, + relay_factor: u8, + now: Instant, + ) -> Result { + self.endpoint.remove_key(key, relay_factor, now) } /// Issue a cluster-wide `list_keys` query to enumerate installed keys. + /// Responses return direct-only; see + /// [`list_keys_with`](Self::list_keys_with) for relayed delivery. #[cfg(encryption)] #[cfg_attr( docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] pub fn list_keys(&mut self, now: Instant) -> Result { - self.endpoint.list_keys(now) + self.list_keys_with(0, now) + } + + /// As [`list_keys`](Self::list_keys), with the responses relayed through + /// `relay_factor` random intermediary nodes for delivery redundancy (`0` = + /// direct-only, the plain form's behavior). + #[cfg(encryption)] + #[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + )] + pub fn list_keys_with(&mut self, relay_factor: u8, now: Instant) -> Result { + self.endpoint.list_keys(relay_factor, now) } /// Answer an inbound key-management request. `req` is the [`KeyRequest`] diff --git a/serf-reactor/src/delegate/keyring_file/mod.rs b/serf-reactor/src/delegate/keyring_file/mod.rs index f51708e2..e616f705 100644 --- a/serf-reactor/src/delegate/keyring_file/mod.rs +++ b/serf-reactor/src/delegate/keyring_file/mod.rs @@ -9,7 +9,7 @@ use std::{ use memberlist_proto::{Keyring, SecretKey}; use serf_proto::{secret_key_from_bytes, secret_key_to_bytes}; -use super::KeyringDelegate; +use super::{KeyringDelegate, KeyringPersistError, KeyringPersistence}; /// A [`KeyringDelegate`] that persists every keyring rotation to a file, plus /// a [`load`](Self::load) to rebuild the ring at construction — the turnkey @@ -26,30 +26,50 @@ use super::KeyringDelegate; /// Rotations are handed to a dedicated persistence thread — `keyring_updated` /// runs inline on the driver pump, which must never block on storage, so the /// callback only clones the ring into an unbounded channel (rotations are -/// rare and small) and the worker does the file I/O. Each write goes to a -/// sibling temp file created with owner-only permissions (`0600` on Unix), -/// synced, then atomically renamed over the destination, so a crash mid-write -/// never truncates the previous ring and a restrictive mode on the key file -/// is never widened by a rotation. `keyring_updated` has no error channel; a -/// failed write is surfaced through `tracing` (a no-op without the `tracing` -/// feature) and the wire keeps the rotated ring regardless — the file is the -/// durable copy, not the live one. +/// rare and small) and the worker does the file I/O, acknowledging each write +/// back through [`KeyringPersistence::Pending`] so the driver can gate the +/// key response on durability. Each write goes through an exclusively-created, +/// owner-only (`0600` on Unix), unpredictably-named sibling temp file, synced, +/// then atomically renamed over the destination: a crash mid-write never +/// truncates the previous ring, a restrictive mode on the key file is never +/// widened by a rotation, and key bytes can never land in a pre-existing +/// inode or behind a planted symlink. On non-Unix platforms there is no mode +/// to assert — the file inherits the destination directory's ACLs, so place +/// it in an owner-restricted directory. Dropping the delegate joins the +/// worker after it drains every queued rotation, so a shutdown cannot discard +/// a write that was already acknowledged toward the wire. pub struct FileKeyringDelegate { path: PathBuf, - /// Hand-off to the persistence thread; dropping the delegate drops the - /// sender, and the worker exits after draining what was queued. - worker: mpsc::Sender, + /// Hand-off to the persistence thread; `None` only during drop, which hangs + /// up first so the worker drains and exits. + worker: Option>, + /// The persistence thread, joined on drop after the hang-up. + handle: Option>, +} + +/// One queued rotation: the ring to write and the acknowledgement sender the +/// pump's parked key response polls. +struct PersistJob { + ring: Keyring, + ack: mpsc::Sender>, } impl FileKeyringDelegate { /// A delegate persisting to `path`. + /// + /// Construction sweeps stale sibling temp files — the fixed-name temp + /// earlier releases wrote (whose permissions predate the owner-only + /// guarantee) and abandoned temps from crashed rotations — before the + /// first write can race one. pub fn new(path: impl Into) -> Self { let path: PathBuf = path.into(); - let (worker, jobs) = mpsc::channel::(); + sweep_stale_temps(&path); + let (worker, jobs) = mpsc::channel::(); let worker_path = path.clone(); - std::thread::spawn(move || { - while let Ok(ring) = jobs.recv() { - if let Err(_err) = persist(&worker_path, &ring) { + let handle = std::thread::spawn(move || { + while let Ok(job) = jobs.recv() { + let res = persist(&worker_path, &job.ring); + if let Err(_err) = &res { #[cfg(feature = "tracing")] tracing::warn!( path = %worker_path.display(), @@ -57,9 +77,18 @@ impl FileKeyringDelegate { "serf keyring rotation could not be persisted; the wire keeps the rotated ring" ); } + // Ignoring Err: the pump dropped this rotation's receiver (teardown, + // or the requester's deadline passed) — the outcome has nowhere to go. + let _ = job + .ack + .send(res.map_err(|e| Box::new(e) as KeyringPersistError)); } }); - Self { path, worker } + Self { + path, + worker: Some(worker), + handle: Some(handle), + } } /// The persistence path. @@ -99,45 +128,150 @@ impl FileKeyringDelegate { } /// Serialize `keyring` into the file format (primary first, one hex line per -/// key) and write it via an owner-only temp file, sync, and atomic rename. +/// key) and write it via an exclusively-created owner-only temp file, sync, +/// and atomic rename. fn persist(path: &Path, keyring: &Keyring) -> Result<(), KeyringFileError> { - use std::io::Write as _; use zeroize::Zeroize as _; let mut out = String::new(); push_key_line(&mut out, keyring.primary_ref()); for key in keyring.secondaries() { push_key_line(&mut out, key); } - let tmp = path.with_extension("tmp"); - let mut opts = std::fs::OpenOptions::new(); - opts.create(true).truncate(true).write(true); - // The file holds raw key material: the temp inode that will REPLACE the - // destination is born owner-only, so a rotation can never widen a - // restrictive mode on the key file (`fs::write` would inherit the umask). - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt as _; - opts.mode(0o600); - } - let res = opts - .open(&tmp) - .and_then(|mut file| { - file.write_all(out.as_bytes())?; - file.sync_all() - }) - .and_then(|()| std::fs::rename(&tmp, path)) - .map_err(KeyringFileError::Io); + let res = write_via_exclusive_temp(path, out.as_bytes()).map_err(KeyringFileError::Io); out.zeroize(); res } +/// Write `contents` to `path` through an exclusively-created, owner-only, +/// unpredictably-named sibling temp file, synced then atomically renamed over +/// the destination. +/// +/// `create_new` (`O_CREAT | O_EXCL`) never reuses an existing inode and never +/// follows a symlink — a file or link already sitting at the temp path fails +/// the attempt instead of receiving the key bytes — and the OS-entropy name +/// keeps such a path from being plantable ahead of time. The inode is born +/// `0600` on Unix and re-asserted on the open handle, so raw key material +/// only ever lands in a fresh owner-only inode this process created; a crash +/// mid-write never truncates the previous ring. On non-Unix platforms there +/// is no mode to assert: the fresh inode inherits the parent directory's +/// ACLs, so the destination directory itself must be access-restricted. +fn write_via_exclusive_temp(path: &Path, contents: &[u8]) -> io::Result<()> { + use std::io::Write as _; + let name = path + .file_name() + .and_then(|n| n.to_str()) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "keyring path has no file name"))?; + let dir = path.parent().filter(|d| !d.as_os_str().is_empty()); + let mut attempts = 0u8; + let (tmp, mut file) = loop { + let tmp_name = format!(".{name}.{:016x}.tmp", temp_nonce()?); + let tmp = match dir { + Some(d) => d.join(&tmp_name), + None => PathBuf::from(&tmp_name), + }; + let mut opts = std::fs::OpenOptions::new(); + opts.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + opts.mode(0o600); + } + match opts.open(&tmp) { + Ok(file) => break (tmp, file), + // A 64-bit OS-entropy collision is practically a squatted path; a + // bounded retry with a fresh nonce outlasts any accidental leftover + // without spinning against a directory an attacker keeps filling. + Err(e) if e.kind() == io::ErrorKind::AlreadyExists && attempts < 16 => attempts += 1, + Err(e) => return Err(e), + } + }; + let res = (|| { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt as _; + file.set_permissions(std::fs::Permissions::from_mode(0o600))?; + } + file.write_all(contents)?; + file.sync_all()?; + std::fs::rename(&tmp, path) + })(); + if res.is_err() { + // Ignoring Err: removing the failed temp (it holds key bytes) is + // best-effort hygiene; the write error itself is what propagates. + let _ = std::fs::remove_file(&tmp); + } + res +} + +/// OS-entropy nonce for a temp-file name: unpredictable, so a +/// directory-writing attacker cannot pre-plant a file or symlink at the next +/// temp path. +fn temp_nonce() -> io::Result { + use rand::TryRng as _; + rand::rngs::SysRng.try_next_u64().map_err(io::Error::other) +} + +/// Remove leftovers a rotation can no longer reuse: the fixed-name sibling +/// temp earlier releases wrote (whose permissions predate the owner-only +/// guarantee and may already hold key material), and any `.{name}.*.tmp` +/// temps a crashed rotation abandoned. Removing a planted symlink unlinks +/// the LINK, never its target. Best-effort: a sweep failure never blocks +/// construction — `create_new` already keeps every future write off any +/// path that survives. +fn sweep_stale_temps(path: &Path) { + // Ignoring Err: nothing to sweep, or no permission — both non-fatal. + let _ = std::fs::remove_file(path.with_extension("tmp")); + let (Some(dir), Some(name)) = ( + path.parent().filter(|d| !d.as_os_str().is_empty()), + path.file_name().and_then(|n| n.to_str()), + ) else { + return; + }; + let prefix = format!(".{name}."); + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let file_name = entry.file_name(); + let Some(f) = file_name.to_str() else { + continue; + }; + if f.starts_with(&prefix) && f.ends_with(".tmp") { + // Ignoring Err: best-effort sweep of abandoned temps. + let _ = std::fs::remove_file(entry.path()); + } + } +} + impl KeyringDelegate for FileKeyringDelegate { - fn keyring_updated(&self, keyring: &Keyring) { - // Non-blocking hand-off: the pump must never wait on storage. Ignoring - // Err: the worker exits only when this sender is dropped, so a send can - // only fail during teardown, where losing the final rotation write is - // acceptable (the wire state is authoritative). - let _ = self.worker.send(keyring.clone()); + fn keyring_updated(&self, keyring: &Keyring) -> KeyringPersistence { + // Non-blocking hand-off: the pump must never wait on storage. A failed + // send (the worker already hung up) drops the job — and with it the ack + // sender — so the returned receiver disconnects and the pump reports the + // rotation unpersisted rather than silently acknowledged. + let (ack, rx) = mpsc::channel(); + if let Some(worker) = &self.worker { + // Ignoring Err: see above — the dropped job's disconnected receiver IS + // the failure signal. + let _ = worker.send(PersistJob { + ring: keyring.clone(), + ack, + }); + } + KeyringPersistence::Pending(rx) + } +} + +impl Drop for FileKeyringDelegate { + fn drop(&mut self) { + // Hang up, then join: the worker drains every queued rotation before it + // exits, so a shutdown cannot discard a write the wire already carries. + self.worker = None; + if let Some(handle) = self.handle.take() { + // Ignoring Err: a panicked worker already surfaced its failure through + // the acknowledgement channel; there is nothing to unwind into here. + let _ = handle.join(); + } } } diff --git a/serf-reactor/src/delegate/keyring_file/tests.rs b/serf-reactor/src/delegate/keyring_file/tests.rs index f4472d36..f23d7818 100644 --- a/serf-reactor/src/delegate/keyring_file/tests.rs +++ b/serf-reactor/src/delegate/keyring_file/tests.rs @@ -1,15 +1,13 @@ use super::*; -/// Bounded wait for the persistence worker to materialize the file state -/// `pred` accepts (rotations hand off to a dedicated thread). -fn await_file(mut probe: impl FnMut() -> Option) -> T { - for _ in 0..200 { - if let Some(v) = probe() { - return v; - } - std::thread::sleep(std::time::Duration::from_millis(5)); +/// Wait for one rotation's persistence acknowledgement. +fn acked(p: KeyringPersistence) -> Result<(), KeyringPersistError> { + match p { + KeyringPersistence::Durable => Ok(()), + KeyringPersistence::Pending(rx) => rx + .recv_timeout(std::time::Duration::from_secs(5)) + .expect("the persistence worker acknowledges within the bound"), } - panic!("the persistence worker did not materialize the expected file state"); } fn tmp_path(name: &str) -> PathBuf { @@ -18,8 +16,32 @@ fn tmp_path(name: &str) -> PathBuf { p } -/// A persisted rotation round-trips: `keyring_updated` writes the ring -/// (primary first), `load` rebuilds it with the same primary and secondaries. +/// Any sibling temp file this delegate could have produced for `path`: the +/// legacy fixed-name temp or a random-suffix one. +fn temp_residue(path: &Path) -> Vec { + let mut residue = Vec::new(); + let legacy = path.with_extension("tmp"); + if legacy.symlink_metadata().is_ok() { + residue.push(legacy); + } + let (Some(dir), Some(name)) = (path.parent(), path.file_name().and_then(|n| n.to_str())) else { + return residue; + }; + let prefix = format!(".{name}."); + for entry in std::fs::read_dir(dir).expect("temp dir listable").flatten() { + if let Some(f) = entry.file_name().to_str() + && f.starts_with(&prefix) + && f.ends_with(".tmp") + { + residue.push(entry.path()); + } + } + residue +} + +/// A persisted rotation round-trips: the acknowledged `keyring_updated` write +/// (primary first) is on disk, and `load` rebuilds it with the same primary +/// and secondaries. #[test] fn rotation_round_trips_through_the_file() { let path = tmp_path("roundtrip"); @@ -34,9 +56,12 @@ fn rotation_round_trips_through_the_file() { ); let ring = Keyring::with_secondaries(primary, [secondary]); - delegate.keyring_updated(&ring); + acked(delegate.keyring_updated(&ring)).expect("the rotation persists"); - let loaded = await_file(|| delegate.load().ok().flatten()); + let loaded = delegate + .load() + .expect("an acknowledged write parses") + .expect("an acknowledged write exists"); assert_eq!(loaded.primary_ref(), ring.primary_ref()); assert_eq!(loaded.secondaries(), ring.secondaries()); @@ -79,7 +104,7 @@ fn malformed_files_are_parse_errors() { } /// A second rotation atomically replaces the file: the newest ring wins and -/// no temp-file residue remains. +/// no temp-file residue remains under either naming scheme. #[test] fn a_second_rotation_replaces_the_first() { let path = tmp_path("replace"); @@ -93,36 +118,38 @@ fn a_second_rotation_replaces_the_first() { SecretKey::ChaCha20Poly1305([4u8; 32]), ); - delegate.keyring_updated(&Keyring::new(first)); - delegate.keyring_updated(&Keyring::new(second)); + acked(delegate.keyring_updated(&Keyring::new(first))).expect("first rotation persists"); + acked(delegate.keyring_updated(&Keyring::new(second))).expect("second rotation persists"); - let loaded = await_file(|| { - delegate - .load() - .ok() - .flatten() - .filter(|r| r.primary_ref() == &second) - }); + let loaded = delegate + .load() + .expect("parses") + .expect("the file exists after two rotations"); assert_eq!(loaded.primary_ref(), &second); assert!(loaded.secondaries().is_empty()); assert!( - !path.with_extension("tmp").exists(), - "the atomic rename must consume the temp file" + temp_residue(&path).is_empty(), + "the atomic rename must consume every temp file" ); // Ignoring Err: best-effort test-file cleanup. let _ = std::fs::remove_file(&path); } -/// A rotation never widens the key file's mode: the replacing temp inode is -/// born owner-only, so a `0600` destination stays `0600` (and a fresh file is -/// created `0600`), even under a permissive umask. +/// A rotation never widens the key file's mode — and it NARROWS a permissive +/// one: the replacing temp inode is born owner-only, so a fresh file is +/// created `0600` and a pre-existing `0644` destination is `0600` after the +/// next rotation, even under a permissive umask. #[cfg(unix)] #[test] -fn rotation_preserves_owner_only_permissions() { +fn rotation_enforces_owner_only_permissions() { use std::os::unix::fs::PermissionsExt as _; let path = tmp_path("perms"); + // A permissive pre-existing destination (an operator's hand-created file). + std::fs::write(&path, "junk\n").expect("pre-create the destination"); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)) + .expect("widen the destination"); let delegate = FileKeyringDelegate::new(&path); #[cfg(feature = "aes-gcm")] @@ -133,22 +160,132 @@ fn rotation_preserves_owner_only_permissions() { SecretKey::ChaCha20Poly1305([6u8; 32]), ); - delegate.keyring_updated(&Keyring::new(first)); - await_file(|| path.exists().then_some(())); + acked(delegate.keyring_updated(&Keyring::new(first))).expect("first rotation persists"); let mode = std::fs::metadata(&path).expect("stat").permissions().mode() & 0o777; - assert_eq!(mode, 0o600, "a fresh key file is born owner-only"); + assert_eq!( + mode, 0o600, + "the first rotation narrows a permissive destination to owner-only" + ); - delegate.keyring_updated(&Keyring::new(second)); - await_file(|| { - delegate - .load() - .ok() - .flatten() - .filter(|r| r.primary_ref() == &second) - }); + acked(delegate.keyring_updated(&Keyring::new(second))).expect("second rotation persists"); let mode = std::fs::metadata(&path).expect("stat").permissions().mode() & 0o777; assert_eq!(mode, 0o600, "a rotation must not widen the key file's mode"); // Ignoring Err: best-effort test-file cleanup. let _ = std::fs::remove_file(&path); } + +/// Construction sweeps both classes of stale sibling temps: the fixed-name +/// temp earlier releases wrote (possibly permissive and already holding key +/// material) and abandoned random-suffix temps from crashed rotations. +#[test] +fn construction_sweeps_stale_temps() { + let path = tmp_path("sweep"); + let legacy = path.with_extension("tmp"); + std::fs::write(&legacy, "stale key bytes at permissive mode\n").expect("plant the legacy temp"); + let name = path.file_name().and_then(|n| n.to_str()).expect("name"); + let abandoned = path.with_file_name(format!(".{name}.00000000deadbeef.tmp")); + std::fs::write(&abandoned, "abandoned partial write\n").expect("plant the abandoned temp"); + + let _delegate = FileKeyringDelegate::new(&path); + assert!( + !legacy.exists(), + "the legacy fixed-name temp must be swept at construction" + ); + assert!( + !abandoned.exists(), + "an abandoned random-suffix temp must be swept at construction" + ); +} + +/// A symlink planted at the legacy temp path is unlinked — the LINK, never +/// its target — and no rotation ever writes through it: the exclusive +/// creation refuses any pre-existing path, so key bytes cannot be redirected +/// into an attacker-chosen file. +#[cfg(unix)] +#[test] +fn a_planted_symlink_never_receives_key_bytes() { + let path = tmp_path("symlink"); + let victim = tmp_path("symlink-victim"); + std::fs::write(&victim, "victim contents\n").expect("create the victim"); + let planted = path.with_extension("tmp"); + // Ignoring Err: a leftover link from a previous run is about to be re-planted. + let _ = std::fs::remove_file(&planted); + std::os::unix::fs::symlink(&victim, &planted).expect("plant the symlink"); + + let delegate = FileKeyringDelegate::new(&path); + assert!( + planted.symlink_metadata().is_err(), + "construction unlinks the planted symlink" + ); + assert_eq!( + std::fs::read_to_string(&victim).expect("victim readable"), + "victim contents\n", + "unlinking removes the LINK, never its target" + ); + + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes128([7u8; 16]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = SecretKey::ChaCha20Poly1305([7u8; 32]); + acked(delegate.keyring_updated(&Keyring::new(key))).expect("rotation persists"); + assert_eq!( + std::fs::read_to_string(&victim).expect("victim readable"), + "victim contents\n", + "no rotation writes through a planted path" + ); + + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_file(&victim); +} + +/// A write failure is acknowledged as an error — the response gate's failure +/// signal — not silently swallowed. +#[test] +fn persistence_failure_is_acknowledged_as_an_error() { + let mut path = std::env::temp_dir(); + path.push(format!("serf-keyring-no-such-dir-{}", std::process::id())); + path.push("ring"); + let delegate = FileKeyringDelegate::new(&path); + + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes128([8u8; 16]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = SecretKey::ChaCha20Poly1305([8u8; 32]); + assert!( + acked(delegate.keyring_updated(&Keyring::new(key))).is_err(), + "a write into a missing directory must acknowledge failure" + ); +} + +/// Dropping the delegate joins the worker after it drains the queue: a +/// rotation handed off immediately before the drop is on disk when `drop` +/// returns, so a shutdown cannot discard a write the wire already carries. +#[test] +fn drop_joins_the_worker_and_flushes_queued_rotations() { + let path = tmp_path("drop-flush"); + let delegate = FileKeyringDelegate::new(&path); + + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes128([9u8; 16]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = SecretKey::ChaCha20Poly1305([9u8; 32]); + + let ack = delegate.keyring_updated(&Keyring::new(key)); + drop(delegate); + + // No waiting: the join inside drop already flushed the queue. + let loaded = FileKeyringDelegate::new(&path) + .load() + .expect("the flushed write parses") + .expect("the flushed write exists"); + assert_eq!(loaded.primary_ref(), &key); + assert!( + matches!(ack, KeyringPersistence::Pending(rx) if matches!(rx.try_recv(), Ok(Ok(())))), + "the queued rotation was acknowledged before the worker exited" + ); + + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} diff --git a/serf-reactor/src/delegate/mod.rs b/serf-reactor/src/delegate/mod.rs index 01b7d09b..aa022cfe 100644 --- a/serf-reactor/src/delegate/mod.rs +++ b/serf-reactor/src/delegate/mod.rs @@ -181,14 +181,18 @@ pub trait Delegate: /// that live keyring directly — install adds a secondary, use promotes the primary, /// remove drops a secondary, every op variant-exact — and answers the originator /// from the post-op live state via `respond_key`. This delegate does NOT author -/// those responses; it only OBSERVES a successful rotation, receiving the new live -/// [`Keyring`] so the application can persist it. A `list` and every refused or -/// no-op request do not fire it. +/// those responses, but it does GATE them: the driver defers a rotated op's +/// response until the returned [`KeyringPersistence`] resolves, and folds a +/// persistence failure into that response (`result = false` carrying the error) +/// exactly as the reference implementation folds a keyring-file write error — +/// with the live wire keyring keeping the rotation either way. A `list` and +/// every refused or no-op request do not fire it. /// /// [`keyring_updated`](Self::keyring_updated) is **synchronous and non-blocking**: -/// it runs on the driver pump. If persistence needs async I/O, hand the ring off to -/// a channel the observer owns and drain it elsewhere. `Send + Sync + 'static` -/// because the driver holds it behind an `Arc` shared across worker threads. +/// it runs on the driver pump. If persistence needs I/O, hand the ring off to a +/// worker and return [`KeyringPersistence::Pending`]; the pump polls the receiver +/// without blocking. `Send + Sync + 'static` because the driver holds it behind +/// an `Arc` shared across worker threads. /// /// Requires the `aes-gcm` or `chacha20-poly1305` feature. #[cfg(encryption)] @@ -199,14 +203,57 @@ pub trait Delegate: pub trait KeyringDelegate: Send + Sync + 'static { /// Called after a key-management request successfully rotated the live wire /// keyring, with the new ring the gossip and reliable planes now encrypt under. - /// Not called for a `list` or any refused or no-op request. The default is a - /// no-op — the rotation is applied to the wire regardless; overriding this only - /// adds out-of-band persistence. - fn keyring_updated(&self, keyring: &Keyring) { + /// Not called for a `list` or any refused or no-op request. The default needs + /// no out-of-band persistence and reports [`KeyringPersistence::Durable`] — + /// the rotation is applied to the wire regardless; overriding this only adds + /// persistence and its acknowledgement. + fn keyring_updated(&self, keyring: &Keyring) -> KeyringPersistence { let _ = keyring; // Unused: default no-op; override to persist the rotation. + KeyringPersistence::Durable } } +/// A persistence failure reported through [`KeyringPersistence::Pending`]. +#[cfg(encryption)] +#[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) +)] +pub type KeyringPersistError = Box; + +/// Receiver half of one rotation's persistence acknowledgement. +#[cfg(encryption)] +#[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) +)] +pub type KeyringPersistRx = std::sync::mpsc::Receiver>; + +/// How one keyring rotation reaches durability, reported back from +/// [`KeyringDelegate::keyring_updated`]. +/// +/// The reference implementation writes its keyring file synchronously inside +/// the key-management query handler and folds a write failure into the +/// response. These drivers keep the pump non-blocking instead: a persisting +/// delegate hands back a receiver, the pump parks the key response, and sends +/// it once the receiver resolves — unchanged on success, downgraded to a +/// failed response carrying the error otherwise (a disconnected sender counts +/// as a failure: the worker vanished without acknowledging). The live wire +/// keyring keeps the rotation in every outcome. +#[cfg(encryption)] +#[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) +)] +pub enum KeyringPersistence { + /// The rotation needs no out-of-band persistence (or completed inline): + /// the key response is sent immediately. + Durable, + /// Persistence runs out-of-band; the pump defers the key response until + /// the receiver resolves. + Pending(KeyringPersistRx), +} + /// The join-merge veto predicate, re-exported from the machine. /// /// Supplied at construction (the `merge_delegate` argument) and installed into diff --git a/serf-reactor/src/driver/quic/mod.rs b/serf-reactor/src/driver/quic/mod.rs index 47bc120f..cb9b7868 100644 --- a/serf-reactor/src/driver/quic/mod.rs +++ b/serf-reactor/src/driver/quic/mod.rs @@ -76,6 +76,10 @@ use crate::command::CachedCoordinateCmd; use crate::command::{KeyCmd, ListKeysCmd}; #[cfg(encryption)] use crate::delegate::KeyringDelegate; +#[cfg(encryption)] +use crate::driver::shared::{ + AppliedKeyRequest, KEYRING_PERSIST_POLL_INTERVAL, PendingKeyResponse, settle_parked_key_response, +}; use crate::{ Channel, command::{ @@ -330,6 +334,11 @@ where /// Delivery accounting for the leave fan-out: the local-failure flag and the /// epoch gate spacing ICMP-class retry passes across distinct instants. farewell: FarewellAccounting, + /// Key-management responses parked until the keyring delegate acknowledges + /// each rotation's persistence, bounded by the requester's response + /// deadline. + #[cfg(encryption)] + pending_key_responses: Vec>, /// Parked `Shutdown` replies — acked only after the UDP socket drops, so a caller /// resuming from `shutdown().await` can rebind the same address. A `Vec` because /// several callers can race `shutdown()`. @@ -410,6 +419,8 @@ where left_cluster_seen: false, leave_drain_deadline: None, farewell: FarewellAccounting::new(), + #[cfg(encryption)] + pending_key_responses: Vec::new(), shutdown_reply: Vec::new(), recv_buf: vec![0u8; buf_len.max(1)], iter_drain_cap: driver_opts.iter_drain_cap().max(1), @@ -1072,16 +1083,12 @@ where // compact past the threshold) — a change is durable once this poll // returns. Restart replay + `load_snapshot` recovers the state. if let Some(snap) = self.snapshotter.as_mut() { - match ev { - Event::Member(me) => { - use serf_proto::event::MemberEventKind as MK; - let alive = matches!(me.kind(), MK::Join | MK::Update); - for m in me.members() { - snap.append_member(alive, m.node()); - } + if let Event::Member(me) = ev { + use serf_proto::event::MemberEventKind as MK; + let alive = matches!(me.kind(), MK::Join | MK::Update); + for m in me.members() { + snap.append_member(alive, m.node()); } - Event::LeftCluster => snap.append_leave(), - _ => {} } if matches!(ev, Event::Member(_) | Event::LeftCluster) { snap.append_clocks( @@ -1089,6 +1096,14 @@ where LamportTime::from(self.endpoint.event_time()), LamportTime::from(self.endpoint.query_time()), ); + // The leave marker is written AFTER the clocks so a clean shutdown + // ends the file at the Leave record — the shape compaction preserves + // and replay expects: under the default no-rejoin posture the Leave + // wipes the accumulated state, and a clock record written after it + // would resurrect a clock the reference implementation zeroes. + if matches!(ev, Event::LeftCluster) { + snap.append_leave(); + } let endpoint = &self.endpoint; snap.flush_and_maybe_compact(|| { endpoint @@ -1139,10 +1154,20 @@ where } #[cfg(encryption)] if let Event::KeyRequest(req) = ev { - let resp = self.apply_key_request_live(req); - // Ignoring Err: `respond_key` fails only when the response cannot be routed; - // the key op has already applied to the live wire keyring. - let _ = self.endpoint.respond_key(req, resp, Instant::now()); + match self.apply_key_request_live(req) { + AppliedKeyRequest::Ready(resp) => { + // Ignoring Err: `respond_key` fails only when the response cannot be + // routed; the key op has already applied to the live wire keyring. + let _ = self.endpoint.respond_key(req, resp, Instant::now()); + } + AppliedKeyRequest::AwaitingPersistence(resp, rx) => { + self.pending_key_responses.push(PendingKeyResponse { + req: req.clone(), + resp, + rx, + }); + } + } } } @@ -1154,27 +1179,82 @@ where /// the coordinator's live `encryption_options`, applies the op variant-exactly /// against the live ring, and on a real mutation publishes the rotated ring back /// via `set_encryption_options` — re-keying the gossip datagram plane (the QUIC - /// reliable path always skips, quinn encrypts the stream) — then notifies the - /// keyring observer for persistence. A node with no keyring configured answers - /// `result = false` and makes no wire change; a read-only `list` or a refused op - /// leaves the wire untouched. + /// reliable path always skips, quinn encrypts the stream) — then hands the + /// ring to the keyring delegate: a durable-inline answer responds + /// immediately, while out-of-band persistence parks the response until its + /// acknowledgement resolves. A node with no keyring configured answers + /// `result = false` and makes no wire change; a read-only `list` or a + /// refused op leaves the wire untouched. #[cfg(encryption)] - fn apply_key_request_live(&mut self, req: &KeyRequest) -> KeyResponseArgs { + fn apply_key_request_live(&mut self, req: &KeyRequest) -> AppliedKeyRequest { let mut encryption = self.endpoint.encryption_options().clone(); let Some(current) = encryption.keyring() else { - return KeyResponseArgs { + return AppliedKeyRequest::Ready(KeyResponseArgs { result: false, message: "no keyring configured on this node".into(), ..Default::default() - }; + }); }; let (resp, rotated) = serf_driver::apply_key_request(current, req.op(), req.key()).into_parts(); - if let Some(new_ring) = rotated { - encryption.set_keyring(new_ring.clone()); - self.endpoint.set_encryption_options(encryption); - self.keyring.keyring_updated(&new_ring); + match rotated { + Some(new_ring) => { + encryption.set_keyring(new_ring.clone()); + self.endpoint.set_encryption_options(encryption); + match self.keyring.keyring_updated(&new_ring) { + crate::KeyringPersistence::Durable => AppliedKeyRequest::Ready(resp), + crate::KeyringPersistence::Pending(rx) => { + AppliedKeyRequest::AwaitingPersistence(resp, rx) + } + } + } + None => AppliedKeyRequest::Ready(resp), + } + } + + /// Settle parked key responses: send those whose persistence + /// acknowledgement resolved (as-is on success, downgraded to a failure + /// carrying the error otherwise), drop those whose requester's response + /// deadline passed while the acknowledgement was still pending (nothing + /// useful can be routed), keep the rest parked. Returns whether any + /// response was sent (the queued transmit needs a follow-up poll to flush). + #[cfg(encryption)] + fn reap_pending_key_responses(&mut self, now: Instant) -> bool { + let mut sent = false; + let mut i = 0; + while i < self.pending_key_responses.len() { + let entry = &self.pending_key_responses[i]; + match settle_parked_key_response(&entry.rx, &entry.resp) { + Some(resp) => { + let entry = self.pending_key_responses.swap_remove(i); + // Ignoring Err: `respond_key` fails only when the response cannot be + // routed; the key op has already applied to the live wire keyring. + let _ = self.endpoint.respond_key(&entry.req, resp, now); + sent = true; + } + None if now >= entry.req.deadline() => { + drop(self.pending_key_responses.swap_remove(i)); + #[cfg(feature = "tracing")] + tracing::warn!( + "a keyring persistence acknowledgement outlived the key request's response deadline; the response was dropped" + ); + } + None => i += 1, + } + } + sent + } + + /// Next instant a parked key response should be re-polled, folded into the + /// idle-arm timer target; `None` when nothing is parked. The persistence + /// acknowledgement arrives on a plain channel with no waker integration, so + /// the interval bounds the wait. + #[cfg(encryption)] + fn next_key_ack_check(&self, now: Instant) -> Option { + if self.pending_key_responses.is_empty() { + None + } else { + Some(now + KEYRING_PERSIST_POLL_INTERVAL) } - resp } /// Reap await-result join waiters on the deadline timer (the reply terminal), @@ -1417,6 +1497,12 @@ where // `ExchangeCompleted` events would be partially skipped, leaving contacted // addresses unaccounted in the `Err` tuple. `account_event` folds every // terminal completion into the matching pending join as it drains. + // Parked key responses settle before the drain so a response sent here + // flushes in this same pass; entries still pending when the pump exits + // drop with it (their requesters' deadlines cover the loss). Ignoring + // the sent flag: the drain below flushes queued transmits regardless. + #[cfg(encryption)] + let _ = this.reap_pending_key_responses(Instant::now()); loop { let (_, drain_more) = this.drain_surfaces(cx); if !drain_more { @@ -1572,6 +1658,14 @@ where more = true; } + // Settle parked key responses whose persistence acknowledgement resolved; + // a sent response queues transmits the follow-up poll flushes. + #[cfg(encryption)] + if this.reap_pending_key_responses(now) { + progress = true; + more = true; + } + // Timer + deadline reaps under the shared-UDP-path quiescence gate. On QUIC // every resolving input (a reliable push/pull completion, a probe Ack) rides // the one UDP recv, so — with no disjoint FIFO to watermark — the non-premature @@ -1665,6 +1759,10 @@ where if let Some(epoch) = this.farewell_retry_deadline() { target = target.min(epoch); } + #[cfg(encryption)] + if let Some(check) = this.next_key_ack_check(now) { + target = target.min(check); + } if this.arm_and_poll_timer(target, now, cx) { more = true; } diff --git a/serf-reactor/src/driver/shared.rs b/serf-reactor/src/driver/shared.rs index 4bd617a5..af236760 100644 --- a/serf-reactor/src/driver/shared.rs +++ b/serf-reactor/src/driver/shared.rs @@ -285,6 +285,76 @@ fn classify_errored_farewell(icmp_errors: u8, err: &io::Error) -> ErroredFarewel pub(crate) const LEAVE_DRAIN_TEARDOWN_BOUND: core::time::Duration = core::time::Duration::from_secs(1); +/// Cadence at which a pump re-polls parked key responses awaiting the keyring +/// delegate's persistence acknowledgement. The acknowledgement arrives on a +/// plain channel with no waker integration, so while any response is parked +/// the idle-arm timer target is bounded by this interval; a file-write +/// acknowledgement resolves in milliseconds, so one rotation costs a handful +/// of extra polls and a quiescent pump pays nothing. +#[cfg(all(any(feature = "tcp", feature = "quic"), encryption))] +pub(crate) const KEYRING_PERSIST_POLL_INTERVAL: core::time::Duration = + core::time::Duration::from_millis(1); + +/// A key-management response parked until the keyring delegate acknowledges +/// the rotation's persistence, bounded by the requester's response deadline. +#[cfg(all(any(feature = "tcp", feature = "quic"), encryption))] +pub(crate) struct PendingKeyResponse { + pub(crate) req: serf_proto::event::KeyRequest, + pub(crate) resp: serf_proto::event::KeyResponseArgs, + pub(crate) rx: crate::KeyringPersistRx, +} + +/// The pump-facing outcome of applying one inbound key-management request to +/// the live wire keyring. +#[cfg(all(any(feature = "tcp", feature = "quic"), encryption))] +pub(crate) enum AppliedKeyRequest { + /// No rotation needed out-of-band persistence (a `list`, a refused op, a + /// node with no keyring, or a delegate durable inline): respond now. + Ready(serf_proto::event::KeyResponseArgs), + /// A rotation was applied to the wire and handed to the keyring delegate: + /// the response waits for the persistence acknowledgement. + AwaitingPersistence(serf_proto::event::KeyResponseArgs, crate::KeyringPersistRx), +} + +/// Fold one acknowledgement poll into a parked key response: `None` keeps it +/// parked; `Some` is the final response to send — unchanged on a persisted +/// rotation, downgraded to a failure carrying the error otherwise. The +/// reference implementation folds its keyring-file write error into the +/// response the same way, with the live wire keyring keeping the rotation. +/// A disconnected sender counts as a failure: the worker vanished without +/// acknowledging. +#[cfg(all(any(feature = "tcp", feature = "quic"), encryption))] +pub(crate) fn settle_parked_key_response( + rx: &crate::KeyringPersistRx, + resp: &serf_proto::event::KeyResponseArgs, +) -> Option { + use std::sync::mpsc::TryRecvError; + match rx.try_recv() { + Ok(Ok(())) => Some(resp.clone()), + Ok(Err(e)) => Some(failed_key_response( + resp, + format!("keyring rotated on the wire but not persisted: {e}"), + )), + Err(TryRecvError::Disconnected) => Some(failed_key_response( + resp, + "keyring rotated on the wire but not persisted: the persistence worker exited without acknowledging".to_string(), + )), + Err(TryRecvError::Empty) => None, + } +} + +/// `resp` downgraded to a failed key response carrying `message`. +#[cfg(all(any(feature = "tcp", feature = "quic"), encryption))] +fn failed_key_response( + resp: &serf_proto::event::KeyResponseArgs, + message: String, +) -> serf_proto::event::KeyResponseArgs { + let mut failed = resp.clone(); + failed.result = false; + failed.message = message.into(); + failed +} + /// Record the outcome of one readiness-based leave-farewell datagram send into /// `retained`: /// - `Ready(Ok)` — it left the socket; nothing to retain. diff --git a/serf-reactor/src/driver/snapshotter/tests.rs b/serf-reactor/src/driver/snapshotter/tests.rs index 191d2e1e..94139d92 100644 --- a/serf-reactor/src/driver/snapshotter/tests.rs +++ b/serf-reactor/src/driver/snapshotter/tests.rs @@ -211,6 +211,67 @@ fn compaction_preserves_the_clean_leave_gate() { cleanup(&o); } +/// A clean-leave tail replays identically before and after compaction, under +/// both rejoin postures. The pumps append the clocks BEFORE the leave marker +/// — the compacted terminal shape — so compaction can never change what a +/// restart recovers: were the order reversed, the original file would replay +/// clock floors the no-rejoin posture is supposed to zero, while its +/// compacted replacement zeroed them. +#[test] +fn leave_tail_replays_identically_across_compaction() { + // Identical production-ordered appends (member, clocks, leave — the + // pumps' account_event order); only the threshold differs, so one flush + // compacts and the other keeps the original records. + let plain = opts("leave-order-plain"); + let compacted = opts("leave-order-compacted").with_compact_threshold(1); + for o in [&plain, &compacted] { + let (mut snap, _) = Snapshotter::::open(o).expect("open"); + snap.append_member(true, &node("peer", 7001)); + snap.append_clocks( + LamportTime::new(8), + LamportTime::new(2), + LamportTime::new(1), + ); + snap.append_leave(); + snap.flush_and_maybe_compact(|| vec![node("peer", 7001)]); + } + + let (_p, plain_records) = Snapshotter::::open(&plain).expect("reopen the original"); + let (_c, compacted_records) = + Snapshotter::::open(&compacted).expect("reopen the compacted"); + for rejoin in [false, true] { + let original = ReplayResult::replay(plain_records.clone(), rejoin); + let rewritten = ReplayResult::replay(compacted_records.clone(), rejoin); + assert_eq!( + original.alive_nodes, rewritten.alive_nodes, + "membership must replay identically across compaction (rejoin: {rejoin})" + ); + assert_eq!( + ( + original.last_clock, + original.last_event_clock, + original.last_query_clock + ), + ( + rewritten.last_clock, + rewritten.last_event_clock, + rewritten.last_query_clock + ), + "clock floors must replay identically across compaction (rejoin: {rejoin})" + ); + } + // And both match the reference semantics: a clean leave zeroes the clocks + // and empties the membership unless the rejoin posture ignores it. + let fresh = ReplayResult::replay(plain_records, false); + assert!(fresh.alive_nodes.is_empty()); + assert_eq!(fresh.last_clock, LamportTime::ZERO); + let rejoined = ReplayResult::replay(compacted_records, true); + assert_eq!(rejoined.alive_nodes, vec![node("peer", 7001)]); + assert_eq!(rejoined.last_clock, LamportTime::new(8)); + cleanup(&plain); + cleanup(&compacted); +} + /// Membership activity after a leave clears the clean-left state: the next /// compaction does not re-emit a stale Leave marker over live members. #[test] diff --git a/serf-reactor/src/driver/stream/mod.rs b/serf-reactor/src/driver/stream/mod.rs index 1dd945d2..1551e60b 100644 --- a/serf-reactor/src/driver/stream/mod.rs +++ b/serf-reactor/src/driver/stream/mod.rs @@ -58,6 +58,10 @@ use crate::command::CachedCoordinateCmd; use crate::command::{KeyCmd, ListKeysCmd}; #[cfg(encryption)] use crate::delegate::KeyringDelegate; +#[cfg(encryption)] +use crate::driver::shared::{ + AppliedKeyRequest, KEYRING_PERSIST_POLL_INTERVAL, PendingKeyResponse, settle_parked_key_response, +}; use crate::{ Channel, command::{ @@ -369,6 +373,11 @@ where /// Delivery accounting for the leave fan-out: the local-failure flag and the /// epoch gate spacing ICMP-class retry passes across distinct instants. farewell: FarewellAccounting, + /// Key-management responses parked until the keyring delegate acknowledges + /// each rotation's persistence, bounded by the requester's response + /// deadline. + #[cfg(encryption)] + pending_key_responses: Vec>, /// Parked `Shutdown` replies — acked only after the bind sockets drop, so a /// caller resuming from `shutdown().await` can rebind the same address. A `Vec` /// because several callers can race `shutdown()`. @@ -515,6 +524,8 @@ where left_cluster_seen: false, leave_drain_deadline: None, farewell: FarewellAccounting::new(), + #[cfg(encryption)] + pending_key_responses: Vec::new(), shutdown_reply: Vec::new(), bridges: HashMap::new(), accepted_rx, @@ -1317,16 +1328,12 @@ where // compact past the threshold) — a change is durable once this poll // returns. Restart replay + `load_snapshot` recovers the state. if let Some(snap) = self.snapshotter.as_mut() { - match ev { - Event::Member(me) => { - use serf_proto::event::MemberEventKind as MK; - let alive = matches!(me.kind(), MK::Join | MK::Update); - for m in me.members() { - snap.append_member(alive, m.node()); - } + if let Event::Member(me) = ev { + use serf_proto::event::MemberEventKind as MK; + let alive = matches!(me.kind(), MK::Join | MK::Update); + for m in me.members() { + snap.append_member(alive, m.node()); } - Event::LeftCluster => snap.append_leave(), - _ => {} } if matches!(ev, Event::Member(_) | Event::LeftCluster) { snap.append_clocks( @@ -1334,6 +1341,14 @@ where LamportTime::from(self.endpoint.event_time()), LamportTime::from(self.endpoint.query_time()), ); + // The leave marker is written AFTER the clocks so a clean shutdown + // ends the file at the Leave record — the shape compaction preserves + // and replay expects: under the default no-rejoin posture the Leave + // wipes the accumulated state, and a clock record written after it + // would resurrect a clock the reference implementation zeroes. + if matches!(ev, Event::LeftCluster) { + snap.append_leave(); + } let endpoint = &self.endpoint; snap.flush_and_maybe_compact(|| { endpoint @@ -1384,10 +1399,20 @@ where } #[cfg(encryption)] if let Event::KeyRequest(req) = ev { - let resp = self.apply_key_request_live(req); - // Ignoring Err: `respond_key` fails only when the response cannot be routed; - // the key op has already applied to the live wire keyring. - let _ = self.endpoint.respond_key(req, resp, Instant::now()); + match self.apply_key_request_live(req) { + AppliedKeyRequest::Ready(resp) => { + // Ignoring Err: `respond_key` fails only when the response cannot be + // routed; the key op has already applied to the live wire keyring. + let _ = self.endpoint.respond_key(req, resp, Instant::now()); + } + AppliedKeyRequest::AwaitingPersistence(resp, rx) => { + self.pending_key_responses.push(PendingKeyResponse { + req: req.clone(), + resp, + rx, + }); + } + } } } @@ -1399,26 +1424,81 @@ where /// the coordinator's live `encryption_options`, applies the op variant-exactly /// against the live ring, and on a real mutation publishes the rotated ring back /// via `set_encryption_options` — so the gossip and reliable planes re-key in - /// lockstep — then notifies the keyring observer for persistence. A node with no - /// keyring configured answers `result = false` and makes no wire change; a - /// read-only `list` or a refused op leaves the wire untouched. + /// lockstep — then hands the ring to the keyring delegate: a durable-inline + /// answer responds immediately, while out-of-band persistence parks the + /// response until its acknowledgement resolves. A node with no keyring + /// configured answers `result = false` and makes no wire change; a read-only + /// `list` or a refused op leaves the wire untouched. #[cfg(encryption)] - fn apply_key_request_live(&mut self, req: &KeyRequest) -> KeyResponseArgs { + fn apply_key_request_live(&mut self, req: &KeyRequest) -> AppliedKeyRequest { let mut encryption = self.endpoint.encryption_options().clone(); let Some(current) = encryption.keyring() else { - return KeyResponseArgs { + return AppliedKeyRequest::Ready(KeyResponseArgs { result: false, message: "no keyring configured on this node".into(), ..Default::default() - }; + }); }; let (resp, rotated) = serf_driver::apply_key_request(current, req.op(), req.key()).into_parts(); - if let Some(new_ring) = rotated { - encryption.set_keyring(new_ring.clone()); - self.endpoint.set_encryption_options(encryption); - self.keyring.keyring_updated(&new_ring); + match rotated { + Some(new_ring) => { + encryption.set_keyring(new_ring.clone()); + self.endpoint.set_encryption_options(encryption); + match self.keyring.keyring_updated(&new_ring) { + crate::KeyringPersistence::Durable => AppliedKeyRequest::Ready(resp), + crate::KeyringPersistence::Pending(rx) => { + AppliedKeyRequest::AwaitingPersistence(resp, rx) + } + } + } + None => AppliedKeyRequest::Ready(resp), + } + } + + /// Settle parked key responses: send those whose persistence + /// acknowledgement resolved (as-is on success, downgraded to a failure + /// carrying the error otherwise), drop those whose requester's response + /// deadline passed while the acknowledgement was still pending (nothing + /// useful can be routed), keep the rest parked. Returns whether any + /// response was sent (the queued transmit needs a follow-up poll to flush). + #[cfg(encryption)] + fn reap_pending_key_responses(&mut self, now: Instant) -> bool { + let mut sent = false; + let mut i = 0; + while i < self.pending_key_responses.len() { + let entry = &self.pending_key_responses[i]; + match settle_parked_key_response(&entry.rx, &entry.resp) { + Some(resp) => { + let entry = self.pending_key_responses.swap_remove(i); + // Ignoring Err: `respond_key` fails only when the response cannot be + // routed; the key op has already applied to the live wire keyring. + let _ = self.endpoint.respond_key(&entry.req, resp, now); + sent = true; + } + None if now >= entry.req.deadline() => { + drop(self.pending_key_responses.swap_remove(i)); + #[cfg(feature = "tracing")] + tracing::warn!( + "a keyring persistence acknowledgement outlived the key request's response deadline; the response was dropped" + ); + } + None => i += 1, + } + } + sent + } + + /// Next instant a parked key response should be re-polled, folded into the + /// idle-arm timer target; `None` when nothing is parked. The persistence + /// acknowledgement arrives on a plain channel with no waker integration, so + /// the interval bounds the wait. + #[cfg(encryption)] + fn next_key_ack_check(&self, now: Instant) -> Option { + if self.pending_key_responses.is_empty() { + None + } else { + Some(now + KEYRING_PERSIST_POLL_INTERVAL) } - resp } /// Reap await-result join waiters on the deadline timer (the reply terminal), @@ -1707,6 +1787,13 @@ where // the queue empties or the deadline wins. Dropping the socket then // closes its UDP FD synchronously. if this.socket.is_some() { + // Parked key responses settle before the drain so a response sent + // here flushes in this same pass; entries still pending when the pump + // exits drop with it (their requesters' deadlines cover the loss). + // Ignoring the sent flag: the drain below flushes queued transmits + // regardless. + #[cfg(encryption)] + let _ = this.reap_pending_key_responses(Instant::now()); loop { let (_, drain_more, _) = this.drain_surfaces(cx); if !drain_more { @@ -1952,6 +2039,14 @@ where more = true; } + // Settle parked key responses whose persistence acknowledgement resolved; + // a sent response queues transmits the follow-up poll flushes. + #[cfg(encryption)] + if this.reap_pending_key_responses(now) { + progress = true; + more = true; + } + // Timer + deadline reaps under two RESIDENCE-SCOPED gates (replacing the old // fixed-count deferral, which fired prematurely at a low `iter_drain_cap` or a // large exchange, and could starve under a flood). Every join/leave-resolving @@ -2070,6 +2165,10 @@ where if let Some(epoch) = this.farewell_retry_deadline() { target = target.min(epoch); } + #[cfg(encryption)] + if let Some(check) = this.next_key_ack_check(now) { + target = target.min(check); + } this.arm_timer(target, now); if let Some(timer) = this.timer.as_mut() && timer.as_mut().poll(cx).is_ready() diff --git a/serf-reactor/src/lib.rs b/serf-reactor/src/lib.rs index 958217c1..f02e03cc 100644 --- a/serf-reactor/src/lib.rs +++ b/serf-reactor/src/lib.rs @@ -167,7 +167,9 @@ pub use delegate::{FileKeyringDelegate, KeyringFileError}; docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] -pub use delegate::{KeyringDelegate, VoidKeyringDelegate}; +pub use delegate::{ + KeyringDelegate, KeyringPersistError, KeyringPersistRx, KeyringPersistence, VoidKeyringDelegate, +}; /// Gossip-encryption config types re-exported from `memberlist-proto`, so a /// caller can build a transport's `with_encryption` keyring without naming diff --git a/serf-reactor/tests/key_rotation.rs b/serf-reactor/tests/key_rotation.rs index 116c9dbe..dff32d1e 100644 --- a/serf-reactor/tests/key_rotation.rs +++ b/serf-reactor/tests/key_rotation.rs @@ -67,12 +67,15 @@ impl RecordingKeyring { } impl KeyringDelegate for RecordingKeyring { - fn keyring_updated(&self, keyring: &Keyring) { + fn keyring_updated(&self, keyring: &Keyring) -> serf_reactor::KeyringPersistence { self .rings .lock() .expect("keyring log not poisoned") .push(keyring.clone()); + // The in-memory record is durable the moment it is pushed, so the key + // response goes out immediately. + serf_reactor::KeyringPersistence::Durable } } @@ -324,6 +327,70 @@ where b.shutdown().await.expect("rot-b shuts down"); } +/// With node B persisting through a [`serf_reactor::FileKeyringDelegate`], an +/// `install_key` from A still collects BOTH nodes' successful responses — B's +/// response is parked until the file write is acknowledged, then routed within +/// the query window — and because the response was gated on that +/// acknowledgement, the persisted file already carries the new key when the +/// response arrives. +async fn file_backed_rotation_gates_the_response_on_persistence() +where + R: Runtime, +{ + let k1 = secret_key(0x33); + let k2 = secret_key(0x44); + + let mut path = std::env::temp_dir(); + // Keyed by runtime as well as pid: the tokio and smol cells run + // concurrently in one test binary and must not share a file. + path.push(format!( + "serf-key-rotation-file-{}-{}", + std::process::id(), + core::any::type_name::().replace("::", "-"), + )); + // Ignoring Err: a leftover file from a previous run is fine to lose. + let _ = std::fs::remove_file(&path); + + let enc = || EncryptionOptions::new().with_keyring(Keyring::new(k1)); + let b = spawn_encrypted_node::( + "file-b", + enc(), + Arc::new(serf_reactor::FileKeyringDelegate::new(&path)), + ) + .await; + let a = spawn_encrypted_node::("file-a", enc(), Arc::new(RecordingKeyring::default())).await; + let b_addr = b.advertise_address(); + + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B over the encrypted reliable plane"); + converge(&a, &b).await; + + let mut a_events = a.events(); + a.install_key(k2).await.expect("install_key dispatched"); + let kr = next_key_response::(&mut a_events).await; + assert!( + kr.num_resp >= 2, + "install_key must collect a response from BOTH nodes, including the one parked on file persistence (num_resp={})", + kr.num_resp + ); + assert_eq!(kr.num_err, 0, "install_key must succeed on every node"); + + let persisted = serf_reactor::FileKeyringDelegate::new(&path) + .load() + .expect("the acknowledged write parses") + .expect("the acknowledged write exists"); + assert!( + persisted.secondaries().contains(&k2) || persisted.primary_ref() == &k2, + "the response was gated on persistence, so the file already holds the installed key" + ); + + a.shutdown().await.expect("file-a shuts down"); + b.shutdown().await.expect("file-b shuts down"); + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} + // The tokio cell: the runtime-generic scenario driven on tokio's multi-thread // runtime. Gated on `tokio` so the `--test key_rotation -- smol` build can drop the // `agnostic/tokio` code path. @@ -335,6 +402,11 @@ mod tokio_cells { async fn key_rotation_across_two_nodes_rotates_both_live_keyrings() { super::key_rotation_across_two_nodes_rotates_both_live_keyrings::().await; } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn file_backed_rotation_gates_the_response_on_persistence() { + super::file_backed_rotation_gates_the_response_on_persistence::().await; + } } // The smol cell: the identical scenario over `SmolRuntime`, driven by smol's @@ -349,4 +421,11 @@ mod smol_cells { super::key_rotation_across_two_nodes_rotates_both_live_keyrings::(), ); } + + #[test] + fn file_backed_rotation_gates_the_response_on_persistence_smol() { + SmolRuntime::block_on( + super::file_backed_rotation_gates_the_response_on_persistence::(), + ); + } } diff --git a/serf-reactor/tests/tcp.rs b/serf-reactor/tests/tcp.rs index a3103768..1f6253c1 100644 --- a/serf-reactor/tests/tcp.rs +++ b/serf-reactor/tests/tcp.rs @@ -677,9 +677,18 @@ where } /// A unique snapshot path under the system temp dir. -fn snapshot_path(name: &str) -> std::path::PathBuf { +fn snapshot_path(name: &str) -> std::path::PathBuf +where + R: Runtime, +{ let mut p = std::env::temp_dir(); - p.push(format!("serf-e2e-snap-{name}-{}", std::process::id())); + // Keyed by runtime as well as pid: the tokio and smol cells run + // concurrently in one test binary and must not share a snapshot file. + p.push(format!( + "serf-e2e-snap-{name}-{}-{}", + std::process::id(), + core::any::type_name::().replace("::", "-"), + )); // Ignoring Err: a leftover file from a previous run is fine to lose. let _ = std::fs::remove_file(&p); p @@ -693,7 +702,7 @@ async fn snapshot_restart_rejoins_the_cluster() where R: Runtime, { - let path = snapshot_path("rejoin"); + let path = snapshot_path::("rejoin"); let a = spawn_node::("snap-a").await; let b = spawn_node_with_snapshot::("snap-b", serf_reactor::SnapshotOptions::new(&path), false).await; @@ -731,7 +740,7 @@ async fn snapshot_leave_gate_controls_rejoin() where R: Runtime, { - let path = snapshot_path("leave-gate"); + let path = snapshot_path::("leave-gate"); let a = spawn_node::("gate-a").await; let b = spawn_node_with_snapshot::("gate-b", serf_reactor::SnapshotOptions::new(&path), false).await; @@ -746,6 +755,30 @@ where b.leave().await.expect("gate-b leaves gracefully"); b.shutdown().await.expect("gate-b shuts down"); + // The pump writes the clock floors BEFORE the leave marker, so a clean + // shutdown ends the file at the Leave record — the terminal shape replay + // expects and compaction preserves. A record written after it would + // resurrect state the default posture is supposed to zero. + { + let bytes = std::fs::read(&path).expect("the snapshot survives the leave"); + let mut records = Vec::new(); + let mut cursor = 0usize; + while cursor < bytes.len() { + let (rec, used) = + serf_proto::snapshot::SnapshotRecord::::decode(&bytes[cursor..]) + .expect("a clean-leave snapshot decodes whole"); + records.push(rec); + cursor += used; + } + assert!( + matches!( + records.last(), + Some(serf_proto::snapshot::SnapshotRecord::Leave) + ), + "a clean shutdown must end the snapshot at the Leave record" + ); + } + // Default posture: the leave clears the recovered state — no auto-rejoin. let b2 = spawn_node_with_snapshot::("gate-b", serf_reactor::SnapshotOptions::new(&path), false).await; From 83a034ec7c035b488b8b4cfb067d09166092075e Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 17:01:09 +1200 Subject: [PATCH 107/128] fix(serf-reactor): keyring sweep must not delete a .tmp-named destination; make the rename durable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A destination whose own extension is tmp is its with_extension image, so the construction sweep was deleting the persisted keyring itself — and under the previous implementation such a destination was written in place, so the file can hold the only copy of legitimate key material. The sweep now skips the legacy path when it equals the destination, and the random-suffix matcher accepts only the exact .name.16-hex.tmp shape so an operator's own sibling files are never this delegate's to delete. A completed rename is not crash-durable until the directory entry is: the acknowledgement — which releases a successful key response to the cluster — now follows a sync of the containing directory, so a crash after the response cannot revert the node to the old ring while peers believe the rotation durable. On non-Unix platforms std cannot open a directory handle; durability is left to the filesystem's metadata journaling there. A rotation into a directory that cannot be synced acknowledges failure. KeyringPersistence is must_use so a future call site cannot silently drop the acknowledgement. --- serf-reactor/src/delegate/keyring_file/mod.rs | 53 ++++++++++++--- .../src/delegate/keyring_file/tests.rs | 67 +++++++++++++++++++ serf-reactor/src/delegate/mod.rs | 1 + 3 files changed, 111 insertions(+), 10 deletions(-) diff --git a/serf-reactor/src/delegate/keyring_file/mod.rs b/serf-reactor/src/delegate/keyring_file/mod.rs index e616f705..e3caecf4 100644 --- a/serf-reactor/src/delegate/keyring_file/mod.rs +++ b/serf-reactor/src/delegate/keyring_file/mod.rs @@ -193,16 +193,35 @@ fn write_via_exclusive_temp(path: &Path, contents: &[u8]) -> io::Result<()> { } file.write_all(contents)?; file.sync_all()?; - std::fs::rename(&tmp, path) + std::fs::rename(&tmp, path)?; + // The rename is not durable until the DIRECTORY entry is: a crash after + // returning `Ok` here must not revert the destination to the old file — + // the acknowledgement built on this return is what releases a successful + // key response to the cluster. + sync_dir(dir.unwrap_or_else(|| Path::new("."))) })(); if res.is_err() { // Ignoring Err: removing the failed temp (it holds key bytes) is - // best-effort hygiene; the write error itself is what propagates. + // best-effort hygiene; the write error itself is what propagates. A + // failure after the rename consumed the temp removes nothing. let _ = std::fs::remove_file(&tmp); } res } +/// Sync a directory so a completed rename of an entry inside it survives a +/// crash. On non-Unix platforms `std` cannot open a directory handle; the +/// rename's durability is left to the filesystem's metadata journaling there. +#[cfg(unix)] +fn sync_dir(dir: &Path) -> io::Result<()> { + std::fs::File::open(dir)?.sync_all() +} + +#[cfg(not(unix))] +fn sync_dir(_dir: &Path) -> io::Result<()> { + Ok(()) +} + /// OS-entropy nonce for a temp-file name: unpredictable, so a /// directory-writing attacker cannot pre-plant a file or symlink at the next /// temp path. @@ -213,14 +232,21 @@ fn temp_nonce() -> io::Result { /// Remove leftovers a rotation can no longer reuse: the fixed-name sibling /// temp earlier releases wrote (whose permissions predate the owner-only -/// guarantee and may already hold key material), and any `.{name}.*.tmp` -/// temps a crashed rotation abandoned. Removing a planted symlink unlinks -/// the LINK, never its target. Best-effort: a sweep failure never blocks -/// construction — `create_new` already keeps every future write off any -/// path that survives. +/// guarantee and may already hold key material), and the exact-shape +/// `.{name}.{16 hex}.tmp` temps a crashed rotation abandoned. Removing a +/// planted symlink unlinks the LINK, never its target. Best-effort: a sweep +/// failure never blocks construction — `create_new` already keeps every +/// future write off any path that survives. fn sweep_stale_temps(path: &Path) { - // Ignoring Err: nothing to sweep, or no permission — both non-fatal. - let _ = std::fs::remove_file(path.with_extension("tmp")); + // A destination whose own extension is `tmp` IS its `with_extension` + // image: sweeping it would delete the persisted keyring at construction + // (the previous implementation wrote such a destination in place, so the + // file can hold the only copy of legitimate key material). + let legacy = path.with_extension("tmp"); + if legacy != path { + // Ignoring Err: nothing to sweep, or no permission — both non-fatal. + let _ = std::fs::remove_file(legacy); + } let (Some(dir), Some(name)) = ( path.parent().filter(|d| !d.as_os_str().is_empty()), path.file_name().and_then(|n| n.to_str()), @@ -236,7 +262,14 @@ fn sweep_stale_temps(path: &Path) { let Some(f) = file_name.to_str() else { continue; }; - if f.starts_with(&prefix) && f.ends_with(".tmp") { + // Exact-shape match only — a sibling file that merely shares the prefix + // and suffix (an operator's own backup, say) is not this delegate's to + // delete. + let matches_temp_shape = f + .strip_prefix(&prefix) + .and_then(|rest| rest.strip_suffix(".tmp")) + .is_some_and(|mid| mid.len() == 16 && mid.bytes().all(|b| b.is_ascii_hexdigit())); + if matches_temp_shape { // Ignoring Err: best-effort sweep of abandoned temps. let _ = std::fs::remove_file(entry.path()); } diff --git a/serf-reactor/src/delegate/keyring_file/tests.rs b/serf-reactor/src/delegate/keyring_file/tests.rs index f23d7818..b6d0ba7e 100644 --- a/serf-reactor/src/delegate/keyring_file/tests.rs +++ b/serf-reactor/src/delegate/keyring_file/tests.rs @@ -240,6 +240,73 @@ fn a_planted_symlink_never_receives_key_bytes() { let _ = std::fs::remove_file(&victim); } +/// A destination whose own extension is `tmp` is NOT its legacy temp: the +/// construction sweep must preserve it — the previous implementation wrote +/// such a destination in place, so the file can hold the only copy of the +/// keyring. +#[test] +fn a_tmp_extension_destination_survives_construction() { + let path = tmp_path("selfnamed").with_extension("tmp"); + // Ignoring Err: a leftover file from a previous run is about to be rewritten. + let _ = std::fs::remove_file(&path); + + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes128([10u8; 16]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = SecretKey::ChaCha20Poly1305([10u8; 32]); + { + let delegate = FileKeyringDelegate::new(&path); + acked(delegate.keyring_updated(&Keyring::new(key))).expect("the rotation persists"); + } + + let reopened = FileKeyringDelegate::new(&path); + let loaded = reopened + .load() + .expect("the persisted keyring parses") + .expect("constructing a delegate must not sweep a .tmp-named destination"); + assert_eq!(loaded.primary_ref(), &key); + + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} + +/// The success acknowledgement is durability: a rotation whose parent +/// directory cannot be synced reports failure, because the completed rename +/// is not crash-durable until the directory entry is. +#[cfg(unix)] +#[test] +fn an_unsyncable_directory_fails_the_acknowledgement() { + use std::os::unix::fs::PermissionsExt as _; + + let mut dir = std::env::temp_dir(); + dir.push(format!("serf-keyring-unsync-{}", std::process::id())); + // Ignoring Err: a leftover directory from a previous run is fine to reuse. + let _ = std::fs::create_dir(&dir); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).expect("open the dir"); + let path = dir.join("ring"); + let delegate = FileKeyringDelegate::new(&path); + + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes128([11u8; 16]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = SecretKey::ChaCha20Poly1305([11u8; 32]); + + // Write+search without read: the temp creation, write, and rename all + // succeed, but the directory handle needed for the durability sync cannot + // be opened — the acknowledgement must report that as a failure. + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o300)) + .expect("make the dir unsyncable"); + let outcome = acked(delegate.keyring_updated(&Keyring::new(key))); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).expect("restore the dir"); + assert!( + outcome.is_err(), + "an un-syncable rename must not acknowledge success" + ); + + // Ignoring Err: best-effort test-tree cleanup. + let _ = std::fs::remove_dir_all(&dir); +} + /// A write failure is acknowledged as an error — the response gate's failure /// signal — not silently swallowed. #[test] diff --git a/serf-reactor/src/delegate/mod.rs b/serf-reactor/src/delegate/mod.rs index aa022cfe..5878bec7 100644 --- a/serf-reactor/src/delegate/mod.rs +++ b/serf-reactor/src/delegate/mod.rs @@ -245,6 +245,7 @@ pub type KeyringPersistRx = std::sync::mpsc::Receiver Date: Sun, 12 Jul 2026 17:11:53 +1200 Subject: [PATCH 108/128] fix(serf-reactor): case-insensitive tmp-alias guard; make FileKeyringDelegate Unix-only MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The legacy-sweep guard compared paths lexically, so a destination named with an uppercase TMP extension differed from its lowercase with_extension image while aliasing the same file on the case-insensitive filesystems that are the default on macOS and Windows — construction could still delete the only persisted keyring. The guard now treats any ASCII case of the tmp extension as aliasing the destination and skips the sweep; the regression rotates into a .TMP-named destination and proves load returns it after reconstruction. The acknowledgement contract is rename durability, and no safe standard API can flush a directory entry on Windows — rather than acknowledge a rotation a power loss could revert, the turnkey delegate is now compile-time Unix-only (the trait and acknowledgement types stay portable; a Windows application implements KeyringDelegate with a platform-durable strategy). The file-backed rotation e2e is gated accordingly. --- serf-reactor/src/delegate/keyring_file/mod.rs | 47 ++++++++++--------- .../src/delegate/keyring_file/tests.rs | 34 ++++++++++++-- serf-reactor/src/delegate/mod.rs | 6 +-- serf-reactor/src/lib.rs | 4 +- serf-reactor/tests/key_rotation.rs | 3 ++ 5 files changed, 63 insertions(+), 31 deletions(-) diff --git a/serf-reactor/src/delegate/keyring_file/mod.rs b/serf-reactor/src/delegate/keyring_file/mod.rs index e3caecf4..fe010d29 100644 --- a/serf-reactor/src/delegate/keyring_file/mod.rs +++ b/serf-reactor/src/delegate/keyring_file/mod.rs @@ -33,11 +33,17 @@ use super::{KeyringDelegate, KeyringPersistError, KeyringPersistence}; /// then atomically renamed over the destination: a crash mid-write never /// truncates the previous ring, a restrictive mode on the key file is never /// widened by a rotation, and key bytes can never land in a pre-existing -/// inode or behind a planted symlink. On non-Unix platforms there is no mode -/// to assert — the file inherits the destination directory's ACLs, so place -/// it in an owner-restricted directory. Dropping the delegate joins the +/// inode or behind a planted symlink. Dropping the delegate joins the /// worker after it drains every queued rotation, so a shutdown cannot discard /// a write that was already acknowledged toward the wire. +/// +/// UNIX-ONLY: the acknowledgement contract is rename durability — the +/// containing directory is synced before a rotation reports success — and no +/// safe standard API can flush a directory entry on Windows, so this turnkey +/// delegate does not exist there rather than acknowledge a rotation a power +/// loss could revert. A Windows application implements [`KeyringDelegate`] +/// itself with a platform-durable strategy (a write-through rename via the +/// platform APIs, or storage with its own durability contract). pub struct FileKeyringDelegate { path: PathBuf, /// Hand-off to the persistence thread; `None` only during drop, which hangs @@ -152,9 +158,7 @@ fn persist(path: &Path, keyring: &Keyring) -> Result<(), KeyringFileError> { /// keeps such a path from being plantable ahead of time. The inode is born /// `0600` on Unix and re-asserted on the open handle, so raw key material /// only ever lands in a fresh owner-only inode this process created; a crash -/// mid-write never truncates the previous ring. On non-Unix platforms there -/// is no mode to assert: the fresh inode inherits the parent directory's -/// ACLs, so the destination directory itself must be access-restricted. +/// mid-write never truncates the previous ring. fn write_via_exclusive_temp(path: &Path, contents: &[u8]) -> io::Result<()> { use std::io::Write as _; let name = path @@ -171,7 +175,6 @@ fn write_via_exclusive_temp(path: &Path, contents: &[u8]) -> io::Result<()> { }; let mut opts = std::fs::OpenOptions::new(); opts.write(true).create_new(true); - #[cfg(unix)] { use std::os::unix::fs::OpenOptionsExt as _; opts.mode(0o600); @@ -186,7 +189,6 @@ fn write_via_exclusive_temp(path: &Path, contents: &[u8]) -> io::Result<()> { } }; let res = (|| { - #[cfg(unix)] { use std::os::unix::fs::PermissionsExt as _; file.set_permissions(std::fs::Permissions::from_mode(0o600))?; @@ -210,18 +212,11 @@ fn write_via_exclusive_temp(path: &Path, contents: &[u8]) -> io::Result<()> { } /// Sync a directory so a completed rename of an entry inside it survives a -/// crash. On non-Unix platforms `std` cannot open a directory handle; the -/// rename's durability is left to the filesystem's metadata journaling there. -#[cfg(unix)] +/// crash. fn sync_dir(dir: &Path) -> io::Result<()> { std::fs::File::open(dir)?.sync_all() } -#[cfg(not(unix))] -fn sync_dir(_dir: &Path) -> io::Result<()> { - Ok(()) -} - /// OS-entropy nonce for a temp-file name: unpredictable, so a /// directory-writing attacker cannot pre-plant a file or symlink at the next /// temp path. @@ -238,14 +233,20 @@ fn temp_nonce() -> io::Result { /// failure never blocks construction — `create_new` already keeps every /// future write off any path that survives. fn sweep_stale_temps(path: &Path) { - // A destination whose own extension is `tmp` IS its `with_extension` - // image: sweeping it would delete the persisted keyring at construction - // (the previous implementation wrote such a destination in place, so the - // file can hold the only copy of legitimate key material). - let legacy = path.with_extension("tmp"); - if legacy != path { + // A destination whose own extension is `tmp` — in ANY case — must keep + // its `with_extension` image: lexically the paths can differ (`ring.TMP` + // versus `ring.tmp`) yet alias the same file on the case-insensitive + // filesystems that are the default on macOS and Windows, and the previous + // implementation wrote such a destination in place, so the file can hold + // the only copy of legitimate key material. Sweeping hygiene is never + // worth risking the persisted keyring. + let legacy_aliases_destination = path + .extension() + .and_then(|e| e.to_str()) + .is_some_and(|e| e.eq_ignore_ascii_case("tmp")); + if !legacy_aliases_destination { // Ignoring Err: nothing to sweep, or no permission — both non-fatal. - let _ = std::fs::remove_file(legacy); + let _ = std::fs::remove_file(path.with_extension("tmp")); } let (Some(dir), Some(name)) = ( path.parent().filter(|d| !d.as_os_str().is_empty()), diff --git a/serf-reactor/src/delegate/keyring_file/tests.rs b/serf-reactor/src/delegate/keyring_file/tests.rs index b6d0ba7e..973b04f7 100644 --- a/serf-reactor/src/delegate/keyring_file/tests.rs +++ b/serf-reactor/src/delegate/keyring_file/tests.rs @@ -140,7 +140,6 @@ fn a_second_rotation_replaces_the_first() { /// one: the replacing temp inode is born owner-only, so a fresh file is /// created `0600` and a pre-existing `0644` destination is `0600` after the /// next rotation, even under a permissive umask. -#[cfg(unix)] #[test] fn rotation_enforces_owner_only_permissions() { use std::os::unix::fs::PermissionsExt as _; @@ -202,7 +201,6 @@ fn construction_sweeps_stale_temps() { /// its target — and no rotation ever writes through it: the exclusive /// creation refuses any pre-existing path, so key bytes cannot be redirected /// into an attacker-chosen file. -#[cfg(unix)] #[test] fn a_planted_symlink_never_receives_key_bytes() { let path = tmp_path("symlink"); @@ -270,10 +268,40 @@ fn a_tmp_extension_destination_survives_construction() { let _ = std::fs::remove_file(&path); } +/// The alias guard is case-insensitive: a destination named with an +/// uppercase `TMP` extension lexically differs from its lowercase +/// `with_extension` image, yet the two alias the same file on the +/// case-insensitive filesystems that are the default on macOS — construction +/// must preserve it. +#[test] +fn an_uppercase_tmp_destination_survives_construction() { + let path = tmp_path("selfnamed-upper").with_extension("TMP"); + // Ignoring Err: a leftover file from a previous run is about to be rewritten. + let _ = std::fs::remove_file(&path); + + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes128([12u8; 16]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = SecretKey::ChaCha20Poly1305([12u8; 32]); + { + let delegate = FileKeyringDelegate::new(&path); + acked(delegate.keyring_updated(&Keyring::new(key))).expect("the rotation persists"); + } + + let reopened = FileKeyringDelegate::new(&path); + let loaded = reopened + .load() + .expect("the persisted keyring parses") + .expect("constructing a delegate must not sweep a case-aliased .TMP destination"); + assert_eq!(loaded.primary_ref(), &key); + + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} + /// The success acknowledgement is durability: a rotation whose parent /// directory cannot be synced reports failure, because the completed rename /// is not crash-durable until the directory entry is. -#[cfg(unix)] #[test] fn an_unsyncable_directory_fails_the_acknowledgement() { use std::os::unix::fs::PermissionsExt as _; diff --git a/serf-reactor/src/delegate/mod.rs b/serf-reactor/src/delegate/mod.rs index 5878bec7..568779de 100644 --- a/serf-reactor/src/delegate/mod.rs +++ b/serf-reactor/src/delegate/mod.rs @@ -13,16 +13,16 @@ //! block), and `MergeDelegate` is the machine's inline admission veto supplied at //! construction rather than an observation hook. -#[cfg(encryption)] +#[cfg(all(encryption, unix))] mod keyring_file; mod void; pub use void::VoidDelegate; -#[cfg(encryption)] +#[cfg(all(encryption, unix))] #[cfg_attr( docsrs, - doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + doc(cfg(all(any(feature = "aes-gcm", feature = "chacha20-poly1305"), unix))) )] pub use keyring_file::{FileKeyringDelegate, KeyringFileError}; #[cfg(encryption)] diff --git a/serf-reactor/src/lib.rs b/serf-reactor/src/lib.rs index f02e03cc..cd810ba6 100644 --- a/serf-reactor/src/lib.rs +++ b/serf-reactor/src/lib.rs @@ -156,10 +156,10 @@ pub use delegate::{ Delegate, MemberDelegate, MergeDelegate, QueryDelegate, UserEventDelegate, VoidDelegate, }; -#[cfg(encryption)] +#[cfg(all(encryption, unix))] #[cfg_attr( docsrs, - doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + doc(cfg(all(any(feature = "aes-gcm", feature = "chacha20-poly1305"), unix))) )] pub use delegate::{FileKeyringDelegate, KeyringFileError}; #[cfg(encryption)] diff --git a/serf-reactor/tests/key_rotation.rs b/serf-reactor/tests/key_rotation.rs index dff32d1e..22d7e91d 100644 --- a/serf-reactor/tests/key_rotation.rs +++ b/serf-reactor/tests/key_rotation.rs @@ -333,6 +333,7 @@ where /// the query window — and because the response was gated on that /// acknowledgement, the persisted file already carries the new key when the /// response arrives. +#[cfg(unix)] async fn file_backed_rotation_gates_the_response_on_persistence() where R: Runtime, @@ -403,6 +404,7 @@ mod tokio_cells { super::key_rotation_across_two_nodes_rotates_both_live_keyrings::().await; } + #[cfg(unix)] #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn file_backed_rotation_gates_the_response_on_persistence() { super::file_backed_rotation_gates_the_response_on_persistence::().await; @@ -422,6 +424,7 @@ mod smol_cells { ); } + #[cfg(unix)] #[test] fn file_backed_rotation_gates_the_response_on_persistence_smol() { SmolRuntime::block_on( From 3ab9b9dff1b428a434a899830a03e28fb271a47d Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 17:20:17 +1200 Subject: [PATCH 109/128] fix(serf-reactor): gate the keyring sweep on filesystem identity, not names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A symlinked destination defeats every name-based alias guard: a configured ring.current pointing at a file that lives at the legacy temp name has an extension that never case-folds to tmp, so the sweep unlinked the symlink's target — the only persisted keyring — and left the destination dangling. Every sweep candidate (the legacy fixed name and the random-suffix temps alike) is now gated on resolved device/inode identity against the destination: a candidate reaching the destination's storage — through lexical identity, a case-folding filesystem, or a symlink on either side — is never touched, a dangling link or a provably distinct file is swept, and unresolvable identity conservatively skips. This subsumes the extension case-fold guard and restores hygiene for the mixed-case corner it over-skipped. The regression seeds a ring at the legacy-named path, links the configured destination to it, and proves construction-then-load preserves the keyring. --- serf-reactor/src/delegate/keyring_file/mod.rs | 44 +++++++++++++------ .../src/delegate/keyring_file/tests.rs | 36 +++++++++++++++ 2 files changed, 66 insertions(+), 14 deletions(-) diff --git a/serf-reactor/src/delegate/keyring_file/mod.rs b/serf-reactor/src/delegate/keyring_file/mod.rs index fe010d29..2560042c 100644 --- a/serf-reactor/src/delegate/keyring_file/mod.rs +++ b/serf-reactor/src/delegate/keyring_file/mod.rs @@ -228,23 +228,16 @@ fn temp_nonce() -> io::Result { /// Remove leftovers a rotation can no longer reuse: the fixed-name sibling /// temp earlier releases wrote (whose permissions predate the owner-only /// guarantee and may already hold key material), and the exact-shape -/// `.{name}.{16 hex}.tmp` temps a crashed rotation abandoned. Removing a +/// `.{name}.{16 hex}.tmp` temps a crashed rotation abandoned. Every removal +/// is gated on FILESYSTEM IDENTITY against the destination — a candidate +/// that resolves to the destination's storage (lexical identity, a +/// case-folding filesystem, or a symlink on either side) is never touched, +/// so sweeping hygiene can never delete the persisted keyring. Removing a /// planted symlink unlinks the LINK, never its target. Best-effort: a sweep /// failure never blocks construction — `create_new` already keeps every /// future write off any path that survives. fn sweep_stale_temps(path: &Path) { - // A destination whose own extension is `tmp` — in ANY case — must keep - // its `with_extension` image: lexically the paths can differ (`ring.TMP` - // versus `ring.tmp`) yet alias the same file on the case-insensitive - // filesystems that are the default on macOS and Windows, and the previous - // implementation wrote such a destination in place, so the file can hold - // the only copy of legitimate key material. Sweeping hygiene is never - // worth risking the persisted keyring. - let legacy_aliases_destination = path - .extension() - .and_then(|e| e.to_str()) - .is_some_and(|e| e.eq_ignore_ascii_case("tmp")); - if !legacy_aliases_destination { + if sweepable(path, &path.with_extension("tmp")) { // Ignoring Err: nothing to sweep, or no permission — both non-fatal. let _ = std::fs::remove_file(path.with_extension("tmp")); } @@ -270,13 +263,36 @@ fn sweep_stale_temps(path: &Path) { .strip_prefix(&prefix) .and_then(|rest| rest.strip_suffix(".tmp")) .is_some_and(|mid| mid.len() == 16 && mid.bytes().all(|b| b.is_ascii_hexdigit())); - if matches_temp_shape { + if matches_temp_shape && sweepable(path, &entry.path()) { // Ignoring Err: best-effort sweep of abandoned temps. let _ = std::fs::remove_file(entry.path()); } } } +/// Whether removing `candidate` cannot touch the keyring the destination +/// `path` reaches. `metadata` FOLLOWS symlinks, so each side resolves to the +/// file a reader would actually open; only a candidate proven to resolve to +/// DIFFERENT storage — or to alias nothing at all — is sweepable. +fn sweepable(path: &Path, candidate: &Path) -> bool { + use std::os::unix::fs::MetadataExt as _; + if std::fs::symlink_metadata(candidate).is_err() { + // Nothing at the candidate name; removal would be a no-op. + return false; + } + match (std::fs::metadata(candidate), std::fs::metadata(path)) { + // A dangling symlink at the candidate name reaches no storage at all. + (Err(e), _) if e.kind() == io::ErrorKind::NotFound => true, + (Ok(c), Ok(d)) => (c.dev(), c.ino()) != (d.dev(), d.ino()), + // The destination resolves to nothing: on any filesystem where the + // candidate name could alias it, resolving the destination would have + // found the candidate's file — the two are genuinely distinct. + (Ok(_), Err(e)) if e.kind() == io::ErrorKind::NotFound => true, + // Identity cannot be established: never delete on uncertainty. + _ => false, + } +} + impl KeyringDelegate for FileKeyringDelegate { fn keyring_updated(&self, keyring: &Keyring) -> KeyringPersistence { // Non-blocking hand-off: the pump must never wait on storage. A failed diff --git a/serf-reactor/src/delegate/keyring_file/tests.rs b/serf-reactor/src/delegate/keyring_file/tests.rs index 973b04f7..44470182 100644 --- a/serf-reactor/src/delegate/keyring_file/tests.rs +++ b/serf-reactor/src/delegate/keyring_file/tests.rs @@ -299,6 +299,42 @@ fn an_uppercase_tmp_destination_survives_construction() { let _ = std::fs::remove_file(&path); } +/// A symlinked destination keeps its target through construction: with the +/// configured path pointing at a file that happens to live at the legacy +/// temp name, the sweep resolves filesystem IDENTITY — not names — and +/// leaves the keyring intact for the follow-up load. +#[test] +fn a_symlinked_destination_keeps_its_target_through_construction() { + let target = tmp_path("linked").with_extension("tmp"); + let link = tmp_path("linked").with_extension("current"); + // Ignoring Err: leftovers from a previous run are about to be recreated. + let _ = std::fs::remove_file(&link); + let _ = std::fs::remove_file(&target); + + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes128([13u8; 16]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = SecretKey::ChaCha20Poly1305([13u8; 32]); + { + // Persist a real ring at the target path (itself a tmp-named + // destination, which construction must already preserve). + let seed = FileKeyringDelegate::new(&target); + acked(seed.keyring_updated(&Keyring::new(key))).expect("the seed rotation persists"); + } + std::os::unix::fs::symlink(&target, &link).expect("plant the destination symlink"); + + let delegate = FileKeyringDelegate::new(&link); + let loaded = delegate + .load() + .expect("the symlinked keyring parses") + .expect("construction must not sweep the storage a symlinked destination resolves to"); + assert_eq!(loaded.primary_ref(), &key); + + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&link); + let _ = std::fs::remove_file(&target); +} + /// The success acknowledgement is durability: a rotation whose parent /// directory cannot be synced reports failure, because the completed rename /// is not crash-durable until the directory entry is. From 3f6f6fca8632cc83f7e2ca3bdb1345e5af0c7616 Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 17:27:22 +1200 Subject: [PATCH 110/128] fix(serf-reactor): name guards precede the sweep's identity gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sequential identity observations race a concurrent rotation: for a candidate whose NAME can itself name the destination — lexically, or equal under the ASCII case folding that aliases names on case-insensitive filesystems — a rename landing between the two metadata reads makes the identities differ and authorizes unlinking the freshly persisted keyring. remove_file unlinks a name, so for those shapes no point-in-time identity observation can ever make the unlink safe: they are refused unconditionally, before any filesystem access. The identity gate remains for genuinely distinct names, where the unlink cannot remove the destination's entry and resolved device/inode still guards the symlink and hard-link shapes. The regression widens the observation gap through a test seam, lands an atomic replacement mid-check exactly as a rotation would, and proves construction never unlinks the destination. --- serf-reactor/src/delegate/keyring_file/mod.rs | 43 ++++++++++-- .../src/delegate/keyring_file/tests.rs | 66 +++++++++++++++++++ 2 files changed, 102 insertions(+), 7 deletions(-) diff --git a/serf-reactor/src/delegate/keyring_file/mod.rs b/serf-reactor/src/delegate/keyring_file/mod.rs index 2560042c..51fe0185 100644 --- a/serf-reactor/src/delegate/keyring_file/mod.rs +++ b/serf-reactor/src/delegate/keyring_file/mod.rs @@ -271,22 +271,51 @@ fn sweep_stale_temps(path: &Path) { } /// Whether removing `candidate` cannot touch the keyring the destination -/// `path` reaches. `metadata` FOLLOWS symlinks, so each side resolves to the -/// file a reader would actually open; only a candidate proven to resolve to -/// DIFFERENT storage — or to alias nothing at all — is sweepable. +/// `path` reaches. +/// +/// `remove_file` unlinks a NAME, so the guards are layered by what a name +/// can do: a candidate whose name can itself name the destination — +/// lexically, or equal under the ASCII case folding that aliases names on +/// case-insensitive filesystems — is NEVER sweepable, because a rotation's +/// rename can land between any identity observation and the unlink, and the +/// unlink would then remove whatever the destination name holds (the freshly +/// persisted keyring). Only for genuinely distinct names — where the unlink +/// cannot remove the destination's entry — is resolved filesystem identity +/// consulted: `metadata` FOLLOWS symlinks, so each side resolves to the file +/// a reader would actually open, and a candidate reaching the destination's +/// storage through a symlink or hard link is skipped. fn sweepable(path: &Path, candidate: &Path) -> bool { use std::os::unix::fs::MetadataExt as _; + if candidate == path { + return false; + } + match ( + candidate.file_name().and_then(|n| n.to_str()), + path.file_name().and_then(|n| n.to_str()), + ) { + (Some(c), Some(p)) if c.eq_ignore_ascii_case(p) => return false, + (Some(_), Some(_)) => {} + // Un-inspectable names: never delete on uncertainty. + _ => return false, + } if std::fs::symlink_metadata(candidate).is_err() { // Nothing at the candidate name; removal would be a no-op. return false; } - match (std::fs::metadata(candidate), std::fs::metadata(path)) { + let candidate_meta = std::fs::metadata(candidate); + // The window between the two observations is where a concurrent rename + // lands; the tests widen it deterministically to prove the name guards + // above — not luck — are what keep a mid-check rename safe. + #[cfg(test)] + tests::between_identity_observations(path); + let destination_meta = std::fs::metadata(path); + match (candidate_meta, destination_meta) { // A dangling symlink at the candidate name reaches no storage at all. (Err(e), _) if e.kind() == io::ErrorKind::NotFound => true, (Ok(c), Ok(d)) => (c.dev(), c.ino()) != (d.dev(), d.ino()), - // The destination resolves to nothing: on any filesystem where the - // candidate name could alias it, resolving the destination would have - // found the candidate's file — the two are genuinely distinct. + // The destination resolves to nothing: with the name-aliasing shapes + // already excluded above, an existing candidate cannot BE the missing + // destination — the two are genuinely distinct. (Ok(_), Err(e)) if e.kind() == io::ErrorKind::NotFound => true, // Identity cannot be established: never delete on uncertainty. _ => false, diff --git a/serf-reactor/src/delegate/keyring_file/tests.rs b/serf-reactor/src/delegate/keyring_file/tests.rs index 44470182..b88addab 100644 --- a/serf-reactor/src/delegate/keyring_file/tests.rs +++ b/serf-reactor/src/delegate/keyring_file/tests.rs @@ -1,5 +1,33 @@ use super::*; +use std::cell::RefCell; + +/// This thread's mid-check hook, when a test installed one. +type ObservationGap = RefCell>>; + +thread_local! { + /// Per-test hook fired between `sweepable`'s two identity observations — + /// the window a concurrent rotation's rename can land in. Thread-local so + /// parallel tests never see each other's hooks. + static BETWEEN_OBSERVATIONS: ObservationGap = const { RefCell::new(None) }; +} + +/// Called by `sweepable` between its two `metadata` observations. +pub(super) fn between_identity_observations(path: &Path) { + BETWEEN_OBSERVATIONS.with(|hook| { + if let Some(f) = hook.borrow_mut().as_mut() { + f(path); + } + }); +} + +/// Install `f` as this thread's mid-check hook for the duration of `run`. +fn with_observation_gap(f: impl FnMut(&Path) + 'static, run: impl FnOnce()) { + BETWEEN_OBSERVATIONS.with(|hook| *hook.borrow_mut() = Some(Box::new(f))); + run(); + BETWEEN_OBSERVATIONS.with(|hook| *hook.borrow_mut() = None); +} + /// Wait for one rotation's persistence acknowledgement. fn acked(p: KeyringPersistence) -> Result<(), KeyringPersistError> { match p { @@ -335,6 +363,44 @@ fn a_symlinked_destination_keeps_its_target_through_construction() { let _ = std::fs::remove_file(&target); } +/// A rename landing between the sweep's two identity observations must +/// never lose the destination: `remove_file` unlinks a NAME, so a candidate +/// whose name can itself name the destination is refused by the name guard +/// BEFORE any identity observation — the widened observation gap here +/// atomically replaces the destination exactly as a concurrent rotation +/// would, and construction must leave the replacement intact. +#[test] +fn a_rename_landing_mid_check_never_loses_the_destination() { + let path = tmp_path("raced").with_extension("tmp"); + std::fs::write(&path, "pre-rotation contents\n").expect("seed the destination"); + + with_observation_gap( + |dest: &Path| { + // The concurrent rotation: a fresh inode atomically renamed over the + // destination, mid-check. + let staged = dest.with_file_name(".raced-replacement"); + std::fs::write(&staged, "freshly persisted contents\n").expect("stage the replacement"); + std::fs::rename(&staged, dest).expect("land the replacement"); + }, + || { + let _delegate = FileKeyringDelegate::new(&path); + }, + ); + + // With the name guard in place the observation gap is never reached for a + // name-aliasing candidate, so the original contents remain; what must hold + // in every world is that the destination NAME was not unlinked. + let contents = + std::fs::read_to_string(&path).expect("the destination survives construction un-unlinked"); + assert!( + contents == "pre-rotation contents\n" || contents == "freshly persisted contents\n", + "the destination holds one of the two written generations, never nothing" + ); + + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} + /// The success acknowledgement is durability: a rotation whose parent /// directory cannot be synced reports failure, because the completed rename /// is not crash-durable until the directory entry is. From 68ce9a8e3f14c9a1e244ad844eb84f55a3384c1f Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 17:36:23 +1200 Subject: [PATCH 111/128] fix(serf-reactor): legacy sweep only for provably distinct destination extensions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ASCII case folding is not the complete filename-alias relation on every supported filesystem: case-insensitive HFS+ ignores certain Unicode scalars when comparing names, so a destination extension carrying one — tmp, say — aliases the generated legacy image while passing both name guards, reopening the mid-check deletion race there. The legacy image differs from the destination only in its extension, so the sweep now runs only when that extension is provably distinct from tmp: pure ASCII and not ASCII-case-folding to tmp, or absent entirely (the image then appends four non-ignorable characters no folding can absorb); any non-ASCII scalar conservatively refuses. Random-suffix candidates need no classifier — their names carry a dot prefix and 16-hex infix the destination's name does not, an excess of non-ignorable ASCII no alias relation can erase. The classifier regression covers the tmp case folds, an HFS+-ignorable scalar, and a non-ASCII extension; the behavioral regression proves an unprovable extension keeps its legacy sibling on every filesystem — exactly the file that IS the persisted keyring where the alias folds. --- serf-reactor/src/delegate/keyring_file/mod.rs | 25 +++++++++- .../src/delegate/keyring_file/tests.rs | 49 +++++++++++++++++++ 2 files changed, 73 insertions(+), 1 deletion(-) diff --git a/serf-reactor/src/delegate/keyring_file/mod.rs b/serf-reactor/src/delegate/keyring_file/mod.rs index 51fe0185..8811fff9 100644 --- a/serf-reactor/src/delegate/keyring_file/mod.rs +++ b/serf-reactor/src/delegate/keyring_file/mod.rs @@ -237,7 +237,7 @@ fn temp_nonce() -> io::Result { /// failure never blocks construction — `create_new` already keeps every /// future write off any path that survives. fn sweep_stale_temps(path: &Path) { - if sweepable(path, &path.with_extension("tmp")) { + if legacy_temp_provably_distinct(path) && sweepable(path, &path.with_extension("tmp")) { // Ignoring Err: nothing to sweep, or no permission — both non-fatal. let _ = std::fs::remove_file(path.with_extension("tmp")); } @@ -270,6 +270,29 @@ fn sweep_stale_temps(path: &Path) { } } +/// Whether the fixed-name legacy temp's name is PROVABLY distinct from the +/// destination's under every supported filename-alias relation. +/// +/// The legacy image differs from the destination only in its extension, so +/// the alias question reduces to the extension replacement. Sweeping is +/// allowed only when the destination's extension is pure ASCII and does not +/// ASCII-case-fold to `tmp` — case-insensitive HFS+ additionally IGNORES +/// certain Unicode scalars when comparing names, so an extension carrying +/// any non-ASCII scalar could fold the two names together and is +/// conservatively refused. No extension at all is safe: the image then +/// APPENDS `.tmp`, four non-ignorable ASCII characters no folding can +/// absorb. Random-suffix candidates need no such classifier — their names +/// carry a dot prefix and a 16-hex infix the destination's name does not, +/// an excess of non-ignorable ASCII no alias relation can erase. +fn legacy_temp_provably_distinct(path: &Path) -> bool { + match path.extension() { + None => true, + Some(e) => e + .to_str() + .is_some_and(|e| e.is_ascii() && !e.eq_ignore_ascii_case("tmp")), + } +} + /// Whether removing `candidate` cannot touch the keyring the destination /// `path` reaches. /// diff --git a/serf-reactor/src/delegate/keyring_file/tests.rs b/serf-reactor/src/delegate/keyring_file/tests.rs index b88addab..11892333 100644 --- a/serf-reactor/src/delegate/keyring_file/tests.rs +++ b/serf-reactor/src/delegate/keyring_file/tests.rs @@ -363,6 +363,55 @@ fn a_symlinked_destination_keeps_its_target_through_construction() { let _ = std::fs::remove_file(&target); } +/// The legacy-sweep classifier admits only extensions provably distinct +/// from `tmp` under every supported filename-alias relation: pure-ASCII +/// non-tmp extensions (or none, where the image appends characters no +/// folding can absorb) pass; any tmp case-fold or any non-ASCII scalar — +/// case-insensitive HFS+ ignores certain Unicode scalars when comparing +/// names, so `ring.t\u{200D}mp` aliases `ring.tmp` there — is refused. +#[test] +fn the_legacy_sweep_classifier_refuses_unprovable_extensions() { + for (destination, distinct) in [ + ("ring.keys", true), + ("ring", true), + ("ring.tmp", false), + ("ring.TMP", false), + ("ring.Tmp", false), + // An HFS+-ignorable scalar (ZERO WIDTH JOINER) inside the extension. + ("ring.t\u{200D}mp", false), + // Any non-ASCII scalar is unprovable, ignorable or not. + ("ring.cl\u{00E9}s", false), + ] { + assert_eq!( + legacy_temp_provably_distinct(Path::new(destination)), + distinct, + "{destination:?}" + ); + } +} + +/// A destination whose extension is not provably distinct keeps its legacy +/// image on EVERY filesystem: whether or not the running filesystem folds +/// the two names together, the conservative refusal leaves the sibling file +/// alone — the fold-aliasing filesystems are exactly where that sibling IS +/// the persisted keyring. +#[test] +fn an_unprovable_extension_keeps_the_legacy_sibling() { + let path = tmp_path("ignorable").with_extension("t\u{200D}mp"); + let sibling = path.with_extension("tmp"); + std::fs::write(&sibling, "possibly the persisted keyring\n").expect("seed the sibling"); + + let _delegate = FileKeyringDelegate::new(&path); + assert!( + sibling.exists(), + "an unprovable extension must refuse the legacy sweep" + ); + + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&sibling); + let _ = std::fs::remove_file(&path); +} + /// A rename landing between the sweep's two identity observations must /// never lose the destination: `remove_file` unlinks a NAME, so a candidate /// whose name can itself name the destination is refused by the name guard From a7cbeaf4760c9542c94a36f58d8145c4c8715079 Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 18:15:22 +1200 Subject: [PATCH 112/128] =?UTF-8?q?fix(serf):=20first-CI=20repairs=20?= =?UTF-8?q?=E2=80=94=20proto=20re-export=20gate,=20qemu=20lock=20pin,=20wa?= =?UTF-8?q?ke-driven=20farewell=20completion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The secret-key codec re-export rode a cipher-only cfg while the bridge module it re-exports from needs a transport: a coordinates-plus-cipher build dangled. The gate now mirrors the module's full compound condition. The QEMU firmware harness is workspace-excluded with its own lockfile, which still pinned memberlist from before the farewell-compound seam — farewell_capacity did not exist there. Re-pinned to current main. The farewell epoch test demanded send completion on the single poll after eligibility, but a fresh socket's writable readiness may not have reached the reactor yet on a loaded runner — the send legitimately returns Pending and production advances on the registered writable wake. The completion half now drives bounded wake iterations; the epoch-gate assertions are unchanged. --- serf-proto/src/lib.rs | 10 ++++++++-- serf-reactor/src/driver/stream/tests.rs | 18 +++++++++++++++--- tests/serf-embassy-qemu/Cargo.lock | 4 ++-- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/serf-proto/src/lib.rs b/serf-proto/src/lib.rs index bf06461d..b7c4a71a 100644 --- a/serf-proto/src/lib.rs +++ b/serf-proto/src/lib.rs @@ -62,10 +62,16 @@ compile_error!("serf-proto requires the `std` or `alloc` feature"); pub(crate) use any::{AnyMessage, EncodeError}; #[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) use bridge::BridgeError; -#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[cfg(all( + any(feature = "aes-gcm", feature = "chacha20-poly1305"), + any(feature = "tcp", feature = "quic") +))] #[cfg_attr( docsrs, - doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) + doc(cfg(all( + any(feature = "aes-gcm", feature = "chacha20-poly1305"), + any(feature = "tcp", feature = "quic") + ))) )] pub use bridge::{BridgeError as SecretKeyCodecError, secret_key_from_bytes, secret_key_to_bytes}; #[cfg(any(feature = "tcp", feature = "quic"))] diff --git a/serf-reactor/src/driver/stream/tests.rs b/serf-reactor/src/driver/stream/tests.rs index 77eab963..b92f8ebd 100644 --- a/serf-reactor/src/driver/stream/tests.rs +++ b/serf-reactor/src/driver/stream/tests.rs @@ -1296,11 +1296,23 @@ async fn farewell_retry_waits_for_its_epoch_across_polls() { ); } - // The epoch elapses: the very next poll's single retry pass sends it. + // The epoch elapses: an eligible retry hands the datagram to the socket. + // The send can legitimately return `Pending` while the fresh socket's + // writable readiness has not yet reached the reactor (production advances + // on the registered writable wake), so drive bounded wake iterations + // rather than demanding completion on one poll. driver.farewell.retry_after = Some(Instant::now()); - let _ = poll_once(&mut driver); + let mut sent = false; + for _ in 0..64 { + let _ = poll_once(&mut driver); + if driver.leave_drain.is_empty() { + sent = true; + break; + } + TokioRuntime::sleep(Duration::from_millis(5)).await; + } assert!( - driver.leave_drain.is_empty(), + sent, "an eligible retry must hand the retained farewell to the socket" ); assert!( diff --git a/tests/serf-embassy-qemu/Cargo.lock b/tests/serf-embassy-qemu/Cargo.lock index 0479f81c..5642a5e2 100644 --- a/tests/serf-embassy-qemu/Cargo.lock +++ b/tests/serf-embassy-qemu/Cargo.lock @@ -590,7 +590,7 @@ checksum = "0ca88d725a0a943b096803bd34e73a4437208b6077654cc4ecb2947a5f91618d" [[package]] name = "memberlist-embedded" version = "0.1.0" -source = "git+https://github.com/al8n/memberlist?branch=main#2c6d2a15b554675bde472f07130817f984a3d4f5" +source = "git+https://github.com/al8n/memberlist?branch=main#6739bc70b768d77b5380ff883ffe3d3d48d32282" dependencies = [ "bytes", "getrandom", @@ -603,7 +603,7 @@ dependencies = [ [[package]] name = "memberlist-proto" version = "0.4.0" -source = "git+https://github.com/al8n/memberlist?branch=main#2c6d2a15b554675bde472f07130817f984a3d4f5" +source = "git+https://github.com/al8n/memberlist?branch=main#6739bc70b768d77b5380ff883ffe3d3d48d32282" dependencies = [ "buffa", "buffa-build", From 8611703bb2d20f087799b0d5aa8a226bf5c0506b Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 18:41:51 +1200 Subject: [PATCH 113/128] test(serf): cover the folded driver paths CI found dark MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first full-matrix coverage run showed the fold work exercised almost entirely through the tcp suite: the quic pump's snapshotter, key-response parking, and replay threading — and the tls constructor's snapshot threading — had no end-to-end coverage at all. Mirror the leave-gate snapshot scenario (including the on-disk tail-shape assertion) onto the quic driver, add a quic file-backed rotation proving the parked response routes within the query window with the key already durable, and run the leave-gate scenario over tls. Unit coverage for the remaining dark arms: every settle outcome of a parked key response (pending, persisted, failed, worker-vanished), the nameless-destination input error, the snapshot coordinate builders and accessors, and the quic endpoint's operator forwarders (health score, merge-delegate install, coordinate resets). --- serf-driver/src/snapshot/tests.rs | 26 ++ serf-proto/src/quic_endpoint/tests.rs | 31 +++ .../src/delegate/keyring_file/tests.rs | 9 + serf-reactor/src/driver/shared/tests.rs | 42 ++++ serf-reactor/tests/quic.rs | 224 ++++++++++++++++++ serf-reactor/tests/tls.rs | 94 ++++++++ 6 files changed, 426 insertions(+) diff --git a/serf-driver/src/snapshot/tests.rs b/serf-driver/src/snapshot/tests.rs index 96029cc1..be75c80d 100644 --- a/serf-driver/src/snapshot/tests.rs +++ b/serf-driver/src/snapshot/tests.rs @@ -256,3 +256,29 @@ fn stats_assembles_counts_and_ops_readings() { assert_eq!(stats.event_clock(), LamportTime::new(8)); assert_eq!(stats.query_clock(), LamportTime::new(9)); } + +/// The coordinate readings ride the snapshot only when the driver attaches +/// them: absent by default, carried through the builders, surfaced on the +/// aggregate. +#[cfg(feature = "coordinates")] +#[test] +fn coordinate_readings_ride_the_snapshot() { + let members = vec![make_member(1, "127.0.0.1:7951", MemberStatus::Alive)]; + let snap = SerfSnapshot::new( + members, + &1u32, + SerfState::Alive, + LamportTime::ZERO, + LamportTime::ZERO, + LamportTime::ZERO, + ); + assert!(snap.coordinate().is_none(), "absent until attached"); + assert!(snap.stats().coordinate_resets().is_none()); + + let coordinate = serf_proto::typed::Coordinate::default(); + let snap = snap + .with_coordinate(Some(coordinate.clone())) + .with_coordinate_resets(Some(3)); + assert_eq!(snap.coordinate(), Some(&coordinate)); + assert_eq!(snap.stats().coordinate_resets(), Some(3)); +} diff --git a/serf-proto/src/quic_endpoint/tests.rs b/serf-proto/src/quic_endpoint/tests.rs index 4e4ac26e..5a7ba603 100644 --- a/serf-proto/src/quic_endpoint/tests.rs +++ b/serf-proto/src/quic_endpoint/tests.rs @@ -383,3 +383,34 @@ fn loopback_push_pull_converges_member_clock() { "the acceptor learned the dialer (node 1) as a member over the exchange" ); } + +/// The operator forwarders reach the inner machine: a fresh endpoint is +/// healthy (score zero), a merge predicate installs through the coordinator, +/// and the coordinate-reset counter reads through to the client when +/// coordinates are enabled. +#[test] +fn operator_forwarders_reach_the_inner_machine() { + let mut e = ep(1, 7952); + assert_eq!(e.health_score(), 0, "a fresh node is healthy"); + + struct RejectAll; + impl memberlist_proto::delegate::MergeDelegate for RejectAll { + fn notify_merge( + &self, + _peers: memberlist_proto::MaybeOwned< + '_, + [memberlist_proto::typed::NodeState], + >, + ) -> bool { + false + } + } + e.set_merge_delegate(RejectAll); + + #[cfg(feature = "coordinates")] + assert_eq!( + e.coordinate_resets(), + Some(0), + "a fresh coordinate client has reset nothing" + ); +} diff --git a/serf-reactor/src/delegate/keyring_file/tests.rs b/serf-reactor/src/delegate/keyring_file/tests.rs index 11892333..a9204e9d 100644 --- a/serf-reactor/src/delegate/keyring_file/tests.rs +++ b/serf-reactor/src/delegate/keyring_file/tests.rs @@ -535,3 +535,12 @@ fn drop_joins_the_worker_and_flushes_queued_rotations() { // Ignoring Err: best-effort test-file cleanup. let _ = std::fs::remove_file(&path); } + +/// A destination with no file name cannot host a sibling temp: the write +/// reports `InvalidInput` instead of panicking or writing anywhere. +#[test] +fn a_nameless_destination_is_an_input_error() { + let err = write_via_exclusive_temp(Path::new("/"), b"contents") + .expect_err("a bare root has no file name to derive a temp from"); + assert_eq!(err.kind(), io::ErrorKind::InvalidInput); +} diff --git a/serf-reactor/src/driver/shared/tests.rs b/serf-reactor/src/driver/shared/tests.rs index 73e22174..ad118b4e 100644 --- a/serf-reactor/src/driver/shared/tests.rs +++ b/serf-reactor/src/driver/shared/tests.rs @@ -301,3 +301,45 @@ fn leave_outcome_maps_the_failure_flag() { Err(crate::error::SerfError::LeaveFarewellUndelivered) )); } + +/// A parked key response settles by acknowledgement outcome: still-pending +/// keeps it parked; a persisted rotation sends the response unchanged; a +/// persistence failure — or a worker that vanished without acknowledging — +/// downgrades it to `result = false` carrying the error, with the live wire +/// keyring keeping the rotation either way. +#[cfg(encryption)] +#[test] +fn parked_key_responses_settle_by_acknowledgement_outcome() { + use std::sync::mpsc; + + let ok_resp = serf_proto::event::KeyResponseArgs { + result: true, + message: "".into(), + ..Default::default() + }; + + // Still pending: stays parked. + let (tx, rx) = mpsc::channel(); + assert!(settle_parked_key_response(&rx, &ok_resp).is_none()); + + // Persisted: the response goes out unchanged. + tx.send(Ok(())).expect("ack sends"); + let settled = settle_parked_key_response(&rx, &ok_resp).expect("resolved"); + assert!(settled.result); + assert!(settled.message.is_empty()); + + // Persistence failure: downgraded, carrying the error. + let (tx, rx) = mpsc::channel(); + tx.send(Err(std::io::Error::other("disk gone").into())) + .expect("ack sends"); + let settled = settle_parked_key_response(&rx, &ok_resp).expect("resolved"); + assert!(!settled.result); + assert!(settled.message.contains("disk gone")); + + // Worker vanished without acknowledging: a failure, not a silent success. + let (tx, rx) = mpsc::channel::>(); + drop(tx); + let settled = settle_parked_key_response(&rx, &ok_resp).expect("resolved"); + assert!(!settled.result); + assert!(settled.message.contains("without acknowledging")); +} diff --git a/serf-reactor/tests/quic.rs b/serf-reactor/tests/quic.rs index 6a456627..8cc61ddb 100644 --- a/serf-reactor/tests/quic.rs +++ b/serf-reactor/tests/quic.rs @@ -745,6 +745,206 @@ where b.shutdown().await.expect("mis-b shuts down"); } +/// A quic node with snapshot persistence configured. +async fn spawn_node_with_snapshot( + id: &str, + snapshot: serf_reactor::SnapshotOptions, + rejoin_after_leave: bool, +) -> Node +where + R: Runtime, +{ + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + Serf::::quic( + QuicTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(bind)) + .with_quic_config(test_quic_options()), + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new().with_rejoin_after_leave(rejoin_after_leave), + None, + None, + Some(snapshot), + #[cfg(encryption)] + std::sync::Arc::new(VoidKeyringDelegate), + ) + .await + .expect("spawn snapshot-backed serf quic node") +} + +/// A unique snapshot path under the system temp dir, keyed by runtime as well +/// as pid: the tokio and smol cells run concurrently in one test binary and +/// must not share a snapshot file. +fn snapshot_path(name: &str) -> std::path::PathBuf +where + R: Runtime, +{ + let mut p = std::env::temp_dir(); + p.push(format!( + "serf-e2e-quic-snap-{name}-{}-{}", + std::process::id(), + core::any::type_name::().replace("::", "-"), + )); + // Ignoring Err: a leftover file from a previous run is fine to lose. + let _ = std::fs::remove_file(&p); + p +} + +/// The clean-leave gate over the quic driver: a graceful leave ends the +/// snapshot at the Leave record, the default posture starts fresh on restart, +/// and the opt-in posture rejoins from the persisted membership. +async fn snapshot_leave_gate_controls_rejoin() +where + R: Runtime, +{ + let path = snapshot_path::("leave-gate"); + let a = spawn_node::("qgate-a").await; + let b = + spawn_node_with_snapshot::("qgate-b", serf_reactor::SnapshotOptions::new(&path), false) + .await; + let a_addr = a.advertise_address(); + + b.join(&SocketAddrResolver, MaybeResolved::Resolved(a_addr), false) + .await + .expect("join reaches node A"); + converge(&a, &b).await; + + b.leave().await.expect("qgate-b leaves gracefully"); + b.shutdown().await.expect("qgate-b shuts down"); + + // The pump writes the clock floors BEFORE the leave marker, so a clean + // shutdown ends the file at the Leave record — the terminal shape replay + // expects and compaction preserves. + { + let bytes = std::fs::read(&path).expect("the snapshot survives the leave"); + let mut records = Vec::new(); + let mut cursor = 0usize; + while cursor < bytes.len() { + let (rec, used) = + serf_proto::snapshot::SnapshotRecord::::decode(&bytes[cursor..]) + .expect("a clean-leave snapshot decodes whole"); + records.push(rec); + cursor += used; + } + assert!( + matches!( + records.last(), + Some(serf_proto::snapshot::SnapshotRecord::Leave) + ), + "a clean shutdown must end the snapshot at the Leave record" + ); + } + + // Default posture: the leave clears the recovered state — no auto-rejoin. + let b2 = + spawn_node_with_snapshot::("qgate-b", serf_reactor::SnapshotOptions::new(&path), false) + .await; + R::sleep(Duration::from_millis(1500)).await; + assert_eq!( + b2.num_members(), + 1, + "a cleanly-left node must not auto-rejoin unless opted in" + ); + b2.shutdown().await.expect("qgate-b2 shuts down"); + + // Opt-in posture: the Leave marker is ignored and the membership recovers. + let b3 = + spawn_node_with_snapshot::("qgate-b", serf_reactor::SnapshotOptions::new(&path), true).await; + converge(&a, &b3).await; + + a.shutdown().await.expect("qgate-a shuts down"); + b3.shutdown().await.expect("qgate-b3 shuts down"); + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} + +/// A key rotation over the quic driver with node B persisting through a +/// [`serf_reactor::FileKeyringDelegate`]: B's response is parked until the +/// file write is acknowledged and still collected within the query window, +/// with the file already holding the installed key when the response arrives. +#[cfg(unix)] +async fn file_backed_rotation_gates_the_response_on_persistence() +where + R: Runtime, +{ + let k1 = test_secret_key(0x55); + let k2 = test_secret_key(0x66); + + let mut path = std::env::temp_dir(); + path.push(format!( + "serf-quic-key-rotation-file-{}-{}", + std::process::id(), + core::any::type_name::().replace("::", "-"), + )); + // Ignoring Err: a leftover file from a previous run is fine to lose. + let _ = std::fs::remove_file(&path); + + let enc = || EncryptionOptions::new().with_keyring(Keyring::new(k1)); + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let b = Serf::::quic( + QuicTransportOptions::::new() + .with_local_id(SmolStr::new("qfile-b")) + .with_advertise_addr(MaybeResolved::Resolved(bind)) + .with_quic_config(test_quic_options()) + .with_encryption(enc()), + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + None, + None, + None, + std::sync::Arc::new(serf_reactor::FileKeyringDelegate::new(&path)), + ) + .await + .expect("spawn file-backed encrypted quic node"); + let a = spawn_encrypted_node::("qfile-a", enc()).await; + let b_addr = b.advertise_address(); + + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B over the encrypted reliable plane"); + converge(&a, &b).await; + + let mut a_events = a.events(); + a.install_key(k2).await.expect("install_key dispatched"); + let kr = R::timeout(Duration::from_secs(20), async { + loop { + match a_events.next().await { + Some(Event::KeyResponse(kr)) => break kr, + Some(_) => {} + None => panic!("the event stream ended before the key response"), + } + } + }) + .await + .expect("A collects the key response within the timeout"); + assert!( + kr.num_resp >= 2, + "install_key must collect a response from BOTH nodes, including the one parked on file persistence (num_resp={})", + kr.num_resp + ); + assert_eq!(kr.num_err, 0, "install_key must succeed on every node"); + + let persisted = serf_reactor::FileKeyringDelegate::new(&path) + .load() + .expect("the acknowledged write parses") + .expect("the acknowledged write exists"); + assert!( + persisted.secondaries().contains(&k2) || persisted.primary_ref() == &k2, + "the response was gated on persistence, so the file already holds the installed key" + ); + + a.shutdown().await.expect("qfile-a shuts down"); + b.shutdown().await.expect("qfile-b shuts down"); + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} + // The tokio cells: the runtime-generic scenarios driven on tokio's multi-thread // runtime. Gated on the `tokio` feature so the `--test quic -- smol` build (which // enables only `smol`) can drop the `agnostic/tokio` code path. @@ -757,6 +957,17 @@ mod tokio_cells { super::two_node_quic_join_converges::().await; } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn snapshot_leave_gate_controls_rejoin() { + super::snapshot_leave_gate_controls_rejoin::().await; + } + + #[cfg(all(unix, encryption))] + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn file_backed_rotation_gates_the_response_on_persistence() { + super::file_backed_rotation_gates_the_response_on_persistence::().await; + } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn user_event_delivered() { super::user_event_delivered::().await; @@ -808,6 +1019,19 @@ mod tokio_cells { mod smol_cells { use agnostic::{RuntimeLite, smol::SmolRuntime}; + #[test] + fn snapshot_leave_gate_controls_rejoin_smol() { + SmolRuntime::block_on(super::snapshot_leave_gate_controls_rejoin::()); + } + + #[cfg(all(unix, encryption))] + #[test] + fn file_backed_rotation_gates_the_response_on_persistence_smol() { + SmolRuntime::block_on( + super::file_backed_rotation_gates_the_response_on_persistence::(), + ); + } + #[test] fn two_node_quic_join_converges_smol() { SmolRuntime::block_on(super::two_node_quic_join_converges::()); diff --git a/serf-reactor/tests/tls.rs b/serf-reactor/tests/tls.rs index ff1d7054..38c53a2b 100644 --- a/serf-reactor/tests/tls.rs +++ b/serf-reactor/tests/tls.rs @@ -344,6 +344,90 @@ where b.shutdown().await.expect("lv-b shuts down"); } +/// A tls node with snapshot persistence configured. +async fn spawn_node_with_snapshot( + id: &str, + snapshot: serf_reactor::SnapshotOptions, + rejoin_after_leave: bool, +) -> Node +where + R: Runtime, +{ + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + Serf::::tls( + TlsTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(bind)) + .with_tls_options(test_tls_options()), + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new().with_rejoin_after_leave(rejoin_after_leave), + None, + None, + Some(snapshot), + #[cfg(encryption)] + std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), + ) + .await + .expect("spawn snapshot-backed serf tls node") +} + +/// The clean-leave gate over the tls transport: the graceful leave persists, +/// the default posture starts fresh on restart, the opt-in posture rejoins. +async fn snapshot_leave_gate_controls_rejoin() +where + R: Runtime, +{ + let mut path = std::env::temp_dir(); + // Keyed by runtime as well as pid: the tokio and smol cells run + // concurrently in one test binary and must not share a snapshot file. + path.push(format!( + "serf-e2e-tls-snap-leave-gate-{}-{}", + std::process::id(), + core::any::type_name::().replace("::", "-"), + )); + // Ignoring Err: a leftover file from a previous run is fine to lose. + let _ = std::fs::remove_file(&path); + + let a = spawn_node::("tgate-a").await; + let b = + spawn_node_with_snapshot::("tgate-b", serf_reactor::SnapshotOptions::new(&path), false) + .await; + let a_addr = a.advertise_address(); + + b.join(&SocketAddrResolver, MaybeResolved::Resolved(a_addr), false) + .await + .expect("join reaches node A"); + converge(&a, &b).await; + + b.leave().await.expect("tgate-b leaves gracefully"); + b.shutdown().await.expect("tgate-b shuts down"); + + // Default posture: the leave clears the recovered state — no auto-rejoin. + let b2 = + spawn_node_with_snapshot::("tgate-b", serf_reactor::SnapshotOptions::new(&path), false) + .await; + R::sleep(Duration::from_millis(1500)).await; + assert_eq!( + b2.num_members(), + 1, + "a cleanly-left node must not auto-rejoin unless opted in" + ); + b2.shutdown().await.expect("tgate-b2 shuts down"); + + // Opt-in posture: the Leave marker is ignored and the membership recovers. + let b3 = + spawn_node_with_snapshot::("tgate-b", serf_reactor::SnapshotOptions::new(&path), true).await; + converge(&a, &b3).await; + + a.shutdown().await.expect("tgate-a shuts down"); + b3.shutdown().await.expect("tgate-b3 shuts down"); + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} + // The tokio cells: the runtime-generic scenarios driven on tokio's multi-thread // runtime. Gated on the `tokio` feature so the `--test tls -- smol` build (which // enables only `smol`) can drop the `agnostic/tokio` code path. @@ -351,6 +435,11 @@ where mod tokio_cells { use agnostic::tokio::TokioRuntime; + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn snapshot_leave_gate_controls_rejoin() { + super::snapshot_leave_gate_controls_rejoin::().await; + } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn two_node_tls_join_converges() { super::two_node_tls_join_converges::().await; @@ -379,6 +468,11 @@ mod tokio_cells { mod smol_cells { use agnostic::{RuntimeLite, smol::SmolRuntime}; + #[test] + fn snapshot_leave_gate_controls_rejoin_smol() { + SmolRuntime::block_on(super::snapshot_leave_gate_controls_rejoin::()); + } + #[test] fn two_node_tls_join_converges_smol() { SmolRuntime::block_on(super::two_node_tls_join_converges::()); From 7113bc6f0deaa98e0e83f44f753dbfdc94855e5e Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 19:18:34 +1200 Subject: [PATCH 114/128] refactor(serf-driver): hoist the snapshotter and keyring-file engine out of serf-reactor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The file snapshotter and the keyring-file persistence engine are pure std file mechanics with no runtime coupling, and the compio driver needs both for operator parity — one shared implementation beats a fork of the hardened logic (exclusive owner-only temps, the identity-gated stale-temp sweep, the directory-synced rename the acknowledgement waits on, the compaction clean-leave gate). They now live in serf-driver alongside the shared key-management apply logic, together with the KeyringPersistence/KeyringPersistRx/KeyringPersistError acknowledgement contract; the snapshotter opens from a bare path plus threshold so it carries no per-crate options type. serf-reactor keeps its public surface through re-exports, with FileKeyringDelegate now a thin wrapper that turns the engine's acknowledgement into KeyringPersistence::Pending. The engine and snapshotter test suites move with the code; the reactor keeps delegate-level round-trip, drop-flush, and first-boot tests. --- serf-driver/Cargo.toml | 17 +- serf-driver/src/keyring/mod.rs | 27 + serf-driver/src/keyring_file/mod.rs | 447 ++++++++++++++ serf-driver/src/keyring_file/tests.rs | 537 +++++++++++++++++ serf-driver/src/lib.rs | 21 +- .../src}/snapshotter/mod.rs | 34 +- .../src}/snapshotter/tests.rs | 71 ++- serf-reactor/Cargo.toml | 3 - serf-reactor/src/delegate/keyring_file/mod.rs | 407 +------------ .../src/delegate/keyring_file/tests.rs | 546 ++---------------- serf-reactor/src/delegate/mod.rs | 39 +- serf-reactor/src/driver/mod.rs | 2 - serf-reactor/src/driver/options/mod.rs | 2 +- serf-reactor/src/driver/quic/mod.rs | 6 +- serf-reactor/src/driver/stream/mod.rs | 6 +- serf-reactor/src/error/mod.rs | 2 +- serf-reactor/src/lib.rs | 2 +- serf-reactor/src/serf/mod.rs | 3 +- serf-reactor/src/transport/runtime.rs | 4 +- 19 files changed, 1183 insertions(+), 993 deletions(-) create mode 100644 serf-driver/src/keyring_file/mod.rs create mode 100644 serf-driver/src/keyring_file/tests.rs rename {serf-reactor/src/driver => serf-driver/src}/snapshotter/mod.rs (92%) rename {serf-reactor/src/driver => serf-driver/src}/snapshotter/tests.rs (83%) diff --git a/serf-driver/Cargo.toml b/serf-driver/Cargo.toml index 7adc06d7..eb0380d2 100644 --- a/serf-driver/Cargo.toml +++ b/serf-driver/Cargo.toml @@ -14,22 +14,33 @@ tls = ["tcp", "serf-proto/tls"] quic = ["serf-proto/quic"] quic-rustls-ring = ["quic", "serf-proto/quic-rustls-ring"] coordinates = ["serf-proto/coordinates"] +tracing = ["dep:tracing"] tag-regex = ["serf-proto/tag-regex"] # Encryption backends — forward to serf-proto (the key-management projection # types) and memberlist-proto (the live `Keyring` the shared apply logic # read-modify-writes). Aggregated to the `encryption` cfg by `build.rs`. -aes-gcm = ["serf-proto/aes-gcm", "memberlist-proto/aes-gcm"] -chacha20-poly1305 = ["serf-proto/chacha20-poly1305", "memberlist-proto/chacha20-poly1305"] +aes-gcm = ["serf-proto/aes-gcm", "memberlist-proto/aes-gcm", "dep:rand", "dep:zeroize"] +chacha20-poly1305 = [ + "serf-proto/chacha20-poly1305", + "memberlist-proto/chacha20-poly1305", + "dep:rand", + "dep:zeroize", +] [dependencies] serf-proto = { path = "../serf-proto", default-features = false } +bytes = { workspace = true } +thiserror = { workspace = true } +tracing = { workspace = true, optional = true } +rand = { workspace = true, optional = true, features = ["sys_rng"] } +zeroize = { version = "1", optional = true, default-features = false } # Pinned to the same git+branch as serf-proto to guarantee a single copy in the # resolver; the shared key-management apply logic rotates memberlist's live # `Keyring` and matches on `SecretKey` under the encryption cfg gate. memberlist-proto = { git = "https://github.com/al8n/memberlist", branch = "main", default-features = false, features = ["std"] } [dev-dependencies] -bytes = { workspace = true } +smol_str = { workspace = true } [package.metadata.docs.rs] all-features = true diff --git a/serf-driver/src/keyring/mod.rs b/serf-driver/src/keyring/mod.rs index b1f35072..ce0104fe 100644 --- a/serf-driver/src/keyring/mod.rs +++ b/serf-driver/src/keyring/mod.rs @@ -30,6 +30,33 @@ //! [`encryption_options`]: serf_proto::StreamEndpoint::encryption_options //! [`set_encryption_options`]: serf_proto::StreamEndpoint::set_encryption_options +/// A persistence failure reported through [`KeyringPersistence::Pending`]. +pub type KeyringPersistError = Box; + +/// Receiver half of one rotation's persistence acknowledgement. +pub type KeyringPersistRx = std::sync::mpsc::Receiver>; + +/// How one keyring rotation reaches durability, reported back from a +/// runtime's `KeyringDelegate::keyring_updated`. +/// +/// The reference implementation writes its keyring file synchronously inside +/// the key-management query handler and folds a write failure into the +/// response. These drivers keep the pump non-blocking instead: a persisting +/// delegate hands back a receiver, the pump parks the key response, and sends +/// it once the receiver resolves — unchanged on success, downgraded to a +/// failed response carrying the error otherwise (a disconnected sender counts +/// as a failure: the worker vanished without acknowledging). The live wire +/// keyring keeps the rotation in every outcome. +#[must_use = "dropping the acknowledgement silently un-gates the key response from persistence"] +pub enum KeyringPersistence { + /// The rotation needs no out-of-band persistence (or completed inline): + /// the key response is sent immediately. + Durable, + /// Persistence runs out-of-band; the pump defers the key response until + /// the receiver resolves. + Pending(KeyringPersistRx), +} + #[cfg(test)] mod tests; diff --git a/serf-driver/src/keyring_file/mod.rs b/serf-driver/src/keyring_file/mod.rs new file mode 100644 index 00000000..1e81a689 --- /dev/null +++ b/serf-driver/src/keyring_file/mod.rs @@ -0,0 +1,447 @@ +//! [`KeyringFilePersistence`] — the shared file-persistence engine behind the +//! per-runtime `FileKeyringDelegate` types. + +use std::{ + io, + path::{Path, PathBuf}, + sync::mpsc, +}; + +use memberlist_proto::{Keyring, SecretKey}; +use serf_proto::{secret_key_from_bytes, secret_key_to_bytes}; + +use crate::keyring::KeyringPersistError; + +/// The file-persistence engine each runtime's `FileKeyringDelegate` wraps: it +/// persists every enqueued keyring rotation to a file and acknowledges each +/// write's durability, plus a [`load`](Self::load) to rebuild the ring at +/// construction — the turnkey replacement for the legacy keyring-file option, +/// with the persistence app-owned like any other delegate. +/// +/// FORMAT: one lowercase-hex line per key, the PRIMARY key first, each line +/// decoding to `[algorithm_tag][raw_key_bytes]`. The leading tag byte keeps +/// the encoding self-describing, so two ciphers sharing a key length (AES-256 +/// and ChaCha20-Poly1305 are both 32 bytes) stay distinguishable. The file +/// holds raw key material by design — protect it with filesystem permissions +/// exactly as the reference implementation's keyring file required. +/// +/// Rotations are handed to a dedicated persistence thread — `keyring_updated` +/// runs inline on the driver pump, which must never block on storage, so the +/// callback only clones the ring into an unbounded channel (rotations are +/// rare and small) and the worker does the file I/O, acknowledging each write +/// back through [`KeyringPersistence::Pending`](crate::keyring::KeyringPersistence::Pending) +/// so the driver can gate the +/// key response on durability. Each write goes through an exclusively-created, +/// owner-only (`0600` on Unix), unpredictably-named sibling temp file, synced, +/// then atomically renamed over the destination: a crash mid-write never +/// truncates the previous ring, a restrictive mode on the key file is never +/// widened by a rotation, and key bytes can never land in a pre-existing +/// inode or behind a planted symlink. Dropping the delegate joins the +/// worker after it drains every queued rotation, so a shutdown cannot discard +/// a write that was already acknowledged toward the wire. +/// +/// UNIX-ONLY: the acknowledgement contract is rename durability — the +/// containing directory is synced before a rotation reports success — and no +/// safe standard API can flush a directory entry on Windows, so this turnkey +/// delegate does not exist there rather than acknowledge a rotation a power +/// loss could revert. A Windows application implements its runtime's +/// `KeyringDelegate` trait itself with a platform-durable strategy (a +/// write-through rename via the platform APIs, or storage with its own +/// durability contract). +pub struct KeyringFilePersistence { + path: PathBuf, + /// Hand-off to the persistence thread; `None` only during drop, which hangs + /// up first so the worker drains and exits. + worker: Option>, + /// The persistence thread, joined on drop after the hang-up. + handle: Option>, +} + +/// One queued rotation: the ring to write and the acknowledgement sender the +/// pump's parked key response polls. +struct PersistJob { + ring: Keyring, + ack: mpsc::Sender>, +} + +impl KeyringFilePersistence { + /// A delegate persisting to `path`. + /// + /// Construction sweeps stale sibling temp files — the fixed-name temp + /// earlier releases wrote (whose permissions predate the owner-only + /// guarantee) and abandoned temps from crashed rotations — before the + /// first write can race one. + pub fn new(path: impl Into) -> Self { + let path: PathBuf = path.into(); + sweep_stale_temps(&path); + let (worker, jobs) = mpsc::channel::(); + let worker_path = path.clone(); + let handle = std::thread::spawn(move || { + while let Ok(job) = jobs.recv() { + let res = persist(&worker_path, &job.ring); + if let Err(_err) = &res { + #[cfg(feature = "tracing")] + tracing::warn!( + path = %worker_path.display(), + error = %_err, + "serf keyring rotation could not be persisted; the wire keeps the rotated ring" + ); + } + // Ignoring Err: the pump dropped this rotation's receiver (teardown, + // or the requester's deadline passed) — the outcome has nowhere to go. + let _ = job + .ack + .send(res.map_err(|e| Box::new(e) as KeyringPersistError)); + } + }); + Self { + path, + worker: Some(worker), + handle: Some(handle), + } + } + + /// The persistence path. + #[must_use] + pub fn path(&self) -> &Path { + &self.path + } + + /// Load a keyring previously persisted by this delegate. + /// + /// Returns `Ok(None)` when the file does not exist (first boot). The first + /// line is the primary key; the rest are secondaries in decrypt-trial order. + /// + /// # Errors + /// + /// [`KeyringFileError::Io`] on a read failure other than not-found; + /// [`KeyringFileError::Parse`] on a malformed line (bad hex, an unknown + /// algorithm tag, a key length not matching its tag, or an empty file). + pub fn load(&self) -> Result, KeyringFileError> { + let raw = match std::fs::read_to_string(&self.path) { + Ok(s) => s, + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(KeyringFileError::Io(e)), + }; + let mut keys = raw + .lines() + .map(str::trim) + .filter(|l| !l.is_empty()) + .map(parse_key_line); + let primary = keys + .next() + .transpose()? + .ok_or_else(|| KeyringFileError::Parse("the keyring file holds no keys".into()))?; + let secondaries = keys.collect::, _>>()?; + Ok(Some(Keyring::with_secondaries(primary, secondaries))) + } +} + +/// Serialize `keyring` into the file format (primary first, one hex line per +/// key) and write it via an exclusively-created owner-only temp file, sync, +/// and atomic rename. +fn persist(path: &Path, keyring: &Keyring) -> Result<(), KeyringFileError> { + use zeroize::Zeroize as _; + let mut out = String::new(); + push_key_line(&mut out, keyring.primary_ref()); + for key in keyring.secondaries() { + push_key_line(&mut out, key); + } + let res = write_via_exclusive_temp(path, out.as_bytes()).map_err(KeyringFileError::Io); + out.zeroize(); + res +} + +/// Write `contents` to `path` through an exclusively-created, owner-only, +/// unpredictably-named sibling temp file, synced then atomically renamed over +/// the destination. +/// +/// `create_new` (`O_CREAT | O_EXCL`) never reuses an existing inode and never +/// follows a symlink — a file or link already sitting at the temp path fails +/// the attempt instead of receiving the key bytes — and the OS-entropy name +/// keeps such a path from being plantable ahead of time. The inode is born +/// `0600` on Unix and re-asserted on the open handle, so raw key material +/// only ever lands in a fresh owner-only inode this process created; a crash +/// mid-write never truncates the previous ring. +fn write_via_exclusive_temp(path: &Path, contents: &[u8]) -> io::Result<()> { + use std::io::Write as _; + let name = path + .file_name() + .and_then(|n| n.to_str()) + .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "keyring path has no file name"))?; + let dir = path.parent().filter(|d| !d.as_os_str().is_empty()); + let mut attempts = 0u8; + let (tmp, mut file) = loop { + let tmp_name = format!(".{name}.{:016x}.tmp", temp_nonce()?); + let tmp = match dir { + Some(d) => d.join(&tmp_name), + None => PathBuf::from(&tmp_name), + }; + let mut opts = std::fs::OpenOptions::new(); + opts.write(true).create_new(true); + { + use std::os::unix::fs::OpenOptionsExt as _; + opts.mode(0o600); + } + match opts.open(&tmp) { + Ok(file) => break (tmp, file), + // A 64-bit OS-entropy collision is practically a squatted path; a + // bounded retry with a fresh nonce outlasts any accidental leftover + // without spinning against a directory an attacker keeps filling. + Err(e) if e.kind() == io::ErrorKind::AlreadyExists && attempts < 16 => attempts += 1, + Err(e) => return Err(e), + } + }; + let res = (|| { + { + use std::os::unix::fs::PermissionsExt as _; + file.set_permissions(std::fs::Permissions::from_mode(0o600))?; + } + file.write_all(contents)?; + file.sync_all()?; + std::fs::rename(&tmp, path)?; + // The rename is not durable until the DIRECTORY entry is: a crash after + // returning `Ok` here must not revert the destination to the old file — + // the acknowledgement built on this return is what releases a successful + // key response to the cluster. + sync_dir(dir.unwrap_or_else(|| Path::new("."))) + })(); + if res.is_err() { + // Ignoring Err: removing the failed temp (it holds key bytes) is + // best-effort hygiene; the write error itself is what propagates. A + // failure after the rename consumed the temp removes nothing. + let _ = std::fs::remove_file(&tmp); + } + res +} + +/// Sync a directory so a completed rename of an entry inside it survives a +/// crash. +fn sync_dir(dir: &Path) -> io::Result<()> { + std::fs::File::open(dir)?.sync_all() +} + +/// OS-entropy nonce for a temp-file name: unpredictable, so a +/// directory-writing attacker cannot pre-plant a file or symlink at the next +/// temp path. +fn temp_nonce() -> io::Result { + use rand::TryRng as _; + rand::rngs::SysRng.try_next_u64().map_err(io::Error::other) +} + +/// Remove leftovers a rotation can no longer reuse: the fixed-name sibling +/// temp earlier releases wrote (whose permissions predate the owner-only +/// guarantee and may already hold key material), and the exact-shape +/// `.{name}.{16 hex}.tmp` temps a crashed rotation abandoned. Every removal +/// is gated on FILESYSTEM IDENTITY against the destination — a candidate +/// that resolves to the destination's storage (lexical identity, a +/// case-folding filesystem, or a symlink on either side) is never touched, +/// so sweeping hygiene can never delete the persisted keyring. Removing a +/// planted symlink unlinks the LINK, never its target. Best-effort: a sweep +/// failure never blocks construction — `create_new` already keeps every +/// future write off any path that survives. +fn sweep_stale_temps(path: &Path) { + if legacy_temp_provably_distinct(path) && sweepable(path, &path.with_extension("tmp")) { + // Ignoring Err: nothing to sweep, or no permission — both non-fatal. + let _ = std::fs::remove_file(path.with_extension("tmp")); + } + let (Some(dir), Some(name)) = ( + path.parent().filter(|d| !d.as_os_str().is_empty()), + path.file_name().and_then(|n| n.to_str()), + ) else { + return; + }; + let prefix = format!(".{name}."); + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let file_name = entry.file_name(); + let Some(f) = file_name.to_str() else { + continue; + }; + // Exact-shape match only — a sibling file that merely shares the prefix + // and suffix (an operator's own backup, say) is not this delegate's to + // delete. + let matches_temp_shape = f + .strip_prefix(&prefix) + .and_then(|rest| rest.strip_suffix(".tmp")) + .is_some_and(|mid| mid.len() == 16 && mid.bytes().all(|b| b.is_ascii_hexdigit())); + if matches_temp_shape && sweepable(path, &entry.path()) { + // Ignoring Err: best-effort sweep of abandoned temps. + let _ = std::fs::remove_file(entry.path()); + } + } +} + +/// Whether the fixed-name legacy temp's name is PROVABLY distinct from the +/// destination's under every supported filename-alias relation. +/// +/// The legacy image differs from the destination only in its extension, so +/// the alias question reduces to the extension replacement. Sweeping is +/// allowed only when the destination's extension is pure ASCII and does not +/// ASCII-case-fold to `tmp` — case-insensitive HFS+ additionally IGNORES +/// certain Unicode scalars when comparing names, so an extension carrying +/// any non-ASCII scalar could fold the two names together and is +/// conservatively refused. No extension at all is safe: the image then +/// APPENDS `.tmp`, four non-ignorable ASCII characters no folding can +/// absorb. Random-suffix candidates need no such classifier — their names +/// carry a dot prefix and a 16-hex infix the destination's name does not, +/// an excess of non-ignorable ASCII no alias relation can erase. +fn legacy_temp_provably_distinct(path: &Path) -> bool { + match path.extension() { + None => true, + Some(e) => e + .to_str() + .is_some_and(|e| e.is_ascii() && !e.eq_ignore_ascii_case("tmp")), + } +} + +/// Whether removing `candidate` cannot touch the keyring the destination +/// `path` reaches. +/// +/// `remove_file` unlinks a NAME, so the guards are layered by what a name +/// can do: a candidate whose name can itself name the destination — +/// lexically, or equal under the ASCII case folding that aliases names on +/// case-insensitive filesystems — is NEVER sweepable, because a rotation's +/// rename can land between any identity observation and the unlink, and the +/// unlink would then remove whatever the destination name holds (the freshly +/// persisted keyring). Only for genuinely distinct names — where the unlink +/// cannot remove the destination's entry — is resolved filesystem identity +/// consulted: `metadata` FOLLOWS symlinks, so each side resolves to the file +/// a reader would actually open, and a candidate reaching the destination's +/// storage through a symlink or hard link is skipped. +fn sweepable(path: &Path, candidate: &Path) -> bool { + use std::os::unix::fs::MetadataExt as _; + if candidate == path { + return false; + } + match ( + candidate.file_name().and_then(|n| n.to_str()), + path.file_name().and_then(|n| n.to_str()), + ) { + (Some(c), Some(p)) if c.eq_ignore_ascii_case(p) => return false, + (Some(_), Some(_)) => {} + // Un-inspectable names: never delete on uncertainty. + _ => return false, + } + if std::fs::symlink_metadata(candidate).is_err() { + // Nothing at the candidate name; removal would be a no-op. + return false; + } + let candidate_meta = std::fs::metadata(candidate); + // The window between the two observations is where a concurrent rename + // lands; the tests widen it deterministically to prove the name guards + // above — not luck — are what keep a mid-check rename safe. + #[cfg(test)] + tests::between_identity_observations(path); + let destination_meta = std::fs::metadata(path); + match (candidate_meta, destination_meta) { + // A dangling symlink at the candidate name reaches no storage at all. + (Err(e), _) if e.kind() == io::ErrorKind::NotFound => true, + (Ok(c), Ok(d)) => (c.dev(), c.ino()) != (d.dev(), d.ino()), + // The destination resolves to nothing: with the name-aliasing shapes + // already excluded above, an existing candidate cannot BE the missing + // destination — the two are genuinely distinct. + (Ok(_), Err(e)) if e.kind() == io::ErrorKind::NotFound => true, + // Identity cannot be established: never delete on uncertainty. + _ => false, + } +} + +impl KeyringFilePersistence { + /// Hand one rotation to the persistence worker, returning the receiver its + /// durability acknowledgement resolves — the value a `keyring_updated` + /// implementation wraps in `KeyringPersistence::Pending`. + /// + /// Non-blocking: the pump must never wait on storage. A failed hand-off + /// (the worker already hung up) drops the job — and with it the ack sender + /// — so the returned receiver disconnects and the pump reports the rotation + /// unpersisted rather than silently acknowledged. + pub fn enqueue(&self, keyring: &Keyring) -> crate::keyring::KeyringPersistRx { + let (ack, rx) = mpsc::channel(); + if let Some(worker) = &self.worker { + // Ignoring Err: see above — the dropped job's disconnected receiver IS + // the failure signal. + let _ = worker.send(PersistJob { + ring: keyring.clone(), + ack, + }); + } + rx + } +} + +impl Drop for KeyringFilePersistence { + fn drop(&mut self) { + // Hang up, then join: the worker drains every queued rotation before it + // exits, so a shutdown cannot discard a write the wire already carries. + self.worker = None; + if let Some(handle) = self.handle.take() { + // Ignoring Err: a panicked worker already surfaced its failure through + // the acknowledgement channel; there is nothing to unwind into here. + let _ = handle.join(); + } + } +} + +/// Errors from [`KeyringFilePersistence::load`]. +#[derive(Debug, thiserror::Error)] +pub enum KeyringFileError { + /// Reading or writing the keyring file failed. + #[error(transparent)] + Io(#[from] io::Error), + /// The file's contents are not a valid keyring serialization. + #[error("malformed keyring file: {0}")] + Parse(String), +} + +/// Append one key as a lowercase-hex `[tag][bytes]` line. The transient +/// tagged buffer is zeroed before it is freed. +fn push_key_line(out: &mut String, key: &SecretKey) { + use core::fmt::Write as _; + let tagged = secret_key_to_bytes(key); + for b in tagged.iter() { + // Ignoring Err: writing hex digits into a String cannot fail. + let _ = write!(out, "{b:02x}"); + } + out.push('\n'); +} + +/// Parse one lowercase-hex `[tag][bytes]` line into a [`SecretKey`]. +fn parse_key_line(line: &str) -> Result { + use zeroize::Zeroize as _; + if line.len() % 2 != 0 { + return Err(KeyringFileError::Parse("odd-length hex key line".into())); + } + let mut buf = Vec::with_capacity(line.len() / 2); + for pair in line.as_bytes().chunks_exact(2) { + let hi = hex_val(pair[0]); + let lo = hex_val(pair[1]); + match (hi, lo) { + (Some(h), Some(l)) => buf.push((h << 4) | l), + _ => { + buf.zeroize(); + return Err(KeyringFileError::Parse( + "non-hex character in key line".into(), + )); + } + } + } + let bytes = bytes::Bytes::copy_from_slice(&buf); + buf.zeroize(); + secret_key_from_bytes(&bytes).map_err(|e| KeyringFileError::Parse(e.to_string())) +} + +/// The value of one lowercase/uppercase hex digit. +fn hex_val(c: u8) -> Option { + match c { + b'0'..=b'9' => Some(c - b'0'), + b'a'..=b'f' => Some(c - b'a' + 10), + b'A'..=b'F' => Some(c - b'A' + 10), + _ => None, + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-driver/src/keyring_file/tests.rs b/serf-driver/src/keyring_file/tests.rs new file mode 100644 index 00000000..ac732b61 --- /dev/null +++ b/serf-driver/src/keyring_file/tests.rs @@ -0,0 +1,537 @@ +use super::*; + +use std::cell::RefCell; + +/// This thread's mid-check hook, when a test installed one. +type ObservationGap = RefCell>>; + +thread_local! { + /// Per-test hook fired between `sweepable`'s two identity observations — + /// the window a concurrent rotation's rename can land in. Thread-local so + /// parallel tests never see each other's hooks. + static BETWEEN_OBSERVATIONS: ObservationGap = const { RefCell::new(None) }; +} + +/// Called by `sweepable` between its two `metadata` observations. +pub(super) fn between_identity_observations(path: &Path) { + BETWEEN_OBSERVATIONS.with(|hook| { + if let Some(f) = hook.borrow_mut().as_mut() { + f(path); + } + }); +} + +/// Install `f` as this thread's mid-check hook for the duration of `run`. +fn with_observation_gap(f: impl FnMut(&Path) + 'static, run: impl FnOnce()) { + BETWEEN_OBSERVATIONS.with(|hook| *hook.borrow_mut() = Some(Box::new(f))); + run(); + BETWEEN_OBSERVATIONS.with(|hook| *hook.borrow_mut() = None); +} + +/// Wait for one rotation's persistence acknowledgement. +fn acked(rx: crate::keyring::KeyringPersistRx) -> Result<(), KeyringPersistError> { + rx.recv_timeout(std::time::Duration::from_secs(5)) + .expect("the persistence worker acknowledges within the bound") +} + +fn tmp_path(name: &str) -> PathBuf { + let mut p = std::env::temp_dir(); + p.push(format!("serf-keyring-file-{name}-{}", std::process::id())); + p +} + +/// Any sibling temp file this engine could have produced for `path`: the +/// legacy fixed-name temp or a random-suffix one. +fn temp_residue(path: &Path) -> Vec { + let mut residue = Vec::new(); + let legacy = path.with_extension("tmp"); + if legacy.symlink_metadata().is_ok() { + residue.push(legacy); + } + let (Some(dir), Some(name)) = (path.parent(), path.file_name().and_then(|n| n.to_str())) else { + return residue; + }; + let prefix = format!(".{name}."); + for entry in std::fs::read_dir(dir).expect("temp dir listable").flatten() { + if let Some(f) = entry.file_name().to_str() + && f.starts_with(&prefix) + && f.ends_with(".tmp") + { + residue.push(entry.path()); + } + } + residue +} + +/// A persisted rotation round-trips: the acknowledged `keyring_updated` write +/// (primary first) is on disk, and `load` rebuilds it with the same primary +/// and secondaries. +#[test] +fn rotation_round_trips_through_the_file() { + let path = tmp_path("roundtrip"); + let engine = KeyringFilePersistence::new(&path); + + #[cfg(feature = "aes-gcm")] + let (primary, secondary) = (SecretKey::Aes128([1u8; 16]), SecretKey::Aes256([2u8; 32])); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let (primary, secondary) = ( + SecretKey::ChaCha20Poly1305([1u8; 32]), + SecretKey::ChaCha20Poly1305([2u8; 32]), + ); + + let ring = Keyring::with_secondaries(primary, [secondary]); + acked(engine.enqueue(&ring)).expect("the rotation persists"); + + let loaded = engine + .load() + .expect("an acknowledged write parses") + .expect("an acknowledged write exists"); + assert_eq!(loaded.primary_ref(), ring.primary_ref()); + assert_eq!(loaded.secondaries(), ring.secondaries()); + + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} + +/// First boot: a missing file loads as `None`, not an error. +#[test] +fn missing_file_loads_as_none() { + let engine = KeyringFilePersistence::new(tmp_path("missing")); + assert!(engine.load().expect("not-found is not an error").is_none()); +} + +/// Malformed contents are a parse error, never a panic: odd-length hex, +/// non-hex characters, an unknown algorithm tag, and an empty file. +#[test] +fn malformed_files_are_parse_errors() { + for (name, contents) in [ + ("odd", "abc\n"), + ("nonhex", "zz\n"), + ("badtag", "ff00112233445566778899aabbccddeeff\n"), + ("empty", "\n"), + ] { + let path = tmp_path(name); + std::fs::write(&path, contents).expect("write test file"); + let engine = KeyringFilePersistence::new(&path); + assert!( + matches!(engine.load(), Err(KeyringFileError::Parse(_))), + "{name} must be a parse error" + ); + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); + } +} + +/// A second rotation atomically replaces the file: the newest ring wins and +/// no temp-file residue remains under either naming scheme. +#[test] +fn a_second_rotation_replaces_the_first() { + let path = tmp_path("replace"); + let engine = KeyringFilePersistence::new(&path); + + #[cfg(feature = "aes-gcm")] + let (first, second) = (SecretKey::Aes128([3u8; 16]), SecretKey::Aes128([4u8; 16])); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let (first, second) = ( + SecretKey::ChaCha20Poly1305([3u8; 32]), + SecretKey::ChaCha20Poly1305([4u8; 32]), + ); + + acked(engine.enqueue(&Keyring::new(first))).expect("first rotation persists"); + acked(engine.enqueue(&Keyring::new(second))).expect("second rotation persists"); + + let loaded = engine + .load() + .expect("parses") + .expect("the file exists after two rotations"); + assert_eq!(loaded.primary_ref(), &second); + assert!(loaded.secondaries().is_empty()); + assert!( + temp_residue(&path).is_empty(), + "the atomic rename must consume every temp file" + ); + + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} + +/// A rotation never widens the key file's mode — and it NARROWS a permissive +/// one: the replacing temp inode is born owner-only, so a fresh file is +/// created `0600` and a pre-existing `0644` destination is `0600` after the +/// next rotation, even under a permissive umask. +#[test] +fn rotation_enforces_owner_only_permissions() { + use std::os::unix::fs::PermissionsExt as _; + + let path = tmp_path("perms"); + // A permissive pre-existing destination (an operator's hand-created file). + std::fs::write(&path, "junk\n").expect("pre-create the destination"); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)) + .expect("widen the destination"); + let engine = KeyringFilePersistence::new(&path); + + #[cfg(feature = "aes-gcm")] + let (first, second) = (SecretKey::Aes128([5u8; 16]), SecretKey::Aes128([6u8; 16])); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let (first, second) = ( + SecretKey::ChaCha20Poly1305([5u8; 32]), + SecretKey::ChaCha20Poly1305([6u8; 32]), + ); + + acked(engine.enqueue(&Keyring::new(first))).expect("first rotation persists"); + let mode = std::fs::metadata(&path).expect("stat").permissions().mode() & 0o777; + assert_eq!( + mode, 0o600, + "the first rotation narrows a permissive destination to owner-only" + ); + + acked(engine.enqueue(&Keyring::new(second))).expect("second rotation persists"); + let mode = std::fs::metadata(&path).expect("stat").permissions().mode() & 0o777; + assert_eq!(mode, 0o600, "a rotation must not widen the key file's mode"); + + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} + +/// Construction sweeps both classes of stale sibling temps: the fixed-name +/// temp earlier releases wrote (possibly permissive and already holding key +/// material) and abandoned random-suffix temps from crashed rotations. +#[test] +fn construction_sweeps_stale_temps() { + let path = tmp_path("sweep"); + let legacy = path.with_extension("tmp"); + std::fs::write(&legacy, "stale key bytes at permissive mode\n").expect("plant the legacy temp"); + let name = path.file_name().and_then(|n| n.to_str()).expect("name"); + let abandoned = path.with_file_name(format!(".{name}.00000000deadbeef.tmp")); + std::fs::write(&abandoned, "abandoned partial write\n").expect("plant the abandoned temp"); + + let _delegate = KeyringFilePersistence::new(&path); + assert!( + !legacy.exists(), + "the legacy fixed-name temp must be swept at construction" + ); + assert!( + !abandoned.exists(), + "an abandoned random-suffix temp must be swept at construction" + ); +} + +/// A symlink planted at the legacy temp path is unlinked — the LINK, never +/// its target — and no rotation ever writes through it: the exclusive +/// creation refuses any pre-existing path, so key bytes cannot be redirected +/// into an attacker-chosen file. +#[test] +fn a_planted_symlink_never_receives_key_bytes() { + let path = tmp_path("symlink"); + let victim = tmp_path("symlink-victim"); + std::fs::write(&victim, "victim contents\n").expect("create the victim"); + let planted = path.with_extension("tmp"); + // Ignoring Err: a leftover link from a previous run is about to be re-planted. + let _ = std::fs::remove_file(&planted); + std::os::unix::fs::symlink(&victim, &planted).expect("plant the symlink"); + + let engine = KeyringFilePersistence::new(&path); + assert!( + planted.symlink_metadata().is_err(), + "construction unlinks the planted symlink" + ); + assert_eq!( + std::fs::read_to_string(&victim).expect("victim readable"), + "victim contents\n", + "unlinking removes the LINK, never its target" + ); + + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes128([7u8; 16]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = SecretKey::ChaCha20Poly1305([7u8; 32]); + acked(engine.enqueue(&Keyring::new(key))).expect("rotation persists"); + assert_eq!( + std::fs::read_to_string(&victim).expect("victim readable"), + "victim contents\n", + "no rotation writes through a planted path" + ); + + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); + let _ = std::fs::remove_file(&victim); +} + +/// A destination whose own extension is `tmp` is NOT its legacy temp: the +/// construction sweep must preserve it — the previous implementation wrote +/// such a destination in place, so the file can hold the only copy of the +/// keyring. +#[test] +fn a_tmp_extension_destination_survives_construction() { + let path = tmp_path("selfnamed").with_extension("tmp"); + // Ignoring Err: a leftover file from a previous run is about to be rewritten. + let _ = std::fs::remove_file(&path); + + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes128([10u8; 16]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = SecretKey::ChaCha20Poly1305([10u8; 32]); + { + let engine = KeyringFilePersistence::new(&path); + acked(engine.enqueue(&Keyring::new(key))).expect("the rotation persists"); + } + + let reopened = KeyringFilePersistence::new(&path); + let loaded = reopened + .load() + .expect("the persisted keyring parses") + .expect("constructing a engine must not sweep a .tmp-named destination"); + assert_eq!(loaded.primary_ref(), &key); + + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} + +/// The alias guard is case-insensitive: a destination named with an +/// uppercase `TMP` extension lexically differs from its lowercase +/// `with_extension` image, yet the two alias the same file on the +/// case-insensitive filesystems that are the default on macOS — construction +/// must preserve it. +#[test] +fn an_uppercase_tmp_destination_survives_construction() { + let path = tmp_path("selfnamed-upper").with_extension("TMP"); + // Ignoring Err: a leftover file from a previous run is about to be rewritten. + let _ = std::fs::remove_file(&path); + + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes128([12u8; 16]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = SecretKey::ChaCha20Poly1305([12u8; 32]); + { + let engine = KeyringFilePersistence::new(&path); + acked(engine.enqueue(&Keyring::new(key))).expect("the rotation persists"); + } + + let reopened = KeyringFilePersistence::new(&path); + let loaded = reopened + .load() + .expect("the persisted keyring parses") + .expect("constructing a engine must not sweep a case-aliased .TMP destination"); + assert_eq!(loaded.primary_ref(), &key); + + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} + +/// A symlinked destination keeps its target through construction: with the +/// configured path pointing at a file that happens to live at the legacy +/// temp name, the sweep resolves filesystem IDENTITY — not names — and +/// leaves the keyring intact for the follow-up load. +#[test] +fn a_symlinked_destination_keeps_its_target_through_construction() { + let target = tmp_path("linked").with_extension("tmp"); + let link = tmp_path("linked").with_extension("current"); + // Ignoring Err: leftovers from a previous run are about to be recreated. + let _ = std::fs::remove_file(&link); + let _ = std::fs::remove_file(&target); + + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes128([13u8; 16]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = SecretKey::ChaCha20Poly1305([13u8; 32]); + { + // Persist a real ring at the target path (itself a tmp-named + // destination, which construction must already preserve). + let seed = KeyringFilePersistence::new(&target); + acked(seed.enqueue(&Keyring::new(key))).expect("the seed rotation persists"); + } + std::os::unix::fs::symlink(&target, &link).expect("plant the destination symlink"); + + let engine = KeyringFilePersistence::new(&link); + let loaded = engine + .load() + .expect("the symlinked keyring parses") + .expect("construction must not sweep the storage a symlinked destination resolves to"); + assert_eq!(loaded.primary_ref(), &key); + + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&link); + let _ = std::fs::remove_file(&target); +} + +/// The legacy-sweep classifier admits only extensions provably distinct +/// from `tmp` under every supported filename-alias relation: pure-ASCII +/// non-tmp extensions (or none, where the image appends characters no +/// folding can absorb) pass; any tmp case-fold or any non-ASCII scalar — +/// case-insensitive HFS+ ignores certain Unicode scalars when comparing +/// names, so `ring.t\u{200D}mp` aliases `ring.tmp` there — is refused. +#[test] +fn the_legacy_sweep_classifier_refuses_unprovable_extensions() { + for (destination, distinct) in [ + ("ring.keys", true), + ("ring", true), + ("ring.tmp", false), + ("ring.TMP", false), + ("ring.Tmp", false), + // An HFS+-ignorable scalar (ZERO WIDTH JOINER) inside the extension. + ("ring.t\u{200D}mp", false), + // Any non-ASCII scalar is unprovable, ignorable or not. + ("ring.cl\u{00E9}s", false), + ] { + assert_eq!( + legacy_temp_provably_distinct(Path::new(destination)), + distinct, + "{destination:?}" + ); + } +} + +/// A destination whose extension is not provably distinct keeps its legacy +/// image on EVERY filesystem: whether or not the running filesystem folds +/// the two names together, the conservative refusal leaves the sibling file +/// alone — the fold-aliasing filesystems are exactly where that sibling IS +/// the persisted keyring. +#[test] +fn an_unprovable_extension_keeps_the_legacy_sibling() { + let path = tmp_path("ignorable").with_extension("t\u{200D}mp"); + let sibling = path.with_extension("tmp"); + std::fs::write(&sibling, "possibly the persisted keyring\n").expect("seed the sibling"); + + let _delegate = KeyringFilePersistence::new(&path); + assert!( + sibling.exists(), + "an unprovable extension must refuse the legacy sweep" + ); + + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&sibling); + let _ = std::fs::remove_file(&path); +} + +/// A rename landing between the sweep's two identity observations must +/// never lose the destination: `remove_file` unlinks a NAME, so a candidate +/// whose name can itself name the destination is refused by the name guard +/// BEFORE any identity observation — the widened observation gap here +/// atomically replaces the destination exactly as a concurrent rotation +/// would, and construction must leave the replacement intact. +#[test] +fn a_rename_landing_mid_check_never_loses_the_destination() { + let path = tmp_path("raced").with_extension("tmp"); + std::fs::write(&path, "pre-rotation contents\n").expect("seed the destination"); + + with_observation_gap( + |dest: &Path| { + // The concurrent rotation: a fresh inode atomically renamed over the + // destination, mid-check. + let staged = dest.with_file_name(".raced-replacement"); + std::fs::write(&staged, "freshly persisted contents\n").expect("stage the replacement"); + std::fs::rename(&staged, dest).expect("land the replacement"); + }, + || { + let _delegate = KeyringFilePersistence::new(&path); + }, + ); + + // With the name guard in place the observation gap is never reached for a + // name-aliasing candidate, so the original contents remain; what must hold + // in every world is that the destination NAME was not unlinked. + let contents = + std::fs::read_to_string(&path).expect("the destination survives construction un-unlinked"); + assert!( + contents == "pre-rotation contents\n" || contents == "freshly persisted contents\n", + "the destination holds one of the two written generations, never nothing" + ); + + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} + +/// The success acknowledgement is durability: a rotation whose parent +/// directory cannot be synced reports failure, because the completed rename +/// is not crash-durable until the directory entry is. +#[test] +fn an_unsyncable_directory_fails_the_acknowledgement() { + use std::os::unix::fs::PermissionsExt as _; + + let mut dir = std::env::temp_dir(); + dir.push(format!("serf-keyring-unsync-{}", std::process::id())); + // Ignoring Err: a leftover directory from a previous run is fine to reuse. + let _ = std::fs::create_dir(&dir); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).expect("open the dir"); + let path = dir.join("ring"); + let engine = KeyringFilePersistence::new(&path); + + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes128([11u8; 16]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = SecretKey::ChaCha20Poly1305([11u8; 32]); + + // Write+search without read: the temp creation, write, and rename all + // succeed, but the directory handle needed for the durability sync cannot + // be opened — the acknowledgement must report that as a failure. + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o300)) + .expect("make the dir unsyncable"); + let outcome = acked(engine.enqueue(&Keyring::new(key))); + std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).expect("restore the dir"); + assert!( + outcome.is_err(), + "an un-syncable rename must not acknowledge success" + ); + + // Ignoring Err: best-effort test-tree cleanup. + let _ = std::fs::remove_dir_all(&dir); +} + +/// A write failure is acknowledged as an error — the response gate's failure +/// signal — not silently swallowed. +#[test] +fn persistence_failure_is_acknowledged_as_an_error() { + let mut path = std::env::temp_dir(); + path.push(format!("serf-keyring-no-such-dir-{}", std::process::id())); + path.push("ring"); + let engine = KeyringFilePersistence::new(&path); + + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes128([8u8; 16]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = SecretKey::ChaCha20Poly1305([8u8; 32]); + assert!( + acked(engine.enqueue(&Keyring::new(key))).is_err(), + "a write into a missing directory must acknowledge failure" + ); +} + +/// Dropping the engine joins the worker after it drains the queue: a +/// rotation handed off immediately before the drop is on disk when `drop` +/// returns, so a shutdown cannot discard a write the wire already carries. +#[test] +fn drop_joins_the_worker_and_flushes_queued_rotations() { + let path = tmp_path("drop-flush"); + let engine = KeyringFilePersistence::new(&path); + + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes128([9u8; 16]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = SecretKey::ChaCha20Poly1305([9u8; 32]); + + let ack = engine.enqueue(&Keyring::new(key)); + drop(engine); + + // No waiting: the join inside drop already flushed the queue. + let loaded = KeyringFilePersistence::new(&path) + .load() + .expect("the flushed write parses") + .expect("the flushed write exists"); + assert_eq!(loaded.primary_ref(), &key); + assert!( + matches!(ack.try_recv(), Ok(Ok(()))), + "the queued rotation was acknowledged before the worker exited" + ); + + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} + +/// A destination with no file name cannot host a sibling temp: the write +/// reports `InvalidInput` instead of panicking or writing anywhere. +#[test] +fn a_nameless_destination_is_an_input_error() { + let err = write_via_exclusive_temp(Path::new("/"), b"contents") + .expect_err("a bare root has no file name to derive a temp from"); + assert_eq!(err.kind(), io::ErrorKind::InvalidInput); +} diff --git a/serf-driver/src/lib.rs b/serf-driver/src/lib.rs index b420deab..9169b1d3 100644 --- a/serf-driver/src/lib.rs +++ b/serf-driver/src/lib.rs @@ -14,10 +14,13 @@ pub mod error; #[cfg(all(encryption, any(feature = "tcp", feature = "quic")))] mod keyring; +#[cfg(all(encryption, any(feature = "tcp", feature = "quic"), unix))] +mod keyring_file; #[cfg(any(feature = "tcp", feature = "quic"))] mod observation; #[cfg(any(feature = "tcp", feature = "quic"))] mod snapshot; +mod snapshotter; #[cfg(all(encryption, any(feature = "tcp", feature = "quic")))] #[cfg_attr( @@ -27,10 +30,26 @@ mod snapshot; any(feature = "tcp", feature = "quic") ))) )] -pub use keyring::{KeyApplyOutcome, apply_key_request, keyring_carries_cross_cipher_twin}; +pub use keyring::{ + KeyApplyOutcome, KeyringPersistError, KeyringPersistRx, KeyringPersistence, apply_key_request, + keyring_carries_cross_cipher_twin, +}; +#[cfg(all(encryption, any(feature = "tcp", feature = "quic"), unix))] +#[cfg_attr( + docsrs, + doc(cfg(all( + any(feature = "aes-gcm", feature = "chacha20-poly1305"), + any(feature = "tcp", feature = "quic"), + unix + ))) +)] +pub use keyring_file::{KeyringFileError, KeyringFilePersistence}; #[cfg(any(feature = "tcp", feature = "quic"))] #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] pub use observation::observation_payload_bytes; #[cfg(any(feature = "tcp", feature = "quic"))] #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] pub use snapshot::{SerfSnapshot, SerfStats}; +pub use snapshotter::{ + DEFAULT_SNAPSHOT_COMPACT_THRESHOLD, OpenedSnapshot, SnapshotOpenError, Snapshotter, +}; diff --git a/serf-reactor/src/driver/snapshotter/mod.rs b/serf-driver/src/snapshotter/mod.rs similarity index 92% rename from serf-reactor/src/driver/snapshotter/mod.rs rename to serf-driver/src/snapshotter/mod.rs index a04d87e2..04689593 100644 --- a/serf-reactor/src/driver/snapshotter/mod.rs +++ b/serf-driver/src/snapshotter/mod.rs @@ -19,12 +19,14 @@ use std::{ use memberlist_proto::Data; use serf_proto::{LamportTime, snapshot::SnapshotRecord}; -use super::options::SnapshotOptions; +/// Default snapshot compaction threshold: the append file is rewritten to the +/// live state once it grows past this many bytes. +pub const DEFAULT_SNAPSHOT_COMPACT_THRESHOLD: u64 = 128 * 1024; /// The snapshot writer paired with the records already on disk, as handed /// from the constructor (which opens and decodes) to the transport body /// (which replays and pumps). -pub(crate) type OpenedSnapshot = (Snapshotter, Vec>); +pub type OpenedSnapshot = (Snapshotter, Vec>); /// The append-side of the snapshot file, held by the driver pump. /// @@ -33,7 +35,7 @@ pub(crate) type OpenedSnapshot = (Snapshotter, Vec { +pub struct Snapshotter { path: PathBuf, file: io::BufWriter, bytes_written: u64, @@ -61,8 +63,11 @@ where /// partial trailing record and appends continue after the last whole one. A /// malformed record BEFORE the tail (an unknown tag or an undecodable node) /// is a hard error — the file is not trustworthy. - pub(crate) fn open(opts: &SnapshotOptions) -> Result, SnapshotOpenError> { - let path = opts.path().to_path_buf(); + pub fn open( + path: impl Into, + compact_threshold: u64, + ) -> Result, SnapshotOpenError> { + let path: PathBuf = path.into(); let raw = match fs::read(&path) { Ok(b) => b, Err(e) if e.kind() == io::ErrorKind::NotFound => Vec::new(), @@ -106,7 +111,7 @@ where path, file: io::BufWriter::new(file), bytes_written: cursor as u64, - compact_threshold: opts.compact_threshold(), + compact_threshold, last_member_clock: LamportTime::ZERO, last_event_clock: LamportTime::ZERO, last_query_clock: LamportTime::ZERO, @@ -155,11 +160,7 @@ where /// Append the membership records for one surfaced member event: `Alive` for /// a joined or updated member, `NotAlive` for a left, failed, or reaped one. - pub(crate) fn append_member( - &mut self, - alive: bool, - node: &memberlist_proto::Node, - ) { + pub fn append_member(&mut self, alive: bool, node: &memberlist_proto::Node) { let record = if alive { SnapshotRecord::Alive(node.clone()) } else { @@ -170,12 +171,7 @@ where } /// Append any clock high-water marks that advanced since the last append. - pub(crate) fn append_clocks( - &mut self, - member: LamportTime, - event: LamportTime, - query: LamportTime, - ) { + pub fn append_clocks(&mut self, member: LamportTime, event: LamportTime, query: LamportTime) { if member > self.last_member_clock { self.append(&SnapshotRecord::Clock(member)); self.last_member_clock = member; @@ -193,7 +189,7 @@ where /// Append the leave marker: the local node cleanly left the cluster. On the /// next start, replay clears the recovered state unless /// `rejoin_after_leave` ignores it. - pub(crate) fn append_leave(&mut self) { + pub fn append_leave(&mut self) { self.clean_left = true; self.append(&SnapshotRecord::Leave); } @@ -201,7 +197,7 @@ where /// Flush the buffered appends to the OS, and — when the file has grown past /// the compaction threshold — rewrite it to just `alive` and the clock /// floors via a sibling temp file and an atomic rename. - pub(crate) fn flush_and_maybe_compact( + pub fn flush_and_maybe_compact( &mut self, alive: impl FnOnce() -> Vec>, ) { diff --git a/serf-reactor/src/driver/snapshotter/tests.rs b/serf-driver/src/snapshotter/tests.rs similarity index 83% rename from serf-reactor/src/driver/snapshotter/tests.rs rename to serf-driver/src/snapshotter/tests.rs index 94139d92..35335b14 100644 --- a/serf-reactor/src/driver/snapshotter/tests.rs +++ b/serf-driver/src/snapshotter/tests.rs @@ -6,12 +6,39 @@ use smol_str::SmolStr; use super::*; -fn opts(name: &str) -> SnapshotOptions { +/// A fresh snapshot path plus the compaction threshold under test. +struct TestSnapshot { + path: PathBuf, + compact_threshold: u64, +} + +impl TestSnapshot { + fn path(&self) -> &std::path::Path { + &self.path + } + + fn with_compact_threshold(mut self, threshold: u64) -> Self { + self.compact_threshold = threshold; + self + } + + fn open(&self) -> Result, SnapshotOpenError> + where + I: memberlist_proto::Data + Clone + Eq + core::hash::Hash, + { + Snapshotter::open(&self.path, self.compact_threshold) + } +} + +fn opts(name: &str) -> TestSnapshot { let mut p = std::env::temp_dir(); p.push(format!("serf-snapshotter-{name}-{}", std::process::id())); // Ignoring Err: a leftover file from a previous run is fine to lose. let _ = fs::remove_file(&p); - SnapshotOptions::new(p) + TestSnapshot { + path: p, + compact_threshold: crate::DEFAULT_SNAPSHOT_COMPACT_THRESHOLD, + } } fn node(id: &str, port: u16) -> Node { @@ -21,7 +48,7 @@ fn node(id: &str, port: u16) -> Node { ) } -fn cleanup(o: &SnapshotOptions) { +fn cleanup(o: &TestSnapshot) { // Ignoring Err: best-effort test-file cleanup. let _ = fs::remove_file(o.path()); } @@ -33,7 +60,7 @@ fn cleanup(o: &SnapshotOptions) { fn appends_replay_across_reopen() { let o = opts("roundtrip"); { - let (mut snap, records) = Snapshotter::::open(&o).expect("first open of a fresh path"); + let (mut snap, records) = o.open::().expect("first open of a fresh path"); assert!(records.is_empty(), "a fresh path replays to nothing"); snap.append_member(true, &node("a", 7001)); snap.append_member(true, &node("b", 7002)); @@ -46,7 +73,7 @@ fn appends_replay_across_reopen() { snap.flush_and_maybe_compact(Vec::new); } - let (_snap, records) = Snapshotter::::open(&o).expect("reopen parses"); + let (_snap, records) = o.open::().expect("reopen parses"); let replay = ReplayResult::replay(records, false); assert_eq!(replay.alive_nodes, vec![node("a", 7001)]); assert_eq!(replay.last_clock, LamportTime::new(5)); @@ -61,14 +88,14 @@ fn appends_replay_across_reopen() { fn leave_marker_gates_the_replay() { let o = opts("leave-gate"); { - let (mut snap, _) = Snapshotter::::open(&o).expect("open"); + let (mut snap, _) = o.open::().expect("open"); snap.append_member(true, &node("a", 7001)); snap.append_clocks(LamportTime::new(9), LamportTime::ZERO, LamportTime::ZERO); snap.append_leave(); snap.flush_and_maybe_compact(Vec::new); } - let (_s, records) = Snapshotter::::open(&o).expect("reopen"); + let (_s, records) = o.open::().expect("reopen"); let fresh = ReplayResult::replay(records.clone(), false); assert!(fresh.alive_nodes.is_empty(), "a clean leave starts fresh"); assert_eq!(fresh.last_clock, LamportTime::ZERO); @@ -90,7 +117,7 @@ fn leave_marker_gates_the_replay() { fn truncated_tail_is_tolerated_and_repaired() { let o = opts("torn-tail"); { - let (mut snap, _) = Snapshotter::::open(&o).expect("open"); + let (mut snap, _) = o.open::().expect("open"); snap.append_member(true, &node("a", 7001)); snap.flush_and_maybe_compact(Vec::new); } @@ -105,7 +132,7 @@ fn truncated_tail_is_tolerated_and_repaired() { f.write_all(&[0x00, 0xff, 0xff]).expect("write torn tail"); } - let (mut snap, records) = Snapshotter::::open(&o).expect("torn tail tolerated"); + let (mut snap, records) = o.open::().expect("torn tail tolerated"); let replay = ReplayResult::replay(records, false); assert_eq!(replay.alive_nodes, vec![node("a", 7001)]); @@ -113,7 +140,7 @@ fn truncated_tail_is_tolerated_and_repaired() { snap.append_member(true, &node("b", 7002)); snap.flush_and_maybe_compact(Vec::new); drop(snap); - let (_s, records) = Snapshotter::::open(&o).expect("reopen after repair"); + let (_s, records) = o.open::().expect("reopen after repair"); let replay = ReplayResult::replay(records, false); assert_eq!(replay.alive_nodes, vec![node("a", 7001), node("b", 7002)]); cleanup(&o); @@ -135,10 +162,7 @@ fn corrupt_middle_record_refuses_to_open() { f.write_all(&rec).expect("write trailing record"); } assert!( - matches!( - Snapshotter::::open(&o), - Err(SnapshotOpenError::Corrupt(_)) - ), + matches!(o.open::(), Err(SnapshotOpenError::Corrupt(_))), "an unknown tag before the tail must refuse the open" ); cleanup(&o); @@ -150,7 +174,7 @@ fn corrupt_middle_record_refuses_to_open() { fn compaction_rewrites_to_the_live_state() { let o = opts("compact").with_compact_threshold(64); { - let (mut snap, _) = Snapshotter::::open(&o).expect("open"); + let (mut snap, _) = o.open::().expect("open"); // Churn well past 64 bytes: many joins and removals of a transient peer. for i in 0..32u16 { snap.append_member(true, &node("transient", 8000 + i)); @@ -168,7 +192,7 @@ fn compaction_rewrites_to_the_live_state() { size < 128, "compaction must shrink the churned file, got {size} bytes" ); - let (_s, records) = Snapshotter::::open(&o).expect("reopen compacted"); + let (_s, records) = o.open::().expect("reopen compacted"); let replay = ReplayResult::replay(records, false); assert_eq!(replay.alive_nodes, vec![node("kept", 7001)]); assert_eq!(replay.last_clock, LamportTime::new(7)); @@ -185,7 +209,7 @@ fn compaction_rewrites_to_the_live_state() { fn compaction_preserves_the_clean_leave_gate() { let o = opts("compact-leave").with_compact_threshold(1); { - let (mut snap, _) = Snapshotter::::open(&o).expect("open"); + let (mut snap, _) = o.open::().expect("open"); snap.append_member(true, &node("peer", 7001)); snap.append_clocks(LamportTime::new(4), LamportTime::ZERO, LamportTime::ZERO); snap.append_leave(); @@ -193,7 +217,7 @@ fn compaction_preserves_the_clean_leave_gate() { snap.flush_and_maybe_compact(|| vec![node("peer", 7001)]); } - let (_s, records) = Snapshotter::::open(&o).expect("reopen compacted"); + let (_s, records) = o.open::().expect("reopen compacted"); let fresh = ReplayResult::replay(records.clone(), false); assert!( fresh.alive_nodes.is_empty(), @@ -225,7 +249,7 @@ fn leave_tail_replays_identically_across_compaction() { let plain = opts("leave-order-plain"); let compacted = opts("leave-order-compacted").with_compact_threshold(1); for o in [&plain, &compacted] { - let (mut snap, _) = Snapshotter::::open(o).expect("open"); + let (mut snap, _) = o.open::().expect("open"); snap.append_member(true, &node("peer", 7001)); snap.append_clocks( LamportTime::new(8), @@ -236,9 +260,8 @@ fn leave_tail_replays_identically_across_compaction() { snap.flush_and_maybe_compact(|| vec![node("peer", 7001)]); } - let (_p, plain_records) = Snapshotter::::open(&plain).expect("reopen the original"); - let (_c, compacted_records) = - Snapshotter::::open(&compacted).expect("reopen the compacted"); + let (_p, plain_records) = plain.open::().expect("reopen the original"); + let (_c, compacted_records) = compacted.open::().expect("reopen the compacted"); for rejoin in [false, true] { let original = ReplayResult::replay(plain_records.clone(), rejoin); let rewritten = ReplayResult::replay(compacted_records.clone(), rejoin); @@ -278,12 +301,12 @@ fn leave_tail_replays_identically_across_compaction() { fn membership_after_a_leave_clears_the_compacted_gate() { let o = opts("compact-rejoined").with_compact_threshold(1); { - let (mut snap, _) = Snapshotter::::open(&o).expect("open"); + let (mut snap, _) = o.open::().expect("open"); snap.append_leave(); snap.append_member(true, &node("peer", 7001)); snap.flush_and_maybe_compact(|| vec![node("peer", 7001)]); } - let (_s, records) = Snapshotter::::open(&o).expect("reopen"); + let (_s, records) = o.open::().expect("reopen"); let fresh = ReplayResult::replay(records, false); assert_eq!( fresh.alive_nodes, diff --git a/serf-reactor/Cargo.toml b/serf-reactor/Cargo.toml index b348f601..65ae9d3c 100644 --- a/serf-reactor/Cargo.toml +++ b/serf-reactor/Cargo.toml @@ -28,13 +28,11 @@ aes-gcm = [ "serf-proto/aes-gcm", "serf-driver/aes-gcm", "memberlist-proto/aes-gcm", - "dep:zeroize", ] chacha20-poly1305 = [ "serf-proto/chacha20-poly1305", "serf-driver/chacha20-poly1305", "memberlist-proto/chacha20-poly1305", - "dep:zeroize", ] # Regex-backed tag-filter matching. tag-regex = ["serf-proto/tag-regex", "serf-driver/tag-regex"] @@ -64,7 +62,6 @@ smol_str.workspace = true smallvec.workspace = true bytes = { workspace = true } thiserror.workspace = true -zeroize = { version = "1", optional = true, default-features = false } futures-channel.workspace = true futures-util = { workspace = true, default-features = false, features = [ "std", diff --git a/serf-reactor/src/delegate/keyring_file/mod.rs b/serf-reactor/src/delegate/keyring_file/mod.rs index 8811fff9..a0f28c69 100644 --- a/serf-reactor/src/delegate/keyring_file/mod.rs +++ b/serf-reactor/src/delegate/keyring_file/mod.rs @@ -1,63 +1,27 @@ -//! [`FileKeyringDelegate`] — turnkey file persistence for keyring rotations. +//! [`FileKeyringDelegate`] — turnkey file persistence for keyring rotations, +//! backed by the shared [`serf_driver::KeyringFilePersistence`] engine. -use std::{ - io, - path::{Path, PathBuf}, - sync::mpsc, -}; +use std::path::{Path, PathBuf}; -use memberlist_proto::{Keyring, SecretKey}; -use serf_proto::{secret_key_from_bytes, secret_key_to_bytes}; +use memberlist_proto::Keyring; +pub use serf_driver::KeyringFileError; +use serf_driver::KeyringFilePersistence; -use super::{KeyringDelegate, KeyringPersistError, KeyringPersistence}; +use super::{KeyringDelegate, KeyringPersistence}; /// A [`KeyringDelegate`] that persists every keyring rotation to a file, plus /// a [`load`](Self::load) to rebuild the ring at construction — the turnkey /// replacement for the legacy keyring-file option, with the persistence /// app-owned like any other delegate. /// -/// FORMAT: one lowercase-hex line per key, the PRIMARY key first, each line -/// decoding to `[algorithm_tag][raw_key_bytes]`. The leading tag byte keeps -/// the encoding self-describing, so two ciphers sharing a key length (AES-256 -/// and ChaCha20-Poly1305 are both 32 bytes) stay distinguishable. The file -/// holds raw key material by design — protect it with filesystem permissions -/// exactly as the reference implementation's keyring file required. -/// -/// Rotations are handed to a dedicated persistence thread — `keyring_updated` -/// runs inline on the driver pump, which must never block on storage, so the -/// callback only clones the ring into an unbounded channel (rotations are -/// rare and small) and the worker does the file I/O, acknowledging each write -/// back through [`KeyringPersistence::Pending`] so the driver can gate the -/// key response on durability. Each write goes through an exclusively-created, -/// owner-only (`0600` on Unix), unpredictably-named sibling temp file, synced, -/// then atomically renamed over the destination: a crash mid-write never -/// truncates the previous ring, a restrictive mode on the key file is never -/// widened by a rotation, and key bytes can never land in a pre-existing -/// inode or behind a planted symlink. Dropping the delegate joins the -/// worker after it drains every queued rotation, so a shutdown cannot discard -/// a write that was already acknowledged toward the wire. -/// -/// UNIX-ONLY: the acknowledgement contract is rename durability — the -/// containing directory is synced before a rotation reports success — and no -/// safe standard API can flush a directory entry on Windows, so this turnkey -/// delegate does not exist there rather than acknowledge a rotation a power -/// loss could revert. A Windows application implements [`KeyringDelegate`] -/// itself with a platform-durable strategy (a write-through rename via the -/// platform APIs, or storage with its own durability contract). +/// The file mechanics — the self-describing hex format, the exclusively +/// created owner-only temps, the identity-gated stale-temp sweep, and the +/// directory-synced rename the acknowledgement waits on — live in the shared +/// [`KeyringFilePersistence`] engine; see its documentation for the format +/// and durability contract (including why the engine, and therefore this +/// delegate, is Unix-only). pub struct FileKeyringDelegate { - path: PathBuf, - /// Hand-off to the persistence thread; `None` only during drop, which hangs - /// up first so the worker drains and exits. - worker: Option>, - /// The persistence thread, joined on drop after the hang-up. - handle: Option>, -} - -/// One queued rotation: the ring to write and the acknowledgement sender the -/// pump's parked key response polls. -struct PersistJob { - ring: Keyring, - ack: mpsc::Sender>, + engine: KeyringFilePersistence, } impl FileKeyringDelegate { @@ -68,39 +32,15 @@ impl FileKeyringDelegate { /// guarantee) and abandoned temps from crashed rotations — before the /// first write can race one. pub fn new(path: impl Into) -> Self { - let path: PathBuf = path.into(); - sweep_stale_temps(&path); - let (worker, jobs) = mpsc::channel::(); - let worker_path = path.clone(); - let handle = std::thread::spawn(move || { - while let Ok(job) = jobs.recv() { - let res = persist(&worker_path, &job.ring); - if let Err(_err) = &res { - #[cfg(feature = "tracing")] - tracing::warn!( - path = %worker_path.display(), - error = %_err, - "serf keyring rotation could not be persisted; the wire keeps the rotated ring" - ); - } - // Ignoring Err: the pump dropped this rotation's receiver (teardown, - // or the requester's deadline passed) — the outcome has nowhere to go. - let _ = job - .ack - .send(res.map_err(|e| Box::new(e) as KeyringPersistError)); - } - }); Self { - path, - worker: Some(worker), - handle: Some(handle), + engine: KeyringFilePersistence::new(path), } } /// The persistence path. #[must_use] pub fn path(&self) -> &Path { - &self.path + self.engine.path() } /// Load a keyring previously persisted by this delegate. @@ -114,324 +54,13 @@ impl FileKeyringDelegate { /// [`KeyringFileError::Parse`] on a malformed line (bad hex, an unknown /// algorithm tag, a key length not matching its tag, or an empty file). pub fn load(&self) -> Result, KeyringFileError> { - let raw = match std::fs::read_to_string(&self.path) { - Ok(s) => s, - Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(None), - Err(e) => return Err(KeyringFileError::Io(e)), - }; - let mut keys = raw - .lines() - .map(str::trim) - .filter(|l| !l.is_empty()) - .map(parse_key_line); - let primary = keys - .next() - .transpose()? - .ok_or_else(|| KeyringFileError::Parse("the keyring file holds no keys".into()))?; - let secondaries = keys.collect::, _>>()?; - Ok(Some(Keyring::with_secondaries(primary, secondaries))) - } -} - -/// Serialize `keyring` into the file format (primary first, one hex line per -/// key) and write it via an exclusively-created owner-only temp file, sync, -/// and atomic rename. -fn persist(path: &Path, keyring: &Keyring) -> Result<(), KeyringFileError> { - use zeroize::Zeroize as _; - let mut out = String::new(); - push_key_line(&mut out, keyring.primary_ref()); - for key in keyring.secondaries() { - push_key_line(&mut out, key); - } - let res = write_via_exclusive_temp(path, out.as_bytes()).map_err(KeyringFileError::Io); - out.zeroize(); - res -} - -/// Write `contents` to `path` through an exclusively-created, owner-only, -/// unpredictably-named sibling temp file, synced then atomically renamed over -/// the destination. -/// -/// `create_new` (`O_CREAT | O_EXCL`) never reuses an existing inode and never -/// follows a symlink — a file or link already sitting at the temp path fails -/// the attempt instead of receiving the key bytes — and the OS-entropy name -/// keeps such a path from being plantable ahead of time. The inode is born -/// `0600` on Unix and re-asserted on the open handle, so raw key material -/// only ever lands in a fresh owner-only inode this process created; a crash -/// mid-write never truncates the previous ring. -fn write_via_exclusive_temp(path: &Path, contents: &[u8]) -> io::Result<()> { - use std::io::Write as _; - let name = path - .file_name() - .and_then(|n| n.to_str()) - .ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "keyring path has no file name"))?; - let dir = path.parent().filter(|d| !d.as_os_str().is_empty()); - let mut attempts = 0u8; - let (tmp, mut file) = loop { - let tmp_name = format!(".{name}.{:016x}.tmp", temp_nonce()?); - let tmp = match dir { - Some(d) => d.join(&tmp_name), - None => PathBuf::from(&tmp_name), - }; - let mut opts = std::fs::OpenOptions::new(); - opts.write(true).create_new(true); - { - use std::os::unix::fs::OpenOptionsExt as _; - opts.mode(0o600); - } - match opts.open(&tmp) { - Ok(file) => break (tmp, file), - // A 64-bit OS-entropy collision is practically a squatted path; a - // bounded retry with a fresh nonce outlasts any accidental leftover - // without spinning against a directory an attacker keeps filling. - Err(e) if e.kind() == io::ErrorKind::AlreadyExists && attempts < 16 => attempts += 1, - Err(e) => return Err(e), - } - }; - let res = (|| { - { - use std::os::unix::fs::PermissionsExt as _; - file.set_permissions(std::fs::Permissions::from_mode(0o600))?; - } - file.write_all(contents)?; - file.sync_all()?; - std::fs::rename(&tmp, path)?; - // The rename is not durable until the DIRECTORY entry is: a crash after - // returning `Ok` here must not revert the destination to the old file — - // the acknowledgement built on this return is what releases a successful - // key response to the cluster. - sync_dir(dir.unwrap_or_else(|| Path::new("."))) - })(); - if res.is_err() { - // Ignoring Err: removing the failed temp (it holds key bytes) is - // best-effort hygiene; the write error itself is what propagates. A - // failure after the rename consumed the temp removes nothing. - let _ = std::fs::remove_file(&tmp); - } - res -} - -/// Sync a directory so a completed rename of an entry inside it survives a -/// crash. -fn sync_dir(dir: &Path) -> io::Result<()> { - std::fs::File::open(dir)?.sync_all() -} - -/// OS-entropy nonce for a temp-file name: unpredictable, so a -/// directory-writing attacker cannot pre-plant a file or symlink at the next -/// temp path. -fn temp_nonce() -> io::Result { - use rand::TryRng as _; - rand::rngs::SysRng.try_next_u64().map_err(io::Error::other) -} - -/// Remove leftovers a rotation can no longer reuse: the fixed-name sibling -/// temp earlier releases wrote (whose permissions predate the owner-only -/// guarantee and may already hold key material), and the exact-shape -/// `.{name}.{16 hex}.tmp` temps a crashed rotation abandoned. Every removal -/// is gated on FILESYSTEM IDENTITY against the destination — a candidate -/// that resolves to the destination's storage (lexical identity, a -/// case-folding filesystem, or a symlink on either side) is never touched, -/// so sweeping hygiene can never delete the persisted keyring. Removing a -/// planted symlink unlinks the LINK, never its target. Best-effort: a sweep -/// failure never blocks construction — `create_new` already keeps every -/// future write off any path that survives. -fn sweep_stale_temps(path: &Path) { - if legacy_temp_provably_distinct(path) && sweepable(path, &path.with_extension("tmp")) { - // Ignoring Err: nothing to sweep, or no permission — both non-fatal. - let _ = std::fs::remove_file(path.with_extension("tmp")); - } - let (Some(dir), Some(name)) = ( - path.parent().filter(|d| !d.as_os_str().is_empty()), - path.file_name().and_then(|n| n.to_str()), - ) else { - return; - }; - let prefix = format!(".{name}."); - let Ok(entries) = std::fs::read_dir(dir) else { - return; - }; - for entry in entries.flatten() { - let file_name = entry.file_name(); - let Some(f) = file_name.to_str() else { - continue; - }; - // Exact-shape match only — a sibling file that merely shares the prefix - // and suffix (an operator's own backup, say) is not this delegate's to - // delete. - let matches_temp_shape = f - .strip_prefix(&prefix) - .and_then(|rest| rest.strip_suffix(".tmp")) - .is_some_and(|mid| mid.len() == 16 && mid.bytes().all(|b| b.is_ascii_hexdigit())); - if matches_temp_shape && sweepable(path, &entry.path()) { - // Ignoring Err: best-effort sweep of abandoned temps. - let _ = std::fs::remove_file(entry.path()); - } - } -} - -/// Whether the fixed-name legacy temp's name is PROVABLY distinct from the -/// destination's under every supported filename-alias relation. -/// -/// The legacy image differs from the destination only in its extension, so -/// the alias question reduces to the extension replacement. Sweeping is -/// allowed only when the destination's extension is pure ASCII and does not -/// ASCII-case-fold to `tmp` — case-insensitive HFS+ additionally IGNORES -/// certain Unicode scalars when comparing names, so an extension carrying -/// any non-ASCII scalar could fold the two names together and is -/// conservatively refused. No extension at all is safe: the image then -/// APPENDS `.tmp`, four non-ignorable ASCII characters no folding can -/// absorb. Random-suffix candidates need no such classifier — their names -/// carry a dot prefix and a 16-hex infix the destination's name does not, -/// an excess of non-ignorable ASCII no alias relation can erase. -fn legacy_temp_provably_distinct(path: &Path) -> bool { - match path.extension() { - None => true, - Some(e) => e - .to_str() - .is_some_and(|e| e.is_ascii() && !e.eq_ignore_ascii_case("tmp")), - } -} - -/// Whether removing `candidate` cannot touch the keyring the destination -/// `path` reaches. -/// -/// `remove_file` unlinks a NAME, so the guards are layered by what a name -/// can do: a candidate whose name can itself name the destination — -/// lexically, or equal under the ASCII case folding that aliases names on -/// case-insensitive filesystems — is NEVER sweepable, because a rotation's -/// rename can land between any identity observation and the unlink, and the -/// unlink would then remove whatever the destination name holds (the freshly -/// persisted keyring). Only for genuinely distinct names — where the unlink -/// cannot remove the destination's entry — is resolved filesystem identity -/// consulted: `metadata` FOLLOWS symlinks, so each side resolves to the file -/// a reader would actually open, and a candidate reaching the destination's -/// storage through a symlink or hard link is skipped. -fn sweepable(path: &Path, candidate: &Path) -> bool { - use std::os::unix::fs::MetadataExt as _; - if candidate == path { - return false; - } - match ( - candidate.file_name().and_then(|n| n.to_str()), - path.file_name().and_then(|n| n.to_str()), - ) { - (Some(c), Some(p)) if c.eq_ignore_ascii_case(p) => return false, - (Some(_), Some(_)) => {} - // Un-inspectable names: never delete on uncertainty. - _ => return false, - } - if std::fs::symlink_metadata(candidate).is_err() { - // Nothing at the candidate name; removal would be a no-op. - return false; - } - let candidate_meta = std::fs::metadata(candidate); - // The window between the two observations is where a concurrent rename - // lands; the tests widen it deterministically to prove the name guards - // above — not luck — are what keep a mid-check rename safe. - #[cfg(test)] - tests::between_identity_observations(path); - let destination_meta = std::fs::metadata(path); - match (candidate_meta, destination_meta) { - // A dangling symlink at the candidate name reaches no storage at all. - (Err(e), _) if e.kind() == io::ErrorKind::NotFound => true, - (Ok(c), Ok(d)) => (c.dev(), c.ino()) != (d.dev(), d.ino()), - // The destination resolves to nothing: with the name-aliasing shapes - // already excluded above, an existing candidate cannot BE the missing - // destination — the two are genuinely distinct. - (Ok(_), Err(e)) if e.kind() == io::ErrorKind::NotFound => true, - // Identity cannot be established: never delete on uncertainty. - _ => false, + self.engine.load() } } impl KeyringDelegate for FileKeyringDelegate { fn keyring_updated(&self, keyring: &Keyring) -> KeyringPersistence { - // Non-blocking hand-off: the pump must never wait on storage. A failed - // send (the worker already hung up) drops the job — and with it the ack - // sender — so the returned receiver disconnects and the pump reports the - // rotation unpersisted rather than silently acknowledged. - let (ack, rx) = mpsc::channel(); - if let Some(worker) = &self.worker { - // Ignoring Err: see above — the dropped job's disconnected receiver IS - // the failure signal. - let _ = worker.send(PersistJob { - ring: keyring.clone(), - ack, - }); - } - KeyringPersistence::Pending(rx) - } -} - -impl Drop for FileKeyringDelegate { - fn drop(&mut self) { - // Hang up, then join: the worker drains every queued rotation before it - // exits, so a shutdown cannot discard a write the wire already carries. - self.worker = None; - if let Some(handle) = self.handle.take() { - // Ignoring Err: a panicked worker already surfaced its failure through - // the acknowledgement channel; there is nothing to unwind into here. - let _ = handle.join(); - } - } -} - -/// Errors from [`FileKeyringDelegate::load`]. -#[derive(Debug, thiserror::Error)] -pub enum KeyringFileError { - /// Reading or writing the keyring file failed. - #[error(transparent)] - Io(#[from] io::Error), - /// The file's contents are not a valid keyring serialization. - #[error("malformed keyring file: {0}")] - Parse(String), -} - -/// Append one key as a lowercase-hex `[tag][bytes]` line. The transient -/// tagged buffer is zeroed before it is freed. -fn push_key_line(out: &mut String, key: &SecretKey) { - use core::fmt::Write as _; - let tagged = secret_key_to_bytes(key); - for b in tagged.iter() { - // Ignoring Err: writing hex digits into a String cannot fail. - let _ = write!(out, "{b:02x}"); - } - out.push('\n'); -} - -/// Parse one lowercase-hex `[tag][bytes]` line into a [`SecretKey`]. -fn parse_key_line(line: &str) -> Result { - use zeroize::Zeroize as _; - if line.len() % 2 != 0 { - return Err(KeyringFileError::Parse("odd-length hex key line".into())); - } - let mut buf = Vec::with_capacity(line.len() / 2); - for pair in line.as_bytes().chunks_exact(2) { - let hi = hex_val(pair[0]); - let lo = hex_val(pair[1]); - match (hi, lo) { - (Some(h), Some(l)) => buf.push((h << 4) | l), - _ => { - buf.zeroize(); - return Err(KeyringFileError::Parse( - "non-hex character in key line".into(), - )); - } - } - } - let bytes = bytes::Bytes::copy_from_slice(&buf); - buf.zeroize(); - secret_key_from_bytes(&bytes).map_err(|e| KeyringFileError::Parse(e.to_string())) -} - -/// The value of one lowercase/uppercase hex digit. -fn hex_val(c: u8) -> Option { - match c { - b'0'..=b'9' => Some(c - b'0'), - b'a'..=b'f' => Some(c - b'a' + 10), - b'A'..=b'F' => Some(c - b'A' + 10), - _ => None, + KeyringPersistence::Pending(self.engine.enqueue(keyring)) } } diff --git a/serf-reactor/src/delegate/keyring_file/tests.rs b/serf-reactor/src/delegate/keyring_file/tests.rs index a9204e9d..1f85dba2 100644 --- a/serf-reactor/src/delegate/keyring_file/tests.rs +++ b/serf-reactor/src/delegate/keyring_file/tests.rs @@ -1,546 +1,84 @@ use super::*; -use std::cell::RefCell; - -/// This thread's mid-check hook, when a test installed one. -type ObservationGap = RefCell>>; - -thread_local! { - /// Per-test hook fired between `sweepable`'s two identity observations — - /// the window a concurrent rotation's rename can land in. Thread-local so - /// parallel tests never see each other's hooks. - static BETWEEN_OBSERVATIONS: ObservationGap = const { RefCell::new(None) }; -} - -/// Called by `sweepable` between its two `metadata` observations. -pub(super) fn between_identity_observations(path: &Path) { - BETWEEN_OBSERVATIONS.with(|hook| { - if let Some(f) = hook.borrow_mut().as_mut() { - f(path); - } - }); -} - -/// Install `f` as this thread's mid-check hook for the duration of `run`. -fn with_observation_gap(f: impl FnMut(&Path) + 'static, run: impl FnOnce()) { - BETWEEN_OBSERVATIONS.with(|hook| *hook.borrow_mut() = Some(Box::new(f))); - run(); - BETWEEN_OBSERVATIONS.with(|hook| *hook.borrow_mut() = None); -} - -/// Wait for one rotation's persistence acknowledgement. -fn acked(p: KeyringPersistence) -> Result<(), KeyringPersistError> { - match p { - KeyringPersistence::Durable => Ok(()), - KeyringPersistence::Pending(rx) => rx - .recv_timeout(std::time::Duration::from_secs(5)) - .expect("the persistence worker acknowledges within the bound"), - } -} +use memberlist_proto::SecretKey; fn tmp_path(name: &str) -> PathBuf { let mut p = std::env::temp_dir(); - p.push(format!("serf-keyring-file-{name}-{}", std::process::id())); + p.push(format!( + "serf-reactor-keyring-{name}-{}", + std::process::id() + )); + // Ignoring Err: a leftover file from a previous run is fine to lose. + let _ = std::fs::remove_file(&p); p } -/// Any sibling temp file this delegate could have produced for `path`: the -/// legacy fixed-name temp or a random-suffix one. -fn temp_residue(path: &Path) -> Vec { - let mut residue = Vec::new(); - let legacy = path.with_extension("tmp"); - if legacy.symlink_metadata().is_ok() { - residue.push(legacy); - } - let (Some(dir), Some(name)) = (path.parent(), path.file_name().and_then(|n| n.to_str())) else { - return residue; - }; - let prefix = format!(".{name}."); - for entry in std::fs::read_dir(dir).expect("temp dir listable").flatten() { - if let Some(f) = entry.file_name().to_str() - && f.starts_with(&prefix) - && f.ends_with(".tmp") - { - residue.push(entry.path()); - } - } - residue +fn test_key(fill: u8) -> SecretKey { + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes128([fill; 16]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = SecretKey::ChaCha20Poly1305([fill; 32]); + key } -/// A persisted rotation round-trips: the acknowledged `keyring_updated` write -/// (primary first) is on disk, and `load` rebuilds it with the same primary -/// and secondaries. +/// A rotation handed through the DELEGATE trait persists and acknowledges: +/// `keyring_updated` returns a pending acknowledgement that resolves once the +/// write is durable, and `load` rebuilds the persisted ring. #[test] -fn rotation_round_trips_through_the_file() { +fn rotation_round_trips_through_the_delegate() { let path = tmp_path("roundtrip"); let delegate = FileKeyringDelegate::new(&path); + assert_eq!(delegate.path(), path.as_path()); - #[cfg(feature = "aes-gcm")] - let (primary, secondary) = (SecretKey::Aes128([1u8; 16]), SecretKey::Aes256([2u8; 32])); - #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] - let (primary, secondary) = ( - SecretKey::ChaCha20Poly1305([1u8; 32]), - SecretKey::ChaCha20Poly1305([2u8; 32]), - ); - - let ring = Keyring::with_secondaries(primary, [secondary]); - acked(delegate.keyring_updated(&ring)).expect("the rotation persists"); + let ring = Keyring::new(test_key(1)); + let KeyringPersistence::Pending(ack) = delegate.keyring_updated(&ring) else { + panic!("a file-backed rotation is never durable inline"); + }; + ack + .recv_timeout(std::time::Duration::from_secs(5)) + .expect("the persistence worker acknowledges within the bound") + .expect("the rotation persists"); let loaded = delegate .load() .expect("an acknowledged write parses") .expect("an acknowledged write exists"); assert_eq!(loaded.primary_ref(), ring.primary_ref()); - assert_eq!(loaded.secondaries(), ring.secondaries()); - - // Ignoring Err: best-effort test-file cleanup. - let _ = std::fs::remove_file(&path); -} - -/// First boot: a missing file loads as `None`, not an error. -#[test] -fn missing_file_loads_as_none() { - let delegate = FileKeyringDelegate::new(tmp_path("missing")); - assert!( - delegate - .load() - .expect("not-found is not an error") - .is_none() - ); -} - -/// Malformed contents are a parse error, never a panic: odd-length hex, -/// non-hex characters, an unknown algorithm tag, and an empty file. -#[test] -fn malformed_files_are_parse_errors() { - for (name, contents) in [ - ("odd", "abc\n"), - ("nonhex", "zz\n"), - ("badtag", "ff00112233445566778899aabbccddeeff\n"), - ("empty", "\n"), - ] { - let path = tmp_path(name); - std::fs::write(&path, contents).expect("write test file"); - let delegate = FileKeyringDelegate::new(&path); - assert!( - matches!(delegate.load(), Err(KeyringFileError::Parse(_))), - "{name} must be a parse error" - ); - // Ignoring Err: best-effort test-file cleanup. - let _ = std::fs::remove_file(&path); - } -} - -/// A second rotation atomically replaces the file: the newest ring wins and -/// no temp-file residue remains under either naming scheme. -#[test] -fn a_second_rotation_replaces_the_first() { - let path = tmp_path("replace"); - let delegate = FileKeyringDelegate::new(&path); - - #[cfg(feature = "aes-gcm")] - let (first, second) = (SecretKey::Aes128([3u8; 16]), SecretKey::Aes128([4u8; 16])); - #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] - let (first, second) = ( - SecretKey::ChaCha20Poly1305([3u8; 32]), - SecretKey::ChaCha20Poly1305([4u8; 32]), - ); - - acked(delegate.keyring_updated(&Keyring::new(first))).expect("first rotation persists"); - acked(delegate.keyring_updated(&Keyring::new(second))).expect("second rotation persists"); - - let loaded = delegate - .load() - .expect("parses") - .expect("the file exists after two rotations"); - assert_eq!(loaded.primary_ref(), &second); - assert!(loaded.secondaries().is_empty()); - assert!( - temp_residue(&path).is_empty(), - "the atomic rename must consume every temp file" - ); - - // Ignoring Err: best-effort test-file cleanup. - let _ = std::fs::remove_file(&path); -} - -/// A rotation never widens the key file's mode — and it NARROWS a permissive -/// one: the replacing temp inode is born owner-only, so a fresh file is -/// created `0600` and a pre-existing `0644` destination is `0600` after the -/// next rotation, even under a permissive umask. -#[test] -fn rotation_enforces_owner_only_permissions() { - use std::os::unix::fs::PermissionsExt as _; - - let path = tmp_path("perms"); - // A permissive pre-existing destination (an operator's hand-created file). - std::fs::write(&path, "junk\n").expect("pre-create the destination"); - std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)) - .expect("widen the destination"); - let delegate = FileKeyringDelegate::new(&path); - - #[cfg(feature = "aes-gcm")] - let (first, second) = (SecretKey::Aes128([5u8; 16]), SecretKey::Aes128([6u8; 16])); - #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] - let (first, second) = ( - SecretKey::ChaCha20Poly1305([5u8; 32]), - SecretKey::ChaCha20Poly1305([6u8; 32]), - ); - - acked(delegate.keyring_updated(&Keyring::new(first))).expect("first rotation persists"); - let mode = std::fs::metadata(&path).expect("stat").permissions().mode() & 0o777; - assert_eq!( - mode, 0o600, - "the first rotation narrows a permissive destination to owner-only" - ); - - acked(delegate.keyring_updated(&Keyring::new(second))).expect("second rotation persists"); - let mode = std::fs::metadata(&path).expect("stat").permissions().mode() & 0o777; - assert_eq!(mode, 0o600, "a rotation must not widen the key file's mode"); - - // Ignoring Err: best-effort test-file cleanup. - let _ = std::fs::remove_file(&path); -} - -/// Construction sweeps both classes of stale sibling temps: the fixed-name -/// temp earlier releases wrote (possibly permissive and already holding key -/// material) and abandoned random-suffix temps from crashed rotations. -#[test] -fn construction_sweeps_stale_temps() { - let path = tmp_path("sweep"); - let legacy = path.with_extension("tmp"); - std::fs::write(&legacy, "stale key bytes at permissive mode\n").expect("plant the legacy temp"); - let name = path.file_name().and_then(|n| n.to_str()).expect("name"); - let abandoned = path.with_file_name(format!(".{name}.00000000deadbeef.tmp")); - std::fs::write(&abandoned, "abandoned partial write\n").expect("plant the abandoned temp"); - - let _delegate = FileKeyringDelegate::new(&path); - assert!( - !legacy.exists(), - "the legacy fixed-name temp must be swept at construction" - ); - assert!( - !abandoned.exists(), - "an abandoned random-suffix temp must be swept at construction" - ); -} - -/// A symlink planted at the legacy temp path is unlinked — the LINK, never -/// its target — and no rotation ever writes through it: the exclusive -/// creation refuses any pre-existing path, so key bytes cannot be redirected -/// into an attacker-chosen file. -#[test] -fn a_planted_symlink_never_receives_key_bytes() { - let path = tmp_path("symlink"); - let victim = tmp_path("symlink-victim"); - std::fs::write(&victim, "victim contents\n").expect("create the victim"); - let planted = path.with_extension("tmp"); - // Ignoring Err: a leftover link from a previous run is about to be re-planted. - let _ = std::fs::remove_file(&planted); - std::os::unix::fs::symlink(&victim, &planted).expect("plant the symlink"); - - let delegate = FileKeyringDelegate::new(&path); - assert!( - planted.symlink_metadata().is_err(), - "construction unlinks the planted symlink" - ); - assert_eq!( - std::fs::read_to_string(&victim).expect("victim readable"), - "victim contents\n", - "unlinking removes the LINK, never its target" - ); - - #[cfg(feature = "aes-gcm")] - let key = SecretKey::Aes128([7u8; 16]); - #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] - let key = SecretKey::ChaCha20Poly1305([7u8; 32]); - acked(delegate.keyring_updated(&Keyring::new(key))).expect("rotation persists"); - assert_eq!( - std::fs::read_to_string(&victim).expect("victim readable"), - "victim contents\n", - "no rotation writes through a planted path" - ); - - // Ignoring Err: best-effort test-file cleanup. - let _ = std::fs::remove_file(&path); - let _ = std::fs::remove_file(&victim); -} - -/// A destination whose own extension is `tmp` is NOT its legacy temp: the -/// construction sweep must preserve it — the previous implementation wrote -/// such a destination in place, so the file can hold the only copy of the -/// keyring. -#[test] -fn a_tmp_extension_destination_survives_construction() { - let path = tmp_path("selfnamed").with_extension("tmp"); - // Ignoring Err: a leftover file from a previous run is about to be rewritten. - let _ = std::fs::remove_file(&path); - - #[cfg(feature = "aes-gcm")] - let key = SecretKey::Aes128([10u8; 16]); - #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] - let key = SecretKey::ChaCha20Poly1305([10u8; 32]); - { - let delegate = FileKeyringDelegate::new(&path); - acked(delegate.keyring_updated(&Keyring::new(key))).expect("the rotation persists"); - } - - let reopened = FileKeyringDelegate::new(&path); - let loaded = reopened - .load() - .expect("the persisted keyring parses") - .expect("constructing a delegate must not sweep a .tmp-named destination"); - assert_eq!(loaded.primary_ref(), &key); - - // Ignoring Err: best-effort test-file cleanup. - let _ = std::fs::remove_file(&path); -} - -/// The alias guard is case-insensitive: a destination named with an -/// uppercase `TMP` extension lexically differs from its lowercase -/// `with_extension` image, yet the two alias the same file on the -/// case-insensitive filesystems that are the default on macOS — construction -/// must preserve it. -#[test] -fn an_uppercase_tmp_destination_survives_construction() { - let path = tmp_path("selfnamed-upper").with_extension("TMP"); - // Ignoring Err: a leftover file from a previous run is about to be rewritten. - let _ = std::fs::remove_file(&path); - - #[cfg(feature = "aes-gcm")] - let key = SecretKey::Aes128([12u8; 16]); - #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] - let key = SecretKey::ChaCha20Poly1305([12u8; 32]); - { - let delegate = FileKeyringDelegate::new(&path); - acked(delegate.keyring_updated(&Keyring::new(key))).expect("the rotation persists"); - } - - let reopened = FileKeyringDelegate::new(&path); - let loaded = reopened - .load() - .expect("the persisted keyring parses") - .expect("constructing a delegate must not sweep a case-aliased .TMP destination"); - assert_eq!(loaded.primary_ref(), &key); - - // Ignoring Err: best-effort test-file cleanup. - let _ = std::fs::remove_file(&path); -} - -/// A symlinked destination keeps its target through construction: with the -/// configured path pointing at a file that happens to live at the legacy -/// temp name, the sweep resolves filesystem IDENTITY — not names — and -/// leaves the keyring intact for the follow-up load. -#[test] -fn a_symlinked_destination_keeps_its_target_through_construction() { - let target = tmp_path("linked").with_extension("tmp"); - let link = tmp_path("linked").with_extension("current"); - // Ignoring Err: leftovers from a previous run are about to be recreated. - let _ = std::fs::remove_file(&link); - let _ = std::fs::remove_file(&target); - - #[cfg(feature = "aes-gcm")] - let key = SecretKey::Aes128([13u8; 16]); - #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] - let key = SecretKey::ChaCha20Poly1305([13u8; 32]); - { - // Persist a real ring at the target path (itself a tmp-named - // destination, which construction must already preserve). - let seed = FileKeyringDelegate::new(&target); - acked(seed.keyring_updated(&Keyring::new(key))).expect("the seed rotation persists"); - } - std::os::unix::fs::symlink(&target, &link).expect("plant the destination symlink"); - - let delegate = FileKeyringDelegate::new(&link); - let loaded = delegate - .load() - .expect("the symlinked keyring parses") - .expect("construction must not sweep the storage a symlinked destination resolves to"); - assert_eq!(loaded.primary_ref(), &key); - - // Ignoring Err: best-effort test-file cleanup. - let _ = std::fs::remove_file(&link); - let _ = std::fs::remove_file(&target); -} - -/// The legacy-sweep classifier admits only extensions provably distinct -/// from `tmp` under every supported filename-alias relation: pure-ASCII -/// non-tmp extensions (or none, where the image appends characters no -/// folding can absorb) pass; any tmp case-fold or any non-ASCII scalar — -/// case-insensitive HFS+ ignores certain Unicode scalars when comparing -/// names, so `ring.t\u{200D}mp` aliases `ring.tmp` there — is refused. -#[test] -fn the_legacy_sweep_classifier_refuses_unprovable_extensions() { - for (destination, distinct) in [ - ("ring.keys", true), - ("ring", true), - ("ring.tmp", false), - ("ring.TMP", false), - ("ring.Tmp", false), - // An HFS+-ignorable scalar (ZERO WIDTH JOINER) inside the extension. - ("ring.t\u{200D}mp", false), - // Any non-ASCII scalar is unprovable, ignorable or not. - ("ring.cl\u{00E9}s", false), - ] { - assert_eq!( - legacy_temp_provably_distinct(Path::new(destination)), - distinct, - "{destination:?}" - ); - } -} - -/// A destination whose extension is not provably distinct keeps its legacy -/// image on EVERY filesystem: whether or not the running filesystem folds -/// the two names together, the conservative refusal leaves the sibling file -/// alone — the fold-aliasing filesystems are exactly where that sibling IS -/// the persisted keyring. -#[test] -fn an_unprovable_extension_keeps_the_legacy_sibling() { - let path = tmp_path("ignorable").with_extension("t\u{200D}mp"); - let sibling = path.with_extension("tmp"); - std::fs::write(&sibling, "possibly the persisted keyring\n").expect("seed the sibling"); - - let _delegate = FileKeyringDelegate::new(&path); - assert!( - sibling.exists(), - "an unprovable extension must refuse the legacy sweep" - ); - - // Ignoring Err: best-effort test-file cleanup. - let _ = std::fs::remove_file(&sibling); - let _ = std::fs::remove_file(&path); -} - -/// A rename landing between the sweep's two identity observations must -/// never lose the destination: `remove_file` unlinks a NAME, so a candidate -/// whose name can itself name the destination is refused by the name guard -/// BEFORE any identity observation — the widened observation gap here -/// atomically replaces the destination exactly as a concurrent rotation -/// would, and construction must leave the replacement intact. -#[test] -fn a_rename_landing_mid_check_never_loses_the_destination() { - let path = tmp_path("raced").with_extension("tmp"); - std::fs::write(&path, "pre-rotation contents\n").expect("seed the destination"); - - with_observation_gap( - |dest: &Path| { - // The concurrent rotation: a fresh inode atomically renamed over the - // destination, mid-check. - let staged = dest.with_file_name(".raced-replacement"); - std::fs::write(&staged, "freshly persisted contents\n").expect("stage the replacement"); - std::fs::rename(&staged, dest).expect("land the replacement"); - }, - || { - let _delegate = FileKeyringDelegate::new(&path); - }, - ); - - // With the name guard in place the observation gap is never reached for a - // name-aliasing candidate, so the original contents remain; what must hold - // in every world is that the destination NAME was not unlinked. - let contents = - std::fs::read_to_string(&path).expect("the destination survives construction un-unlinked"); - assert!( - contents == "pre-rotation contents\n" || contents == "freshly persisted contents\n", - "the destination holds one of the two written generations, never nothing" - ); // Ignoring Err: best-effort test-file cleanup. let _ = std::fs::remove_file(&path); } -/// The success acknowledgement is durability: a rotation whose parent -/// directory cannot be synced reports failure, because the completed rename -/// is not crash-durable until the directory entry is. -#[test] -fn an_unsyncable_directory_fails_the_acknowledgement() { - use std::os::unix::fs::PermissionsExt as _; - - let mut dir = std::env::temp_dir(); - dir.push(format!("serf-keyring-unsync-{}", std::process::id())); - // Ignoring Err: a leftover directory from a previous run is fine to reuse. - let _ = std::fs::create_dir(&dir); - std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).expect("open the dir"); - let path = dir.join("ring"); - let delegate = FileKeyringDelegate::new(&path); - - #[cfg(feature = "aes-gcm")] - let key = SecretKey::Aes128([11u8; 16]); - #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] - let key = SecretKey::ChaCha20Poly1305([11u8; 32]); - - // Write+search without read: the temp creation, write, and rename all - // succeed, but the directory handle needed for the durability sync cannot - // be opened — the acknowledgement must report that as a failure. - std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o300)) - .expect("make the dir unsyncable"); - let outcome = acked(delegate.keyring_updated(&Keyring::new(key))); - std::fs::set_permissions(&dir, std::fs::Permissions::from_mode(0o700)).expect("restore the dir"); - assert!( - outcome.is_err(), - "an un-syncable rename must not acknowledge success" - ); - - // Ignoring Err: best-effort test-tree cleanup. - let _ = std::fs::remove_dir_all(&dir); -} - -/// A write failure is acknowledged as an error — the response gate's failure -/// signal — not silently swallowed. -#[test] -fn persistence_failure_is_acknowledged_as_an_error() { - let mut path = std::env::temp_dir(); - path.push(format!("serf-keyring-no-such-dir-{}", std::process::id())); - path.push("ring"); - let delegate = FileKeyringDelegate::new(&path); - - #[cfg(feature = "aes-gcm")] - let key = SecretKey::Aes128([8u8; 16]); - #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] - let key = SecretKey::ChaCha20Poly1305([8u8; 32]); - assert!( - acked(delegate.keyring_updated(&Keyring::new(key))).is_err(), - "a write into a missing directory must acknowledge failure" - ); -} - -/// Dropping the delegate joins the worker after it drains the queue: a -/// rotation handed off immediately before the drop is on disk when `drop` -/// returns, so a shutdown cannot discard a write the wire already carries. +/// Dropping the delegate flushes the queue through the engine's worker join: +/// a rotation enqueued immediately before the drop is on disk when `drop` +/// returns. #[test] -fn drop_joins_the_worker_and_flushes_queued_rotations() { +fn drop_flushes_queued_rotations() { let path = tmp_path("drop-flush"); let delegate = FileKeyringDelegate::new(&path); - - #[cfg(feature = "aes-gcm")] - let key = SecretKey::Aes128([9u8; 16]); - #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] - let key = SecretKey::ChaCha20Poly1305([9u8; 32]); - - let ack = delegate.keyring_updated(&Keyring::new(key)); + let key = test_key(2); + // The acknowledgement is intentionally not awaited: the drop is the barrier. + let _pending = delegate.keyring_updated(&Keyring::new(key)); drop(delegate); - // No waiting: the join inside drop already flushed the queue. let loaded = FileKeyringDelegate::new(&path) .load() .expect("the flushed write parses") .expect("the flushed write exists"); assert_eq!(loaded.primary_ref(), &key); - assert!( - matches!(ack, KeyringPersistence::Pending(rx) if matches!(rx.try_recv(), Ok(Ok(())))), - "the queued rotation was acknowledged before the worker exited" - ); // Ignoring Err: best-effort test-file cleanup. let _ = std::fs::remove_file(&path); } -/// A destination with no file name cannot host a sibling temp: the write -/// reports `InvalidInput` instead of panicking or writing anywhere. +/// First boot: a missing file loads as `None`, not an error. #[test] -fn a_nameless_destination_is_an_input_error() { - let err = write_via_exclusive_temp(Path::new("/"), b"contents") - .expect_err("a bare root has no file name to derive a temp from"); - assert_eq!(err.kind(), io::ErrorKind::InvalidInput); +fn missing_file_loads_as_none() { + let delegate = FileKeyringDelegate::new(tmp_path("missing")); + assert!( + delegate + .load() + .expect("not-found is not an error") + .is_none() + ); } diff --git a/serf-reactor/src/delegate/mod.rs b/serf-reactor/src/delegate/mod.rs index 568779de..5e513850 100644 --- a/serf-reactor/src/delegate/mod.rs +++ b/serf-reactor/src/delegate/mod.rs @@ -213,47 +213,14 @@ pub trait KeyringDelegate: Send + Sync + 'static { } } -/// A persistence failure reported through [`KeyringPersistence::Pending`]. +/// The rotation-durability acknowledgement contract, shared with the other +/// runtime drivers through `serf-driver`. #[cfg(encryption)] #[cfg_attr( docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] -pub type KeyringPersistError = Box; - -/// Receiver half of one rotation's persistence acknowledgement. -#[cfg(encryption)] -#[cfg_attr( - docsrs, - doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) -)] -pub type KeyringPersistRx = std::sync::mpsc::Receiver>; - -/// How one keyring rotation reaches durability, reported back from -/// [`KeyringDelegate::keyring_updated`]. -/// -/// The reference implementation writes its keyring file synchronously inside -/// the key-management query handler and folds a write failure into the -/// response. These drivers keep the pump non-blocking instead: a persisting -/// delegate hands back a receiver, the pump parks the key response, and sends -/// it once the receiver resolves — unchanged on success, downgraded to a -/// failed response carrying the error otherwise (a disconnected sender counts -/// as a failure: the worker vanished without acknowledging). The live wire -/// keyring keeps the rotation in every outcome. -#[cfg(encryption)] -#[cfg_attr( - docsrs, - doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) -)] -#[must_use = "dropping the acknowledgement silently un-gates the key response from persistence"] -pub enum KeyringPersistence { - /// The rotation needs no out-of-band persistence (or completed inline): - /// the key response is sent immediately. - Durable, - /// Persistence runs out-of-band; the pump defers the key response until - /// the receiver resolves. - Pending(KeyringPersistRx), -} +pub use serf_driver::{KeyringPersistError, KeyringPersistRx, KeyringPersistence}; /// The join-merge veto predicate, re-exported from the machine. /// diff --git a/serf-reactor/src/driver/mod.rs b/serf-reactor/src/driver/mod.rs index 4bb77442..50558a55 100644 --- a/serf-reactor/src/driver/mod.rs +++ b/serf-reactor/src/driver/mod.rs @@ -7,8 +7,6 @@ pub(crate) mod options; #[cfg(any(feature = "tcp", feature = "quic"))] -pub(crate) mod snapshotter; - #[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) mod shared; diff --git a/serf-reactor/src/driver/options/mod.rs b/serf-reactor/src/driver/options/mod.rs index 3235ad17..032322de 100644 --- a/serf-reactor/src/driver/options/mod.rs +++ b/serf-reactor/src/driver/options/mod.rs @@ -604,7 +604,7 @@ impl Default for StreamTransportOptions { /// Default snapshot compaction threshold: the append file is rewritten to the /// live state once it grows past this many bytes. -pub const DEFAULT_SNAPSHOT_COMPACT_THRESHOLD: u64 = 128 * 1024; +pub use serf_driver::DEFAULT_SNAPSHOT_COMPACT_THRESHOLD; /// Snapshot persistence wiring: where the driver appends membership records /// and when it compacts the file. Supplied as a constructor argument (`None` diff --git a/serf-reactor/src/driver/quic/mod.rs b/serf-reactor/src/driver/quic/mod.rs index cb9b7868..ab00080d 100644 --- a/serf-reactor/src/driver/quic/mod.rs +++ b/serf-reactor/src/driver/quic/mod.rs @@ -372,7 +372,7 @@ where recv_force_pending: bool, /// Snapshot persistence: appends a record for every surfaced membership /// change and the advancing clock floors, `None` when persistence is off. - snapshotter: Option>, + snapshotter: Option>, /// The driver's keyring delegate: applies inbound key-management ops and produces /// the `respond_key` answer. Present only under an encryption backend. #[cfg(encryption)] @@ -399,7 +399,7 @@ where obs_payload_budget: Option, driver_opts: RuntimeOptions, label: Option, - snapshotter: Option>, + snapshotter: Option>, #[cfg(encryption)] keyring: Arc, ) -> Self { let buf_len = recv_buf_len_for(endpoint.gossip_mtu(), quic_max_udp_payload); @@ -1873,7 +1873,7 @@ pub(crate) fn spawn_quic_driver( delegate: D, driver_opts: RuntimeOptions, label: Option, - snapshotter: Option>, + snapshotter: Option>, #[cfg(encryption)] keyring: Arc, ) -> QuicDriver where diff --git a/serf-reactor/src/driver/stream/mod.rs b/serf-reactor/src/driver/stream/mod.rs index 1551e60b..03145225 100644 --- a/serf-reactor/src/driver/stream/mod.rs +++ b/serf-reactor/src/driver/stream/mod.rs @@ -467,7 +467,7 @@ where bridge_recv_buf_len: usize, /// Snapshot persistence: appends a record for every surfaced membership /// change and the advancing clock floors, `None` when persistence is off. - snapshotter: Option>, + snapshotter: Option>, /// The driver's keyring delegate: applies inbound key-management ops and /// produces the `respond_key` answer. Present only under an encryption backend. #[cfg(encryption)] @@ -499,7 +499,7 @@ where stream_opts: StreamTransportOptions, label: Option, stream_timeout: Duration, - snapshotter: Option>, + snapshotter: Option>, #[cfg(encryption)] keyring: Arc, ) -> Self { let buf_len = endpoint @@ -2360,7 +2360,7 @@ pub(crate) fn spawn_stream_driver( stream_opts: StreamTransportOptions, label: Option, stream_timeout: Duration, - snapshotter: Option>, + snapshotter: Option>, #[cfg(encryption)] keyring: Arc, ) -> StreamDriver where diff --git a/serf-reactor/src/error/mod.rs b/serf-reactor/src/error/mod.rs index a282e570..f1cb1517 100644 --- a/serf-reactor/src/error/mod.rs +++ b/serf-reactor/src/error/mod.rs @@ -157,7 +157,7 @@ pub enum SerfError { #[cfg(any(feature = "tcp", feature = "quic"))] #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] #[error(transparent)] - SnapshotOpen(#[from] crate::driver::snapshotter::SnapshotOpenError), + SnapshotOpen(#[from] serf_driver::SnapshotOpenError), /// The driver task has shut down and is no longer accepting commands. #[error("driver shut down")] diff --git a/serf-reactor/src/lib.rs b/serf-reactor/src/lib.rs index cd810ba6..57154656 100644 --- a/serf-reactor/src/lib.rs +++ b/serf-reactor/src/lib.rs @@ -196,4 +196,4 @@ pub use driver::options::{ }; #[cfg(any(feature = "tcp", feature = "quic"))] #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] -pub use driver::snapshotter::SnapshotOpenError; +pub use serf_driver::SnapshotOpenError; diff --git a/serf-reactor/src/serf/mod.rs b/serf-reactor/src/serf/mod.rs index b744311a..14c18836 100644 --- a/serf-reactor/src/serf/mod.rs +++ b/serf-reactor/src/serf/mod.rs @@ -194,7 +194,8 @@ where let snapshot = match snapshot { Some(opts) => { let (writer, records) = - crate::driver::snapshotter::Snapshotter::open(&opts).map_err(SerfError::from)?; + serf_driver::Snapshotter::open(opts.path(), opts.compact_threshold()) + .map_err(SerfError::from)?; Some((writer, records)) } None => None, diff --git a/serf-reactor/src/transport/runtime.rs b/serf-reactor/src/transport/runtime.rs index 4b10f293..ced35929 100644 --- a/serf-reactor/src/transport/runtime.rs +++ b/serf-reactor/src/transport/runtime.rs @@ -52,7 +52,7 @@ pub struct TransportRuntime { /// Snapshot persistence, opened (and its file decoded) by the `Serf` /// constructor so a corrupt file fails construction: `T::run` replays the /// records into the endpoint and hands the writer to the pump. - pub(crate) snapshot: Option>, + pub(crate) snapshot: Option>, /// The driver's keyring delegate, applied to inbound key-management requests. /// Present only under an encryption backend. #[cfg(encryption)] @@ -74,7 +74,7 @@ impl TransportRuntime { member_drop: ReactorDropCounter, reconnect_delegate: Option>>, merge_delegate: Option>>, - snapshot: Option>, + snapshot: Option>, #[cfg(encryption)] keyring: Arc, ) -> Self { Self { From ce55b1314c46ee29c03aa0ad9295e801851a6eaf Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 19:29:38 +1200 Subject: [PATCH 115/128] feat(serf-compio): persistence-acknowledged keyring delegates and the machine's merge predicate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit keyring_updated now returns the shared KeyringPersistence acknowledgement (default Durable, so observation-only delegates are unchanged): a rotated op's key response is parked until the acknowledgement resolves and a persistence failure folds into the response exactly as the reference implementation folds a keyring-file write error, bounded by the requester's response deadline. Both pumps reap parked responses ahead of every output drain (so a sent response flushes in the same pass) and fold the re-poll cadence into their timer targets only while something is parked; the settle logic itself is the serf-driver implementation the reactor already runs. FileKeyringDelegate arrives as a thin wrapper over the shared engine, Unix-only per its durability contract. The async MergeDelegate is retired: it had no call site — the machine merges push/pull state before any event surfaces, so a driver-side veto can never run — and its replacement is the machine's own synchronous push/pull filter, re-exported with docs stating exactly what it bounds (one exchange's bulk admission, not durable exclusion). --- serf-compio/src/delegate/keyring_file/mod.rs | 70 +++++++++ .../src/delegate/keyring_file/tests.rs | 81 +++++++++++ serf-compio/src/delegate/mod.rs | 108 ++++++++------ serf-compio/src/delegate/void.rs | 22 +-- serf-compio/src/driver/quic/mod.rs | 133 ++++++++++++++--- serf-compio/src/driver/stream/mod.rs | 134 +++++++++++++++--- serf-compio/src/lib.rs | 17 ++- serf-compio/src/serf/tests.rs | 5 +- serf-driver/src/keyring/mod.rs | 68 +++++++++ serf-driver/src/lib.rs | 5 +- serf-reactor/src/driver/shared.rs | 73 +--------- 11 files changed, 545 insertions(+), 171 deletions(-) create mode 100644 serf-compio/src/delegate/keyring_file/mod.rs create mode 100644 serf-compio/src/delegate/keyring_file/tests.rs diff --git a/serf-compio/src/delegate/keyring_file/mod.rs b/serf-compio/src/delegate/keyring_file/mod.rs new file mode 100644 index 00000000..a949de63 --- /dev/null +++ b/serf-compio/src/delegate/keyring_file/mod.rs @@ -0,0 +1,70 @@ +//! [`FileKeyringDelegate`] — turnkey file persistence for keyring rotations, +//! backed by the shared [`serf_driver::KeyringFilePersistence`] engine. + +use std::path::{Path, PathBuf}; + +use memberlist_proto::Keyring; +pub use serf_driver::KeyringFileError; +use serf_driver::{KeyringFilePersistence, KeyringPersistence}; + +use super::KeyringDelegate; + +/// A [`KeyringDelegate`] that persists every keyring rotation to a file, plus +/// a [`load`](Self::load) to rebuild the ring at construction — the turnkey +/// replacement for the legacy keyring-file option, with the persistence +/// app-owned like any other delegate. +/// +/// The file mechanics — the self-describing hex format, the exclusively +/// created owner-only temps, the identity-gated stale-temp sweep, and the +/// directory-synced rename the acknowledgement waits on — live in the shared +/// [`KeyringFilePersistence`] engine; see its documentation for the format +/// and durability contract (including why the engine, and therefore this +/// delegate, is Unix-only). The engine's persistence worker is a plain +/// thread, so the delegate drops into compio's single-threaded driver world +/// without carrying any `Send` requirement onto the pump. +pub struct FileKeyringDelegate { + engine: KeyringFilePersistence, +} + +impl FileKeyringDelegate { + /// A delegate persisting to `path`. + /// + /// Construction sweeps stale sibling temp files — the fixed-name temp + /// earlier releases wrote (whose permissions predate the owner-only + /// guarantee) and abandoned temps from crashed rotations — before the + /// first write can race one. + pub fn new(path: impl Into) -> Self { + Self { + engine: KeyringFilePersistence::new(path), + } + } + + /// The persistence path. + #[must_use] + pub fn path(&self) -> &Path { + self.engine.path() + } + + /// Load a keyring previously persisted by this delegate. + /// + /// Returns `Ok(None)` when the file does not exist (first boot). The first + /// line is the primary key; the rest are secondaries in decrypt-trial order. + /// + /// # Errors + /// + /// [`KeyringFileError::Io`] on a read failure other than not-found; + /// [`KeyringFileError::Parse`] on a malformed line (bad hex, an unknown + /// algorithm tag, a key length not matching its tag, or an empty file). + pub fn load(&self) -> Result, KeyringFileError> { + self.engine.load() + } +} + +impl KeyringDelegate for FileKeyringDelegate { + fn keyring_updated(&self, keyring: &Keyring) -> KeyringPersistence { + KeyringPersistence::Pending(self.engine.enqueue(keyring)) + } +} + +#[cfg(test)] +mod tests; diff --git a/serf-compio/src/delegate/keyring_file/tests.rs b/serf-compio/src/delegate/keyring_file/tests.rs new file mode 100644 index 00000000..133aef5a --- /dev/null +++ b/serf-compio/src/delegate/keyring_file/tests.rs @@ -0,0 +1,81 @@ +use super::*; + +use memberlist_proto::SecretKey; + +fn tmp_path(name: &str) -> PathBuf { + let mut p = std::env::temp_dir(); + p.push(format!("serf-compio-keyring-{name}-{}", std::process::id())); + // Ignoring Err: a leftover file from a previous run is fine to lose. + let _ = std::fs::remove_file(&p); + p +} + +fn test_key(fill: u8) -> SecretKey { + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes128([fill; 16]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = SecretKey::ChaCha20Poly1305([fill; 32]); + key +} + +/// A rotation handed through the DELEGATE trait persists and acknowledges: +/// `keyring_updated` returns a pending acknowledgement that resolves once the +/// write is durable, and `load` rebuilds the persisted ring. +#[test] +fn rotation_round_trips_through_the_delegate() { + let path = tmp_path("roundtrip"); + let delegate = FileKeyringDelegate::new(&path); + assert_eq!(delegate.path(), path.as_path()); + + let ring = Keyring::new(test_key(1)); + let KeyringPersistence::Pending(ack) = delegate.keyring_updated(&ring) else { + panic!("a file-backed rotation is never durable inline"); + }; + ack + .recv_timeout(std::time::Duration::from_secs(5)) + .expect("the persistence worker acknowledges within the bound") + .expect("the rotation persists"); + + let loaded = delegate + .load() + .expect("an acknowledged write parses") + .expect("an acknowledged write exists"); + assert_eq!(loaded.primary_ref(), ring.primary_ref()); + + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} + +/// Dropping the delegate flushes the queue through the engine's worker join: +/// a rotation enqueued immediately before the drop is on disk when `drop` +/// returns. +#[test] +fn drop_flushes_queued_rotations() { + let path = tmp_path("drop-flush"); + let delegate = FileKeyringDelegate::new(&path); + let key = test_key(2); + // The acknowledgement is intentionally not awaited: the drop is the barrier. + let _pending = delegate.keyring_updated(&Keyring::new(key)); + drop(delegate); + + let loaded = FileKeyringDelegate::new(&path) + .load() + .expect("the flushed write parses") + .expect("the flushed write exists"); + assert_eq!(loaded.primary_ref(), &key); + + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} + +/// First boot: a missing file loads as `None`, not an error. +#[test] +fn missing_file_loads_as_none() { + let delegate = FileKeyringDelegate::new(tmp_path("missing")); + assert!( + delegate + .load() + .expect("not-found is not an error") + .is_none() + ); +} diff --git a/serf-compio/src/delegate/mod.rs b/serf-compio/src/delegate/mod.rs index a6a8001b..33f31b59 100644 --- a/serf-compio/src/delegate/mod.rs +++ b/serf-compio/src/delegate/mod.rs @@ -8,12 +8,22 @@ //! //! `KeyringDelegate` and `MergeDelegate` are separate from the observation //! `Delegate` composite: `KeyringDelegate` is sync (keyring ops must not -//! block), and `MergeDelegate` is an async admission veto supplied at -//! construction rather than an observation hook. +//! block) and acknowledges each rotation's persistence, and `MergeDelegate` +//! is the machine's synchronous push/pull filter, supplied at construction +//! rather than an observation hook. +#[cfg(all(encryption, unix))] +mod keyring_file; mod void; -pub use void::{NoopMergeDelegate, VoidDelegate}; +pub use void::VoidDelegate; + +#[cfg(all(encryption, unix))] +#[cfg_attr( + docsrs, + doc(cfg(all(any(feature = "aes-gcm", feature = "chacha20-poly1305"), unix))) +)] +pub use keyring_file::{FileKeyringDelegate, KeyringFileError}; #[cfg(encryption)] pub use void::VoidKeyringDelegate; @@ -150,8 +160,14 @@ pub trait Delegate: /// no-op request do not fire it. /// /// [`keyring_updated`](Self::keyring_updated) is **synchronous and non-blocking**: -/// it runs on the driver pump. If persistence needs async I/O, hand the ring off to -/// a channel the observer owns and drain it elsewhere. +/// it runs on the driver pump. If persistence needs I/O, hand the ring off to a +/// worker and return +/// [`KeyringPersistence::Pending`](serf_driver::KeyringPersistence::Pending); +/// the pump polls the receiver without blocking and defers the rotated op's +/// key response until it resolves, folding a persistence failure into that +/// response (`result = false` carrying the error) exactly as the reference +/// implementation folds a keyring-file write error — with the live wire +/// keyring keeping the rotation either way. /// /// Requires the `aes-gcm` or `chacha20-poly1305` feature. #[cfg(encryption)] @@ -162,49 +178,38 @@ pub trait Delegate: pub trait KeyringDelegate: 'static { /// Called after a key-management request successfully rotated the live wire /// keyring, with the new ring the gossip and reliable planes now encrypt under. - /// Not called for a `list` or any refused or no-op request. The default is a - /// no-op — the rotation is applied to the wire regardless; overriding this only - /// adds out-of-band persistence. - fn keyring_updated(&self, keyring: &Keyring) { + /// Not called for a `list` or any refused or no-op request. The default needs + /// no out-of-band persistence and reports + /// [`KeyringPersistence::Durable`](serf_driver::KeyringPersistence::Durable) + /// — the rotation is applied to the wire regardless; overriding this only + /// adds persistence and its acknowledgement. + fn keyring_updated(&self, keyring: &Keyring) -> serf_driver::KeyringPersistence { let _ = keyring; // Unused: default no-op; override to persist the rotation. + serf_driver::KeyringPersistence::Durable } } -/// Async veto hook invoked by the driver on the join path before accepting -/// remote member state from a push-pull exchange. -/// -/// `Ok(())` permits the merge; `Err(Self::Error)` cancels it. The driver wraps -/// the concrete error into [`SerfError`](crate::SerfError) before forwarding it -/// to the join caller. +/// The join-merge veto predicate, re-exported from the machine. /// -/// The hook is **async and driver-side** deliberately: the application may need -/// to consult an ACL service or other async resource before deciding whether to -/// accept a batch of remote peers. A synchronous (Sans-I/O) filter would -/// preclude that. +/// Supplied at construction (the `merge_delegate` argument) and installed into +/// the memberlist machine, which consults it INLINE for every push/pull merge +/// — a join and a periodic anti-entropy refresh alike — before applying the +/// remote member state. Returning `false` cancels that merge: the vetoed peer +/// set is not applied from the exchange. /// -/// `!Send`-agnostic: neither the trait object nor the future returned by -/// `notify_merge` carry a `Send` bound, so compio's `!Send` driver can -/// implement it without wrapping. +/// This is a PUSH/PULL FILTER, not an admission-control boundary: a rejected +/// peer can still enter membership moments later through gossiped Alive +/// messages, exactly as in the reference implementation. Do not rely on it +/// for durable exclusion or as an ACL — it bounds what a single state +/// exchange can bulk-admit, nothing more. The predicate is synchronous by +/// design: it runs inside the machine's drain, so an application needing +/// async I/O (an ACL service, say) resolves its policy ahead of time and +/// answers from that resolved state here. /// /// Requires a stream or QUIC transport feature (`tcp` or `quic`). #[cfg(any(feature = "tcp", feature = "quic"))] #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] -#[allow(async_fn_in_trait)] -pub trait MergeDelegate: 'static { - /// The veto/error type this delegate reports when a merge is cancelled. - type Error; - - /// Called before the driver accepts inbound push-pull peer state. - /// - /// `peers` is the slice of remote [`Member`]s the cluster is about to merge. - /// Return `Ok(())` to proceed, or `Err(e)` to cancel the merge. - /// - /// The default implementation always permits the merge. - async fn notify_merge(&self, peers: &[Arc>]) -> Result<(), Self::Error> { - let _ = peers; // Unused in the default permit-all impl; an overriding delegate inspects it. - Ok(()) - } -} +pub use memberlist_proto::delegate::MergeDelegate; #[cfg(test)] mod tests { @@ -224,15 +229,28 @@ mod tests { assert_delegate(&v); } - /// Verify `NoopMergeDelegate` satisfies `MergeDelegate` with `Error = - /// Infallible` — a type-level check; no I/O needed. + /// The re-exported merge predicate is the machine's synchronous push/pull + /// filter — a type-level check that a plain permit-all impl satisfies it. #[cfg(any(feature = "tcp", feature = "quic"))] #[test] - fn noop_merge_delegate_satisfies_trait() { - fn assert_merge>( - _: &T, - ) { + fn a_sync_predicate_satisfies_the_merge_delegate() { + struct PermitAll; + impl MergeDelegate for PermitAll { + fn notify_merge( + &self, + _peers: memberlist_proto::MaybeOwned< + '_, + [memberlist_proto::typed::NodeState], + >, + ) -> bool { + true + } + } + fn assert_merge(_: &T) + where + T: MergeDelegate, + { } - assert_merge(&NoopMergeDelegate); + assert_merge(&PermitAll); } } diff --git a/serf-compio/src/delegate/void.rs b/serf-compio/src/delegate/void.rs index 230d08ce..581895d6 100644 --- a/serf-compio/src/delegate/void.rs +++ b/serf-compio/src/delegate/void.rs @@ -5,7 +5,7 @@ use core::marker::PhantomData; #[cfg(any(feature = "tcp", feature = "quic"))] -use super::{Delegate, MemberDelegate, MergeDelegate, QueryDelegate, UserEventDelegate}; +use super::{Delegate, MemberDelegate, QueryDelegate, UserEventDelegate}; #[cfg(encryption)] use super::KeyringDelegate; @@ -79,26 +79,6 @@ where type Address = A; } -/// A merge delegate that always permits merges. -/// -/// The default delegate for drivers that do not need join admission control. -/// Its associated error type is [`core::convert::Infallible`], reflecting that -/// `notify_merge` can never fail. -/// -/// Requires a stream or QUIC transport feature (`tcp` or `quic`). -#[cfg(any(feature = "tcp", feature = "quic"))] -#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] -pub struct NoopMergeDelegate; - -#[cfg(any(feature = "tcp", feature = "quic"))] -impl MergeDelegate for NoopMergeDelegate -where - I: 'static, - A: 'static, -{ - type Error = core::convert::Infallible; -} - /// A keyring delegate that persists nothing. /// /// The default for nodes that do not need to observe key rotations. The driver diff --git a/serf-compio/src/driver/quic/mod.rs b/serf-compio/src/driver/quic/mod.rs index 76cc25c0..c80431d5 100644 --- a/serf-compio/src/driver/quic/mod.rs +++ b/serf-compio/src/driver/quic/mod.rs @@ -68,6 +68,10 @@ use crate::{ snapshot::{SerfSnapshot, SnapshotCell}, }; #[cfg(encryption)] +use serf_driver::{ + AppliedKeyRequest, KEYRING_PERSIST_POLL_INTERVAL, PendingKeyResponse, settle_parked_key_response, +}; +#[cfg(encryption)] use serf_proto::{KeyResponseArgs, event::KeyRequest}; /// Driver-side state for one outstanding await-result join call. @@ -343,6 +347,8 @@ pub(crate) async fn quic_driver_loop( // closes the gossip socket so the bound port is free when the caller resumes // from `shutdown.await`. let mut shutdown_reply: Option>> = None; + #[cfg(encryption)] + let mut pending_key_responses: Vec> = Vec::new(); let mut pending = PendingCommands { joins: Vec::new(), leave: None, @@ -418,6 +424,8 @@ pub(crate) async fn quic_driver_loop( // `Dead`-self notices that must reach the wire before the socket drops), // reap, publish, break. if exit { + #[cfg(encryption)] + reap_pending_key_responses(&mut endpoint, &mut pending_key_responses, Instant::now()); drain_outputs::( &mut endpoint, &gossip_socket, @@ -429,6 +437,8 @@ pub(crate) async fn quic_driver_loop( &mut pending, #[cfg(encryption)] &*keyring, + #[cfg(encryption)] + &mut pending_key_responses, ) .await; reap_pending_joins(&mut endpoint, &mut pending.joins, Instant::now()).await; @@ -449,6 +459,10 @@ pub(crate) async fn quic_driver_loop( Some(endpoint_deadline), min_pending_join_deadline(&pending.joins), min_pending_leave_deadline(&pending.leave), + #[cfg(encryption)] + next_key_ack_check(&pending_key_responses, setup_now), + #[cfg(not(encryption))] + None, ] .into_iter() .flatten() @@ -478,6 +492,9 @@ pub(crate) async fn quic_driver_loop( dirty = true; } + #[cfg(encryption)] + reap_pending_key_responses(&mut endpoint, &mut pending_key_responses, Instant::now()); + let terminal = drain_outputs::( &mut endpoint, &gossip_socket, @@ -489,6 +506,8 @@ pub(crate) async fn quic_driver_loop( &mut pending, #[cfg(encryption)] &*keyring, + #[cfg(encryption)] + &mut pending_key_responses, ) .await; reap_pending_joins(&mut endpoint, &mut pending.joins, Instant::now()).await; @@ -508,6 +527,8 @@ pub(crate) async fn quic_driver_loop( // flush their outputs before entering the select so a snapshot observer sees // the post-input state promptly. if dirty { + #[cfg(encryption)] + reap_pending_key_responses(&mut endpoint, &mut pending_key_responses, Instant::now()); let terminal = drain_outputs::( &mut endpoint, &gossip_socket, @@ -519,6 +540,8 @@ pub(crate) async fn quic_driver_loop( &mut pending, #[cfg(encryption)] &*keyring, + #[cfg(encryption)] + &mut pending_key_responses, ) .await; reap_pending_joins(&mut endpoint, &mut pending.joins, Instant::now()).await; @@ -613,6 +636,8 @@ pub(crate) async fn quic_driver_loop( // Post-select drain (sends + sync accounting only — no second recv SQE). A // conflict `Event::Shutdown` drained here is terminal — fold it into `exit`. + #[cfg(encryption)] + reap_pending_key_responses(&mut endpoint, &mut pending_key_responses, Instant::now()); if drain_outputs::( &mut endpoint, &gossip_socket, @@ -624,6 +649,8 @@ pub(crate) async fn quic_driver_loop( &mut pending, #[cfg(encryption)] &*keyring, + #[cfg(encryption)] + &mut pending_key_responses, ) .await { @@ -1210,6 +1237,7 @@ async fn drain_events( pending: &mut PendingCommands, terminal: &mut bool, #[cfg(encryption)] keyring: &dyn KeyringDelegate, + #[cfg(encryption)] pending_key_responses: &mut Vec>, ) -> bool where I: memberlist_proto::Id + Clone, @@ -1259,11 +1287,21 @@ where // `respond_key`) ahead of the observation hand-off below. #[cfg(encryption)] if let Event::KeyRequest(req) = &ev { - let resp = apply_key_request_live(endpoint, keyring, req); - // Ignoring Err: `respond_key` fails only when the response cannot be routed - // (originator gone / relay dropped); the key op has already applied to the - // live wire keyring. - let _ = endpoint.respond_key(req, resp, Instant::now()); + match apply_key_request_live(endpoint, keyring, req) { + AppliedKeyRequest::Ready(resp) => { + // Ignoring Err: `respond_key` fails only when the response cannot be + // routed (originator gone / relay dropped); the key op has already + // applied to the live wire keyring. + let _ = endpoint.respond_key(req, resp, Instant::now()); + } + AppliedKeyRequest::AwaitingPersistence(resp, rx) => { + pending_key_responses.push(PendingKeyResponse { + req: req.clone(), + resp, + rx, + }); + } + } } let payload_bytes = observation_payload_bytes(&ev); @@ -1319,6 +1357,7 @@ async fn drain_outputs( obs_payload_budget: Option, pending: &mut PendingCommands, #[cfg(encryption)] keyring: &dyn KeyringDelegate, + #[cfg(encryption)] pending_key_responses: &mut Vec>, ) -> bool where I: memberlist_proto::Id + Clone, @@ -1340,6 +1379,8 @@ where &mut terminal, #[cfg(encryption)] keyring, + #[cfg(encryption)] + pending_key_responses, ) .await; if !(did_ingress || did_transmits || did_quic || did_events) { @@ -1357,15 +1398,17 @@ where /// against the live ring, and on a real mutation publishes the rotated ring back /// via `set_encryption_options` — re-keying the gossip datagram plane (the QUIC /// reliable path always skips, quinn encrypts the stream) — then notifies the -/// keyring observer for persistence. A node with no keyring configured answers -/// `result = false` and makes no wire change; a read-only `list` or a refused op -/// leaves the wire untouched. +/// keyring delegate: a durable-inline answer responds immediately, while +/// out-of-band persistence parks the response until its acknowledgement +/// resolves. A node with no keyring configured answers `result = false` and +/// makes no wire change; a read-only `list` or a refused op leaves the wire +/// untouched. #[cfg(encryption)] fn apply_key_request_live( endpoint: &mut QuicEndpoint, delegate: &dyn KeyringDelegate, req: &KeyRequest, -) -> KeyResponseArgs +) -> AppliedKeyRequest where I: memberlist_proto::Id + Clone, G: Rng, @@ -1373,19 +1416,26 @@ where { let mut encryption = endpoint.encryption_options().clone(); let Some(current) = encryption.keyring() else { - return KeyResponseArgs { + return AppliedKeyRequest::Ready(KeyResponseArgs { result: false, message: "no keyring configured on this node".into(), ..Default::default() - }; + }); }; let (resp, rotated) = serf_driver::apply_key_request(current, req.op(), req.key()).into_parts(); - if let Some(new_ring) = rotated { - encryption.set_keyring(new_ring.clone()); - endpoint.set_encryption_options(encryption); - delegate.keyring_updated(&new_ring); + match rotated { + Some(new_ring) => { + encryption.set_keyring(new_ring.clone()); + endpoint.set_encryption_options(encryption); + match delegate.keyring_updated(&new_ring) { + serf_driver::KeyringPersistence::Durable => AppliedKeyRequest::Ready(resp), + serf_driver::KeyringPersistence::Pending(rx) => { + AppliedKeyRequest::AwaitingPersistence(resp, rx) + } + } + } + None => AppliedKeyRequest::Ready(resp), } - resp } /// Per-driver observation task: dispatch each event's [`Delegate`] hook, then fan @@ -1476,6 +1526,57 @@ async fn reap_pending_joins( /// Earliest pending-join deadline, if any — folded into the per-iteration /// `timeout_deadline` so the timer fires by the first expiring join's deadline. +/// Settle parked key responses: send those whose persistence acknowledgement +/// resolved (as-is on success, downgraded to a failure carrying the error +/// otherwise), drop those whose requester's response deadline passed while +/// the acknowledgement was still pending (nothing useful can be routed), keep +/// the rest parked. Called ahead of each output drain so a sent response +/// flushes in the same pass. +#[cfg(encryption)] +fn reap_pending_key_responses( + endpoint: &mut QuicEndpoint, + parked: &mut Vec>, + now: Instant, +) where + I: memberlist_proto::Id + Clone, + G: Rng, + R: Rng + SeedableRng, +{ + let mut i = 0; + while i < parked.len() { + let entry = &parked[i]; + match settle_parked_key_response(&entry.rx, &entry.resp) { + Some(resp) => { + let entry = parked.swap_remove(i); + // Ignoring Err: `respond_key` fails only when the response cannot be + // routed; the key op has already applied to the live wire keyring. + let _ = endpoint.respond_key(&entry.req, resp, now); + } + None if now >= entry.req.deadline() => { + drop(parked.swap_remove(i)); + #[cfg(feature = "tracing")] + tracing::warn!( + "a keyring persistence acknowledgement outlived the key request's response deadline; the response was dropped" + ); + } + None => i += 1, + } + } +} + +/// Next instant a parked key response should be re-polled, folded into the +/// loop's timer target; `None` when nothing is parked. The persistence +/// acknowledgement arrives on a plain channel with no waker integration, so +/// the interval bounds the wait. +#[cfg(encryption)] +fn next_key_ack_check(parked: &[PendingKeyResponse], now: Instant) -> Option { + if parked.is_empty() { + None + } else { + Some(now + KEYRING_PERSIST_POLL_INTERVAL) + } +} + fn min_pending_join_deadline(pending_joins: &[PendingJoin]) -> Option { pending_joins .iter() diff --git a/serf-compio/src/driver/stream/mod.rs b/serf-compio/src/driver/stream/mod.rs index 37226b3f..70f2be31 100644 --- a/serf-compio/src/driver/stream/mod.rs +++ b/serf-compio/src/driver/stream/mod.rs @@ -76,6 +76,10 @@ use crate::{ snapshot::{SerfSnapshot, SnapshotCell}, }; #[cfg(encryption)] +use serf_driver::{ + AppliedKeyRequest, KEYRING_PERSIST_POLL_INTERVAL, PendingKeyResponse, settle_parked_key_response, +}; +#[cfg(encryption)] use serf_proto::{KeyResponseArgs, event::KeyRequest}; /// Driver-side state for one outstanding await-result join call. @@ -472,6 +476,8 @@ pub(crate) async fn stream_driver_loop( // cleanup closes the listener and gossip socket so the bound ports are free // when the caller resumes from `shutdown.await`. let mut shutdown_reply: Option>> = None; + #[cfg(encryption)] + let mut pending_key_responses: Vec> = Vec::new(); let mut pending = PendingCommands { joins: Vec::new(), leave: None, @@ -617,6 +623,8 @@ pub(crate) async fn stream_driver_loop( // Honor `exit` from the iter-top cmd drain before the select so a quiet // shutdown lands promptly. Flush, reap, publish, break. if exit { + #[cfg(encryption)] + reap_pending_key_responses(&mut endpoint, &mut pending_key_responses, Instant::now()); drain_outputs::( &mut endpoint, &mut bridges, @@ -631,6 +639,8 @@ pub(crate) async fn stream_driver_loop( &mut pending, #[cfg(encryption)] &*keyring, + #[cfg(encryption)] + &mut pending_key_responses, ) .await; reap_pending_joins(&mut endpoint, &mut pending.joins, Instant::now()).await; @@ -651,6 +661,10 @@ pub(crate) async fn stream_driver_loop( Some(endpoint_deadline), min_pending_join_deadline(&pending.joins), min_pending_leave_deadline(&pending.leave), + #[cfg(encryption)] + next_key_ack_check(&pending_key_responses, setup_now), + #[cfg(not(encryption))] + None, ] .into_iter() .flatten() @@ -685,6 +699,9 @@ pub(crate) async fn stream_driver_loop( dirty = true; } + #[cfg(encryption)] + reap_pending_key_responses(&mut endpoint, &mut pending_key_responses, Instant::now()); + let terminal = drain_outputs::( &mut endpoint, &mut bridges, @@ -699,6 +716,8 @@ pub(crate) async fn stream_driver_loop( &mut pending, #[cfg(encryption)] &*keyring, + #[cfg(encryption)] + &mut pending_key_responses, ) .await; reap_pending_joins(&mut endpoint, &mut pending.joins, Instant::now()).await; @@ -718,6 +737,8 @@ pub(crate) async fn stream_driver_loop( // flush their outputs before entering the select so a snapshot observer // sees the post-input state promptly. if dirty { + #[cfg(encryption)] + reap_pending_key_responses(&mut endpoint, &mut pending_key_responses, Instant::now()); let terminal = drain_outputs::( &mut endpoint, &mut bridges, @@ -732,6 +753,8 @@ pub(crate) async fn stream_driver_loop( &mut pending, #[cfg(encryption)] &*keyring, + #[cfg(encryption)] + &mut pending_key_responses, ) .await; reap_pending_joins(&mut endpoint, &mut pending.joins, Instant::now()).await; @@ -868,6 +891,8 @@ pub(crate) async fn stream_driver_loop( // A conflict `Event::Shutdown` drained here is terminal — fold it into // `exit` so the loop breaks into teardown after delivering the event. + #[cfg(encryption)] + reap_pending_key_responses(&mut endpoint, &mut pending_key_responses, Instant::now()); if drain_outputs::( &mut endpoint, &mut bridges, @@ -882,6 +907,8 @@ pub(crate) async fn stream_driver_loop( &mut pending, #[cfg(encryption)] &*keyring, + #[cfg(encryption)] + &mut pending_key_responses, ) .await { @@ -1592,15 +1619,17 @@ where /// the coordinator's live `encryption_options`, applies the op variant-exactly /// against the live ring, and on a real mutation publishes the rotated ring back /// via `set_encryption_options` — so the gossip and reliable planes re-key in -/// lockstep — then notifies the keyring observer for persistence. A node with no -/// keyring configured answers `result = false` and makes no wire change; a -/// read-only `list` or a refused op leaves the wire untouched. +/// lockstep — then hands the ring to the keyring delegate: a durable-inline +/// answer responds immediately, while out-of-band persistence parks the +/// response until its acknowledgement resolves. A node with no keyring +/// configured answers `result = false` and makes no wire change; a read-only +/// `list` or a refused op leaves the wire untouched. #[cfg(encryption)] fn apply_key_request_live( endpoint: &mut StreamEndpoint, delegate: &dyn KeyringDelegate, req: &KeyRequest, -) -> KeyResponseArgs +) -> AppliedKeyRequest where I: memberlist_proto::Id + Clone, RT: StreamTransport, @@ -1609,19 +1638,26 @@ where { let mut encryption = endpoint.encryption_options().clone(); let Some(current) = encryption.keyring() else { - return KeyResponseArgs { + return AppliedKeyRequest::Ready(KeyResponseArgs { result: false, message: "no keyring configured on this node".into(), ..Default::default() - }; + }); }; let (resp, rotated) = serf_driver::apply_key_request(current, req.op(), req.key()).into_parts(); - if let Some(new_ring) = rotated { - encryption.set_keyring(new_ring.clone()); - endpoint.set_encryption_options(encryption); - delegate.keyring_updated(&new_ring); + match rotated { + Some(new_ring) => { + encryption.set_keyring(new_ring.clone()); + endpoint.set_encryption_options(encryption); + match delegate.keyring_updated(&new_ring) { + serf_driver::KeyringPersistence::Durable => AppliedKeyRequest::Ready(resp), + serf_driver::KeyringPersistence::Pending(rx) => { + AppliedKeyRequest::AwaitingPersistence(resp, rx) + } + } + } + None => AppliedKeyRequest::Ready(resp), } - resp } /// Drain every queued serf [`Event`]: synchronous protocol accounting (leave @@ -1642,6 +1678,7 @@ async fn drain_events( pending: &mut PendingCommands, terminal: &mut bool, #[cfg(encryption)] keyring: &dyn KeyringDelegate, + #[cfg(encryption)] pending_key_responses: &mut Vec>, ) -> bool where I: memberlist_proto::Id + Clone, @@ -1692,11 +1729,21 @@ where // `respond_key`) ahead of the observation hand-off below. #[cfg(encryption)] if let Event::KeyRequest(req) = &ev { - let resp = apply_key_request_live(endpoint, keyring, req); - // Ignoring Err: `respond_key` fails only when the response cannot be routed - // (originator gone / relay dropped); the key op has already applied to the - // live wire keyring. - let _ = endpoint.respond_key(req, resp, Instant::now()); + match apply_key_request_live(endpoint, keyring, req) { + AppliedKeyRequest::Ready(resp) => { + // Ignoring Err: `respond_key` fails only when the response cannot be + // routed (originator gone / relay dropped); the key op has already + // applied to the live wire keyring. + let _ = endpoint.respond_key(req, resp, Instant::now()); + } + AppliedKeyRequest::AwaitingPersistence(resp, rx) => { + pending_key_responses.push(PendingKeyResponse { + req: req.clone(), + resp, + rx, + }); + } + } } let payload_bytes = observation_payload_bytes(&ev); @@ -1755,6 +1802,7 @@ async fn drain_outputs( obs_payload_budget: Option, pending: &mut PendingCommands, #[cfg(encryption)] keyring: &dyn KeyringDelegate, + #[cfg(encryption)] pending_key_responses: &mut Vec>, ) -> bool where I: memberlist_proto::Id + Clone, @@ -1778,6 +1826,8 @@ where &mut terminal, #[cfg(encryption)] keyring, + #[cfg(encryption)] + pending_key_responses, ) .await; if !(did_actions || did_transports || did_transmits || did_events) { @@ -1875,6 +1925,58 @@ async fn reap_pending_joins( /// Earliest pending-join deadline, if any — folded into the per-iteration /// `timeout_deadline` so the timer fires by the first expiring join's deadline. +/// Settle parked key responses: send those whose persistence acknowledgement +/// resolved (as-is on success, downgraded to a failure carrying the error +/// otherwise), drop those whose requester's response deadline passed while +/// the acknowledgement was still pending (nothing useful can be routed), keep +/// the rest parked. Called ahead of each output drain so a sent response +/// flushes in the same pass. +#[cfg(encryption)] +fn reap_pending_key_responses( + endpoint: &mut StreamEndpoint, + parked: &mut Vec>, + now: Instant, +) where + I: memberlist_proto::Id + Clone, + RT: StreamTransport, + G: rand::Rng, + R: rand::Rng + SeedableRng, +{ + let mut i = 0; + while i < parked.len() { + let entry = &parked[i]; + match settle_parked_key_response(&entry.rx, &entry.resp) { + Some(resp) => { + let entry = parked.swap_remove(i); + // Ignoring Err: `respond_key` fails only when the response cannot be + // routed; the key op has already applied to the live wire keyring. + let _ = endpoint.respond_key(&entry.req, resp, now); + } + None if now >= entry.req.deadline() => { + drop(parked.swap_remove(i)); + #[cfg(feature = "tracing")] + tracing::warn!( + "a keyring persistence acknowledgement outlived the key request's response deadline; the response was dropped" + ); + } + None => i += 1, + } + } +} + +/// Next instant a parked key response should be re-polled, folded into the +/// loop's timer target; `None` when nothing is parked. The persistence +/// acknowledgement arrives on a plain channel with no waker integration, so +/// the interval bounds the wait. +#[cfg(encryption)] +fn next_key_ack_check(parked: &[PendingKeyResponse], now: Instant) -> Option { + if parked.is_empty() { + None + } else { + Some(now + KEYRING_PERSIST_POLL_INTERVAL) + } +} + fn min_pending_join_deadline(pending_joins: &[PendingJoin]) -> Option { pending_joins .iter() diff --git a/serf-compio/src/lib.rs b/serf-compio/src/lib.rs index 1bd35a8e..de98b52c 100644 --- a/serf-compio/src/lib.rs +++ b/serf-compio/src/lib.rs @@ -101,16 +101,29 @@ pub use error::{ #[cfg(any(feature = "tcp", feature = "quic"))] #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] pub use delegate::{ - Delegate, MemberDelegate, MergeDelegate, NoopMergeDelegate, QueryDelegate, UserEventDelegate, - VoidDelegate, + Delegate, MemberDelegate, MergeDelegate, QueryDelegate, UserEventDelegate, VoidDelegate, }; +#[cfg(all(encryption, unix))] +#[cfg_attr( + docsrs, + doc(cfg(all(any(feature = "aes-gcm", feature = "chacha20-poly1305"), unix))) +)] +pub use delegate::{FileKeyringDelegate, KeyringFileError}; #[cfg(encryption)] #[cfg_attr( docsrs, doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) )] pub use delegate::{KeyringDelegate, VoidKeyringDelegate}; +/// The rotation-durability acknowledgement contract, shared with the other +/// runtime drivers through `serf-driver`. +#[cfg(encryption)] +#[cfg_attr( + docsrs, + doc(cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))) +)] +pub use serf_driver::{KeyringPersistError, KeyringPersistRx, KeyringPersistence}; /// Gossip-encryption config types re-exported from `memberlist-proto`, so a /// caller can build a transport's `with_encryption` keyring without naming diff --git a/serf-compio/src/serf/tests.rs b/serf-compio/src/serf/tests.rs index b7486eb8..e0c9a9be 100644 --- a/serf-compio/src/serf/tests.rs +++ b/serf-compio/src/serf/tests.rs @@ -1048,8 +1048,11 @@ impl RecordingKeyring { #[cfg(encryption)] impl KeyringDelegate for RecordingKeyring { - fn keyring_updated(&self, keyring: &Keyring) { + fn keyring_updated(&self, keyring: &Keyring) -> serf_driver::KeyringPersistence { self.rings.borrow_mut().push(keyring.clone()); + // The in-memory record is durable the moment it is pushed, so the key + // response goes out immediately. + serf_driver::KeyringPersistence::Durable } } diff --git a/serf-driver/src/keyring/mod.rs b/serf-driver/src/keyring/mod.rs index ce0104fe..15e83813 100644 --- a/serf-driver/src/keyring/mod.rs +++ b/serf-driver/src/keyring/mod.rs @@ -57,6 +57,74 @@ pub enum KeyringPersistence { Pending(KeyringPersistRx), } +/// Cadence at which a pump re-polls parked key responses awaiting the keyring +/// delegate's persistence acknowledgement. The acknowledgement arrives on a +/// plain channel with no waker integration, so while any response is parked +/// the pump's timer target is bounded by this interval; a file-write +/// acknowledgement resolves in milliseconds, so one rotation costs a handful +/// of extra polls and a quiescent pump pays nothing. +pub const KEYRING_PERSIST_POLL_INTERVAL: core::time::Duration = + core::time::Duration::from_millis(1); + +/// A key-management response parked until the keyring delegate acknowledges +/// the rotation's persistence, bounded by the requester's response deadline. +pub struct PendingKeyResponse { + /// The originating request, kept for routing the deferred response. + pub req: serf_proto::event::KeyRequest, + /// The response built from the post-rotation live state. + pub resp: serf_proto::event::KeyResponseArgs, + /// The persistence acknowledgement the response waits on. + pub rx: KeyringPersistRx, +} + +/// The pump-facing outcome of applying one inbound key-management request to +/// the live wire keyring. +pub enum AppliedKeyRequest { + /// No rotation needed out-of-band persistence (a `list`, a refused op, a + /// node with no keyring, or a delegate durable inline): respond now. + Ready(serf_proto::event::KeyResponseArgs), + /// A rotation was applied to the wire and handed to the keyring delegate: + /// the response waits for the persistence acknowledgement. + AwaitingPersistence(serf_proto::event::KeyResponseArgs, KeyringPersistRx), +} + +/// Fold one acknowledgement poll into a parked key response: `None` keeps it +/// parked; `Some` is the final response to send — unchanged on a persisted +/// rotation, downgraded to a failure carrying the error otherwise. The +/// reference implementation folds its keyring-file write error into the +/// response the same way, with the live wire keyring keeping the rotation. +/// A disconnected sender counts as a failure: the worker vanished without +/// acknowledging. +pub fn settle_parked_key_response( + rx: &KeyringPersistRx, + resp: &serf_proto::event::KeyResponseArgs, +) -> Option { + use std::sync::mpsc::TryRecvError; + match rx.try_recv() { + Ok(Ok(())) => Some(resp.clone()), + Ok(Err(e)) => Some(failed_key_response( + resp, + format!("keyring rotated on the wire but not persisted: {e}"), + )), + Err(TryRecvError::Disconnected) => Some(failed_key_response( + resp, + "keyring rotated on the wire but not persisted: the persistence worker exited without acknowledging".to_string(), + )), + Err(TryRecvError::Empty) => None, + } +} + +/// `resp` downgraded to a failed key response carrying `message`. +fn failed_key_response( + resp: &serf_proto::event::KeyResponseArgs, + message: String, +) -> serf_proto::event::KeyResponseArgs { + let mut failed = resp.clone(); + failed.result = false; + failed.message = message.into(); + failed +} + #[cfg(test)] mod tests; diff --git a/serf-driver/src/lib.rs b/serf-driver/src/lib.rs index 9169b1d3..03018620 100644 --- a/serf-driver/src/lib.rs +++ b/serf-driver/src/lib.rs @@ -31,8 +31,9 @@ mod snapshotter; ))) )] pub use keyring::{ - KeyApplyOutcome, KeyringPersistError, KeyringPersistRx, KeyringPersistence, apply_key_request, - keyring_carries_cross_cipher_twin, + AppliedKeyRequest, KEYRING_PERSIST_POLL_INTERVAL, KeyApplyOutcome, KeyringPersistError, + KeyringPersistRx, KeyringPersistence, PendingKeyResponse, apply_key_request, + keyring_carries_cross_cipher_twin, settle_parked_key_response, }; #[cfg(all(encryption, any(feature = "tcp", feature = "quic"), unix))] #[cfg_attr( diff --git a/serf-reactor/src/driver/shared.rs b/serf-reactor/src/driver/shared.rs index af236760..3dd9f816 100644 --- a/serf-reactor/src/driver/shared.rs +++ b/serf-reactor/src/driver/shared.rs @@ -285,75 +285,12 @@ fn classify_errored_farewell(icmp_errors: u8, err: &io::Error) -> ErroredFarewel pub(crate) const LEAVE_DRAIN_TEARDOWN_BOUND: core::time::Duration = core::time::Duration::from_secs(1); -/// Cadence at which a pump re-polls parked key responses awaiting the keyring -/// delegate's persistence acknowledgement. The acknowledgement arrives on a -/// plain channel with no waker integration, so while any response is parked -/// the idle-arm timer target is bounded by this interval; a file-write -/// acknowledgement resolves in milliseconds, so one rotation costs a handful -/// of extra polls and a quiescent pump pays nothing. +/// The parked key-response machinery, shared with the other runtime drivers +/// through `serf-driver`. #[cfg(all(any(feature = "tcp", feature = "quic"), encryption))] -pub(crate) const KEYRING_PERSIST_POLL_INTERVAL: core::time::Duration = - core::time::Duration::from_millis(1); - -/// A key-management response parked until the keyring delegate acknowledges -/// the rotation's persistence, bounded by the requester's response deadline. -#[cfg(all(any(feature = "tcp", feature = "quic"), encryption))] -pub(crate) struct PendingKeyResponse { - pub(crate) req: serf_proto::event::KeyRequest, - pub(crate) resp: serf_proto::event::KeyResponseArgs, - pub(crate) rx: crate::KeyringPersistRx, -} - -/// The pump-facing outcome of applying one inbound key-management request to -/// the live wire keyring. -#[cfg(all(any(feature = "tcp", feature = "quic"), encryption))] -pub(crate) enum AppliedKeyRequest { - /// No rotation needed out-of-band persistence (a `list`, a refused op, a - /// node with no keyring, or a delegate durable inline): respond now. - Ready(serf_proto::event::KeyResponseArgs), - /// A rotation was applied to the wire and handed to the keyring delegate: - /// the response waits for the persistence acknowledgement. - AwaitingPersistence(serf_proto::event::KeyResponseArgs, crate::KeyringPersistRx), -} - -/// Fold one acknowledgement poll into a parked key response: `None` keeps it -/// parked; `Some` is the final response to send — unchanged on a persisted -/// rotation, downgraded to a failure carrying the error otherwise. The -/// reference implementation folds its keyring-file write error into the -/// response the same way, with the live wire keyring keeping the rotation. -/// A disconnected sender counts as a failure: the worker vanished without -/// acknowledging. -#[cfg(all(any(feature = "tcp", feature = "quic"), encryption))] -pub(crate) fn settle_parked_key_response( - rx: &crate::KeyringPersistRx, - resp: &serf_proto::event::KeyResponseArgs, -) -> Option { - use std::sync::mpsc::TryRecvError; - match rx.try_recv() { - Ok(Ok(())) => Some(resp.clone()), - Ok(Err(e)) => Some(failed_key_response( - resp, - format!("keyring rotated on the wire but not persisted: {e}"), - )), - Err(TryRecvError::Disconnected) => Some(failed_key_response( - resp, - "keyring rotated on the wire but not persisted: the persistence worker exited without acknowledging".to_string(), - )), - Err(TryRecvError::Empty) => None, - } -} - -/// `resp` downgraded to a failed key response carrying `message`. -#[cfg(all(any(feature = "tcp", feature = "quic"), encryption))] -fn failed_key_response( - resp: &serf_proto::event::KeyResponseArgs, - message: String, -) -> serf_proto::event::KeyResponseArgs { - let mut failed = resp.clone(); - failed.result = false; - failed.message = message.into(); - failed -} +pub(crate) use serf_driver::{ + AppliedKeyRequest, KEYRING_PERSIST_POLL_INTERVAL, PendingKeyResponse, settle_parked_key_response, +}; /// Record the outcome of one readiness-based leave-farewell datagram send into /// `retained`: From c3a376e854b5d555c317e2b50ab0be2a09c8bb4e Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 19:40:52 +1200 Subject: [PATCH 116/128] feat(serf-compio): constructor merge predicate and snapshot persistence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Serf::new gains the two remaining construction hooks: merge_delegate installs the machine's synchronous push/pull filter into the endpoint each transport builds, and snapshot opens the shared file snapshotter BEFORE any socket binds — a corrupt snapshot refuses construction loudly — with the decoded records riding the runtime bundle into T::run, which replays them under the serf option rejoin_after_leave and hands the writer to the pump. Both pump loops append at their event chokepoint: member records, then the advancing clock floors, with the clean-leave marker written last so a clean shutdown ends the file at the Leave record (the terminal shape compaction preserves and replay expects), flushing — and compacting past the threshold — before the drain returns. SnapshotOptions joins the driver options (constructor argument, not a runtime knob) and the snapshot-open failure surfaces as SerfError::SnapshotOpen. --- serf-compio/src/driver/options/mod.rs | 45 +++++++++++++++++++++++++ serf-compio/src/driver/quic/mod.rs | 47 +++++++++++++++++++++++++++ serf-compio/src/driver/stream/mod.rs | 47 +++++++++++++++++++++++++++ serf-compio/src/error/mod.rs | 7 ++++ serf-compio/src/lib.rs | 4 ++- serf-compio/src/quic/mod.rs | 17 +++++++++- serf-compio/src/quic/tests.rs | 8 +++++ serf-compio/src/serf/mod.rs | 23 +++++++++++++ serf-compio/src/serf/tests.rs | 14 ++++++++ serf-compio/src/tcp/mod.rs | 17 +++++++++- serf-compio/src/tls/mod.rs | 17 +++++++++- serf-compio/src/tls/tests.rs | 6 ++++ serf-compio/src/transport/runtime.rs | 12 +++++++ 13 files changed, 260 insertions(+), 4 deletions(-) diff --git a/serf-compio/src/driver/options/mod.rs b/serf-compio/src/driver/options/mod.rs index 332893b1..e7e14193 100644 --- a/serf-compio/src/driver/options/mod.rs +++ b/serf-compio/src/driver/options/mod.rs @@ -639,5 +639,50 @@ impl Default for StreamTransportOptions { } } +/// Default snapshot compaction threshold: the append file is rewritten to the +/// live state once it grows past this many bytes. +pub use serf_driver::DEFAULT_SNAPSHOT_COMPACT_THRESHOLD; + +/// Snapshot persistence wiring: where the driver appends membership records +/// and when it compacts the file. Supplied as a constructor argument (`None` +/// disables persistence); the replay-vs-fresh-start decision after a clean +/// leave is the serf option `rejoin_after_leave`. +#[derive(Debug, Clone)] +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +pub struct SnapshotOptions { + path: std::path::PathBuf, + compact_threshold: u64, +} + +impl SnapshotOptions { + /// Persist to `path`, compacting at the default threshold. + pub fn new(path: impl Into) -> Self { + Self { + path: path.into(), + compact_threshold: DEFAULT_SNAPSHOT_COMPACT_THRESHOLD, + } + } + + /// The snapshot file path. + #[must_use] + pub fn path(&self) -> &std::path::Path { + &self.path + } + + /// Rewrite the file to the live state once it grows past `bytes` + /// (`0` = never compact). + #[must_use] + pub const fn with_compact_threshold(mut self, bytes: u64) -> Self { + self.compact_threshold = bytes; + self + } + + /// The compaction threshold in bytes (`0` = never compact). + #[must_use] + pub const fn compact_threshold(&self) -> u64 { + self.compact_threshold + } +} + #[cfg(test)] mod tests; diff --git a/serf-compio/src/driver/quic/mod.rs b/serf-compio/src/driver/quic/mod.rs index c80431d5..67c2c00c 100644 --- a/serf-compio/src/driver/quic/mod.rs +++ b/serf-compio/src/driver/quic/mod.rs @@ -312,6 +312,9 @@ pub(crate) async fn quic_driver_loop( label: Option, // The driver's keyring delegate: applies inbound key-management ops and // produces the `respond_key` answer. Present only under an encryption backend. + // The snapshot writer the pump appends membership records to; `None` + // disables persistence. + mut snapshotter: Option>, #[cfg(encryption)] keyring: Rc, ) where D: Delegate, @@ -435,6 +438,7 @@ pub(crate) async fn quic_driver_loop( &obs_payload_bytes, obs_payload_budget, &mut pending, + &mut snapshotter, #[cfg(encryption)] &*keyring, #[cfg(encryption)] @@ -504,6 +508,7 @@ pub(crate) async fn quic_driver_loop( &obs_payload_bytes, obs_payload_budget, &mut pending, + &mut snapshotter, #[cfg(encryption)] &*keyring, #[cfg(encryption)] @@ -538,6 +543,7 @@ pub(crate) async fn quic_driver_loop( &obs_payload_bytes, obs_payload_budget, &mut pending, + &mut snapshotter, #[cfg(encryption)] &*keyring, #[cfg(encryption)] @@ -647,6 +653,7 @@ pub(crate) async fn quic_driver_loop( &obs_payload_bytes, obs_payload_budget, &mut pending, + &mut snapshotter, #[cfg(encryption)] &*keyring, #[cfg(encryption)] @@ -1235,6 +1242,7 @@ async fn drain_events( obs_payload_bytes: &Cell, obs_payload_budget: Option, pending: &mut PendingCommands, + snapshotter: &mut Option>, terminal: &mut bool, #[cfg(encryption)] keyring: &dyn KeyringDelegate, #[cfg(encryption)] pending_key_responses: &mut Vec>, @@ -1247,6 +1255,43 @@ where let mut drained = false; while let Some(ev) = endpoint.poll_event() { drained = true; + // Snapshot persistence: append the surfaced membership change, the + // advancing clock floors, and the clean-leave marker, then flush (and + // compact past the threshold) — a change is durable once this drain + // returns. Restart replay + `load_snapshot` recovers the state. + if let Some(snap) = snapshotter.as_mut() { + if let Event::Member(me) = &ev { + use serf_proto::event::MemberEventKind as MK; + let alive = matches!(me.kind(), MK::Join | MK::Update); + for m in me.members() { + snap.append_member(alive, m.node()); + } + } + if matches!(ev, Event::Member(_) | Event::LeftCluster) { + snap.append_clocks( + LamportTime::from(endpoint.member_time()), + LamportTime::from(endpoint.event_time()), + LamportTime::from(endpoint.query_time()), + ); + // The leave marker is written AFTER the clocks so a clean shutdown + // ends the file at the Leave record — the terminal shape compaction + // preserves and replay expects: under the default no-rejoin posture + // the Leave wipes the accumulated state, and a clock record written + // after it would resurrect a clock the reference implementation + // zeroes. + if matches!(ev, Event::LeftCluster) { + snap.append_leave(); + } + snap.flush_and_maybe_compact(|| { + endpoint + .members_snapshot() + .iter() + .filter(|m| m.status() == serf_proto::members::MemberStatus::Alive) + .map(|m| m.node().clone()) + .collect() + }); + } + } // Await-result join resolution. `ExchangeCompleted` fires for every outbound // bridge kind; an await-join waiter consumes only `PushPull` completions. // `complete_join_exchange` drives both decoupled terminals: it resolves the @@ -1356,6 +1401,7 @@ async fn drain_outputs( obs_payload_bytes: &Cell, obs_payload_budget: Option, pending: &mut PendingCommands, + snapshotter: &mut Option>, #[cfg(encryption)] keyring: &dyn KeyringDelegate, #[cfg(encryption)] pending_key_responses: &mut Vec>, ) -> bool @@ -1376,6 +1422,7 @@ where obs_payload_bytes, obs_payload_budget, pending, + snapshotter, &mut terminal, #[cfg(encryption)] keyring, diff --git a/serf-compio/src/driver/stream/mod.rs b/serf-compio/src/driver/stream/mod.rs index 70f2be31..84a45aed 100644 --- a/serf-compio/src/driver/stream/mod.rs +++ b/serf-compio/src/driver/stream/mod.rs @@ -433,6 +433,9 @@ pub(crate) async fn stream_driver_loop( // Cluster label applied to both gossip encode and decode. `None` accepts // datagrams from any cluster. label: Option, + // The snapshot writer the pump appends membership records to; `None` + // disables persistence. + mut snapshotter: Option>, // The driver's keyring delegate: applies inbound key-management ops and // produces the `respond_key` answer. Present only under an encryption backend. #[cfg(encryption)] keyring: Rc, @@ -637,6 +640,7 @@ pub(crate) async fn stream_driver_loop( &obs_payload_bytes, obs_payload_budget, &mut pending, + &mut snapshotter, #[cfg(encryption)] &*keyring, #[cfg(encryption)] @@ -714,6 +718,7 @@ pub(crate) async fn stream_driver_loop( &obs_payload_bytes, obs_payload_budget, &mut pending, + &mut snapshotter, #[cfg(encryption)] &*keyring, #[cfg(encryption)] @@ -751,6 +756,7 @@ pub(crate) async fn stream_driver_loop( &obs_payload_bytes, obs_payload_budget, &mut pending, + &mut snapshotter, #[cfg(encryption)] &*keyring, #[cfg(encryption)] @@ -905,6 +911,7 @@ pub(crate) async fn stream_driver_loop( &obs_payload_bytes, obs_payload_budget, &mut pending, + &mut snapshotter, #[cfg(encryption)] &*keyring, #[cfg(encryption)] @@ -1676,6 +1683,7 @@ async fn drain_events( obs_payload_bytes: &Cell, obs_payload_budget: Option, pending: &mut PendingCommands, + snapshotter: &mut Option>, terminal: &mut bool, #[cfg(encryption)] keyring: &dyn KeyringDelegate, #[cfg(encryption)] pending_key_responses: &mut Vec>, @@ -1689,6 +1697,43 @@ where let mut drained = false; while let Some(ev) = endpoint.poll_event() { drained = true; + // Snapshot persistence: append the surfaced membership change, the + // advancing clock floors, and the clean-leave marker, then flush (and + // compact past the threshold) — a change is durable once this drain + // returns. Restart replay + `load_snapshot` recovers the state. + if let Some(snap) = snapshotter.as_mut() { + if let Event::Member(me) = &ev { + use serf_proto::event::MemberEventKind as MK; + let alive = matches!(me.kind(), MK::Join | MK::Update); + for m in me.members() { + snap.append_member(alive, m.node()); + } + } + if matches!(ev, Event::Member(_) | Event::LeftCluster) { + snap.append_clocks( + LamportTime::from(endpoint.member_time()), + LamportTime::from(endpoint.event_time()), + LamportTime::from(endpoint.query_time()), + ); + // The leave marker is written AFTER the clocks so a clean shutdown + // ends the file at the Leave record — the terminal shape compaction + // preserves and replay expects: under the default no-rejoin posture + // the Leave wipes the accumulated state, and a clock record written + // after it would resurrect a clock the reference implementation + // zeroes. + if matches!(ev, Event::LeftCluster) { + snap.append_leave(); + } + snap.flush_and_maybe_compact(|| { + endpoint + .members_snapshot() + .iter() + .filter(|m| m.status() == serf_proto::members::MemberStatus::Alive) + .map(|m| m.node().clone()) + .collect() + }); + } + } // Await-result join resolution. `ExchangeCompleted` fires for every outbound // bridge kind; an await-join waiter consumes only `PushPull` completions. // `complete_join_exchange` drives both decoupled terminals: it resolves the @@ -1801,6 +1846,7 @@ async fn drain_outputs( obs_payload_bytes: &Cell, obs_payload_budget: Option, pending: &mut PendingCommands, + snapshotter: &mut Option>, #[cfg(encryption)] keyring: &dyn KeyringDelegate, #[cfg(encryption)] pending_key_responses: &mut Vec>, ) -> bool @@ -1823,6 +1869,7 @@ where obs_payload_bytes, obs_payload_budget, pending, + snapshotter, &mut terminal, #[cfg(encryption)] keyring, diff --git a/serf-compio/src/error/mod.rs b/serf-compio/src/error/mod.rs index cf800e0c..059b1a22 100644 --- a/serf-compio/src/error/mod.rs +++ b/serf-compio/src/error/mod.rs @@ -125,6 +125,13 @@ pub enum SerfError { #[error(transparent)] Proto(#[from] serf_proto::endpoint::Error), + /// The snapshot file could not be opened or decoded at construction — the + /// node refuses to start against membership state it cannot trust. + #[cfg(any(feature = "tcp", feature = "quic"))] + #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] + #[error(transparent)] + SnapshotOpen(#[from] serf_driver::SnapshotOpenError), + /// Encryption codec error from memberlist-wire. #[cfg(encryption)] #[cfg_attr( diff --git a/serf-compio/src/lib.rs b/serf-compio/src/lib.rs index de98b52c..56864dcc 100644 --- a/serf-compio/src/lib.rs +++ b/serf-compio/src/lib.rs @@ -182,5 +182,7 @@ pub use driver::options::{ Channel, DEFAULT_BRIDGE_INBOUND_CAP, DEFAULT_BRIDGE_RECV_BUF_LEN, DEFAULT_CLOSE_TIMEOUT, DEFAULT_CMD_FAIRNESS_BUDGET, DEFAULT_DIAL_TIMEOUT, DEFAULT_EVENT_QUEUE_CAP, DEFAULT_IDLE_WAKE_INTERVAL, DEFAULT_ITER_DRAIN_CAP, DEFAULT_LEAVE_TIMEOUT, - DEFAULT_OBSERVATION_CHANNEL, ParseChannelError, RuntimeOptions, StreamTransportOptions, + DEFAULT_OBSERVATION_CHANNEL, DEFAULT_SNAPSHOT_COMPACT_THRESHOLD, ParseChannelError, + RuntimeOptions, SnapshotOptions, StreamTransportOptions, }; +pub use serf_driver::SnapshotOpenError; diff --git a/serf-compio/src/quic/mod.rs b/serf-compio/src/quic/mod.rs index 2bce0526..541d96da 100644 --- a/serf-compio/src/quic/mod.rs +++ b/serf-compio/src/quic/mod.rs @@ -329,7 +329,8 @@ where // Serf's core RNG is seeded from its own OS-drawn entropy (`self.serf_rng`), // independent of the coordinator's gossip RNG, so two nodes never share the // query-ID / relay-selection stream. - let endpoint = serf_proto::QuicEndpoint::< + let rejoin_after_leave = runtime.serf_options.rejoin_after_leave(); + let mut endpoint = serf_proto::QuicEndpoint::< Self::Id, G, StdRng, @@ -342,6 +343,19 @@ where runtime.member_drop, ) .with_reconnect_delegate(runtime.reconnect_delegate); + if let Some(md) = runtime.merge_delegate { + endpoint.set_merge_delegate(md); + } + let snapshotter = match runtime.snapshot_file { + Some((writer, records)) => { + let replay = serf_proto::snapshot::ReplayResult::replay(records, rejoin_after_leave); + // Ignoring Err: load_snapshot refuses only on a machine that already + // lost an id-conflict vote; a freshly built endpoint is Alive. + let _ = endpoint.load_snapshot(replay, memberlist_proto::Instant::now()); + Some(writer) + } + None => None, + }; crate::driver::quic::quic_driver_loop::( endpoint, @@ -356,6 +370,7 @@ where runtime.driver_options, runtime.delegate, None, + snapshotter, #[cfg(encryption)] runtime.keyring, ) diff --git a/serf-compio/src/quic/tests.rs b/serf-compio/src/quic/tests.rs index cf50fc8f..deb6f893 100644 --- a/serf-compio/src/quic/tests.rs +++ b/serf-compio/src/quic/tests.rs @@ -199,6 +199,8 @@ async fn try_spawn_node_at( SerfOptions::new(), gossip_rng().expect("seed gossip rng"), None, + None, + None, #[cfg(encryption)] std::rc::Rc::new(VoidKeyringDelegate), ) @@ -226,6 +228,8 @@ async fn assert_quic_new_rejects(runtime: RuntimeOptions) { SerfOptions::new(), gossip_rng().expect("seed gossip rng"), None, + None, + None, #[cfg(encryption)] std::rc::Rc::new(VoidKeyringDelegate), ) @@ -548,6 +552,8 @@ async fn spawn_encrypted_node(id: &str, encryption: EncryptionOptions) -> Serf Serf( options: T::Options, @@ -154,9 +161,12 @@ where serf_options: SerfOptions, gossip_rng: G, reconnect_delegate: Option>>, + merge_delegate: Option>>, + snapshot: Option, #[cfg(encryption)] keyring: Rc, ) -> core::result::Result where + I: memberlist_proto::Id, T: Transport, RES: Resolver
, AR: AdvertiseAddrResolver, @@ -177,6 +187,17 @@ where .validate() .map_err(|e| SerfError::InvalidOption(InvalidOption::new("serf_options", e.to_string())))?; + // Open and decode the snapshot BEFORE binding any socket: a corrupt file + // refuses construction loudly, and the decoded records ride the runtime + // bundle into `T::run`, which replays them into the endpoint it builds. + let snapshot_file = match &snapshot { + Some(opts) => Some( + serf_driver::Snapshotter::open(opts.path(), opts.compact_threshold()) + .map_err(SerfError::from)?, + ), + None => None, + }; + // Cache the join deadline on the handle BEFORE `runtime_options` is moved // into the driver bundle, so each await-result join can stamp its absolute // `WaitForCompletion` deadline from `Instant::now() + join_deadline`. @@ -224,6 +245,8 @@ where runtime_options, serf_options, reconnect_delegate, + merge_delegate, + snapshot_file, #[cfg(encryption)] keyring, ); diff --git a/serf-compio/src/serf/tests.rs b/serf-compio/src/serf/tests.rs index e0c9a9be..e80b4d0e 100644 --- a/serf-compio/src/serf/tests.rs +++ b/serf-compio/src/serf/tests.rs @@ -68,6 +68,8 @@ async fn try_spawn_node_at(id: &str, bind: SocketAddr) -> Result, SerfOptions::new(), gossip_rng().expect("seed gossip rng"), None, + None, + None, #[cfg(encryption)] std::rc::Rc::new(VoidKeyringDelegate), ) @@ -185,6 +187,8 @@ async fn assert_tcp_new_rejects(runtime: RuntimeOptions) { SerfOptions::new(), gossip_rng().expect("seed gossip rng"), None, + None, + None, #[cfg(encryption)] std::rc::Rc::new(VoidKeyringDelegate), ) @@ -230,6 +234,8 @@ async fn tcp_new_rejects_over_ceiling_user_event_size() { serf, gossip_rng().expect("seed gossip rng"), None, + None, + None, #[cfg(encryption)] std::rc::Rc::new(VoidKeyringDelegate), ) @@ -351,6 +357,8 @@ async fn spawn_encrypted_node(id: &str, encryption: EncryptionOptions) -> Serf S SerfOptions::new(), gossip_rng().expect("seed gossip rng"), None, + None, + None, #[cfg(encryption)] std::rc::Rc::new(VoidKeyringDelegate), ) @@ -569,6 +579,8 @@ async fn spawn_node_with_serf_options(id: &str, serf_options: SerfOptions) -> Se serf_options, gossip_rng().expect("seed gossip rng"), None, + None, + None, #[cfg(encryption)] std::rc::Rc::new(VoidKeyringDelegate), ) @@ -1078,6 +1090,8 @@ async fn spawn_encrypted_node_with_keyring( SerfOptions::new(), gossip_rng().expect("seed gossip rng"), None, + None, + None, keyring, ) .await diff --git a/serf-compio/src/tcp/mod.rs b/serf-compio/src/tcp/mod.rs index dd2bc02f..24cc4620 100644 --- a/serf-compio/src/tcp/mod.rs +++ b/serf-compio/src/tcp/mod.rs @@ -353,7 +353,8 @@ where // Serf's core RNG is seeded from its own OS-drawn entropy (`self.serf_rng`), // independent of the coordinator's gossip RNG, so two nodes never share the // query-ID / relay-selection stream. - let endpoint = serf_proto::StreamEndpoint::< + let rejoin_after_leave = runtime.serf_options.rejoin_after_leave(); + let mut endpoint = serf_proto::StreamEndpoint::< Self::Id, SocketAddr, RawRecords, @@ -368,6 +369,19 @@ where runtime.member_drop, ) .with_reconnect_delegate(runtime.reconnect_delegate); + if let Some(md) = runtime.merge_delegate { + endpoint.set_merge_delegate(md); + } + let snapshotter = match runtime.snapshot_file { + Some((writer, records)) => { + let replay = serf_proto::snapshot::ReplayResult::replay(records, rejoin_after_leave); + // Ignoring Err: load_snapshot refuses only on a machine that already + // lost an id-conflict vote; a freshly built endpoint is Alive. + let _ = endpoint.load_snapshot(replay, memberlist_proto::Instant::now()); + Some(writer) + } + None => None, + }; crate::driver::stream::stream_driver_loop::( endpoint, @@ -383,6 +397,7 @@ where self.stream_options, runtime.delegate, None, + snapshotter, #[cfg(encryption)] runtime.keyring, ) diff --git a/serf-compio/src/tls/mod.rs b/serf-compio/src/tls/mod.rs index 255f9791..69627eca 100644 --- a/serf-compio/src/tls/mod.rs +++ b/serf-compio/src/tls/mod.rs @@ -439,7 +439,8 @@ where // Serf's core RNG is seeded from its own OS-drawn entropy (`self.serf_rng`), // independent of the coordinator's gossip RNG, so two nodes never share the // query-ID / relay-selection stream. - let endpoint = serf_proto::StreamEndpoint::< + let rejoin_after_leave = runtime.serf_options.rejoin_after_leave(); + let mut endpoint = serf_proto::StreamEndpoint::< Self::Id, SocketAddr, Labeled, @@ -454,6 +455,19 @@ where runtime.member_drop, ) .with_reconnect_delegate(runtime.reconnect_delegate); + if let Some(md) = runtime.merge_delegate { + endpoint.set_merge_delegate(md); + } + let snapshotter = match runtime.snapshot_file { + Some((writer, records)) => { + let replay = serf_proto::snapshot::ReplayResult::replay(records, rejoin_after_leave); + // Ignoring Err: load_snapshot refuses only on a machine that already + // lost an id-conflict vote; a freshly built endpoint is Alive. + let _ = endpoint.load_snapshot(replay, memberlist_proto::Instant::now()); + Some(writer) + } + None => None, + }; crate::driver::stream::stream_driver_loop::, D, G, StdRng>( endpoint, @@ -469,6 +483,7 @@ where self.stream_options, runtime.delegate, None, + snapshotter, #[cfg(encryption)] runtime.keyring, ) diff --git a/serf-compio/src/tls/tests.rs b/serf-compio/src/tls/tests.rs index b309e73e..5f76ff8c 100644 --- a/serf-compio/src/tls/tests.rs +++ b/serf-compio/src/tls/tests.rs @@ -128,6 +128,8 @@ async fn try_spawn_node_at(id: &str, bind: SocketAddr) -> Result, SerfOptions::new(), gossip_rng().expect("seed gossip rng"), None, + None, + None, #[cfg(encryption)] std::rc::Rc::new(VoidKeyringDelegate), ) @@ -178,6 +180,8 @@ async fn tls_new_rejects_zero_observation_channel() { SerfOptions::new(), gossip_rng().expect("seed gossip rng"), None, + None, + None, #[cfg(encryption)] std::rc::Rc::new(VoidKeyringDelegate), ) @@ -413,6 +417,8 @@ async fn spawn_encrypted_node(id: &str, encryption: EncryptionOptions) -> Serf>>, + /// The machine's synchronous push/pull merge filter, installed into the + /// endpoint by `T::run`. `None` admits every exchange. + pub(crate) merge_delegate: + Option>>, + /// The opened snapshot writer plus the records already on disk: `T::run` + /// replays the records into the endpoint it builds and hands the writer to + /// the pump for appending. `None` disables persistence. + pub(crate) snapshot_file: Option>, /// The driver's keyring delegate, applied to inbound key-management requests. /// Present only under an encryption backend. #[cfg(encryption)] @@ -90,6 +98,8 @@ where driver_options: RuntimeOptions, serf_options: SerfOptions, reconnect_delegate: Option>>, + merge_delegate: Option>>, + snapshot_file: Option>, #[cfg(encryption)] keyring: Rc, ) -> Self { Self { @@ -105,6 +115,8 @@ where driver_options, serf_options, reconnect_delegate, + merge_delegate, + snapshot_file, #[cfg(encryption)] keyring, } From cd96b79e7c5de35b8cbce40a5920518eaed3a3a4 Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 19:44:59 +1200 Subject: [PATCH 117/128] feat(serf-compio): operator accessors over the enriched snapshot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both pumps now attach the driver-side live readings when publishing the membership snapshot — health score, broadcast queue depth, the encryption flag, and (under the coordinates feature) the local Vivaldi coordinate and reset count — and the handle reads them lock-free: stats(), encryption_enabled(), health_score(), and coordinate(). cached_coordinate rides a read-only command answered from the endpoint's coordinate cache in every lifecycle state, with the shutdown drain failing a parked probe like any other queued command. --- serf-compio/src/command/mod.rs | 19 ++++++++++ serf-compio/src/driver/quic/mod.rs | 24 ++++++++++++ serf-compio/src/driver/stream/mod.rs | 24 ++++++++++++ serf-compio/src/serf/mod.rs | 55 ++++++++++++++++++++++++++++ 4 files changed, 122 insertions(+) diff --git a/serf-compio/src/command/mod.rs b/serf-compio/src/command/mod.rs index ca048fa3..5876db9f 100644 --- a/serf-compio/src/command/mod.rs +++ b/serf-compio/src/command/mod.rs @@ -281,6 +281,19 @@ pub(crate) struct ListKeysCmd { pub(crate) reply: Sender>, } +/// Payload for [`Command::CachedCoordinate`]. +/// +/// Requires the `coordinates` feature. +#[cfg(feature = "coordinates")] +pub(crate) struct CachedCoordinateCmd { + /// The peer whose most-recently-observed coordinate is requested. + pub(crate) id: I, + /// One-shot reply channel delivering the peer's cached coordinate, `None` + /// when coordinates are disabled or no RTT sample has arrived from the peer. + #[cfg(any(feature = "tcp", feature = "quic"))] + pub(crate) reply: Sender>>, +} + /// Payload for [`Command::Shutdown`]. pub(crate) struct ShutdownCmd { /// One-shot reply channel for the shutdown acknowledgement. @@ -368,6 +381,12 @@ pub(crate) enum Command { )] ListKeys(ListKeysCmd), + /// A read-only probe of the peer-coordinate cache. + /// + /// Requires the `coordinates` feature. + #[cfg(feature = "coordinates")] + #[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] + CachedCoordinate(CachedCoordinateCmd), /// Signal the driver task to shut down gracefully. Shutdown(ShutdownCmd), } diff --git a/serf-compio/src/driver/quic/mod.rs b/serf-compio/src/driver/quic/mod.rs index 67c2c00c..9af2e24a 100644 --- a/serf-compio/src/driver/quic/mod.rs +++ b/serf-compio/src/driver/quic/mod.rs @@ -749,6 +749,10 @@ fn reply_shutdown(c: Command) { Command::ListKeys(ListKeysCmd { reply, .. }) => { let _ = reply.send(Err(SerfError::Shutdown)); } + #[cfg(feature = "coordinates")] + Command::CachedCoordinate(crate::command::CachedCoordinateCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } } } @@ -1007,6 +1011,13 @@ async fn dispatch_command( // Ignoring Err: caller dropped the reply receiver. let _ = reply.send(res); } + #[cfg(feature = "coordinates")] + Command::CachedCoordinate(crate::command::CachedCoordinateCmd { id, reply }) => { + // A read-only probe of the coordinate cache: answerable in every + // lifecycle state (post-leave introspection stays valid). + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(Ok(endpoint.cached_coordinate(&id))); + } Command::Shutdown(ShutdownCmd { reply }) => { // Do NOT ack the caller here — the socket is still bound; stash the reply // and let the post-loop cleanup ack AFTER it drops. @@ -1678,6 +1689,19 @@ fn refresh_snapshot( LamportTime::from(endpoint.event_time()), LamportTime::from(endpoint.query_time()), ); + #[cfg(feature = "coordinates")] + let snap = snap + .with_coordinate(endpoint.get_coordinate()) + .with_coordinate_resets(endpoint.coordinate_resets()); + #[cfg(encryption)] + let encrypted = endpoint.encryption_options().keyring().is_some(); + #[cfg(not(encryption))] + let encrypted = false; + let snap = snap.with_ops_stats( + endpoint.health_score(), + endpoint.user_broadcast_queue_len(), + encrypted, + ); *snapshot.borrow_mut() = Rc::new(snap); } diff --git a/serf-compio/src/driver/stream/mod.rs b/serf-compio/src/driver/stream/mod.rs index 84a45aed..333c510a 100644 --- a/serf-compio/src/driver/stream/mod.rs +++ b/serf-compio/src/driver/stream/mod.rs @@ -1023,6 +1023,10 @@ fn reply_shutdown(c: Command) { Command::ListKeys(ListKeysCmd { reply, .. }) => { let _ = reply.send(Err(SerfError::Shutdown)); } + #[cfg(feature = "coordinates")] + Command::CachedCoordinate(crate::command::CachedCoordinateCmd { reply, .. }) => { + let _ = reply.send(Err(SerfError::Shutdown)); + } } } @@ -1308,6 +1312,13 @@ async fn dispatch_command( // Ignoring Err: caller dropped the reply receiver. let _ = reply.send(res); } + #[cfg(feature = "coordinates")] + Command::CachedCoordinate(crate::command::CachedCoordinateCmd { id, reply }) => { + // A read-only probe of the coordinate cache: answerable in every + // lifecycle state (post-leave introspection stays valid). + // Ignoring Err: caller dropped the reply receiver. + let _ = reply.send(Ok(endpoint.cached_coordinate(&id))); + } Command::Shutdown(ShutdownCmd { reply }) => { // Drain every live bridge so the byte-movers observe the close and exit. // Do NOT ack the caller here — the sockets are still bound; stash the @@ -2180,6 +2191,19 @@ fn refresh_snapshot( LamportTime::from(endpoint.event_time()), LamportTime::from(endpoint.query_time()), ); + #[cfg(feature = "coordinates")] + let snap = snap + .with_coordinate(endpoint.get_coordinate()) + .with_coordinate_resets(endpoint.coordinate_resets()); + #[cfg(encryption)] + let encrypted = endpoint.encryption_options().keyring().is_some(); + #[cfg(not(encryption))] + let encrypted = false; + let snap = snap.with_ops_stats( + endpoint.health_score(), + endpoint.user_broadcast_queue_len(), + encrypted, + ); *snapshot.borrow_mut() = Rc::new(snap); } diff --git a/serf-compio/src/serf/mod.rs b/serf-compio/src/serf/mod.rs index 272b772f..6eed72c7 100644 --- a/serf-compio/src/serf/mod.rs +++ b/serf-compio/src/serf/mod.rs @@ -273,6 +273,61 @@ where }) } + /// The operator aggregate — member/failed/left counts, clock floors, and + /// the driver-attached live readings (health score, broadcast queue depth, + /// encryption flag) — read lock-free from the latest published snapshot. + #[must_use] + pub fn stats(&self) -> serf_driver::SerfStats { + self.shared.snapshot.borrow().stats() + } + + /// Whether a gossip/reliable encryption keyring is configured on this node, + /// read lock-free from the latest published snapshot. The keyring's presence + /// is fixed at construction (key rotation replaces its contents, never adds + /// or removes the ring itself), so this answer is stable for the node's + /// lifetime once the driver's first snapshot lands. + #[must_use] + pub fn encryption_enabled(&self) -> bool { + self.shared.snapshot.borrow().stats().encrypted() + } + + /// The node-awareness health score, read lock-free from the latest published + /// snapshot: `0` = healthy; higher values stretch the failure-detection + /// timeouts (the node believes itself degraded). + #[must_use] + pub fn health_score(&self) -> usize { + self.shared.snapshot.borrow().stats().health_score() + } + + /// The local node's current Vivaldi network coordinate, read lock-free from + /// the latest published snapshot. + /// + /// `None` when coordinates are disabled + /// (`Options::with_disable_coordinates(true)`) or before the driver's first + /// snapshot publish. Coordinates converge as probe round-trips accumulate; + /// estimate inter-node RTT by comparing two nodes' coordinates. + #[cfg(feature = "coordinates")] + #[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] + #[must_use] + pub fn coordinate(&self) -> Option { + self.shared.snapshot.borrow().coordinate().cloned() + } + + /// The most-recently-observed Vivaldi coordinate of the peer `id`, updated on + /// each successful probe round-trip from that peer. + /// + /// Resolves `None` when coordinates are disabled or no RTT sample has been + /// received from `id` yet. + #[cfg(feature = "coordinates")] + #[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] + pub async fn cached_coordinate(&self, id: I) -> Result> { + let (tx, rx) = oneshot::channel(); + self.send(Command::CachedCoordinate( + crate::command::CachedCoordinateCmd { id, reply: tx }, + ))?; + await_reply(rx).await + } + /// The local node identifier. #[inline] pub fn local_id(&self) -> &I { From 77ea743f1f5dbf589e22f4d9781feb8b7bf630d7 Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 19:56:01 +1200 Subject: [PATCH 118/128] test(serf-compio): end-to-end coverage for the parity surface Mirror the reactor's scenario classes onto the compio driver: the snapshot leave-gate (on-disk tail ends at the Leave record; the default posture starts fresh on restart while the opt-in posture rejoins), the recording merge predicate consulted on a join push/pull with the joining peer's state, the operator aggregate and coordinate surfaces on the handle, and a file-backed rotation whose parked response is still collected within the query window with the key already durable when it arrives. --- serf-compio/src/serf/tests.rs | 300 ++++++++++++++++++++++++++++++++++ 1 file changed, 300 insertions(+) diff --git a/serf-compio/src/serf/tests.rs b/serf-compio/src/serf/tests.rs index e80b4d0e..fe69287b 100644 --- a/serf-compio/src/serf/tests.rs +++ b/serf-compio/src/serf/tests.rs @@ -1291,3 +1291,303 @@ async fn two_node_tcp_key_rotation_rotates_both_live_keyrings() { a.shutdown().await.expect("rot-a shuts down"); b.shutdown().await.expect("rot-b shuts down"); } + +/// Build a TCP node persisting membership to `snapshot`. +async fn spawn_node_with_snapshot( + id: &str, + snapshot: crate::SnapshotOptions, + rejoin_after_leave: bool, +) -> Serf { + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved( + "127.0.0.1:0".parse().expect("loopback addr"), + )); + Serf::new::, SocketAddrResolver, FirstAddrResolver, _, _>( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new().with_rejoin_after_leave(rejoin_after_leave), + gossip_rng().expect("seed gossip rng"), + None, + None, + Some(snapshot), + #[cfg(encryption)] + std::rc::Rc::new(VoidKeyringDelegate), + ) + .await + .expect("spawn snapshot-backed serf node") +} + +/// A unique snapshot path under the system temp dir. +fn snapshot_path(name: &str) -> std::path::PathBuf { + let mut p = std::env::temp_dir(); + p.push(format!("serf-compio-snap-{name}-{}", std::process::id())); + // Ignoring Err: a leftover file from a previous run is fine to lose. + let _ = std::fs::remove_file(&p); + p +} + +/// Poll both nodes until each reports the full two-member cluster. +async fn converge(a: &Serf, b: &Serf) { + compio::time::timeout(Duration::from_secs(20), async { + loop { + if a.num_members() == 2 && b.num_members() == 2 { + break; + } + compio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("both nodes converge to a 2-member cluster"); +} + +/// The clean-leave gate over the compio driver: a graceful leave ends the +/// snapshot at the Leave record, the default posture starts fresh on restart, +/// and the opt-in posture rejoins from the persisted membership. +#[compio::test] +async fn snapshot_leave_gate_controls_rejoin() { + let path = snapshot_path("leave-gate"); + let a = spawn_node("cgate-a").await; + let b = spawn_node_with_snapshot("cgate-b", crate::SnapshotOptions::new(&path), false).await; + let a_addr = a.advertise_address(); + + b.join(&SocketAddrResolver, MaybeResolved::Resolved(a_addr), false) + .await + .expect("join reaches node A"); + converge(&a, &b).await; + + b.leave().await.expect("cgate-b leaves gracefully"); + b.shutdown().await.expect("cgate-b shuts down"); + + // The pump writes the clock floors BEFORE the leave marker, so a clean + // shutdown ends the file at the Leave record — the terminal shape replay + // expects and compaction preserves. + { + let bytes = std::fs::read(&path).expect("the snapshot survives the leave"); + let mut records = Vec::new(); + let mut cursor = 0usize; + while cursor < bytes.len() { + let (rec, used) = + serf_proto::snapshot::SnapshotRecord::::decode(&bytes[cursor..]) + .expect("a clean-leave snapshot decodes whole"); + records.push(rec); + cursor += used; + } + assert!( + matches!( + records.last(), + Some(serf_proto::snapshot::SnapshotRecord::Leave) + ), + "a clean shutdown must end the snapshot at the Leave record" + ); + } + + // Default posture: the leave clears the recovered state — no auto-rejoin. + let b2 = spawn_node_with_snapshot("cgate-b", crate::SnapshotOptions::new(&path), false).await; + compio::time::sleep(Duration::from_millis(1500)).await; + assert_eq!( + b2.num_members(), + 1, + "a cleanly-left node must not auto-rejoin unless opted in" + ); + b2.shutdown().await.expect("cgate-b2 shuts down"); + + // Opt-in posture: the Leave marker is ignored and the membership recovers. + let b3 = spawn_node_with_snapshot("cgate-b", crate::SnapshotOptions::new(&path), true).await; + converge(&a, &b3).await; + + a.shutdown().await.expect("cgate-a shuts down"); + b3.shutdown().await.expect("cgate-b3 shuts down"); + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} + +/// The constructor-supplied merge delegate is the predicate the machine +/// consults: with a recording accept-all delegate installed on B, A's join +/// push-pull drives at least one `notify_merge` on B carrying A's node state. +/// (A veto here gates only the push/pull application — a rejected peer can +/// still be admitted moments later through gossip Alives, exactly as in the +/// reference implementation, so the stable assertion is consultation, not +/// permanent exclusion.) +#[compio::test] +async fn merge_delegate_is_consulted_on_join() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct RecordingMerge { + hits: std::sync::Arc, + saw_peer: std::sync::Arc, + } + impl crate::MergeDelegate for RecordingMerge { + fn notify_merge( + &self, + peers: memberlist_proto::MaybeOwned< + '_, + [memberlist_proto::typed::NodeState], + >, + ) -> bool { + self.hits.fetch_add(1, Ordering::Relaxed); + if peers.iter().any(|p| p.id_ref().as_str() == "cmerge-a") { + self.saw_peer.fetch_add(1, Ordering::Relaxed); + } + true + } + } + + let hits = std::sync::Arc::new(AtomicUsize::new(0)); + let saw_peer = std::sync::Arc::new(AtomicUsize::new(0)); + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new("cmerge-b")) + .with_advertise_addr(MaybeResolved::Resolved( + "127.0.0.1:0".parse().expect("loopback addr"), + )); + let b = + Serf::new::, SocketAddrResolver, FirstAddrResolver, _, _>( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + gossip_rng().expect("seed gossip rng"), + None, + Some(Box::new(RecordingMerge { + hits: hits.clone(), + saw_peer: saw_peer.clone(), + })), + None, + #[cfg(encryption)] + std::rc::Rc::new(VoidKeyringDelegate), + ) + .await + .expect("spawn merge-recording serf node"); + let a = spawn_node("cmerge-a").await; + let b_addr = b.advertise_address(); + + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + converge(&a, &b).await; + + assert!( + hits.load(Ordering::Relaxed) >= 1, + "the join push/pull must consult the installed merge delegate" + ); + assert!( + saw_peer.load(Ordering::Relaxed) >= 1, + "the consulted merge must carry the joining peer's node state" + ); + + a.shutdown().await.expect("cmerge-a shuts down"); + b.shutdown().await.expect("cmerge-b shuts down"); +} + +/// The operator aggregate reflects the converged view and the live endpoint +/// readings on the compio handle. +#[compio::test] +async fn operator_accessors_surface_on_the_handle() { + let b = spawn_node("cstat-b").await; + let a = spawn_node("cstat-a").await; + let b_addr = b.advertise_address(); + + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + converge(&a, &b).await; + + let stats = a.stats(); + assert_eq!(stats.members(), 2); + assert_eq!(stats.failed(), 0); + assert_eq!(stats.left(), 0); + assert_eq!(a.health_score(), 0, "a healthy node scores 0"); + assert!(!a.encryption_enabled(), "no keyring is configured"); + + // Coordinates converge as probe round-trips accumulate: both the local + // coordinate and B's cached coordinate must surface within the window. + #[cfg(feature = "coordinates")] + { + let deadline = std::time::Instant::now() + Duration::from_secs(20); + loop { + let local = a.coordinate(); + let cached = a + .cached_coordinate(SmolStr::new("cstat-b")) + .await + .expect("cached_coordinate round-trips through the driver"); + if local.is_some() && cached.is_some() { + break; + } + assert!( + std::time::Instant::now() < deadline, + "coordinates must surface on the handle: local={local:?} cached={cached:?}" + ); + compio::time::sleep(Duration::from_millis(100)).await; + } + } + + a.shutdown().await.expect("cstat-a shuts down"); + b.shutdown().await.expect("cstat-b shuts down"); +} + +/// With node B persisting through a [`crate::FileKeyringDelegate`], an +/// `install_key` from A still collects BOTH nodes' successful responses — B's +/// response is parked until the file write is acknowledged, then routed within +/// the query window — and because the response was gated on that +/// acknowledgement, the persisted file already carries the new key when the +/// response arrives. +#[cfg(all(encryption, unix))] +#[compio::test] +async fn file_backed_rotation_gates_the_response_on_persistence() { + let k1 = test_secret_key(0x33); + let k2 = test_secret_key(0x44); + + let mut path = std::env::temp_dir(); + path.push(format!("serf-compio-key-file-{}", std::process::id())); + // Ignoring Err: a leftover file from a previous run is fine to lose. + let _ = std::fs::remove_file(&path); + + let enc = || EncryptionOptions::new().with_keyring(Keyring::new(k1)); + let b = spawn_encrypted_node_with_keyring( + "cfile-b", + enc(), + std::rc::Rc::new(crate::FileKeyringDelegate::new(&path)), + ) + .await; + let a = spawn_encrypted_node_with_keyring( + "cfile-a", + enc(), + std::rc::Rc::new(RecordingKeyring::default()), + ) + .await; + let b_addr = b.advertise_address(); + + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B over the encrypted reliable plane"); + converge(&a, &b).await; + + let mut a_events = a.events(); + a.install_key(k2).await.expect("install_key dispatched"); + let kr = next_key_response(&mut a_events).await; + assert!( + kr.num_resp >= 2, + "install_key must collect a response from BOTH nodes, including the one parked on file persistence (num_resp={})", + kr.num_resp + ); + assert_eq!(kr.num_err, 0, "install_key must succeed on every node"); + + let persisted = crate::FileKeyringDelegate::new(&path) + .load() + .expect("the acknowledged write parses") + .expect("the acknowledged write exists"); + assert!( + persisted.secondaries().contains(&k2) || persisted.primary_ref() == &k2, + "the response was gated on persistence, so the file already holds the installed key" + ); + + a.shutdown().await.expect("cfile-a shuts down"); + b.shutdown().await.expect("cfile-b shuts down"); + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} From ef17326e867ccff95b882914c7f70d15e389b66f Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 20:05:43 +1200 Subject: [PATCH 119/128] fix(serf): forward serf-driver/tracing from both runtime tracing features MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hoist moved the persistence-failure warnings — snapshot append, flush, compaction, and keyring-write — behind serf-driver's tracing feature, which neither runtime crate's tracing feature forwarded: an operator enabling tracing on serf-reactor or serf-compio would have compiled none of the only diagnostics for silently dropped snapshot records or unpersisted rotations. Both manifests forward it now, and each runtime carries a wiring test asserting serf-driver's TRACING_WIRED probe so a dropped forward fails loudly instead of silencing telemetry. --- serf-compio/Cargo.toml | 2 +- serf-compio/src/driver/options/tests.rs | 13 +++++++++++++ serf-driver/src/lib.rs | 7 +++++++ serf-reactor/Cargo.toml | 2 +- serf-reactor/src/driver/options/tests.rs | 13 +++++++++++++ 5 files changed, 35 insertions(+), 2 deletions(-) diff --git a/serf-compio/Cargo.toml b/serf-compio/Cargo.toml index 8b0c0692..98cc0d2f 100644 --- a/serf-compio/Cargo.toml +++ b/serf-compio/Cargo.toml @@ -29,7 +29,7 @@ chacha20-poly1305 = [ # Regex-backed tag-filter matching. tag-regex = ["serf-proto/tag-regex", "serf-driver/tag-regex"] # Emit `tracing` spans around the public driver operations. -tracing = ["dep:tracing"] +tracing = ["dep:tracing", "serf-driver/tracing"] # Optional config layering: `serde` adds Serialize/Deserialize; `clap` adds CLI flags. serde = ["dep:serde", "dep:humantime-serde"] clap = ["dep:clap", "dep:humantime"] diff --git a/serf-compio/src/driver/options/tests.rs b/serf-compio/src/driver/options/tests.rs index 7e6ae97d..fff2d356 100644 --- a/serf-compio/src/driver/options/tests.rs +++ b/serf-compio/src/driver/options/tests.rs @@ -459,3 +459,16 @@ fn runtime_options_update_applies_explicit_override() { .expect("explicit override parses"); assert_eq!(cli.o.leave_timeout(), Duration::from_secs(3)); } + +/// The crate's `tracing` feature must reach the shared engines in +/// `serf-driver`: their persistence-failure warnings (snapshot append, +/// flush, compaction, keyring write) are the only diagnostics for silently +/// dropped records, and they compile only under `serf-driver/tracing`. +#[cfg(feature = "tracing")] +#[test] +fn tracing_forwards_to_the_shared_engines() { + assert!( + serf_driver::TRACING_WIRED, + "the tracing feature must forward serf-driver/tracing" + ); +} diff --git a/serf-driver/src/lib.rs b/serf-driver/src/lib.rs index 03018620..d28af2bc 100644 --- a/serf-driver/src/lib.rs +++ b/serf-driver/src/lib.rs @@ -51,6 +51,13 @@ pub use observation::observation_payload_bytes; #[cfg(any(feature = "tcp", feature = "quic"))] #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] pub use snapshot::{SerfSnapshot, SerfStats}; +/// Whether this build of the shared engines compiles their `tracing` +/// telemetry — the persistence-failure warnings in the snapshotter and the +/// keyring-file engine. The runtime crates' `tracing` features must forward +/// here; their wiring tests assert this constant so a dropped forward fails +/// loudly instead of silencing operator diagnostics. +pub const TRACING_WIRED: bool = cfg!(feature = "tracing"); + pub use snapshotter::{ DEFAULT_SNAPSHOT_COMPACT_THRESHOLD, OpenedSnapshot, SnapshotOpenError, Snapshotter, }; diff --git a/serf-reactor/Cargo.toml b/serf-reactor/Cargo.toml index 65ae9d3c..70046d0e 100644 --- a/serf-reactor/Cargo.toml +++ b/serf-reactor/Cargo.toml @@ -37,7 +37,7 @@ chacha20-poly1305 = [ # Regex-backed tag-filter matching. tag-regex = ["serf-proto/tag-regex", "serf-driver/tag-regex"] # Emit `tracing` spans around the public driver operations. -tracing = ["dep:tracing"] +tracing = ["dep:tracing", "serf-driver/tracing"] # Runtime conveniences: pull in the concrete agnostic runtime impls. tokio = ["agnostic/tokio"] smol = ["agnostic/smol"] diff --git a/serf-reactor/src/driver/options/tests.rs b/serf-reactor/src/driver/options/tests.rs index 2b8535a8..ab5e8dcb 100644 --- a/serf-reactor/src/driver/options/tests.rs +++ b/serf-reactor/src/driver/options/tests.rs @@ -412,3 +412,16 @@ fn runtime_options_update_applies_explicit_override() { .expect("explicit override parses"); assert_eq!(cli.o.leave_timeout(), Duration::from_secs(3)); } + +/// The crate's `tracing` feature must reach the shared engines in +/// `serf-driver`: their persistence-failure warnings (snapshot append, +/// flush, compaction, keyring write) are the only diagnostics for silently +/// dropped records, and they compile only under `serf-driver/tracing`. +#[cfg(feature = "tracing")] +#[test] +fn tracing_forwards_to_the_shared_engines() { + assert!( + serf_driver::TRACING_WIRED, + "the tracing feature must forward serf-driver/tracing" + ); +} From cfc6a4087a2d5542df573eb21ef9d7c469665ada Mon Sep 17 00:00:00 2001 From: al8n Date: Sun, 12 Jul 2026 20:11:59 +1200 Subject: [PATCH 120/128] fix(serf): const-evaluate the tracing wiring pins The runtime crates' all-targets clippy gates deny assertions on constants, which the wiring tests' runtime asserts tripped. The pin now const-evaluates inside each test, which both satisfies the lint and strengthens the check: a dropped serf-driver/tracing forward fails every tracing-featured compile of the test target outright rather than waiting for the test run. --- serf-compio/src/driver/options/tests.rs | 12 ++++++++---- serf-reactor/src/driver/options/tests.rs | 12 ++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/serf-compio/src/driver/options/tests.rs b/serf-compio/src/driver/options/tests.rs index fff2d356..2d9ef3ec 100644 --- a/serf-compio/src/driver/options/tests.rs +++ b/serf-compio/src/driver/options/tests.rs @@ -467,8 +467,12 @@ fn runtime_options_update_applies_explicit_override() { #[cfg(feature = "tracing")] #[test] fn tracing_forwards_to_the_shared_engines() { - assert!( - serf_driver::TRACING_WIRED, - "the tracing feature must forward serf-driver/tracing" - ); + // A dropped forward fails this build outright: the pin is const-evaluated, + // so every tracing-featured compile of the test target enforces it. + const { + assert!( + serf_driver::TRACING_WIRED, + "the tracing feature must forward serf-driver/tracing" + ) + }; } diff --git a/serf-reactor/src/driver/options/tests.rs b/serf-reactor/src/driver/options/tests.rs index ab5e8dcb..7a48654c 100644 --- a/serf-reactor/src/driver/options/tests.rs +++ b/serf-reactor/src/driver/options/tests.rs @@ -420,8 +420,12 @@ fn runtime_options_update_applies_explicit_override() { #[cfg(feature = "tracing")] #[test] fn tracing_forwards_to_the_shared_engines() { - assert!( - serf_driver::TRACING_WIRED, - "the tracing feature must forward serf-driver/tracing" - ); + // A dropped forward fails this build outright: the pin is const-evaluated, + // so every tracing-featured compile of the test target enforces it. + const { + assert!( + serf_driver::TRACING_WIRED, + "the tracing feature must forward serf-driver/tracing" + ) + }; } From c6d3be3d7070f0087ead7895d8199bfa950cf556 Mon Sep 17 00:00:00 2001 From: Al Liu Date: Sun, 12 Jul 2026 21:56:05 +1200 Subject: [PATCH 121/128] =?UTF-8?q?test(serf-reactor):=20legacy=20cluster-?= =?UTF-8?q?scenario=20ports=20=E2=80=94=20force-leave,=20remove,=20rejoin,?= =?UTF-8?q?=20join-cancel=20(#85)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- serf-reactor/src/quic/mod.rs | 29 ++ serf-reactor/src/tcp/mod.rs | 29 ++ serf-reactor/src/tls/mod.rs | 29 ++ serf-reactor/tests/cluster/mod.rs | 85 ++++- serf-reactor/tests/tcp.rs | 506 ++++++++++++++++++++++++++++++ 5 files changed, 673 insertions(+), 5 deletions(-) diff --git a/serf-reactor/src/quic/mod.rs b/serf-reactor/src/quic/mod.rs index 7df778ac..8513f2fb 100644 --- a/serf-reactor/src/quic/mod.rs +++ b/serf-reactor/src/quic/mod.rs @@ -92,6 +92,10 @@ pub struct QuicTransportOptions> { /// SWIM suspicion multiplier override. `None` keeps the coordinator default. See /// [`with_suspicion_mult`](Self::with_suspicion_mult). suspicion_mult: Option, + /// Reclaim window for a same-name member returning at a NEW address: a + /// dead member older than this is revived in place of a conflict. See + /// [`with_dead_node_reclaim_time`](Self::with_dead_node_reclaim_time). + dead_node_reclaim_time: Option, /// SWIM suspicion max-timeout multiplier override. `None` keeps the coordinator /// default. See [`with_suspicion_max_timeout_mult`](Self::with_suspicion_max_timeout_mult). suspicion_max_timeout_mult: Option, @@ -121,6 +125,7 @@ impl QuicTransportOptions { probe_timeout: None, gossip_interval: None, suspicion_mult: None, + dead_node_reclaim_time: None, suspicion_max_timeout_mult: None, #[cfg(encryption)] encryption: EncryptionOptions::new(), @@ -216,6 +221,18 @@ impl QuicTransportOptions { self } + /// Builder: allow a dead member to be revived under the SAME id at a NEW + /// address once it has been dead longer than `window` — the reference + /// implementation's dead-node reclaim. Left unset (the default), a + /// same-name Alive from a different address is a name conflict, never a + /// revival. + #[must_use] + #[inline] + pub const fn with_dead_node_reclaim_time(mut self, window: Duration) -> Self { + self.dead_node_reclaim_time = Some(window); + self + } + /// Builder: override the memberlist SWIM suspicion max-timeout multiplier — the /// upper bound on the suspicion timeout as a multiple of the minimum. /// @@ -295,6 +312,13 @@ impl QuicTransportOptions { self.suspicion_mult } + /// The configured dead-node reclaim window, if overridden. + #[must_use] + #[inline] + pub const fn dead_node_reclaim_time(&self) -> Option { + self.dead_node_reclaim_time + } + /// The SWIM suspicion max-timeout-multiplier override, if set. #[inline] pub const fn suspicion_max_timeout_mult(&self) -> Option { @@ -347,6 +371,7 @@ where probe_timeout: Option, gossip_interval: Option, suspicion_mult: Option, + dead_node_reclaim_time: Option, suspicion_max_timeout_mult: Option, /// Independent OS-seeded seed for the serf core's RNG, drawn once per node in /// [`Transport::new`] and consumed when [`Transport::run`] builds the endpoint via @@ -453,6 +478,7 @@ where probe_timeout: options.probe_timeout, gossip_interval: options.gossip_interval, suspicion_mult: options.suspicion_mult, + dead_node_reclaim_time: options.dead_node_reclaim_time, suspicion_max_timeout_mult: options.suspicion_max_timeout_mult, serf_rng, #[cfg(encryption)] @@ -509,6 +535,9 @@ where if let Some(v) = self.suspicion_mult { inner_opts = inner_opts.with_suspicion_mult(v); } + if let Some(v) = self.dead_node_reclaim_time { + inner_opts = inner_opts.with_dead_node_reclaim_time(v); + } if let Some(v) = self.suspicion_max_timeout_mult { inner_opts = inner_opts.with_suspicion_max_timeout_mult(v); } diff --git a/serf-reactor/src/tcp/mod.rs b/serf-reactor/src/tcp/mod.rs index 5a86d2f9..b846e194 100644 --- a/serf-reactor/src/tcp/mod.rs +++ b/serf-reactor/src/tcp/mod.rs @@ -69,6 +69,10 @@ pub struct TcpTransportOptions> { /// SWIM suspicion multiplier override. `None` keeps the coordinator default. See /// [`with_suspicion_mult`](Self::with_suspicion_mult). suspicion_mult: Option, + /// Reclaim window for a same-name member returning at a NEW address: a + /// dead member older than this is revived in place of a conflict. See + /// [`with_dead_node_reclaim_time`](Self::with_dead_node_reclaim_time). + dead_node_reclaim_time: Option, /// SWIM suspicion max-timeout multiplier override. `None` keeps the coordinator /// default. See [`with_suspicion_max_timeout_mult`](Self::with_suspicion_max_timeout_mult). suspicion_max_timeout_mult: Option, @@ -96,6 +100,7 @@ impl TcpTransportOptions { probe_timeout: None, gossip_interval: None, suspicion_mult: None, + dead_node_reclaim_time: None, suspicion_max_timeout_mult: None, #[cfg(encryption)] encryption: EncryptionOptions::new(), @@ -188,6 +193,18 @@ impl TcpTransportOptions { self } + /// Builder: allow a dead member to be revived under the SAME id at a NEW + /// address once it has been dead longer than `window` — the reference + /// implementation's dead-node reclaim. Left unset (the default), a + /// same-name Alive from a different address is a name conflict, never a + /// revival. + #[must_use] + #[inline] + pub const fn with_dead_node_reclaim_time(mut self, window: Duration) -> Self { + self.dead_node_reclaim_time = Some(window); + self + } + /// Builder: override the memberlist SWIM suspicion max-timeout multiplier — the /// upper bound on the suspicion timeout as a multiple of the minimum. /// @@ -260,6 +277,13 @@ impl TcpTransportOptions { self.suspicion_mult } + /// The configured dead-node reclaim window, if overridden. + #[must_use] + #[inline] + pub const fn dead_node_reclaim_time(&self) -> Option { + self.dead_node_reclaim_time + } + /// The SWIM suspicion max-timeout-multiplier override, if set. #[inline] pub const fn suspicion_max_timeout_mult(&self) -> Option { @@ -313,6 +337,7 @@ where probe_timeout: Option, gossip_interval: Option, suspicion_mult: Option, + dead_node_reclaim_time: Option, suspicion_max_timeout_mult: Option, /// Independent OS-seeded seed for the serf core's RNG, drawn once per node in /// [`Transport::new`] and consumed when [`Transport::run`] builds the endpoint. @@ -439,6 +464,7 @@ where probe_timeout: options.probe_timeout, gossip_interval: options.gossip_interval, suspicion_mult: options.suspicion_mult, + dead_node_reclaim_time: options.dead_node_reclaim_time, suspicion_max_timeout_mult: options.suspicion_max_timeout_mult, serf_rng, #[cfg(encryption)] @@ -494,6 +520,9 @@ where if let Some(v) = self.suspicion_mult { inner_opts = inner_opts.with_suspicion_mult(v); } + if let Some(v) = self.dead_node_reclaim_time { + inner_opts = inner_opts.with_dead_node_reclaim_time(v); + } if let Some(v) = self.suspicion_max_timeout_mult { inner_opts = inner_opts.with_suspicion_max_timeout_mult(v); } diff --git a/serf-reactor/src/tls/mod.rs b/serf-reactor/src/tls/mod.rs index 809dfa44..38276428 100644 --- a/serf-reactor/src/tls/mod.rs +++ b/serf-reactor/src/tls/mod.rs @@ -124,6 +124,10 @@ pub struct TlsTransportOptions> { /// SWIM suspicion multiplier override. `None` keeps the coordinator default. See /// [`with_suspicion_mult`](Self::with_suspicion_mult). suspicion_mult: Option, + /// Reclaim window for a same-name member returning at a NEW address: a + /// dead member older than this is revived in place of a conflict. See + /// [`with_dead_node_reclaim_time`](Self::with_dead_node_reclaim_time). + dead_node_reclaim_time: Option, /// SWIM suspicion max-timeout multiplier override. `None` keeps the coordinator /// default. See [`with_suspicion_max_timeout_mult`](Self::with_suspicion_max_timeout_mult). suspicion_max_timeout_mult: Option, @@ -157,6 +161,7 @@ impl TlsTransportOptions { probe_timeout: None, gossip_interval: None, suspicion_mult: None, + dead_node_reclaim_time: None, suspicion_max_timeout_mult: None, #[cfg(encryption)] encryption: EncryptionOptions::new(), @@ -270,6 +275,18 @@ impl TlsTransportOptions { self } + /// Builder: allow a dead member to be revived under the SAME id at a NEW + /// address once it has been dead longer than `window` — the reference + /// implementation's dead-node reclaim. Left unset (the default), a + /// same-name Alive from a different address is a name conflict, never a + /// revival. + #[must_use] + #[inline] + pub const fn with_dead_node_reclaim_time(mut self, window: Duration) -> Self { + self.dead_node_reclaim_time = Some(window); + self + } + /// Builder: override the memberlist SWIM suspicion max-timeout multiplier — the /// upper bound on the suspicion timeout as a multiple of the minimum. /// @@ -349,6 +366,13 @@ impl TlsTransportOptions { self.suspicion_mult } + /// The configured dead-node reclaim window, if overridden. + #[must_use] + #[inline] + pub const fn dead_node_reclaim_time(&self) -> Option { + self.dead_node_reclaim_time + } + /// The SWIM suspicion max-timeout-multiplier override, if set. #[inline] pub const fn suspicion_max_timeout_mult(&self) -> Option { @@ -417,6 +441,7 @@ where probe_timeout: Option, gossip_interval: Option, suspicion_mult: Option, + dead_node_reclaim_time: Option, suspicion_max_timeout_mult: Option, /// Independent OS-seeded seed for the serf core's RNG, drawn once per node in /// [`Transport::new`] and consumed when [`Transport::run`] builds the endpoint @@ -554,6 +579,7 @@ where probe_timeout: options.probe_timeout, gossip_interval: options.gossip_interval, suspicion_mult: options.suspicion_mult, + dead_node_reclaim_time: options.dead_node_reclaim_time, suspicion_max_timeout_mult: options.suspicion_max_timeout_mult, serf_rng, #[cfg(encryption)] @@ -609,6 +635,9 @@ where if let Some(v) = self.suspicion_mult { inner_opts = inner_opts.with_suspicion_mult(v); } + if let Some(v) = self.dead_node_reclaim_time { + inner_opts = inner_opts.with_dead_node_reclaim_time(v); + } if let Some(v) = self.suspicion_max_timeout_mult { inner_opts = inner_opts.with_suspicion_max_timeout_mult(v); } diff --git a/serf-reactor/tests/cluster/mod.rs b/serf-reactor/tests/cluster/mod.rs index e9d39088..e7aa0558 100644 --- a/serf-reactor/tests/cluster/mod.rs +++ b/serf-reactor/tests/cluster/mod.rs @@ -64,6 +64,13 @@ pub struct ClusterTiming { reconnect_interval: Duration, reconnect_timeout: Duration, tombstone_timeout: Duration, + /// Dead-node reclaim window (`None` keeps revival-at-a-new-address a + /// conflict, the coordinator default). + pub dead_node_reclaim: Option, + /// Periodic anti-entropy push/pull override (`None` keeps the coordinator + /// default; `Duration::ZERO` disables it, leaving gossiped intents as the + /// only dissemination path — the exclusivity a causal clock fence needs). + pub push_pull_interval: Option, leave_propagate_delay: Duration, } @@ -90,6 +97,8 @@ impl ClusterTiming { reconnect_interval: Duration::from_millis(100), reconnect_timeout: Duration::from_millis(1), tombstone_timeout: Duration::from_millis(1), + dead_node_reclaim: None, + push_pull_interval: None, // Short enough to keep a graceful leave sub-second, long enough to give // in-flight probes a gossip cycle to observe the leave intent. leave_propagate_delay: Duration::from_millis(100), @@ -110,6 +119,18 @@ impl ClusterTiming { /// in the tombstone view instead of reaping it, so a graceful-leave assertion /// observes `[Join, Leave]` without a trailing `Reap`. #[must_use] + pub fn with_dead_node_reclaim(mut self, v: Duration) -> Self { + self.dead_node_reclaim = Some(v); + self + } + + /// Override (or, at zero, disable) the periodic anti-entropy push/pull. + pub fn with_push_pull_interval(mut self, v: Duration) -> Self { + self.push_pull_interval = Some(v); + self + } + + /// Override the tombstone retention window. pub fn with_tombstone_timeout(mut self, v: Duration) -> Self { self.tombstone_timeout = v; self @@ -122,13 +143,20 @@ impl ClusterTiming { id: &str, advertise: SocketAddr, ) -> TcpTransportOptions { - TcpTransportOptions::::new() + let mut opts = TcpTransportOptions::::new() .with_local_id(SmolStr::new(id)) .with_advertise_addr(MaybeResolved::Resolved(advertise)) .with_probe_interval(self.probe_interval) .with_probe_timeout(self.probe_timeout) .with_gossip_interval(self.gossip_interval) - .with_suspicion_mult(self.suspicion_mult) + .with_suspicion_mult(self.suspicion_mult); + if let Some(v) = self.dead_node_reclaim { + opts = opts.with_dead_node_reclaim_time(v); + } + if let Some(v) = self.push_pull_interval { + opts = opts.with_push_pull_interval(v); + } + opts } /// The serf `Options` for every node: the fast reap / reconnect timing. @@ -292,6 +320,48 @@ where self.slots[i].serf = Some(serf); } + /// Restart a previously-killed node `i` at the SAME id but a FRESH ephemeral + /// port on the same interface, re-attaching a collector to the slot's + /// existing log and updating the recorded address. Unlike + /// [`restart`](Self::restart), the serf reconnect re-dial cannot reach the + /// node (it targets the old address), so the caller re-joins explicitly — + /// exercising the same-name-new-address revival path. + pub async fn restart_at_ephemeral(&mut self, i: usize) { + assert!( + self.slots[i].serf.is_none(), + "node {i} must be killed before restart" + ); + let id = self.slots[i].id.clone(); + let old_addr = self.slots[i].addr; + // An ephemeral bind guarantees an AVAILABLE port, not a DIFFERENT one: + // the OS can hand the just-released port straight back, which would + // silently degrade a new-address scenario into a same-address one. + // Rebind until the address genuinely differs. + const DISTINCT_PORT_RETRIES: usize = 25; + let mut attempt = 0usize; + let serf = loop { + let serf = build_node::( + id.as_str(), + "127.0.0.1:0".parse().expect("loopback addr"), + &self.timing, + ) + .await + .expect("an ephemeral rebind cannot collide"); + if serf.advertise_address() != old_addr { + break serf; + } + assert!( + attempt + 1 < DISTINCT_PORT_RETRIES, + "the OS kept re-issuing the released port {old_addr}" + ); + attempt += 1; + serf.shutdown().await.expect("same-port rebind shuts down"); + }; + self.slots[i].addr = serf.advertise_address(); + attach_collector::(&serf, self.slots[i].log.clone()); + self.slots[i].serf = Some(serf); + } + /// Poll every live node until each reports exactly `expect` members, or fail on /// the poll timeout. pub async fn converge(&self, expect: usize) { @@ -343,7 +413,7 @@ where } }) .await - .expect("observer holds the subject as a Left tombstone"); + .unwrap_or_else(|_| panic!("node {observer} never holds {subject:?} as a Left tombstone")); } /// Poll until `observer`'s log records a member event of `kind` naming @@ -358,7 +428,12 @@ where } }) .await - .expect("observer records the expected member event"); + .unwrap_or_else(|_| { + panic!( + "node {observer} never records {kind:?} for {subject:?} (saw {:?})", + self.member_event_kinds(observer, subject) + ) + }); } /// Poll until the ordered member-event kinds `observer` recorded about `subject` @@ -399,7 +474,7 @@ where } /// The ordered member-event kinds `observer` recorded about `subject`. - fn member_event_kinds(&self, observer: usize, subject: &str) -> Vec { + pub fn member_event_kinds(&self, observer: usize, subject: &str) -> Vec { self.slots[observer] .log .lock() diff --git a/serf-reactor/tests/tcp.rs b/serf-reactor/tests/tcp.rs index 1f6253c1..16b0b3e9 100644 --- a/serf-reactor/tests/tcp.rs +++ b/serf-reactor/tests/tcp.rs @@ -960,6 +960,442 @@ where cluster.shutdown_all().await; } +/// Port of legacy `serf_force_leave_failed` (Go `TestSerf_ForceLeaveFailed`): +/// an operator force-leaving a FAILED member transitions it to Left on every +/// surviving node, rather than leaving it to linger Failed until the reap. +async fn serf_force_leave_failed() +where + R: Runtime, +{ + // A long tombstone keeps the force-left member observable as Left for the + // whole assertion window, and a long reconnect timeout keeps the FAILED + // member from being reaped out of the views before the intent lands (the + // reference tests run with the default day-scale reconnect timeout). + let mut cluster = cluster::Cluster::::spawn( + &["fleave-a", "fleave-b", "fleave-c"], + cluster::ClusterTiming::fast() + .with_tombstone_timeout(Duration::from_secs(120)) + .with_reconnect_timeout(Duration::from_secs(120)), + ) + .await; + let subject = cluster.id(2); + + cluster.kill_abrupt(2).await; + cluster + .await_member_event(0, subject.as_str(), MemberEventKind::Failed) + .await; + cluster + .await_member_event(1, subject.as_str(), MemberEventKind::Failed) + .await; + + cluster + .node(0) + .force_leave(subject.clone(), false) + .await + .expect("force_leave dispatches for a failed member"); + + // The force-leave propagates: BOTH survivors converge the failed member to + // a Left tombstone, and the observer's event log records the full + // Join -> Failed -> Leave lifecycle. + cluster.await_left_tombstone(0, subject.as_str()).await; + cluster.await_left_tombstone(1, subject.as_str()).await; + cluster + .assert_member_events( + 0, + subject.as_str(), + &[ + MemberEventKind::Join, + MemberEventKind::Failed, + MemberEventKind::Leave, + ], + ) + .await; + + cluster.shutdown_all().await; +} + +/// Port of legacy `serf_force_leave_left` (Go `TestSerf_ForceLeaveLeft`, +/// folding `TestSerf_ForceLeaveLeaving`): force-leaving a member that already +/// departed gracefully is an accepted no-op — it stays a Left tombstone and +/// the membership view is unchanged. The transient Leaving window itself is +/// machine-covered (`endpoint::force_leave_transitions_alive_member_to_leaving`); +/// a driver cannot deterministically observe it mid-flight. +async fn serf_force_leave_left_is_idempotent() +where + R: Runtime, +{ + // A long tombstone keeps the departed member observable as Left for the + // whole assertion window, and push/pull is DISABLED so the gossiped intent + // is the only path that can advance the survivor's member clock — the + // exclusivity the causal fence below relies on (anti-entropy also + // witnesses remote clocks and would replay the Left state, masking the + // fresh intent). + let mut cluster = cluster::Cluster::::spawn( + &["fleft-a", "fleft-b", "fleft-c"], + cluster::ClusterTiming::fast() + .with_tombstone_timeout(Duration::from_secs(120)) + .with_push_pull_interval(Duration::ZERO), + ) + .await; + let subject = cluster.id(2); + + cluster.leave_graceful(2).await; + cluster.await_left_tombstone(0, subject.as_str()).await; + cluster.await_left_tombstone(1, subject.as_str()).await; + // FENCE the baselines on the COLLECTORS, not the membership snapshot: the + // tombstone proves the state flipped, while the graceful leave's member + // event may still be in flight to a detached collector. Awaiting the exact + // sequence pins each baseline at [Join, Leave]. + let expected = [MemberEventKind::Join, MemberEventKind::Leave]; + cluster + .assert_member_events(0, subject.as_str(), &expected) + .await; + cluster + .assert_member_events(1, subject.as_str(), &expected) + .await; + + let clock_before = cluster.node(0).stats().member_clock(); + cluster + .node(0) + .force_leave(subject.clone(), false) + .await + .expect("force_leave on an already-left member is an accepted no-op"); + + // CAUSAL FENCE: the force-leave stamps the member clock and the intent + // carries that ltime, which every receiver witnesses BEFORE deciding + // whether the intent applies — so the non-issuing survivor's clock + // reaching the issuer's post-command value proves THIS intent was + // received and processed there, not merely that unrelated traffic flowed. + let stamp = R::timeout(Duration::from_secs(20), async { + loop { + let c = cluster.node(0).stats().member_clock(); + if c > clock_before { + break c; + } + R::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("the issuer stamps the member clock for the no-op intent"); + R::timeout(Duration::from_secs(20), async { + loop { + if cluster.node(1).stats().member_clock() >= stamp { + break; + } + R::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("the non-issuing survivor witnesses the no-op intent's clock"); + + // A short settle lets the survivors reprocess any rebroadcast echo of the + // intent (a stale re-receipt must also stay a no-op), then both views must + // be unchanged — still the Left tombstone, still three members, and NO + // additional member event. + R::sleep(Duration::from_millis(500)).await; + cluster.await_left_tombstone(0, subject.as_str()).await; + cluster.await_left_tombstone(1, subject.as_str()).await; + for observer in [0usize, 1] { + assert_eq!( + cluster.node(observer).num_members(), + 3, + "node {observer}: the Left tombstone is retained, not pruned, by a plain force_leave" + ); + } + // With propagation causally fenced above, BARRIER the collector drainage: + // a fresh sentinel node joins and both collectors must record ITS Join + // before the subject's sequences are compared — each collector's stream is + // ordered, so any duplicate event the (already-processed) intent emitted + // is queued ahead of the sentinel and would already be visible. + let sentinel = spawn_node::("fleft-sentinel").await; + let a_addr = cluster.node(0).advertise_address(); + sentinel + .join(&SocketAddrResolver, MaybeResolved::Resolved(a_addr), false) + .await + .expect("the sentinel joins through the issuing survivor"); + cluster + .await_member_event(0, "fleft-sentinel", MemberEventKind::Join) + .await; + cluster + .await_member_event(1, "fleft-sentinel", MemberEventKind::Join) + .await; + assert_eq!( + cluster.member_event_kinds(0, subject.as_str()), + expected, + "the issuing survivor records no additional member event for the no-op" + ); + assert_eq!( + cluster.member_event_kinds(1, subject.as_str()), + expected, + "the non-issuing survivor records no additional member event for the no-op" + ); + + sentinel.shutdown().await.expect("the sentinel shuts down"); + cluster.shutdown_all().await; +} + +/// Port of legacy `serf_remove_failed_node` + `serf_remove_failed_events_leave` +/// (Go `TestSerf_RemoveFailedNode` / `TestSerfRemoveFailedEventsLeave`): after +/// a failure, `remove_failed_node` on one survivor propagates — the OTHER +/// survivor also observes a Leave member event for the failed node and holds +/// it as a Left tombstone. +async fn serf_remove_failed_node_propagates() +where + R: Runtime, +{ + // A long tombstone keeps the removed member observable as Left for the + // whole assertion window. + let mut cluster = cluster::Cluster::::spawn( + &["remove-a", "remove-b", "remove-c"], + cluster::ClusterTiming::fast() + .with_tombstone_timeout(Duration::from_secs(120)) + .with_reconnect_timeout(Duration::from_secs(120)), + ) + .await; + let subject = cluster.id(2); + + cluster.kill_abrupt(2).await; + cluster + .await_member_event(0, subject.as_str(), MemberEventKind::Failed) + .await; + cluster + .await_member_event(1, subject.as_str(), MemberEventKind::Failed) + .await; + + cluster + .node(0) + .remove_failed_node(subject.clone()) + .await + .expect("remove_failed_node dispatches"); + + // Propagation: the NON-issuing survivor holds the tombstone too. + cluster.await_left_tombstone(0, subject.as_str()).await; + cluster.await_left_tombstone(1, subject.as_str()).await; + cluster + .await_member_event(1, subject.as_str(), MemberEventKind::Leave) + .await; + + cluster.shutdown_all().await; +} + +/// Port of legacy `serf_remove_failed_node_prune` +/// (Go `TestSerf_RemoveFailedNode_prune`): removing with prune erases the +/// failed member outright on every survivor — the membership count drops +/// without waiting out the Left tombstone. +async fn serf_remove_failed_node_prune_erases() +where + R: Runtime, +{ + // Hold the failed member (no reap, no reconnect eviction) so the prune — + // not the reaper — is what erases it. + let mut cluster = cluster::Cluster::::spawn( + &["prune-a", "prune-b", "prune-c"], + cluster::ClusterTiming::fast().with_reconnect_timeout(Duration::from_secs(120)), + ) + .await; + let subject = cluster.id(2); + + cluster.kill_abrupt(2).await; + cluster + .await_member_event(0, subject.as_str(), MemberEventKind::Failed) + .await; + cluster + .await_member_event(1, subject.as_str(), MemberEventKind::Failed) + .await; + + cluster + .node(0) + .remove_failed_node_prune(subject.clone()) + .await + .expect("remove_failed_node_prune dispatches"); + + cluster.await_num_members(0, 2).await; + cluster.await_num_members(1, 2).await; + + cluster.shutdown_all().await; +} + +/// Port of the legacy `serf_remove_failed_node` absent-member edge +/// (Go `TestSerf_RemoveFailedNode_ourself` shape): removing a name that is +/// not a member reports success as a no-op. +async fn remove_failed_node_absent_is_a_noop() +where + R: Runtime, +{ + let a = spawn_node::("absent-a").await; + a.remove_failed_node(SmolStr::new("no-such-node")) + .await + .expect("removing an absent member is an accepted no-op"); + assert_eq!(a.num_members(), 1, "the membership view is unchanged"); + a.shutdown().await.expect("absent-a shuts down"); +} + +/// Port of legacy `serf_reconnect_same_ip` (Go `TestSerf_Reconnect_SameIP`): +/// the failed node returns at the SAME IP but a DIFFERENT port and re-joins — +/// the same-name member revives at its new address (Join, Failed, Join), and +/// the observer's view carries the updated port. +async fn serf_reconnect_same_ip() +where + R: Runtime, +{ + // The reclaim window is what allows a SAME-name member to revive at a NEW + // address at all — without it a different-address Alive is a name conflict, + // exactly as in the reference implementation's dead-node reclaim. + let mut cluster = cluster::Cluster::::spawn( + &["sameip-a", "sameip-b"], + cluster::ClusterTiming::fast() + .with_reconnect_timeout(Duration::from_secs(30)) + .with_dead_node_reclaim(Duration::from_millis(1)), + ) + .await; + let subject = cluster.id(1); + let old_addr = cluster.node(1).advertise_address(); + + cluster.kill_abrupt(1).await; + cluster + .await_member_event(0, subject.as_str(), MemberEventKind::Failed) + .await; + // The reclaim admits a new-address revival only once the failed state is + // STRICTLY older than the window; observing the Failed event is not that + // fence, so wait comfortably past the (1ms) window before rejoining. + R::sleep(Duration::from_millis(100)).await; + + cluster.restart_at_ephemeral(1).await; + let new_addr = cluster.node(1).advertise_address(); + assert_ne!(old_addr, new_addr, "the restart rebinds a fresh port"); + let a_addr = cluster.node(0).advertise_address(); + cluster + .node(1) + .join(&SocketAddrResolver, MaybeResolved::Resolved(a_addr), false) + .await + .expect("the restarted node rejoins through the survivor"); + + // The failed tombstone still counts toward the member view, so the revival + // signal is the SECOND Join event; the sequence assertion below polls until + // the third event lands. + cluster + .assert_member_events( + 0, + subject.as_str(), + &[ + MemberEventKind::Join, + MemberEventKind::Failed, + MemberEventKind::Join, + ], + ) + .await; + // The revived member is tracked at its NEW address. + let seen = cluster + .node(0) + .members() + .iter() + .find(|m| m.node().id_ref().as_str() == subject.as_str()) + .map(|m| *m.node().addr_ref()) + .expect("the observer tracks the revived member"); + assert_eq!( + seen, new_addr, + "the same-name revival updates the tracked address" + ); + + cluster.shutdown_all().await; +} + +/// Port of legacy `serf_join_cancel` (Go `TestSerf_Join_Cancel`): with a +/// vetoing merge predicate on BOTH nodes, a join attempt admits nothing — +/// each side's delegate is consulted with the peer's state and each +/// membership view stays at one. (The machine's veto is a push/pull FILTER: +/// with neither side ever admitting the other, no gossip path exists either, +/// so the exclusion here is total and deterministic.) +async fn serf_join_cancel() +where + R: Runtime, +{ + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct VetoAll { + hits: std::sync::Arc, + } + impl serf_reactor::MergeDelegate for VetoAll { + fn notify_merge( + &self, + _peers: memberlist_proto::MaybeOwned< + '_, + [memberlist_proto::typed::NodeState], + >, + ) -> bool { + self.hits.fetch_add(1, Ordering::Relaxed); + false + } + } + + async fn spawn_vetoing(id: &str, hits: std::sync::Arc) -> Node + where + R: Runtime, + { + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + Serf::::tcp( + TcpTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(bind)), + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + None, + Some(Box::new(VetoAll { hits })), + None, + #[cfg(encryption)] + std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), + ) + .await + .expect("spawn vetoing serf node") + } + + let a_hits = std::sync::Arc::new(AtomicUsize::new(0)); + let b_hits = std::sync::Arc::new(AtomicUsize::new(0)); + let b = spawn_vetoing::("cancel-b", b_hits.clone()).await; + let a = spawn_vetoing::("cancel-a", a_hits.clone()).await; + let b_addr = b.advertise_address(); + + let outcome = a + .join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await; + + // The SEED's predicate runs and refuses; its machine then closes the + // exchange WITHOUT sending its own state — a vetoed filter must not leak + // the peer's state — so the joiner's predicate never receives anything to + // judge. (The reference implementation consults both sides because it + // ships its state before the remote verdict; the machine here deliberately + // tightens that.) + R::timeout(Duration::from_secs(20), async { + loop { + if b_hits.load(Ordering::Relaxed) >= 1 { + break; + } + R::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("the seed's merge predicate is consulted by the join push/pull"); + assert_eq!( + a_hits.load(Ordering::Relaxed), + 0, + "no state reaches the joiner's predicate once the seed refused" + ); + // Nothing admitted anywhere: with no member ever merged there is no gossip + // path either, so the exclusion holds on both views. + assert_eq!(a.num_members(), 1, "the joiner admitted nothing"); + assert_eq!(b.num_members(), 1, "the seed admitted nothing"); + assert!( + outcome.is_err(), + "a fully vetoed join reports failure to the caller (got {outcome:?})" + ); + + a.shutdown().await.expect("cancel-a shuts down"); + b.shutdown().await.expect("cancel-b shuts down"); +} + /// A deterministic test secret key, selecting whichever AEAD cipher this build /// compiled so the encrypted tests work under either backend. #[cfg(encryption)] @@ -1203,6 +1639,41 @@ mod tokio_cells { super::serf_reconnect::().await; } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_force_leave_failed() { + super::serf_force_leave_failed::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_force_leave_left_is_idempotent() { + super::serf_force_leave_left_is_idempotent::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_remove_failed_node_propagates() { + super::serf_remove_failed_node_propagates::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_remove_failed_node_prune_erases() { + super::serf_remove_failed_node_prune_erases::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn remove_failed_node_absent_is_a_noop() { + super::remove_failed_node_absent_is_a_noop::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_reconnect_same_ip() { + super::serf_reconnect_same_ip::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_join_cancel() { + super::serf_join_cancel::().await; + } + #[cfg(encryption)] #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn two_node_join_converges_encrypted() { @@ -1315,6 +1786,41 @@ mod smol_cells { SmolRuntime::block_on(super::serf_reconnect::()); } + #[test] + fn serf_force_leave_failed_smol() { + SmolRuntime::block_on(super::serf_force_leave_failed::()); + } + + #[test] + fn serf_force_leave_left_is_idempotent_smol() { + SmolRuntime::block_on(super::serf_force_leave_left_is_idempotent::()); + } + + #[test] + fn serf_remove_failed_node_propagates_smol() { + SmolRuntime::block_on(super::serf_remove_failed_node_propagates::()); + } + + #[test] + fn serf_remove_failed_node_prune_erases_smol() { + SmolRuntime::block_on(super::serf_remove_failed_node_prune_erases::()); + } + + #[test] + fn remove_failed_node_absent_is_a_noop_smol() { + SmolRuntime::block_on(super::remove_failed_node_absent_is_a_noop::()); + } + + #[test] + fn serf_reconnect_same_ip_smol() { + SmolRuntime::block_on(super::serf_reconnect_same_ip::()); + } + + #[test] + fn serf_join_cancel_smol() { + SmolRuntime::block_on(super::serf_join_cancel::()); + } + #[cfg(encryption)] #[test] fn two_node_join_converges_encrypted_smol() { From 87887e6bf6661bf46a2871b6aae8ddc50d6ba25b Mon Sep 17 00:00:00 2001 From: Al Liu Date: Sun, 12 Jul 2026 23:15:58 +1200 Subject: [PATCH 122/128] =?UTF-8?q?test(serf-reactor):=20legacy=20scenario?= =?UTF-8?q?=20ports=20=E2=80=94=20tag=20propagation,=20restart-update,=20q?= =?UTF-8?q?uery=20filter=20(#87)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- serf-reactor/tests/cluster/mod.rs | 86 +++++- serf-reactor/tests/tcp.rs | 464 ++++++++++++++++++++++++++++++ 2 files changed, 542 insertions(+), 8 deletions(-) diff --git a/serf-reactor/tests/cluster/mod.rs b/serf-reactor/tests/cluster/mod.rs index e7aa0558..167788db 100644 --- a/serf-reactor/tests/cluster/mod.rs +++ b/serf-reactor/tests/cluster/mod.rs @@ -105,19 +105,37 @@ impl ClusterTiming { } } - /// Override the reconnect timeout — the age at which the reaper removes a Failed - /// member. Raise it well beyond the test's kill-to-restart window to hold a - /// failed peer for reconnection instead of reaping it. + /// Override the probe interval — the failure-detection cadence. Raise it + /// well beyond the test window to PARK failure detection entirely, so a + /// killed-and-restarted peer is never declared Failed in between and the + /// observer holds it Alive across the whole cycle. #[must_use] + pub fn with_probe_interval(mut self, v: Duration) -> Self { + self.probe_interval = v; + self + } + + /// Override the reconnect re-dial cadence — how often a survivor attempts to + /// re-establish contact with a Failed member. Raise it beyond the test window + /// to park the re-dial (and the push/pull merge it runs) out of the scenario. + #[must_use] + pub fn with_reconnect_interval(mut self, v: Duration) -> Self { + self.reconnect_interval = v; + self + } + + /// Override the failed-member retention window — the age at which the reaper + /// removes a Failed member. Raise it well beyond the test's kill-to-restart + /// window to hold a failed peer for reconnection instead of reaping it. pub fn with_reconnect_timeout(mut self, v: Duration) -> Self { self.reconnect_timeout = v; self } - /// Override the tombstone timeout — the age at which the reaper removes a - /// gracefully-Left member. Raise it beyond the test window to HOLD a left peer - /// in the tombstone view instead of reaping it, so a graceful-leave assertion - /// observes `[Join, Leave]` without a trailing `Reap`. + /// Override the dead-node reclaim age — how long a dead member's identity + /// must age before a same-name claim at a NEW address is admitted. Set it + /// near zero to let a restarted node rebind at a fresh port without a name + /// conflict. #[must_use] pub fn with_dead_node_reclaim(mut self, v: Duration) -> Self { self.dead_node_reclaim = Some(v); @@ -130,7 +148,10 @@ impl ClusterTiming { self } - /// Override the tombstone retention window. + /// Override the tombstone timeout — the age at which the reaper removes a + /// gracefully-Left member. Raise it beyond the test window to HOLD a left + /// peer in the tombstone view instead of reaping it, so a graceful-leave + /// assertion observes `[Join, Leave]` without a trailing `Reap`. pub fn with_tombstone_timeout(mut self, v: Duration) -> Self { self.tombstone_timeout = v; self @@ -175,6 +196,8 @@ impl ClusterTiming { struct MemberRec { kind: MemberEventKind, ids: Vec, + /// Each member's tags at the time the event surfaced, aligned with `ids`. + tags: Vec, } /// One cluster node: its stable id and advertise address, the live handle (taken @@ -484,6 +507,51 @@ where .map(|rec| rec.kind) .collect() } + + /// Poll until `observer`'s log holds a `kind` event for `subject` whose + /// member payload carries `tag` = `want`, then return the subject's ordered + /// event kinds THROUGH that record (inclusive). Fencing on the collector's + /// log — rather than on the membership view, which publishes independently + /// of the event stream — lets a caller assert the exact event prefix that + /// produced an observed state without racing still-in-flight events. + pub async fn await_member_event_with_tag( + &self, + observer: usize, + subject: &str, + kind: MemberEventKind, + tag: &str, + want: &str, + ) -> Vec { + let prefix_through_match = || -> Option> { + let log = self.slots[observer].log.lock().expect("event log lock"); + let mut prefix = Vec::new(); + for rec in log.iter() { + let Some(i) = rec.ids.iter().position(|id| id.as_str() == subject) else { + continue; + }; + prefix.push(rec.kind); + if rec.kind == kind && rec.tags[i].0.get(tag).map(SmolStr::as_str) == Some(want) { + return Some(prefix); + } + } + None + }; + R::timeout(POLL_TIMEOUT, async { + loop { + if let Some(prefix) = prefix_through_match() { + break prefix; + } + R::sleep(POLL_STEP).await; + } + }) + .await + .unwrap_or_else(|_| { + panic!( + "node {observer} never records {kind:?} for {subject:?} carrying {tag}={want:?} (saw {:?})", + self.member_event_kinds(observer, subject) + ) + }) + } } /// An ephemeral loopback bind address (`127.0.0.1:0`). @@ -534,9 +602,11 @@ where .iter() .map(|m| m.node().id_ref().clone()) .collect(); + let tags = me.members().iter().map(|m| m.tags().clone()).collect(); log.lock().expect("event log lock").push(MemberRec { kind: me.kind(), ids, + tags, }); } } diff --git a/serf-reactor/tests/tcp.rs b/serf-reactor/tests/tcp.rs index 16b0b3e9..e32cf9fa 100644 --- a/serf-reactor/tests/tcp.rs +++ b/serf-reactor/tests/tcp.rs @@ -1396,6 +1396,428 @@ where b.shutdown().await.expect("cancel-b shuts down"); } +/// Port of legacy `serf_set_tags` folding `serf_role` (Go `TestSerf_SetTags` +/// / `TestSerf_Role`): a tag change on one node propagates to the peer in +/// BOTH directions — the observer records an Update member event for the +/// setter and its member view carries the new tag value. +async fn serf_set_tags_propagates() +where + R: Runtime, +{ + let mut cluster = + cluster::Cluster::::spawn(&["tags-a", "tags-b"], cluster::ClusterTiming::fast()).await; + let a_id = cluster.id(0); + let b_id = cluster.id(1); + + let mut tags_b = serf_proto::Tags::new(); + tags_b + .0 + .insert(SmolStr::new("role"), SmolStr::new("worker")); + cluster + .node(1) + .set_tags(tags_b) + .await + .expect("B re-tags itself"); + cluster + .await_member_event(0, b_id.as_str(), MemberEventKind::Update) + .await; + let seen = cluster + .node(0) + .members() + .iter() + .find(|m| m.node().id_ref().as_str() == b_id.as_str()) + .map(|m| m.tags().0.get("role").cloned()) + .expect("A tracks B"); + assert_eq!( + seen.as_deref(), + Some("worker"), + "A's view of B carries the propagated role tag" + ); + + // And the reverse direction. + let mut tags_a = serf_proto::Tags::new(); + tags_a.0.insert(SmolStr::new("role"), SmolStr::new("lead")); + cluster + .node(0) + .set_tags(tags_a) + .await + .expect("A re-tags itself"); + cluster + .await_member_event(1, a_id.as_str(), MemberEventKind::Update) + .await; + let seen = cluster + .node(1) + .members() + .iter() + .find(|m| m.node().id_ref().as_str() == a_id.as_str()) + .map(|m| m.tags().0.get("role").cloned()) + .expect("B tracks A"); + assert_eq!( + seen.as_deref(), + Some("lead"), + "B's view of A carries the propagated role tag" + ); + + cluster.shutdown_all().await; +} + +/// A rejoin-cycle regression guard: after a fail/restart/rejoin cycle, a tag +/// change still propagates as an Update with the refreshed value visible — +/// pinning the clock-sync and event-fencing subtleties of the revival path. +/// +/// This is the retag-AFTER-revival half of the legacy update scenario: a +/// revival folds merged tags into the Join event itself (matching the +/// reference `handleNodeJoin`), so the retag must land after the revival +/// fence to surface a distinct Update. The restart-with-changed-tags half — +/// where the Update arrives from the rejoin exchange while the observer +/// still holds the node Alive — is +/// `serf_update_after_restart_with_changed_tags`. +async fn serf_update_after_rejoin() +where + R: Runtime, +{ + let mut cluster = cluster::Cluster::::spawn( + &["upd-a", "upd-b"], + // The explicit rejoin below is the single revival path: the survivor's + // own reconnect re-dial is parked out of the window so its push/pull + // merge cannot race the retag (a revival merge folds the tags into its + // Join rather than a distinct Update), and the failed member is retained + // throughout. + cluster::ClusterTiming::fast() + .with_reconnect_interval(Duration::from_secs(600)) + .with_reconnect_timeout(Duration::from_secs(600)), + ) + .await; + let subject = cluster.id(1); + + cluster.kill_abrupt(1).await; + cluster + .await_member_event(0, subject.as_str(), MemberEventKind::Failed) + .await; + cluster.restart(1).await; + // The restarted node's serf clock begins fresh; an explicit join runs the + // push/pull that witnesses the survivor's clocks, so the tag update minted + // below stamps ABOVE the observer's recorded status time instead of + // arriving stale (the reference test also rejoins explicitly). + let a_addr = cluster.node(0).advertise_address(); + cluster + .node(1) + .join(&SocketAddrResolver, MaybeResolved::Resolved(a_addr), false) + .await + .expect("the restarted node rejoins through the survivor"); + // Fence the REVIVAL on the observer's event sequence — the member count is + // vacuous here (the failed tombstone still counts toward it), and a retag + // racing ahead of the join would ride the join itself, leaving no separate + // Update to observe. + cluster + .assert_member_events( + 0, + subject.as_str(), + &[ + MemberEventKind::Join, + MemberEventKind::Failed, + MemberEventKind::Join, + ], + ) + .await; + + let mut tags = serf_proto::Tags::new(); + tags.0.insert(SmolStr::new("version"), SmolStr::new("v2")); + cluster + .node(1) + .set_tags(tags) + .await + .expect("the rejoined node re-tags itself"); + cluster + .await_member_event(0, subject.as_str(), MemberEventKind::Update) + .await; + let seen = cluster + .node(0) + .members() + .iter() + .find(|m| m.node().id_ref().as_str() == subject.as_str()) + .map(|m| m.tags().0.get("version").cloned()) + .expect("A tracks the rejoined B"); + assert_eq!( + seen.as_deref(), + Some("v2"), + "the failure/rejoin cycle ends with the refreshed tag visible" + ); + + cluster.shutdown_all().await; +} + +/// Poll `observer`'s member view until `subject`'s `version` tag equals +/// `want` — the propagation fence for a retag. +async fn await_version_tag( + cluster: &cluster::Cluster, + observer: usize, + subject: &str, + want: &str, +) where + R: Runtime, +{ + R::timeout(Duration::from_secs(20), async { + loop { + let seen = cluster + .node(observer) + .members() + .iter() + .find(|m| m.node().id_ref().as_str() == subject) + .and_then(|m| m.tags().0.get("version").cloned()); + if seen.as_deref() == Some(want) { + break; + } + R::sleep(Duration::from_millis(20)).await; + } + }) + .await + .unwrap_or_else(|_| panic!("the observer never sees {subject}'s version tag reach {want:?}")); +} + +/// Port of legacy `serf_update` (Go `TestSerf_Update`): a node restarts with +/// CHANGED tags and rejoins while the observer still holds it Alive, and the +/// observer surfaces the change as a member Update — never a Failed/revival +/// detour (whose Join would fold the tags in without a distinct Update). +/// +/// The mechanism underneath is the restart self-refutation: a stale claim +/// about the restarted node (its pre-restart incarnation, the old tags) +/// reaches it, and it refutes by re-broadcasting a bumped Alive carrying its +/// CURRENT tags, which the observer applies as an alive-to-alive metadata +/// change — `NodeUpdated` → `Member(Update)`. +/// +/// Staging makes the refutation's incarnation BUMP load-bearing end-to-end. +/// A retag bumps the local incarnation, so a pre-kill retag raises the +/// observer's held incarnation to exactly the value the restarted node +/// reaches after its own post-restart retag (fresh incarnation + one bump). +/// Every Alive the restarted node can originate on its own is therefore +/// equal-incarnation at the observer and stale-dropped; with failure +/// detection parked (the observer holds the subject Alive across the whole +/// cycle, the reference's restart-beats-detection timing) and periodic +/// push/pull disabled, NO schedule delivers the new tags unless a +/// refutation first lifts the restarted node past the observer's held +/// incarnation. +/// +/// WHICH message then carries the new tags is timing-dependent at driver +/// level: the refutation may be triggered by the rejoin exchange itself +/// (its bumped Alive carries the tags directly) or slightly earlier by a +/// leftover gossip retransmit of the stale claim (an empty-meta refutation +/// lifts the incarnation and the retag's own broadcast is then admitted +/// above it, surfacing one extra Update). Both routes are the same +/// mechanism and converge on the same view; the log assertion therefore +/// pins the event-kind shape rather than an exact count, and the +/// carrier-level causality — an equal-incarnation different-meta self-claim +/// refutes with a broadcast carrying the CURRENT metadata — is pinned +/// deterministically at the machine layer +/// (`alive_node_refute_equal_incarnation_carries_current_meta` in +/// memberlist-proto's SWIM parity suite), where no scheduler is involved. +async fn serf_update_after_restart_with_changed_tags() +where + R: Runtime, +{ + let mut cluster = cluster::Cluster::::spawn( + &["updm-a", "updm-b"], + cluster::ClusterTiming::fast() + .with_probe_interval(Duration::from_secs(600)) + .with_push_pull_interval(Duration::ZERO), + ) + .await; + let subject = cluster.id(1); + + cluster + .assert_member_events(0, subject.as_str(), &[MemberEventKind::Join]) + .await; + + // The incarnation-equalizing retag: after this propagates, the observer + // holds the subject at fresh-incarnation-plus-one — the same value the + // subject reaches below after restarting and retagging once. + let mut tags = serf_proto::Tags::new(); + tags.0.insert(SmolStr::new("version"), SmolStr::new("v1")); + cluster + .node(1) + .set_tags(tags) + .await + .expect("the subject tags itself before the restart"); + await_version_tag(&cluster, 0, subject.as_str(), "v1").await; + + cluster.kill_abrupt(1).await; + cluster.restart(1).await; + + // Present the changed tags BEFORE rejoining, mirroring the reference's + // restart-with-new-tags configuration. This lands the restarted node at + // the observer's held incarnation, so only the refutation below can carry + // the new value. + let mut tags = serf_proto::Tags::new(); + tags.0.insert(SmolStr::new("version"), SmolStr::new("v2")); + cluster + .node(1) + .set_tags(tags) + .await + .expect("the restarted node presents changed tags"); + + let a_addr = cluster.node(0).advertise_address(); + cluster + .node(1) + .join(&SocketAddrResolver, MaybeResolved::Resolved(a_addr), false) + .await + .expect("the restarted node rejoins through the survivor"); + + // The completion fence on the VIEW: a refutation-lifted Alive delivers v2. + await_version_tag(&cluster, 0, subject.as_str(), "v2").await; + + // Fence the COLLECTOR too — the view publishes independently of the event + // stream — by waiting for the logged Update that carries v2, then assert + // the exact prefix through that event: one admission followed by nothing + // but Updates. No Failed (probing parked), no second Join (the observer + // never saw the subject leave), both retags surfaced, and an early + // empty-meta refutation may add one benign extra Update (see the doc), so + // the shape is pinned rather than an exact count. + let prefix = cluster + .await_member_event_with_tag( + 0, + subject.as_str(), + MemberEventKind::Update, + "version", + "v2", + ) + .await; + assert!( + prefix.len() >= 3 + && prefix[0] == MemberEventKind::Join + && prefix[1..].iter().all(|k| *k == MemberEventKind::Update), + "the restart cycle through the v2 Update must surface as one Join then only Updates (got {prefix:?})" + ); + + cluster.shutdown_all().await; +} + +/// Port of legacy `serf_query_filter` (Go `TestSerf_Query_Filter`): an +/// Id-filtered query surfaces on the FILTERED node only, and the originator +/// collects exactly that node's response. Standalone nodes rather than the +/// cluster fixture: the fixture's collector round-robins the event stream +/// away from scenario subscribers. +/// +/// `relay_factor = 1` matches the legacy parameters, but a relayed duplicate +/// is not FORCED to reach the originator here (the responder's relay pick may +/// select the originator itself, whose self-relay is dropped, and relay +/// forwarding is best-effort), so duplicate suppression is NOT this +/// scenario's claim — it is pinned deterministically at the machine layer by +/// serf-proto's `duplicate_query_response_is_deduped`. +async fn serf_query_filter() +where + R: Runtime, +{ + let a = spawn_node::("qf-a").await; + let b = spawn_node::("qf-b").await; + let c = spawn_node::("qf-c").await; + let a_addr = a.advertise_address(); + + // Subscribe before any join so no event races the subscriptions. + let mut a_events = a.events(); + let mut b_events = b.events(); + let mut c_events = c.events(); + + b.join(&SocketAddrResolver, MaybeResolved::Resolved(a_addr), false) + .await + .expect("B joins through A"); + c.join(&SocketAddrResolver, MaybeResolved::Resolved(a_addr), false) + .await + .expect("C joins through A"); + R::timeout(Duration::from_secs(20), async { + loop { + if a.num_members() == 3 && b.num_members() == 3 && c.num_members() == 3 { + break; + } + R::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("the three nodes converge"); + + // An explicit timeout makes the query LIFETIME known, so the exclusivity + // drains below can cover it whole (the default would be computed from the + // gossip cadence and member count). + let query_lifetime = Duration::from_secs(3); + let mut params = a.default_query_param(); + params.filters = vec![serf_proto::typed::Filter::Id(vec![SmolStr::new("qf-b")])]; + params.relay_factor = 1; + params.timeout = query_lifetime; + a.query("who", Bytes::from_static(b"filtered"), params) + .await + .expect("the filtered query dispatches"); + + // B — the filtered target — surfaces the query and answers it. + let responded = R::timeout(Duration::from_secs(20), async { + loop { + match b_events.next().await { + Some(Event::Query(qe)) if qe.name() == "who" => { + b.respond(qe, Bytes::from_static(b"b-here")) + .await + .expect("B responds to the filtered query"); + break true; + } + Some(_) => {} + None => break false, + } + } + }) + .await + .expect("B surfaces the filtered query within the timeout"); + assert!(responded, "the filtered target answers"); + + // Drain the originator across the WHOLE query lifetime plus a margin: + // exactly one matching response — the single responder inside the filter — + // and no surfaced `Event::Query`, the originator being outside its own + // Id filter too. + let drain_window = query_lifetime + Duration::from_secs(1); + let mut responses: Vec<(SmolStr, Bytes)> = Vec::new(); + // Ignoring Err: the timeout IS the drain bound; events collected until it + // elapses are what the assertions below examine. + let _ = R::timeout(drain_window, async { + loop { + match a_events.next().await { + Some(Event::Query(qe)) if qe.name() == "who" => { + panic!("the originator is outside the Id filter and must not surface the query"); + } + Some(Event::QueryResponse(qr)) => { + responses.push((qr.from().id_ref().clone(), qr.payload().clone())); + } + Some(_) => {} + None => break, + } + } + }) + .await; + assert_eq!( + responses.len(), + 1, + "exactly one responder sits inside the Id filter (got {responses:?})" + ); + assert_eq!(responses[0].0.as_str(), "qf-b"); + assert_eq!(responses[0].1, Bytes::from_static(b"b-here")); + + // C — filtered out — must stay silent across the same whole lifetime. + let saw_query = R::timeout(drain_window, async { + loop { + match c_events.next().await { + Some(Event::Query(qe)) if qe.name() == "who" => break true, + Some(_) => {} + None => break false, + } + } + }) + .await + .unwrap_or(false); + assert!( + !saw_query, + "a node outside the Id filter must not surface the query" + ); + + a.shutdown().await.expect("qf-a shuts down"); + b.shutdown().await.expect("qf-b shuts down"); + c.shutdown().await.expect("qf-c shuts down"); +} + /// A deterministic test secret key, selecting whichever AEAD cipher this build /// compiled so the encrypted tests work under either backend. #[cfg(encryption)] @@ -1674,6 +2096,26 @@ mod tokio_cells { super::serf_join_cancel::().await; } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_set_tags_propagates() { + super::serf_set_tags_propagates::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_update_after_rejoin() { + super::serf_update_after_rejoin::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_update_after_restart_with_changed_tags() { + super::serf_update_after_restart_with_changed_tags::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_query_filter() { + super::serf_query_filter::().await; + } + #[cfg(encryption)] #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn two_node_join_converges_encrypted() { @@ -1821,6 +2263,28 @@ mod smol_cells { SmolRuntime::block_on(super::serf_join_cancel::()); } + #[test] + fn serf_set_tags_propagates_smol() { + SmolRuntime::block_on(super::serf_set_tags_propagates::()); + } + + #[test] + fn serf_update_after_rejoin_smol() { + SmolRuntime::block_on(super::serf_update_after_rejoin::()); + } + + #[test] + fn serf_update_after_restart_with_changed_tags_smol() { + SmolRuntime::block_on(super::serf_update_after_restart_with_changed_tags::< + SmolRuntime, + >()); + } + + #[test] + fn serf_query_filter_smol() { + SmolRuntime::block_on(super::serf_query_filter::()); + } + #[cfg(encryption)] #[test] fn two_node_join_converges_encrypted_smol() { From e984c7fc5011548847edb825b5b70bd1a0c69373 Mon Sep 17 00:00:00 2001 From: Al Liu Date: Mon, 13 Jul 2026 00:51:24 +1200 Subject: [PATCH 123/128] test(serf-reactor): port the join/leave membership-lifecycle scenarios (#89) --- serf-reactor/tests/cluster/mod.rs | 35 ++++++ serf-reactor/tests/tcp.rs | 180 +++++++++++++++++++++++++++++- 2 files changed, 214 insertions(+), 1 deletion(-) diff --git a/serf-reactor/tests/cluster/mod.rs b/serf-reactor/tests/cluster/mod.rs index 167788db..b663d34d 100644 --- a/serf-reactor/tests/cluster/mod.rs +++ b/serf-reactor/tests/cluster/mod.rs @@ -301,6 +301,21 @@ where elapsed } + /// Gracefully leave node `i` but keep its handle LIVE — unlike + /// [`leave_graceful`](Self::leave_graceful), no shutdown follows. Lets an + /// assertion observe the leaver's OWN post-leave convergence before teardown. + /// Read that convergence from the event log, not `members()`: the leaver + /// reaps its self Left tombstone, but the published snapshot then freezes + /// (`refresh_snapshot` will not publish a view missing the local id), so the + /// event stream — not the membership view — is the source of truth here. + pub async fn leave_in_place(&self, i: usize) { + self + .node(i) + .leave() + .await + .expect("node leaves gracefully in place"); + } + /// Gracefully leave node `i` with a shutdown racing the leave: both commands /// are issued concurrently, so they typically land in the same driver command /// batch and the teardown itself must egress the still-queued farewell before @@ -439,6 +454,26 @@ where .unwrap_or_else(|_| panic!("node {observer} never holds {subject:?} as a Left tombstone")); } + /// Poll until `observer`'s membership view holds `subject` with `status`, or + /// fail on the poll timeout. + pub async fn await_member_status(&self, observer: usize, subject: &str, status: MemberStatus) { + R::timeout(POLL_TIMEOUT, async { + loop { + if self + .node(observer) + .members() + .iter() + .any(|m| m.node().id_ref().as_str() == subject && m.status() == status) + { + break; + } + R::sleep(POLL_STEP).await; + } + }) + .await + .unwrap_or_else(|_| panic!("node {observer} never holds {subject:?} as {status:?}")); + } + /// Poll until `observer`'s log records a member event of `kind` naming /// `subject`. pub async fn await_member_event(&self, observer: usize, subject: &str, kind: MemberEventKind) { diff --git a/serf-reactor/tests/tcp.rs b/serf-reactor/tests/tcp.rs index e32cf9fa..872fb29e 100644 --- a/serf-reactor/tests/tcp.rs +++ b/serf-reactor/tests/tcp.rs @@ -25,7 +25,7 @@ use bytes::Bytes; use futures_util::{StreamExt, future}; use serf_proto::{ event::{Event, MemberEventKind}, - members::SerfState, + members::{MemberStatus, SerfState}, options::Options as SerfOptions, }; #[cfg(encryption)] @@ -1818,6 +1818,154 @@ where c.shutdown().await.expect("qf-c shuts down"); } +/// Port of legacy `serf_join_leave` (Go `TestSerf_JoinLeave`): after a peer +/// leaves gracefully, the departure settles on BOTH sides under the DEFAULT +/// tombstone timeout. The other leave e2es raise the tombstone timeout to HOLD +/// the tombstone and pin the event sequence; this one exercises the plain +/// default-tombstone reap that none of them cover, and — like the legacy body — +/// checks the leaver's own side, not just the observer's. +/// +/// The leaver is kept running rather than shut down so its convergence is +/// observable, but its side must be read from the EVENT LOG, not the membership +/// snapshot. The leaver reaps its own self Left tombstone (matching the legacy +/// leaver) — `handle_node_leave` adds self to `left_members` and `fire_reap` +/// removes it — but once self is gone from the machine, `refresh_snapshot` +/// refuses to publish a view missing the local id, so `members()` / +/// `num_members()` FREEZE at the pre-reap `[peer, self:Left]` state. The event +/// stream stays truthful: the leaver emits `Leave` then `Reap` for itself and +/// only a `Join` for the peer. (The frozen-snapshot-vs-event divergence for a +/// left-in-place node is tracked as a separate machine-side issue, al8n/serf#88.) +async fn serf_join_leave() +where + R: Runtime, +{ + let mut cluster = + cluster::Cluster::::spawn(&["jl-a", "jl-b"], cluster::ClusterTiming::fast()).await; + let peer = cluster.id(0); + let leaver = cluster.id(1); + + // Leave in place — keep B's handle live so its own convergence is observable. + cluster.leave_in_place(1).await; + + // A (observer) reaps the departed B under the fast profile's 1ms tombstone + + // 100ms reap ticks, returning to just itself (A's own snapshot stays live). + cluster.await_num_members(0, 1).await; + + // B (the leaver) processes its own departure end to end: its event log shows + // Join → Leave → Reap for itself — self IS reaped under the default tombstone. + cluster + .assert_member_events( + 1, + leaver.as_str(), + &[ + MemberEventKind::Join, + MemberEventKind::Leave, + MemberEventKind::Reap, + ], + ) + .await; + // And it never fails or reaps the still-live peer — only the Join. + cluster + .assert_member_events(1, peer.as_str(), &[MemberEventKind::Join]) + .await; + + cluster.shutdown_all().await; +} + +/// Port of legacy `serf_join_leave_join` (Go `TestSerf_JoinLeaveJoin`): a peer +/// leaves — the observer holds it as a Left tombstone — then the same node +/// restarts and rejoins, and the observer transitions it Left → Alive. The +/// tombstone timeout is raised so the Left state is observable before the +/// rejoin rather than reaped away first. +async fn serf_join_leave_join() +where + R: Runtime, +{ + let mut cluster = cluster::Cluster::::spawn( + &["jlj-a", "jlj-b"], + cluster::ClusterTiming::fast().with_tombstone_timeout(Duration::from_secs(30)), + ) + .await; + let subject = cluster.id(1); + let seed = cluster.node(0).advertise_address(); + + cluster.leave_graceful(1).await; + cluster.await_left_tombstone(0, subject.as_str()).await; + + cluster.restart(1).await; + cluster + .node(1) + .join(&SocketAddrResolver, MaybeResolved::Resolved(seed), false) + .await + .expect("the restarted node rejoins the seed"); + + // A transitions B from its Left tombstone back to Alive. + cluster + .await_member_status(0, subject.as_str(), MemberStatus::Alive) + .await; + + cluster.shutdown_all().await; +} + +/// Port of legacy `serf_leave_rejoin_different_role` (Go +/// `TestSerf_LeaveRejoin_DifferentRole`): a node leaves, then a fresh node +/// rejoins at the SAME id and address carrying a DIFFERENT role tag, and the +/// observer's view reflects the new role. The legacy test sets the role at +/// construction; the Sans-I/O stack has no start-time tag surface, so the +/// restarted node applies it via `set_tags` before the rejoin — the revival +/// folds the tag into its Join (matching the reference `handleNodeJoin`), and +/// the observer ends up seeing the new value. +async fn serf_leave_rejoin_different_role() +where + R: Runtime, +{ + let mut cluster = cluster::Cluster::::spawn( + &["lrr-a", "lrr-b"], + cluster::ClusterTiming::fast().with_tombstone_timeout(Duration::from_secs(30)), + ) + .await; + let subject = cluster.id(1); + let seed = cluster.node(0).advertise_address(); + + cluster.leave_graceful(1).await; + cluster.await_left_tombstone(0, subject.as_str()).await; + + cluster.restart(1).await; + let mut tags = serf_proto::Tags::new(); + tags.0.insert(SmolStr::new("role"), SmolStr::new("bar")); + cluster + .node(1) + .set_tags(tags) + .await + .expect("the restarted node adopts the new role before rejoining"); + cluster + .node(1) + .join(&SocketAddrResolver, MaybeResolved::Resolved(seed), false) + .await + .expect("the re-roled node rejoins the seed"); + + // A must see B back as Alive AND carrying the new role — the two ride the + // same revival Join, so poll them together to avoid reading the view between + // the status flip and the tag landing. + R::timeout(Duration::from_secs(20), async { + loop { + let seen = cluster.node(0).members().iter().any(|m| { + m.node().id_ref().as_str() == subject.as_str() + && m.status() == MemberStatus::Alive + && m.tags().0.get("role").map(SmolStr::as_str) == Some("bar") + }); + if seen { + break; + } + R::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("A never sees the rejoined B as Alive carrying role=bar"); + + cluster.shutdown_all().await; +} + /// A deterministic test secret key, selecting whichever AEAD cipher this build /// compiled so the encrypted tests work under either backend. #[cfg(encryption)] @@ -2116,6 +2264,21 @@ mod tokio_cells { super::serf_query_filter::().await; } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_join_leave() { + super::serf_join_leave::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_join_leave_join() { + super::serf_join_leave_join::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_leave_rejoin_different_role() { + super::serf_leave_rejoin_different_role::().await; + } + #[cfg(encryption)] #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn two_node_join_converges_encrypted() { @@ -2285,6 +2448,21 @@ mod smol_cells { SmolRuntime::block_on(super::serf_query_filter::()); } + #[test] + fn serf_join_leave_smol() { + SmolRuntime::block_on(super::serf_join_leave::()); + } + + #[test] + fn serf_join_leave_join_smol() { + SmolRuntime::block_on(super::serf_join_leave_join::()); + } + + #[test] + fn serf_leave_rejoin_different_role_smol() { + SmolRuntime::block_on(super::serf_leave_rejoin_different_role::()); + } + #[cfg(encryption)] #[test] fn two_node_join_converges_encrypted_smol() { From f02836cf26d6a27c3fb38cd68533ff12b445b2dd Mon Sep 17 00:00:00 2001 From: Al Liu Date: Mon, 13 Jul 2026 15:09:24 +1200 Subject: [PATCH 124/128] fix(serf-proto): local-node reap fix + the remaining legacy driver-e2e ports + a test-only MessageDropper (#91) --- .github/workflows/ci-smol.yml | 4 +- .github/workflows/ci-tokio.yml | 2 +- serf-proto/Cargo.toml | 5 + serf-proto/src/endpoint/mod.rs | 114 ++++- serf-proto/src/endpoint/tests.rs | 71 ++++ serf-proto/src/lib.rs | 3 + serf-proto/src/quic_endpoint/mod.rs | 6 +- serf-proto/src/stream_endpoint/mod.rs | 14 +- serf-reactor/Cargo.toml | 4 + serf-reactor/src/delegate/mod.rs | 9 + serf-reactor/src/delegate/void.rs | 25 ++ serf-reactor/src/tcp/mod.rs | 5 + serf-reactor/tests/cluster/mod.rs | 9 +- serf-reactor/tests/tcp.rs | 580 ++++++++++++++++++++++++-- 14 files changed, 786 insertions(+), 65 deletions(-) diff --git a/.github/workflows/ci-smol.yml b/.github/workflows/ci-smol.yml index 76637ef8..89eb8cb7 100644 --- a/.github/workflows/ci-smol.yml +++ b/.github/workflows/ci-smol.yml @@ -33,7 +33,7 @@ on: schedule: [cron: "0 1 */30 * *"] env: - FEATURES: "smol,tcp,tls-rustls-ring,quic-rustls-ring,aes-gcm,chacha20-poly1305,coordinates,tag-regex,tracing,serde,clap,dns,getifs" + FEATURES: "smol,tcp,tls-rustls-ring,quic-rustls-ring,aes-gcm,chacha20-poly1305,coordinates,tag-regex,tracing,serde,clap,dns,getifs,test" jobs: test: @@ -85,6 +85,6 @@ jobs: # `-- smol` selects only the `_smol` cells in each suite binary; the tokio # cells over the same runtime-generic scenarios run in ci-tokio. run: | - cargo test -p serf-reactor --features "smol,tcp,tls-rustls-ring,quic-rustls-ring,aes-gcm,chacha20-poly1305,coordinates,tag-regex,dns,getifs" --test tcp -- smol + cargo test -p serf-reactor --features "smol,tcp,tls-rustls-ring,quic-rustls-ring,aes-gcm,chacha20-poly1305,coordinates,tag-regex,dns,getifs,test" --test tcp -- smol cargo test -p serf-reactor --features "smol,tcp,tls-rustls-ring,quic-rustls-ring,aes-gcm,chacha20-poly1305,coordinates,tag-regex,dns,getifs" --test tls -- smol cargo test -p serf-reactor --features "smol,tcp,tls-rustls-ring,quic-rustls-ring,aes-gcm,chacha20-poly1305,coordinates,tag-regex,dns,getifs" --test quic -- smol diff --git a/.github/workflows/ci-tokio.yml b/.github/workflows/ci-tokio.yml index 6c7db465..4530d14c 100644 --- a/.github/workflows/ci-tokio.yml +++ b/.github/workflows/ci-tokio.yml @@ -30,7 +30,7 @@ on: schedule: [cron: "0 1 */30 * *"] env: - FEATURES: "tokio,tcp,tls-rustls-ring,quic-rustls-ring,aes-gcm,chacha20-poly1305,coordinates,tag-regex,tracing,serde,clap,dns,getifs" + FEATURES: "tokio,tcp,tls-rustls-ring,quic-rustls-ring,aes-gcm,chacha20-poly1305,coordinates,tag-regex,tracing,serde,clap,dns,getifs,test" jobs: test: diff --git a/serf-proto/Cargo.toml b/serf-proto/Cargo.toml index aa7e4083..69131c3f 100644 --- a/serf-proto/Cargo.toml +++ b/serf-proto/Cargo.toml @@ -26,6 +26,11 @@ std = [ ] alloc = ["memberlist-proto/alloc", "rand/alloc"] +# Test-only fault-injection surface (a `MessageDropper` that drops inbound +# membership messages before the machine processes them). Compiled out of every +# non-test build — it has no production use. +test = ["alloc"] + # Encryption key management messages (KeyRequestMessage / KeyResponseMessage). # Pulls in the memberlist-proto encryption backend (at least one AEAD cipher # must be selected; both are forwarded here so the caller can pick). diff --git a/serf-proto/src/endpoint/mod.rs b/serf-proto/src/endpoint/mod.rs index a0967239..d82195e3 100644 --- a/serf-proto/src/endpoint/mod.rs +++ b/serf-proto/src/endpoint/mod.rs @@ -680,6 +680,28 @@ fn next_ltime(clock: &mut u64) -> u64 { /// The serf-logic core of the Sans-I/O super-machine. /// +/// The inbound membership-message kinds a [`MessageDropper`] can drop. +#[cfg(any(test, feature = "test"))] +#[cfg_attr(docsrs, doc(cfg(feature = "test")))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DropKind { + /// An inbound join — a memberlist `NodeJoined` inner event. + Join, + /// An inbound push/pull merge — a memberlist `RemoteStateReceived` inner event. + PushPull, +} + +/// A test-only fault-injection hook that drops selected inbound membership +/// messages before the machine processes them, so a test can force an +/// out-of-order delivery. Gated behind the `test` feature — it has NO production +/// use (silently dropping membership messages breaks convergence). +#[cfg(any(test, feature = "test"))] +#[cfg_attr(docsrs, doc(cfg(feature = "test")))] +pub trait MessageDropper: Send + Sync { + /// Returns `true` if an inbound message of `kind` should be dropped. + fn should_drop(&self, kind: DropKind) -> bool; +} + /// Holds all serf state — the three Lamport clocks, membership store, options, /// event ring, query bookkeeping, deadlines, and (feature-gated) the coordinate /// client — and **no** transport reference. It reaches a memberlist reliable @@ -708,6 +730,18 @@ where { /// serf configuration knobs. opts: Options, + /// This endpoint's own node id, cached from the transport at construction. + /// + /// The machine is otherwise decoupled from its identity (the id lives on the + /// transport), but the reaper needs it: the local node must never be reaped + /// from its own view — a running node always knows itself, and reaping its + /// self `Left` tombstone would drop it from the published snapshot (which + /// requires the local member) and freeze the view. + local_id: I, + /// Test-only inbound message-drop hook (see [`MessageDropper`]); `None` in + /// every real build. + #[cfg(any(test, feature = "test"))] + message_dropper: Option>, /// Member (SWIM membership) Lamport clock — plain `u64`, no atomics. /// Single-threaded machine; no concurrent writers. clock: u64, @@ -910,11 +944,11 @@ where /// default `u64`). A driver that must observe the counts from a detached /// handle injects a shared backing via /// [`new_with_rng_in`](Self::new_with_rng_in) instead. - pub fn new_with_rng(opts: Options, rng: R) -> Self + pub fn new_with_rng(local_id: I, opts: Options, rng: R) -> Self where D: Default, { - Self::new_with_rng_in(opts, rng, D::default(), D::default()) + Self::new_with_rng_in(local_id, opts, rng, D::default(), D::default()) } /// Construct a serf `Endpoint` core injecting the two coalescer shed counters @@ -925,7 +959,7 @@ where /// clones here so the handle observes every coalescer shed WITHOUT the endpoint /// publishing a copy each pump iteration. The single-owner drivers use the /// `u64` default via [`new_with_rng`](Self::new_with_rng). - pub fn new_with_rng_in(opts: Options, rng: R, user_drop: D, member_drop: D) -> Self { + pub fn new_with_rng_in(local_id: I, opts: Options, rng: R, user_drop: D, member_drop: D) -> Self { // Arm the first reap/reconnect/queue-check deadlines relative to the ORIGIN instant. // The driver calls handle_timeout(now) and the deadlines fire when now >= deadline. let first_reap = Instant::ORIGIN + opts.reap_interval(); @@ -961,6 +995,9 @@ where Self { opts, + local_id, + #[cfg(any(test, feature = "test"))] + message_dropper: None, clock: 0, event_clock: 0, query_clock: 0, @@ -1009,11 +1046,19 @@ where /// Suitable for tests and environments where determinism or an explicit seed /// is acceptable. Production drivers should use `new_with_rng` and seed from /// a cryptographically-secure source. - pub fn new(opts: Options) -> Self + pub fn new(local_id: I, opts: Options) -> Self where D: Default, { - Self::new_with_rng(opts, R::seed_from_u64(0)) + Self::new_with_rng(local_id, opts, R::seed_from_u64(0)) + } + + /// Install a test-only [`MessageDropper`] that drops selected inbound + /// membership messages before they are processed. Test fault injection only. + #[cfg(any(test, feature = "test"))] + #[cfg_attr(docsrs, doc(cfg(feature = "test")))] + pub fn set_message_dropper(&mut self, dropper: std::sync::Arc) { + self.message_dropper = Some(dropper); } // ── read accessors ──────────────────────────────────────────────────────── @@ -1456,6 +1501,15 @@ where match ev { // ── membership ─────────────────────────────────────────────────────── IE::NodeJoined(node) => { + // Test-only: drop the inbound join before processing (fault injection). + #[cfg(any(test, feature = "test"))] + if self + .message_dropper + .as_ref() + .is_some_and(|d| d.should_drop(DropKind::Join)) + { + return; + } let now = self.drain_now; self.handle_node_join(&node, now); } @@ -1486,6 +1540,16 @@ where self.handle_user_packet(t, from, data, now); } IE::RemoteStateReceived(r) => { + // Test-only: drop the inbound push/pull before processing (fault + // injection). + #[cfg(any(test, feature = "test"))] + if self + .message_dropper + .as_ref() + .is_some_and(|d| d.should_drop(DropKind::PushPull)) + { + return; + } // Correlate the merge to the exchange that produced it by its // `originating_stream_id` — for an outbound join this is exactly the // `StreamId` `start_push_pull` returned and that the driver recorded. @@ -2466,17 +2530,23 @@ where let mut i = 0; while i < self.members.left_members.len() { let id = self.members.left_members[i].clone(); - let expired = match self.members.states.get(&id) { - Some(ms) => { - let timeout = match &self.reconnect_delegate { - Some(d) => d.reconnect_timeout(ms.member(), tombstone_timeout), - None => tombstone_timeout, - }; - ms.leave_time() - .is_some_and(|lt| now.duration_since(lt) > timeout) - } - None => false, - }; + // Never reap the local node from its own view. A node that leaves in + // place (leaves but keeps running) tombstones itself as `Left`; reaping + // that self tombstone would drop the local member, which the published + // snapshot requires, and freeze the membership view. Hold it instead — + // a running node always knows itself. + let expired = id != self.local_id + && match self.members.states.get(&id) { + Some(ms) => { + let timeout = match &self.reconnect_delegate { + Some(d) => d.reconnect_timeout(ms.member(), tombstone_timeout), + None => tombstone_timeout, + }; + ms.leave_time() + .is_some_and(|lt| now.duration_since(lt) > timeout) + } + None => false, + }; if expired { self.members.left_members.swap_remove(i); if let Some(ms) = self.members.states.remove(&id) { @@ -3068,6 +3138,18 @@ where } } AnyMessage::Join(join) => { + // Test-only: drop the inbound join intent before it witnesses the + // member clock / advances status_time (fault injection). This is the + // gossiped-intent companion to the `IE::NodeJoined` gate — the legacy + // `DropJoins` drops both. + #[cfg(any(test, feature = "test"))] + if self + .message_dropper + .as_ref() + .is_some_and(|d| d.should_drop(DropKind::Join)) + { + return; + } // handle_node_join_intent takes ltime + id reference. let rebroadcast = self.handle_node_join_intent(join.ltime, &join.id.clone(), now); if rebroadcast { diff --git a/serf-proto/src/endpoint/tests.rs b/serf-proto/src/endpoint/tests.rs index c5cf43c7..c30bb212 100644 --- a/serf-proto/src/endpoint/tests.rs +++ b/serf-proto/src/endpoint/tests.rs @@ -876,6 +876,32 @@ fn reap_left_removes_after_tombstone_timeout() { assert!(reaped, "a Member(Reap) event should have been emitted"); } +/// The local node is never reaped from its own view. A node that leaves in +/// place tombstones itself as `Left`; reaping that self tombstone would drop +/// the local member from the published snapshot (which requires it) and freeze +/// the view. Seed both the local id (`1`, the `ep()` local) and a peer (`2`) as +/// `Left`, then reap well past the tombstone timeout: the peer is reaped, the +/// local node is held. +#[test] +fn reap_left_never_reaps_the_local_node() { + let mut e = ep(); + let t0 = memberlist_proto::Instant::ORIGIN; + e.test_seed_left_member_by_status(1, LamportTime::new(3), t0); + e.test_seed_left_member_by_status(2, LamportTime::new(3), t0); + let past_timeout = t0 + core::time::Duration::from_secs(3600 * 25); + e.test_fire_reap(past_timeout); + assert_eq!( + e.test_member_status(2), + None, + "the peer's Left tombstone is reaped past the tombstone timeout" + ); + assert_eq!( + e.test_member_status(1), + Some(MemberStatus::Left), + "the local node is never reaped from its own view" + ); +} + #[test] fn reap_intents_removes_stale_intents() { let mut e = ep(); @@ -1247,6 +1273,51 @@ fn user_event_arrives_over_user_packet_and_surfaces() { ); } +/// An installed [`MessageDropper`](crate::MessageDropper) dropping +/// [`DropKind::Join`](crate::DropKind) drops an inbound gossiped join INTENT +/// before it witnesses the clock — the intent path (`AnyMessage::Join` in +/// `handle_user_packet`), distinct from the `IE::NodeJoined` membership gate. A +/// dropped intent must not advance the member's `status_time`; the same intent +/// without the dropper advances it. +#[test] +fn message_dropper_drops_inbound_join_intent() { + struct DropJoins; + impl crate::MessageDropper for DropJoins { + fn should_drop(&self, kind: crate::DropKind) -> bool { + matches!(kind, crate::DropKind::Join) + } + } + + let join_bytes = AnyMessage::::Join(crate::typed::JoinMessage::new( + 8u64.into(), + 2u32, + )) + .encode() + .unwrap(); + let from: core::net::SocketAddr = "127.0.0.1:1002".parse().unwrap(); + + // With the dropper the join intent is dropped: status_time stays at 5. + let mut dropped = ep(); + dropped.test_seed_member(2, MemberStatus::Alive, LamportTime::new(5)); + dropped.set_message_dropper(std::sync::Arc::new(DropJoins)); + dropped.test_inject_user_packet(from, join_bytes.clone(), memberlist_proto::Instant::ORIGIN); + assert_eq!( + dropped.test_member_status_time(2), + Some(LamportTime::new(5)), + "a dropped join intent must not advance status_time" + ); + + // Without the dropper the same intent advances status_time to 8. + let mut kept = ep(); + kept.test_seed_member(2, MemberStatus::Alive, LamportTime::new(5)); + kept.test_inject_user_packet(from, join_bytes, memberlist_proto::Instant::ORIGIN); + assert_eq!( + kept.test_member_status_time(2), + Some(LamportTime::new(8)), + "an un-dropped join intent advances status_time" + ); +} + #[test] fn duplicate_user_event_over_user_packet_is_deduped() { // Same UserEvent injected twice → second one is silently dropped. diff --git a/serf-proto/src/lib.rs b/serf-proto/src/lib.rs index b7c4a71a..b89720ff 100644 --- a/serf-proto/src/lib.rs +++ b/serf-proto/src/lib.rs @@ -150,6 +150,9 @@ pub(crate) mod coalesce; pub use coalesce::DropCounter; #[cfg(any(feature = "tcp", feature = "quic"))] pub(crate) mod reconnect_delegate; +#[cfg(all(any(feature = "tcp", feature = "quic"), any(test, feature = "test")))] +#[cfg_attr(docsrs, doc(cfg(feature = "test")))] +pub use endpoint::{DropKind, MessageDropper}; #[cfg(any(feature = "tcp", feature = "quic"))] #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] pub use reconnect_delegate::ReconnectDelegate; diff --git a/serf-proto/src/quic_endpoint/mod.rs b/serf-proto/src/quic_endpoint/mod.rs index ec611490..cd8fa9af 100644 --- a/serf-proto/src/quic_endpoint/mod.rs +++ b/serf-proto/src/quic_endpoint/mod.rs @@ -104,8 +104,9 @@ where where D: Default, { + let local_id = transport.endpoint_ref().local_id_ref().clone(); Self { - core: Endpoint::new_with_rng(opts, rng), + core: Endpoint::new_with_rng(local_id, opts, rng), transport, } } @@ -122,8 +123,9 @@ where user_drop: D, member_drop: D, ) -> Self { + let local_id = transport.endpoint_ref().local_id_ref().clone(); Self { - core: Endpoint::new_with_rng_in(opts, rng, user_drop, member_drop), + core: Endpoint::new_with_rng_in(local_id, opts, rng, user_drop, member_drop), transport, } } diff --git a/serf-proto/src/stream_endpoint/mod.rs b/serf-proto/src/stream_endpoint/mod.rs index 9a3720fc..ab9993f4 100644 --- a/serf-proto/src/stream_endpoint/mod.rs +++ b/serf-proto/src/stream_endpoint/mod.rs @@ -103,8 +103,9 @@ where where D: Default, { + let local_id = transport.endpoint_ref().local_id_ref().clone(); Self { - core: Endpoint::new_with_rng(opts, rng), + core: Endpoint::new_with_rng(local_id, opts, rng), transport, } } @@ -121,8 +122,9 @@ where user_drop: D, member_drop: D, ) -> Self { + let local_id = transport.endpoint_ref().local_id_ref().clone(); Self { - core: Endpoint::new_with_rng_in(opts, rng, user_drop, member_drop), + core: Endpoint::new_with_rng_in(local_id, opts, rng, user_drop, member_drop), transport, } } @@ -353,6 +355,14 @@ where self.transport.set_merge_delegate(delegate); } + /// Install a test-only [`MessageDropper`](crate::MessageDropper) on the inner + /// serf machine. Test fault injection only — see [`Endpoint::set_message_dropper`]. + #[cfg(any(test, feature = "test"))] + #[cfg_attr(docsrs, doc(cfg(feature = "test")))] + pub fn set_message_dropper(&mut self, dropper: std::sync::Arc) { + self.core.set_message_dropper(dropper); + } + /// Forwards to [`Endpoint::coordinate_resets`]. #[cfg(feature = "coordinates")] #[cfg_attr(docsrs, doc(cfg(feature = "coordinates")))] diff --git a/serf-reactor/Cargo.toml b/serf-reactor/Cargo.toml index 70046d0e..cd8666f2 100644 --- a/serf-reactor/Cargo.toml +++ b/serf-reactor/Cargo.toml @@ -36,6 +36,10 @@ chacha20-poly1305 = [ ] # Regex-backed tag-filter matching. tag-regex = ["serf-proto/tag-regex", "serf-driver/tag-regex"] +# Test-only fault-injection surface (forwards serf-proto's `test` feature: a +# `MessageDropper` installed via `VoidDelegate::with_message_dropper`). Compiled +# out of every non-test build. +test = ["serf-proto/test"] # Emit `tracing` spans around the public driver operations. tracing = ["dep:tracing", "serf-driver/tracing"] # Runtime conveniences: pull in the concrete agnostic runtime impls. diff --git a/serf-reactor/src/delegate/mod.rs b/serf-reactor/src/delegate/mod.rs index 5e513850..14d3ef8a 100644 --- a/serf-reactor/src/delegate/mod.rs +++ b/serf-reactor/src/delegate/mod.rs @@ -170,6 +170,15 @@ pub trait Delegate: type Id; /// Address type — always `SocketAddr` in the reactor driver. type Address; + + /// Test-only inbound message-drop hook. The driver installs the returned + /// [`MessageDropper`](serf_proto::MessageDropper) on the machine so a test can + /// drop selected inbound membership messages; `None` (the default) drops + /// nothing. Gated behind the `test` feature — no production use. + #[cfg(feature = "test")] + fn message_dropper(&self) -> Option> { + None + } } /// Observer the driver notifies after it rotates the LIVE wire keyring, so an diff --git a/serf-reactor/src/delegate/void.rs b/serf-reactor/src/delegate/void.rs index 39864b1e..54862ad8 100644 --- a/serf-reactor/src/delegate/void.rs +++ b/serf-reactor/src/delegate/void.rs @@ -21,6 +21,10 @@ use super::KeyringDelegate; #[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] pub struct VoidDelegate { _phantom: PhantomData, + /// Test-only inbound message-drop hook returned via + /// [`Delegate::message_dropper`]; `None` in every real build. + #[cfg(feature = "test")] + message_dropper: Option>, } #[cfg(any(feature = "tcp", feature = "quic"))] @@ -30,8 +34,24 @@ impl VoidDelegate { pub const fn new() -> Self { Self { _phantom: PhantomData, + #[cfg(feature = "test")] + message_dropper: None, } } + + /// Attach a test-only [`MessageDropper`](serf_proto::MessageDropper) surfaced + /// through [`Delegate::message_dropper`], so a test node drops selected + /// inbound membership messages. Test fault injection only. + #[cfg(feature = "test")] + #[cfg_attr(docsrs, doc(cfg(feature = "test")))] + #[must_use] + pub fn with_message_dropper( + mut self, + dropper: std::sync::Arc, + ) -> Self { + self.message_dropper = Some(dropper); + self + } } #[cfg(any(feature = "tcp", feature = "quic"))] @@ -77,6 +97,11 @@ where { type Id = I; type Address = A; + + #[cfg(feature = "test")] + fn message_dropper(&self) -> Option> { + self.message_dropper.clone() + } } /// A keyring delegate that persists nothing. diff --git a/serf-reactor/src/tcp/mod.rs b/serf-reactor/src/tcp/mod.rs index b846e194..0040e125 100644 --- a/serf-reactor/src/tcp/mod.rs +++ b/serf-reactor/src/tcp/mod.rs @@ -572,6 +572,11 @@ where if let Some(md) = runtime.merge_delegate { endpoint.set_merge_delegate(md); } + // Test-only: install the delegate's inbound message-dropper on the machine. + #[cfg(feature = "test")] + if let Some(dropper) = runtime.delegate.message_dropper() { + endpoint.set_message_dropper(dropper); + } let snapshotter = match runtime.snapshot { Some((writer, records)) => { let replay = serf_proto::snapshot::ReplayResult::replay(records, rejoin_after_leave); diff --git a/serf-reactor/tests/cluster/mod.rs b/serf-reactor/tests/cluster/mod.rs index b663d34d..9461e6c9 100644 --- a/serf-reactor/tests/cluster/mod.rs +++ b/serf-reactor/tests/cluster/mod.rs @@ -303,11 +303,10 @@ where /// Gracefully leave node `i` but keep its handle LIVE — unlike /// [`leave_graceful`](Self::leave_graceful), no shutdown follows. Lets an - /// assertion observe the leaver's OWN post-leave convergence before teardown. - /// Read that convergence from the event log, not `members()`: the leaver - /// reaps its self Left tombstone, but the published snapshot then freezes - /// (`refresh_snapshot` will not publish a view missing the local id), so the - /// event stream — not the membership view — is the source of truth here. + /// assertion observe the leaver's OWN post-leave convergence before teardown: + /// the leaver holds its self `Left` tombstone (the local node is never reaped + /// from its own view), so both its live membership view and its event log + /// remain readable and truthful. pub async fn leave_in_place(&self, i: usize) { self .node(i) diff --git a/serf-reactor/tests/tcp.rs b/serf-reactor/tests/tcp.rs index 872fb29e..801e1a70 100644 --- a/serf-reactor/tests/tcp.rs +++ b/serf-reactor/tests/tcp.rs @@ -650,13 +650,13 @@ where /// so failure detection inside the scenario window stays sub-second). async fn spawn_node_with_snapshot( id: &str, + bind: SocketAddr, snapshot: serf_reactor::SnapshotOptions, rejoin_after_leave: bool, -) -> Node +) -> Result, serf_reactor::SerfError> where R: Runtime, { - let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); Serf::::tcp( TcpTransportOptions::::new() .with_local_id(SmolStr::new(id)) @@ -673,7 +673,11 @@ where std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), ) .await - .expect("spawn snapshot-backed serf tcp node") +} + +/// An ephemeral loopback bind (`127.0.0.1:0`). +fn ephemeral_bind() -> SocketAddr { + "127.0.0.1:0".parse().expect("loopback addr") } /// A unique snapshot path under the system temp dir. @@ -704,8 +708,14 @@ where { let path = snapshot_path::("rejoin"); let a = spawn_node::("snap-a").await; - let b = - spawn_node_with_snapshot::("snap-b", serf_reactor::SnapshotOptions::new(&path), false).await; + let b = spawn_node_with_snapshot::( + "snap-b", + ephemeral_bind(), + serf_reactor::SnapshotOptions::new(&path), + false, + ) + .await + .expect("spawn snapshot-backed serf tcp node"); let a_addr = a.advertise_address(); b.join(&SocketAddrResolver, MaybeResolved::Resolved(a_addr), false) @@ -717,8 +727,14 @@ where b.shutdown().await.expect("snap-b shuts down"); // A fresh B from the same snapshot auto-rejoins A (no join call). - let b2 = - spawn_node_with_snapshot::("snap-b", serf_reactor::SnapshotOptions::new(&path), false).await; + let b2 = spawn_node_with_snapshot::( + "snap-b", + ephemeral_bind(), + serf_reactor::SnapshotOptions::new(&path), + false, + ) + .await + .expect("spawn snapshot-backed serf tcp node"); converge(&a, &b2).await; assert_eq!( b2.num_members(), @@ -742,8 +758,14 @@ where { let path = snapshot_path::("leave-gate"); let a = spawn_node::("gate-a").await; - let b = - spawn_node_with_snapshot::("gate-b", serf_reactor::SnapshotOptions::new(&path), false).await; + let b = spawn_node_with_snapshot::( + "gate-b", + ephemeral_bind(), + serf_reactor::SnapshotOptions::new(&path), + false, + ) + .await + .expect("spawn snapshot-backed serf tcp node"); let a_addr = a.advertise_address(); b.join(&SocketAddrResolver, MaybeResolved::Resolved(a_addr), false) @@ -780,8 +802,14 @@ where } // Default posture: the leave clears the recovered state — no auto-rejoin. - let b2 = - spawn_node_with_snapshot::("gate-b", serf_reactor::SnapshotOptions::new(&path), false).await; + let b2 = spawn_node_with_snapshot::( + "gate-b", + ephemeral_bind(), + serf_reactor::SnapshotOptions::new(&path), + false, + ) + .await + .expect("spawn snapshot-backed serf tcp node"); R::sleep(Duration::from_millis(1500)).await; assert_eq!( b2.num_members(), @@ -791,8 +819,14 @@ where b2.shutdown().await.expect("gate-b2 shuts down"); // Opt-in posture: the Leave marker is ignored and the membership recovers. - let b3 = - spawn_node_with_snapshot::("gate-b", serf_reactor::SnapshotOptions::new(&path), true).await; + let b3 = spawn_node_with_snapshot::( + "gate-b", + ephemeral_bind(), + serf_reactor::SnapshotOptions::new(&path), + true, + ) + .await + .expect("spawn snapshot-backed serf tcp node"); converge(&a, &b3).await; a.shutdown().await.expect("gate-a shuts down"); @@ -1825,16 +1859,16 @@ where /// default-tombstone reap that none of them cover, and — like the legacy body — /// checks the leaver's own side, not just the observer's. /// -/// The leaver is kept running rather than shut down so its convergence is -/// observable, but its side must be read from the EVENT LOG, not the membership -/// snapshot. The leaver reaps its own self Left tombstone (matching the legacy -/// leaver) — `handle_node_leave` adds self to `left_members` and `fire_reap` -/// removes it — but once self is gone from the machine, `refresh_snapshot` -/// refuses to publish a view missing the local id, so `members()` / -/// `num_members()` FREEZE at the pre-reap `[peer, self:Left]` state. The event -/// stream stays truthful: the leaver emits `Leave` then `Reap` for itself and -/// only a `Join` for the peer. (The frozen-snapshot-vs-event divergence for a -/// left-in-place node is tracked as a separate machine-side issue, al8n/serf#88.) +/// The leaver is kept running rather than shut down so its own convergence is +/// observable. The local node is exempt from reaping (a running node always +/// knows itself), so the leaver HOLDS its self `Left` tombstone rather than +/// reaping it: its own event log is exactly `Join → Leave` — no self `Reap` — +/// and its live membership view still shows itself `Left` beside the `Alive` +/// peer. This is the al8n/serf#88 fix: before the exemption the machine reaped +/// self and `refresh_snapshot` then froze the view (it will not publish a +/// snapshot missing the local id). The ABSENCE of the self `Reap` is what +/// distinguishes the held (correct) case from the reaped-then-frozen (buggy) +/// one — the frozen view carried the same member values. async fn serf_join_leave() where R: Runtime, @@ -1848,25 +1882,24 @@ where cluster.leave_in_place(1).await; // A (observer) reaps the departed B under the fast profile's 1ms tombstone + - // 100ms reap ticks, returning to just itself (A's own snapshot stays live). + // 100ms reap ticks, returning to just itself. This also fences past the reap + // window, so any (regressed) self-reap on B would have surfaced by now. cluster.await_num_members(0, 1).await; - // B (the leaver) processes its own departure end to end: its event log shows - // Join → Leave → Reap for itself — self IS reaped under the default tombstone. + // B (the leaver) holds its self tombstone: its own event log is exactly + // Join → Leave, with NO self Reap. + assert_eq!( + cluster.member_event_kinds(1, leaver.as_str()), + vec![MemberEventKind::Join, MemberEventKind::Leave], + "the leaver holds its self tombstone: Join then Leave, never a self Reap" + ); + // Its live view still shows itself Left beside the still-Alive peer — never + // frozen, never dropping the live peer. cluster - .assert_member_events( - 1, - leaver.as_str(), - &[ - MemberEventKind::Join, - MemberEventKind::Leave, - MemberEventKind::Reap, - ], - ) + .await_member_status(1, leaver.as_str(), MemberStatus::Left) .await; - // And it never fails or reaps the still-live peer — only the Join. cluster - .assert_member_events(1, peer.as_str(), &[MemberEventKind::Join]) + .await_member_status(1, peer.as_str(), MemberStatus::Alive) .await; cluster.shutdown_all().await; @@ -1966,6 +1999,437 @@ where cluster.shutdown_all().await; } +/// Port of legacy `serf_per_node_reconnect_timeout` (Go +/// `TestSerf_PerNodeReconnectTimeout`): a per-member `ReconnectDelegate` +/// override drives the FAILED-member reap timeout, not just the graceful-LEFT +/// tombstone. Node A carries a delegate that zeroes node B's reconnect timeout +/// while A's flat `reconnect_timeout` stays at 24h; after B is abruptly killed +/// and A detects it Failed, A reaps B early — which can only happen if the +/// reaper consulted the delegate on the FAILED path (the flat 24h timeout would +/// otherwise hold B for the whole test). The companion +/// `reconnect_delegate_reaps_left_member` covers the LEFT/tombstone path; this +/// covers the FAILED/reconnect path. +async fn serf_per_node_reconnect_timeout() +where + R: Runtime, +{ + let b = spawn_node::("prt-b").await; + let b_addr = b.advertise_address(); + + // A: fast SWIM so it detects B's kill sub-second, a long flat reconnect + // timeout so a default reap never fires within the test, and a delegate + // zeroing ONLY B's reconnect timeout. + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new("prt-a")) + .with_advertise_addr(MaybeResolved::Resolved(bind)) + .with_probe_interval(Duration::from_millis(100)) + .with_probe_timeout(Duration::from_millis(50)) + .with_gossip_interval(Duration::from_millis(20)) + .with_suspicion_mult(3); + let serf_opts = SerfOptions::new() + .with_reap_interval(Duration::from_millis(100)) + .with_reconnect_timeout(Duration::from_secs(86_400)); + let a = Serf::::tcp( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + serf_opts, + Some(Box::new(ReapImmediately { + target: SmolStr::new("prt-b"), + })), + None, + None, + #[cfg(encryption)] + std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), + ) + .await + .expect("spawn serf tcp node A with a reconnect delegate"); + + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + converge(&a, &b).await; + + // Abruptly kill B (no farewell) so A detects a probe-timeout Failed rather + // than a graceful Leave. + b.shutdown().await.expect("prt-b shuts down abruptly"); + + // A detects B Failed, then the delegate zeroes B's reconnect timeout so A's + // next reap tick drops B — back to a single member. Without the delegate + // consult, A would hold the failed B for the flat 24h. + R::timeout(Duration::from_secs(20), async { + loop { + if a.num_members() == 1 { + break; + } + R::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("A reaps the failed member B early via the reconnect-delegate override"); + + a.shutdown().await.expect("prt-a shuts down"); +} + +/// Port of legacy `serf_snapshot_recovery` (Go `TestSerf_SnapshotRecovery`): a +/// snapshot-backed node that fails, is force-removed by the survivor, then +/// restarts from its snapshot and rejoins — WITHOUT replaying the pre-failure +/// user event onto its fresh event channel. Distinguishes itself from +/// `snapshot_restart_rejoins_the_cluster` by the explicit `remove_failed_node` +/// before the restart (the survivor tombstones B as Left, not merely Failed) +/// and by asserting the recovered node surfaces zero user/query events. +async fn serf_snapshot_recovery() +where + R: Runtime, +{ + let path = snapshot_path::("recovery"); + let a = spawn_node::("sr-a").await; + let b = spawn_node_with_snapshot::( + "sr-b", + ephemeral_bind(), + serf_reactor::SnapshotOptions::new(&path), + false, + ) + .await + .expect("spawn snapshot-backed serf tcp node"); + let a_addr = a.advertise_address(); + let b_id = b.local_id(); + // Capture B's address: the restart must rebind it EXACTLY, because A holds B + // as a tombstone at this address (no dead-node reclaim window is configured), + // so a restart at a fresh port would be a same-id-new-address name conflict — + // exercising conflict resolution instead of snapshot recovery. + let b_addr = b.advertise_address(); + + let mut b_events = b.events(); + b.join(&SocketAddrResolver, MaybeResolved::Resolved(a_addr), false) + .await + .expect("join reaches node A"); + converge(&a, &b).await; + + // A fires a user event; fence on the pre-failure B receiving it, so the + // recovery below is genuinely tested for NOT replaying it. + a.user_event("event!", Bytes::from_static(b"test"), false) + .await + .expect("user event dispatched"); + R::timeout(Duration::from_secs(20), async { + loop { + match b_events.next().await { + Some(Event::User(u)) if u.name.as_str() == "event!" => break, + Some(_) => {} + None => break, + } + } + }) + .await + .expect("pre-failure B observes A's user event"); + + // Abruptly kill B; A detects it Failed. + b.shutdown().await.expect("sr-b shuts down"); + R::timeout(Duration::from_secs(20), async { + loop { + let failed = a + .members() + .iter() + .any(|m| m.node().id_ref() == &b_id && m.status() == MemberStatus::Failed); + if failed { + break; + } + R::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("A detects B failed"); + + // Force-remove the failed B: A tombstones it Left (not merely Failed), so the + // restart below revives against a Left tombstone. + a.remove_failed_node(b_id.clone()) + .await + .expect("A force-removes the failed B"); + R::timeout(Duration::from_secs(20), async { + loop { + let left = a + .members() + .iter() + .any(|m| m.node().id_ref() == &b_id && m.status() == MemberStatus::Left); + if left { + break; + } + R::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("A holds B as a Left tombstone after the force-remove"); + + // Restart B from the snapshot with a FRESH event channel, at its ORIGINAL + // address so the revival is a clean recovery, not an address conflict. The + // freed port may not be instantly rebindable, so retry a bounded number of + // times. + let b2 = { + let mut attempt = 0usize; + loop { + match spawn_node_with_snapshot::( + "sr-b", + b_addr, + serf_reactor::SnapshotOptions::new(&path), + false, + ) + .await + { + Ok(node) => break node, + // Ignoring Err: a transient rebind race on the just-freed port is + // retried; only the final attempt's failure is fatal. + Err(_) if attempt + 1 < 25 => { + attempt += 1; + R::sleep(Duration::from_millis(20)).await; + } + Err(e) => panic!("restart rebind for sr-b at {b_addr} failed: {e}"), + } + } + }; + let mut b2_events = b2.events(); + converge(&a, &b2).await; + + // The recovery must NOT replay the pre-failure user event (nor any query) onto + // the restarted node's channel — only membership events are permitted. Drain a + // settle window and fail on any surfaced user/query event. + let quiet = R::timeout(Duration::from_secs(2), async { + loop { + match b2_events.next().await { + Some(Event::User(u)) => break Some(format!("user:{}", u.name)), + Some(Event::Query(q)) => break Some(format!("query:{}", q.name())), + Some(_) => {} + None => break None, + } + } + }) + .await; + assert!( + quiet.is_err(), + "the recovered node must replay no user/query events, saw {quiet:?}" + ); + + a.shutdown().await.expect("sr-a shuts down"); + b2.shutdown().await.expect("sr-b2 shuts down"); + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} + +/// Spawn a standalone reactor TCP node with fast SWIM timing, so failure +/// detection and query resolution run sub-second (the plain [`spawn_node`] uses +/// default, slower timing). +async fn spawn_fast_node(id: &str) -> Node +where + R: Runtime, +{ + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(bind)) + .with_probe_interval(Duration::from_millis(100)) + .with_probe_timeout(Duration::from_millis(50)) + .with_gossip_interval(Duration::from_millis(20)) + .with_suspicion_mult(3); + Serf::::tcp( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + None, + None, + None, + #[cfg(encryption)] + std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), + ) + .await + .expect("spawn fast serf tcp node") +} + +/// Port of legacy `serf_name_resolution` (Go `TestSerf_NameResolution`): a +/// third node claiming an id already held in the cluster loses the conflict +/// vote and shuts down, while the incumbent survives. `nr-dup` is spawned with +/// the SAME id as the incumbent `nr-1`; after both are joined, the incumbent +/// (which the rest of the cluster already knows) wins the name-resolution query +/// and the newcomer transitions itself to Shutdown. +async fn serf_name_resolution() +where + R: Runtime, +{ + let s1 = spawn_fast_node::("nr-1").await; + let s2 = spawn_fast_node::("nr-2").await; + let s3 = spawn_fast_node::("nr-1").await; // duplicate of s1's id + let s2_addr = s2.advertise_address(); + let s3_addr = s3.advertise_address(); + + // Join the incumbent to s2 first, so the cluster knows nr-1 at s1's address + // and will vote for it in the conflict. + s1.join(&SocketAddrResolver, MaybeResolved::Resolved(s2_addr), false) + .await + .expect("s1 joins s2"); + converge(&s1, &s2).await; + + // Introduce the duplicate: joining nr-1@s3 into a cluster that already holds + // nr-1@s1 triggers the name-resolution conflict. + // Ignoring Err: the join may itself surface the conflict as an error; the + // resolution below is what the test asserts. + let _ = s1 + .join(&SocketAddrResolver, MaybeResolved::Resolved(s3_addr), false) + .await; + + // The newcomer loses the vote and shuts itself down; the incumbent survives. + R::timeout(Duration::from_secs(30), async { + loop { + if s3.state() == SerfState::Shutdown { + break; + } + R::sleep(Duration::from_millis(50)).await; + } + }) + .await + .expect("the duplicate-id newcomer loses the conflict and shuts down"); + assert_eq!( + s1.state(), + SerfState::Alive, + "the incumbent survives the conflict" + ); + + s1.shutdown().await.expect("nr-1 shuts down"); + s2.shutdown().await.expect("nr-2 shuts down"); + // s3 already shut itself down on the conflict loss. +} + +/// A test-only [`MessageDropper`](serf_proto::MessageDropper) that drops inbound +/// joins and push/pulls while its shared flag is set — the reactor analog of +/// the legacy `DropJoins`. +#[cfg(feature = "test")] +#[derive(Clone)] +struct DropJoins { + drop: std::sync::Arc, +} + +#[cfg(feature = "test")] +impl serf_proto::MessageDropper for DropJoins { + fn should_drop(&self, kind: serf_proto::DropKind) -> bool { + matches!( + kind, + serf_proto::DropKind::Join | serf_proto::DropKind::PushPull + ) && self.drop.load(std::sync::atomic::Ordering::SeqCst) + } +} + +/// Spawn a fast-SWIM node bound to `bind`, holding Left tombstones for the whole +/// test, and — when `drop_flag` is `Some` — dropping inbound joins/push-pulls +/// while that flag is set. +#[cfg(feature = "test")] +async fn spawn_air_node( + id: &str, + bind: SocketAddr, + drop_flag: Option>, +) -> Result, serf_reactor::SerfError> +where + R: Runtime, +{ + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(bind)) + .with_probe_interval(Duration::from_millis(100)) + .with_probe_timeout(Duration::from_millis(50)) + .with_gossip_interval(Duration::from_millis(20)) + .with_suspicion_mult(3); + let delegate = match drop_flag { + Some(flag) => VoidDelegate::::new() + .with_message_dropper(std::sync::Arc::new(DropJoins { drop: flag })), + None => VoidDelegate::::new(), + }; + Serf::::tcp( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + delegate, + RuntimeOptions::new(), + // Hold Left tombstones so every node's post-leave view is observable, and + // never auto-reap during the double-leave. + SerfOptions::new().with_tombstone_timeout(Duration::from_secs(30)), + None, + None, + None, + #[cfg(encryption)] + std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), + ) + .await +} + +/// Poll until every node in `nodes` reports `expect` members, or panic on the +/// bound. +#[cfg(feature = "test")] +async fn await_all_num_members(nodes: &[&Node], expect: usize) +where + R: Runtime, +{ + R::timeout(Duration::from_secs(30), async { + loop { + if nodes.iter().all(|n| n.num_members() == expect) { + break; + } + R::sleep(Duration::from_millis(25)).await; + } + }) + .await + .expect("all nodes reach the expected member count"); +} + +/// Exercises the test-only [`MessageDropper`](serf_proto::MessageDropper) +/// infrastructure end to end: a node whose delegate drops every inbound join / +/// push-pull never learns its peer, while the peer — dropping nothing — learns +/// it. This proves the reactor installs the delegate's dropper on the machine +/// and the machine consults it at ingress. (It is NOT the legacy +/// avoid-infinite-rebroadcast scenario, whose anti-rebroadcast property is +/// pinned deterministically at the machine layer by +/// `handle_node_leave_intent_updates_status_time_for_leaving` + +/// `stale_leave_intent_is_dropped`; the reactor cannot observe that property +/// non-vacuously.) +#[cfg(feature = "test")] +async fn message_dropper_drops_inbound_joins() +where + R: Runtime, +{ + // B drops every inbound join/push-pull from the start. + let flag = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(true)); + let a = spawn_air_node::("md-a", ephemeral_bind(), None) + .await + .expect("spawn md-a"); + let b = spawn_air_node::("md-b", ephemeral_bind(), Some(flag.clone())) + .await + .expect("spawn md-b"); + let b_addr = b.advertise_address(); + + // A joins B: A dials B and learns it from the exchange. B drops A's inbound + // push-pull AND every gossip-induced NodeJoined, so B never learns A. + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("md-a joins md-b"); + + // A converges to the 2-member cluster (it sees B). + await_all_num_members(&[&a], 2).await; + + // B stays at a single member across a settle window spanning many gossip + // rounds — it drops every inbound join, so it never learns A. Without the + // dropper B would reach 2. + R::sleep(Duration::from_secs(2)).await; + assert_eq!( + b.num_members(), + 1, + "the dropper node never learns its peer — every inbound join is dropped" + ); + + a.shutdown().await.expect("md-a shuts down"); + b.shutdown().await.expect("md-b shuts down"); +} + /// A deterministic test secret key, selecting whichever AEAD cipher this build /// compiled so the encrypted tests work under either backend. #[cfg(encryption)] @@ -2279,6 +2743,27 @@ mod tokio_cells { super::serf_leave_rejoin_different_role::().await; } + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_per_node_reconnect_timeout() { + super::serf_per_node_reconnect_timeout::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_snapshot_recovery() { + super::serf_snapshot_recovery::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_name_resolution() { + super::serf_name_resolution::().await; + } + + #[cfg(feature = "test")] + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn message_dropper_drops_inbound_joins() { + super::message_dropper_drops_inbound_joins::().await; + } + #[cfg(encryption)] #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn two_node_join_converges_encrypted() { @@ -2463,6 +2948,27 @@ mod smol_cells { SmolRuntime::block_on(super::serf_leave_rejoin_different_role::()); } + #[test] + fn serf_per_node_reconnect_timeout_smol() { + SmolRuntime::block_on(super::serf_per_node_reconnect_timeout::()); + } + + #[test] + fn serf_snapshot_recovery_smol() { + SmolRuntime::block_on(super::serf_snapshot_recovery::()); + } + + #[test] + fn serf_name_resolution_smol() { + SmolRuntime::block_on(super::serf_name_resolution::()); + } + + #[cfg(feature = "test")] + #[test] + fn message_dropper_drops_inbound_joins_smol() { + SmolRuntime::block_on(super::message_dropper_drops_inbound_joins::()); + } + #[cfg(encryption)] #[test] fn two_node_join_converges_encrypted_smol() { From c5ab2257b774a3e1ff55b9c31c985524119d7807 Mon Sep 17 00:00:00 2001 From: Al Liu Date: Mon, 13 Jul 2026 15:37:05 +1200 Subject: [PATCH 125/128] chore: remove the preserved legacy/ serf-core folder (#93) --- .github/workflows/ci-core.yml | 39 +- .github/workflows/fuzz.yml | 51 - Cargo.toml | 2 +- legacy/Cargo.toml | 45 - legacy/README.md | 1 - legacy/examples/toyconsul/Cargo.toml | 16 - legacy/examples/toyconsul/README.md | 68 - legacy/examples/toyconsul/src/main.rs | 584 ----- legacy/fuzz/.gitignore | 4 - legacy/fuzz/Cargo.toml | 20 - legacy/fuzz/fuzz_targets/messages.rs | 15 - legacy/serf-core/Cargo.toml | 100 - legacy/serf-core/README.md | 1 - legacy/serf-core/src/broadcast.rs | 79 - legacy/serf-core/src/coalesce.rs | 155 -- legacy/serf-core/src/coalesce/member.rs | 392 ---- legacy/serf-core/src/coalesce/user.rs | 232 -- legacy/serf-core/src/delegate.rs | 23 - legacy/serf-core/src/delegate/composite.rs | 114 - legacy/serf-core/src/delegate/merge.rs | 55 - legacy/serf-core/src/delegate/reconnect.rs | 56 - legacy/serf-core/src/error.rs | 390 ---- legacy/serf-core/src/event.rs | 512 ----- legacy/serf-core/src/event/crate_event.rs | 225 -- legacy/serf-core/src/key_manager.rs | 348 --- legacy/serf-core/src/lib.rs | 125 -- legacy/serf-core/src/options.rs | 590 ------ legacy/serf-core/src/serf.rs | 255 --- legacy/serf-core/src/serf/api.rs | 602 ------ legacy/serf-core/src/serf/base.rs | 1873 ----------------- legacy/serf-core/src/serf/base/tests.rs | 459 ---- legacy/serf-core/src/serf/base/tests/serf.rs | 950 --------- .../src/serf/base/tests/serf/delegate.rs | 350 --- .../src/serf/base/tests/serf/event.rs | 1106 ---------- .../src/serf/base/tests/serf/join.rs | 613 ------ .../src/serf/base/tests/serf/leave.rs | 516 ----- .../src/serf/base/tests/serf/reap.rs | 171 -- .../src/serf/base/tests/serf/reconnect.rs | 282 --- .../src/serf/base/tests/serf/remove.rs | 222 -- .../src/serf/base/tests/serf/snapshot.rs | 788 ------- legacy/serf-core/src/serf/delegate.rs | 843 -------- legacy/serf-core/src/serf/internal_query.rs | 486 ----- legacy/serf-core/src/serf/query.rs | 602 ------ legacy/serf-core/src/snapshot.rs | 885 -------- legacy/serf-core/src/types.rs | 250 --- legacy/serf-core/src/types/arbitrary_impl.rs | 190 -- legacy/serf-core/src/types/clock.rs | 191 -- legacy/serf-core/src/types/conflict.rs | 161 -- legacy/serf-core/src/types/coordinate.rs | 1282 ----------- legacy/serf-core/src/types/filter.rs | 272 --- .../serf-core/src/types/filter/tag_filter.rs | 228 -- legacy/serf-core/src/types/fuzzy.rs | 200 -- legacy/serf-core/src/types/join.rs | 159 -- legacy/serf-core/src/types/key.rs | 452 ---- legacy/serf-core/src/types/leave.rs | 196 -- legacy/serf-core/src/types/member.rs | 497 ----- legacy/serf-core/src/types/message.rs | 757 ------- legacy/serf-core/src/types/push_pull.rs | 587 ------ legacy/serf-core/src/types/query.rs | 527 ----- legacy/serf-core/src/types/query/response.rs | 303 --- legacy/serf-core/src/types/quickcheck_impl.rs | 324 --- legacy/serf-core/src/types/tags.rs | 156 -- legacy/serf-core/src/types/tests.rs | 310 --- legacy/serf-core/src/types/user_event.rs | 155 -- .../serf-core/src/types/user_event/message.rs | 272 --- .../src/types/user_event/user_events.rs | 181 -- legacy/serf-core/src/types/version.rs | 151 -- legacy/serf/Cargo.toml | 74 - legacy/serf/README.md | 1 - legacy/serf/src/lib.rs | 27 - legacy/serf/src/smol.rs | 19 - legacy/serf/src/tokio.rs | 19 - legacy/serf/test/main.rs | 22 - legacy/serf/test/main/net.rs | 57 - legacy/serf/test/main/net/coordinates.rs | 75 - legacy/serf/test/main/net/delegate.rs | 11 - .../test/main/net/delegate/local_state.rs | 66 - .../serf/test/main/net/delegate/nodemeta.rs | 91 - .../test/main/net/delegate/ping_delegate.rs | 109 - .../test/main/net/delegate/remote_state.rs | 59 - legacy/serf/test/main/net/event.rs | 53 - .../serf/test/main/net/event/default_query.rs | 59 - legacy/serf/test/main/net/event/event_user.rs | 67 - .../main/net/event/event_user_size_limit.rs | 59 - .../serf/test/main/net/event/events_failed.rs | 67 - .../serf/test/main/net/event/events_join.rs | 67 - .../serf/test/main/net/event/events_leave.rs | 67 - ...events_leave_avoid_infinite_rebroadcast.rs | 93 - legacy/serf/test/main/net/event/query.rs | 67 - .../test/main/net/event/query_deduplicate.rs | 59 - .../serf/test/main/net/event/query_filter.rs | 75 - .../test/main/net/event/query_old_message.rs | 59 - .../test/main/net/event/query_same_clock.rs | 59 - .../test/main/net/event/query_size_limit.rs | 59 - .../net/event/query_size_limit_increased.rs | 60 - .../net/event/remove_failed_events_leave.rs | 67 - .../test/main/net/event/should_process.rs | 59 - .../main/net/event/user_event_old_message.rs | 59 - .../main/net/event/user_event_same_clock.rs | 59 - legacy/serf/test/main/net/get_queue_max.rs | 63 - legacy/serf/test/main/net/join.rs | 32 - .../test/main/net/join/intent_buffer_early.rs | 59 - .../serf/test/main/net/join/intent_newer.rs | 59 - .../test/main/net/join/intent_old_message.rs | 59 - .../main/net/join/intent_reset_leaving.rs | 59 - legacy/serf/test/main/net/join/join_cancel.rs | 67 - .../test/main/net/join/join_ignore_old.rs | 67 - legacy/serf/test/main/net/join/join_leave.rs | 67 - .../test/main/net/join/join_leave_join.rs | 67 - legacy/serf/test/main/net/join/leave_ltime.rs | 67 - .../serf/test/main/net/join/pending_intent.rs | 59 - .../test/main/net/join/pending_intents.rs | 59 - legacy/serf/test/main/net/leave.rs | 23 - .../test/main/net/leave/force_leave_failed.rs | 75 - .../main/net/leave/force_leave_leaving.rs | 75 - .../test/main/net/leave/force_leave_left.rs | 75 - .../main/net/leave/intent_buffer_early.rs | 59 - .../serf/test/main/net/leave/intent_newer.rs | 59 - .../test/main/net/leave/intent_old_message.rs | 59 - .../main/net/leave/rejoin_different_role.rs | 67 - .../test/main/net/leave/snapshot_recovery.rs | 77 - legacy/serf/test/main/net/local_member.rs | 59 - legacy/serf/test/main/net/name_resolution.rs | 75 - legacy/serf/test/main/net/num_nodes.rs | 67 - legacy/serf/test/main/net/reap.rs | 9 - legacy/serf/test/main/net/reap/handler.rs | 59 - .../test/main/net/reap/handler_shutdown.rs | 59 - legacy/serf/test/main/net/reap/reap.rs | 59 - legacy/serf/test/main/net/reconnect.rs | 9 - .../serf/test/main/net/reconnect/reconnect.rs | 77 - .../serf/test/main/net/reconnect/same_ip.rs | 75 - .../serf/test/main/net/reconnect/timeout.rs | 67 - legacy/serf/test/main/net/remove.rs | 8 - .../serf/test/main/net/remove/failed_node.rs | 75 - .../main/net/remove/failed_node_ourself.rs | 59 - .../test/main/net/remove/failed_node_prune.rs | 75 - legacy/serf/test/main/net/role.rs | 67 - legacy/serf/test/main/net/set_tags.rs | 67 - legacy/serf/test/main/net/snapshot.rs | 14 - .../serf/test/main/net/snapshot/snapshoter.rs | 59 - .../net/snapshot/snapshoter_force_compact.rs | 59 - .../main/net/snapshot/snapshoter_leave.rs | 59 - .../net/snapshot/snapshoter_leave_rejoin.rs | 59 - .../main/net/snapshot/snapshoter_recovery.rs | 77 - legacy/serf/test/main/net/state.rs | 59 - legacy/serf/test/main/net/stats.rs | 59 - legacy/serf/test/main/net/update.rs | 77 - .../serf/test/main/net/write_keyring_file.rs | 63 - 148 files changed, 2 insertions(+), 27792 deletions(-) delete mode 100644 .github/workflows/fuzz.yml delete mode 100644 legacy/Cargo.toml delete mode 120000 legacy/README.md delete mode 100644 legacy/examples/toyconsul/Cargo.toml delete mode 100644 legacy/examples/toyconsul/README.md delete mode 100644 legacy/examples/toyconsul/src/main.rs delete mode 100644 legacy/fuzz/.gitignore delete mode 100644 legacy/fuzz/Cargo.toml delete mode 100644 legacy/fuzz/fuzz_targets/messages.rs delete mode 100644 legacy/serf-core/Cargo.toml delete mode 120000 legacy/serf-core/README.md delete mode 100644 legacy/serf-core/src/broadcast.rs delete mode 100644 legacy/serf-core/src/coalesce.rs delete mode 100644 legacy/serf-core/src/coalesce/member.rs delete mode 100644 legacy/serf-core/src/coalesce/user.rs delete mode 100644 legacy/serf-core/src/delegate.rs delete mode 100644 legacy/serf-core/src/delegate/composite.rs delete mode 100644 legacy/serf-core/src/delegate/merge.rs delete mode 100644 legacy/serf-core/src/delegate/reconnect.rs delete mode 100644 legacy/serf-core/src/error.rs delete mode 100644 legacy/serf-core/src/event.rs delete mode 100644 legacy/serf-core/src/event/crate_event.rs delete mode 100644 legacy/serf-core/src/key_manager.rs delete mode 100644 legacy/serf-core/src/lib.rs delete mode 100644 legacy/serf-core/src/options.rs delete mode 100644 legacy/serf-core/src/serf.rs delete mode 100644 legacy/serf-core/src/serf/api.rs delete mode 100644 legacy/serf-core/src/serf/base.rs delete mode 100644 legacy/serf-core/src/serf/base/tests.rs delete mode 100644 legacy/serf-core/src/serf/base/tests/serf.rs delete mode 100644 legacy/serf-core/src/serf/base/tests/serf/delegate.rs delete mode 100644 legacy/serf-core/src/serf/base/tests/serf/event.rs delete mode 100644 legacy/serf-core/src/serf/base/tests/serf/join.rs delete mode 100644 legacy/serf-core/src/serf/base/tests/serf/leave.rs delete mode 100644 legacy/serf-core/src/serf/base/tests/serf/reap.rs delete mode 100644 legacy/serf-core/src/serf/base/tests/serf/reconnect.rs delete mode 100644 legacy/serf-core/src/serf/base/tests/serf/remove.rs delete mode 100644 legacy/serf-core/src/serf/base/tests/serf/snapshot.rs delete mode 100644 legacy/serf-core/src/serf/delegate.rs delete mode 100644 legacy/serf-core/src/serf/internal_query.rs delete mode 100644 legacy/serf-core/src/serf/query.rs delete mode 100644 legacy/serf-core/src/snapshot.rs delete mode 100644 legacy/serf-core/src/types.rs delete mode 100644 legacy/serf-core/src/types/arbitrary_impl.rs delete mode 100644 legacy/serf-core/src/types/clock.rs delete mode 100644 legacy/serf-core/src/types/conflict.rs delete mode 100644 legacy/serf-core/src/types/coordinate.rs delete mode 100644 legacy/serf-core/src/types/filter.rs delete mode 100644 legacy/serf-core/src/types/filter/tag_filter.rs delete mode 100644 legacy/serf-core/src/types/fuzzy.rs delete mode 100644 legacy/serf-core/src/types/join.rs delete mode 100644 legacy/serf-core/src/types/key.rs delete mode 100644 legacy/serf-core/src/types/leave.rs delete mode 100644 legacy/serf-core/src/types/member.rs delete mode 100644 legacy/serf-core/src/types/message.rs delete mode 100644 legacy/serf-core/src/types/push_pull.rs delete mode 100644 legacy/serf-core/src/types/query.rs delete mode 100644 legacy/serf-core/src/types/query/response.rs delete mode 100644 legacy/serf-core/src/types/quickcheck_impl.rs delete mode 100644 legacy/serf-core/src/types/tags.rs delete mode 100644 legacy/serf-core/src/types/tests.rs delete mode 100644 legacy/serf-core/src/types/user_event.rs delete mode 100644 legacy/serf-core/src/types/user_event/message.rs delete mode 100644 legacy/serf-core/src/types/user_event/user_events.rs delete mode 100644 legacy/serf-core/src/types/version.rs delete mode 100644 legacy/serf/Cargo.toml delete mode 120000 legacy/serf/README.md delete mode 100644 legacy/serf/src/lib.rs delete mode 100644 legacy/serf/src/smol.rs delete mode 100644 legacy/serf/src/tokio.rs delete mode 100644 legacy/serf/test/main.rs delete mode 100644 legacy/serf/test/main/net.rs delete mode 100644 legacy/serf/test/main/net/coordinates.rs delete mode 100644 legacy/serf/test/main/net/delegate.rs delete mode 100644 legacy/serf/test/main/net/delegate/local_state.rs delete mode 100644 legacy/serf/test/main/net/delegate/nodemeta.rs delete mode 100644 legacy/serf/test/main/net/delegate/ping_delegate.rs delete mode 100644 legacy/serf/test/main/net/delegate/remote_state.rs delete mode 100644 legacy/serf/test/main/net/event.rs delete mode 100644 legacy/serf/test/main/net/event/default_query.rs delete mode 100644 legacy/serf/test/main/net/event/event_user.rs delete mode 100644 legacy/serf/test/main/net/event/event_user_size_limit.rs delete mode 100644 legacy/serf/test/main/net/event/events_failed.rs delete mode 100644 legacy/serf/test/main/net/event/events_join.rs delete mode 100644 legacy/serf/test/main/net/event/events_leave.rs delete mode 100644 legacy/serf/test/main/net/event/events_leave_avoid_infinite_rebroadcast.rs delete mode 100644 legacy/serf/test/main/net/event/query.rs delete mode 100644 legacy/serf/test/main/net/event/query_deduplicate.rs delete mode 100644 legacy/serf/test/main/net/event/query_filter.rs delete mode 100644 legacy/serf/test/main/net/event/query_old_message.rs delete mode 100644 legacy/serf/test/main/net/event/query_same_clock.rs delete mode 100644 legacy/serf/test/main/net/event/query_size_limit.rs delete mode 100644 legacy/serf/test/main/net/event/query_size_limit_increased.rs delete mode 100644 legacy/serf/test/main/net/event/remove_failed_events_leave.rs delete mode 100644 legacy/serf/test/main/net/event/should_process.rs delete mode 100644 legacy/serf/test/main/net/event/user_event_old_message.rs delete mode 100644 legacy/serf/test/main/net/event/user_event_same_clock.rs delete mode 100644 legacy/serf/test/main/net/get_queue_max.rs delete mode 100644 legacy/serf/test/main/net/join.rs delete mode 100644 legacy/serf/test/main/net/join/intent_buffer_early.rs delete mode 100644 legacy/serf/test/main/net/join/intent_newer.rs delete mode 100644 legacy/serf/test/main/net/join/intent_old_message.rs delete mode 100644 legacy/serf/test/main/net/join/intent_reset_leaving.rs delete mode 100644 legacy/serf/test/main/net/join/join_cancel.rs delete mode 100644 legacy/serf/test/main/net/join/join_ignore_old.rs delete mode 100644 legacy/serf/test/main/net/join/join_leave.rs delete mode 100644 legacy/serf/test/main/net/join/join_leave_join.rs delete mode 100644 legacy/serf/test/main/net/join/leave_ltime.rs delete mode 100644 legacy/serf/test/main/net/join/pending_intent.rs delete mode 100644 legacy/serf/test/main/net/join/pending_intents.rs delete mode 100644 legacy/serf/test/main/net/leave.rs delete mode 100644 legacy/serf/test/main/net/leave/force_leave_failed.rs delete mode 100644 legacy/serf/test/main/net/leave/force_leave_leaving.rs delete mode 100644 legacy/serf/test/main/net/leave/force_leave_left.rs delete mode 100644 legacy/serf/test/main/net/leave/intent_buffer_early.rs delete mode 100644 legacy/serf/test/main/net/leave/intent_newer.rs delete mode 100644 legacy/serf/test/main/net/leave/intent_old_message.rs delete mode 100644 legacy/serf/test/main/net/leave/rejoin_different_role.rs delete mode 100644 legacy/serf/test/main/net/leave/snapshot_recovery.rs delete mode 100644 legacy/serf/test/main/net/local_member.rs delete mode 100644 legacy/serf/test/main/net/name_resolution.rs delete mode 100644 legacy/serf/test/main/net/num_nodes.rs delete mode 100644 legacy/serf/test/main/net/reap.rs delete mode 100644 legacy/serf/test/main/net/reap/handler.rs delete mode 100644 legacy/serf/test/main/net/reap/handler_shutdown.rs delete mode 100644 legacy/serf/test/main/net/reap/reap.rs delete mode 100644 legacy/serf/test/main/net/reconnect.rs delete mode 100644 legacy/serf/test/main/net/reconnect/reconnect.rs delete mode 100644 legacy/serf/test/main/net/reconnect/same_ip.rs delete mode 100644 legacy/serf/test/main/net/reconnect/timeout.rs delete mode 100644 legacy/serf/test/main/net/remove.rs delete mode 100644 legacy/serf/test/main/net/remove/failed_node.rs delete mode 100644 legacy/serf/test/main/net/remove/failed_node_ourself.rs delete mode 100644 legacy/serf/test/main/net/remove/failed_node_prune.rs delete mode 100644 legacy/serf/test/main/net/role.rs delete mode 100644 legacy/serf/test/main/net/set_tags.rs delete mode 100644 legacy/serf/test/main/net/snapshot.rs delete mode 100644 legacy/serf/test/main/net/snapshot/snapshoter.rs delete mode 100644 legacy/serf/test/main/net/snapshot/snapshoter_force_compact.rs delete mode 100644 legacy/serf/test/main/net/snapshot/snapshoter_leave.rs delete mode 100644 legacy/serf/test/main/net/snapshot/snapshoter_leave_rejoin.rs delete mode 100644 legacy/serf/test/main/net/snapshot/snapshoter_recovery.rs delete mode 100644 legacy/serf/test/main/net/state.rs delete mode 100644 legacy/serf/test/main/net/stats.rs delete mode 100644 legacy/serf/test/main/net/update.rs delete mode 100644 legacy/serf/test/main/net/write_keyring_file.rs diff --git a/.github/workflows/ci-core.yml b/.github/workflows/ci-core.yml index c70a39e1..f5b49eac 100644 --- a/.github/workflows/ci-core.yml +++ b/.github/workflows/ci-core.yml @@ -2,8 +2,7 @@ name: ci-core # The runtime-independent core: the serf-proto wire codec + Sans-I/O machine # and its runtime-agnostic glue crate serf-driver. Both are pure-CPU and bind no -# sockets. This job also builds and tests the preserved `legacy/` serf-core crate -# (excluded from the workspace) so the old implementation can't silently rot. +# sockets. on: push: @@ -167,39 +166,3 @@ jobs: - name: Test run: cargo test -p serf-driver --all-features - - # The preserved pre-Sans-I/O implementation (legacy/serf-core). It is excluded - # from the workspace, so build + test it from its own directory on the feature - # set the old serf CI used, to keep it from silently rotting. - legacy: - name: legacy - serf-core - runs-on: ubuntu-latest - defaults: - run: - shell: bash - steps: - - uses: actions/checkout@v7 - - - name: Install Rust - uses: dtolnay/rust-toolchain@master - with: - toolchain: nightly - components: clippy, rustfmt - - - name: Cache Cargo registry - uses: actions/cache@v6 - with: - path: | - ~/.cargo/registry - ~/.cargo/git - legacy/target - key: ${{ runner.os }}-cargo-${{ hashFiles('legacy/**/Cargo.lock') }}-legacy - - - name: Install protoc - uses: arduino/setup-protoc@v3 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Test legacy serf-core - run: cargo test --no-default-features --features "test,encryption,serde,quickcheck" - working-directory: legacy/serf-core diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml deleted file mode 100644 index d1059e47..00000000 --- a/.github/workflows/fuzz.yml +++ /dev/null @@ -1,51 +0,0 @@ -name: Fuzz Testing - -# Fuzz the preserved legacy serf-core wire codec. The fuzz crate (serf-types-fuzz, -# target `messages`) lives in the excluded `legacy/` workspace and fuzzes serf's -# message (de)serialization; it is built + briefly run from that workspace root. - -on: - push: - branches: - - main - - refactor/sansio - paths-ignore: - - "README.md" - - "COPYRIGHT" - - "LICENSE*" - - "**.md" - - "**.txt" - - "art" - pull_request: - paths-ignore: - - "README.md" - - "COPYRIGHT" - - "LICENSE*" - - "**.md" - - "**.txt" - - "art" - schedule: - - cron: "0 1 */30 * *" - -jobs: - fuzz: - runs-on: ubuntu-latest - defaults: - run: - shell: bash - working-directory: legacy - steps: - - uses: actions/checkout@v7 - - - name: Install Rust - uses: dtolnay/rust-toolchain@master - with: - toolchain: nightly - - - name: Install cargo-fuzz - run: cargo install cargo-fuzz - - - name: Run fuzzing - run: | - cargo fuzz build - cargo fuzz run messages -- -max_len=4096 -max_total_time=300 diff --git a/Cargo.toml b/Cargo.toml index 7300091b..f289341f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ members = [ # driver. It pins a foreign default target (a Cortex-M machine) and a cortex-m-rt # runtime, so it cannot participate in the host `--workspace` build; it is built # and run on its own via `cd tests/serf-embassy-qemu && cargo run`. -exclude = ["legacy", "tests/serf-embassy-qemu"] +exclude = ["tests/serf-embassy-qemu"] resolver = "3" [workspace.package] diff --git a/legacy/Cargo.toml b/legacy/Cargo.toml deleted file mode 100644 index 68e9b09c..00000000 --- a/legacy/Cargo.toml +++ /dev/null @@ -1,45 +0,0 @@ -[workspace] -members = [ - "serf-core", - "serf", - "examples/toyconsul", - "fuzz", -] -resolver = "3" - -[workspace.package] -version = "0.5.2" -edition = "2024" -rust-version = "1.85.0" -description = "A decentralized solution for service discovery and orchestration that is lightweight, highly available, and fault tolerant." -repository = "https://github.com/al8n/serf" -homepage = "https://github.com/al8n/serf" -license = "MPL-2.0" -keywords = ["gossip", "service-discovery", "cluster", "membership", "distributed"] -categories = ["network-programming", "asynchronous", "algorithms"] -readme = "README.md" - -[workspace.dependencies] -bytes = "1" -byteorder = "1" -derive_more = { version = "2", features = ["full"] } -futures = { version = "0.3", default-features = false } -serde = { version = "1", features = ["derive"] } -humantime-serde = "1" -indexmap = "2" -thiserror = { version = "2", default-features = false } -viewit = "0.1.5" -regex = "1" -smol_str = "0.3" -smallvec = "1" -rand = "0.10" - -arbitrary = { version = "1", default-features = false, features = ["derive"] } -quickcheck = { version = "1", default-features = false } - -memberlist-proto = { version = "0.3", default-features = false } -memberlist-core = { version = "0.8.5", default-features = false } -memberlist = { version = "0.8.5", default-features = false } - -serf-core = { path = "serf-core", version = "0.5", default-features = false } -serf = { path = "serf", version = "0.5", default-features = false } diff --git a/legacy/README.md b/legacy/README.md deleted file mode 120000 index 32d46ee8..00000000 --- a/legacy/README.md +++ /dev/null @@ -1 +0,0 @@ -../README.md \ No newline at end of file diff --git a/legacy/examples/toyconsul/Cargo.toml b/legacy/examples/toyconsul/Cargo.toml deleted file mode 100644 index 2b8a51fb..00000000 --- a/legacy/examples/toyconsul/Cargo.toml +++ /dev/null @@ -1,16 +0,0 @@ -[package] -name = "toyconsul" -rust-version = "1.85.0" -edition = "2024" -publish = false - -[dependencies] -bincode = { version = "2", features = ["serde"] } -clap = { version = "4", features = ["derive"] } -crossbeam-skiplist = "0.1" -serf = { workspace = true, features = ["default", "tokio", "tcp", "serde"] } -serde = { version = "1", features = ["derive"] } -tracing = "0.1" -tracing-subscriber = { version = "0.3", features = ["env-filter"] } -tokio = { version = "1", features = ["full"] } -scopeguard = "1" diff --git a/legacy/examples/toyconsul/README.md b/legacy/examples/toyconsul/README.md deleted file mode 100644 index baf260e3..00000000 --- a/legacy/examples/toyconsul/README.md +++ /dev/null @@ -1,68 +0,0 @@ -# ToyConsul - -A toy eventually consistent distributed registry. - -## Installation - -```bash -cargo install --path . -``` - -## Run - -- In the first terminal - - ```bash - toyconsul start --id instance1 --addr 127.0.0.1:7001 --rpc-addr toyconsul.instance1.sock - ``` - -- In the second terminal - - - Start instance 2 - - ```bash - toyconsul start --id instance2 --addr 127.0.0.1:7002 --rpc-addr toyconsul.instance2.sock - ``` - - - Send the join command to instance2 and let it join to instance1 - - ```bash - toyconsul join --id instance1 --addr 127.0.0.1:7001 --rpc-addr toyconsul.instance2.sock - ``` - -- In the third terminal - - - Start instance 3 - - ```bash - toyconsul start --id instance3 --addr 127.0.0.1:7003 --rpc-addr toyconsul.instance3.sock - ``` - - - Send the join command to instance3 and let it join to instance1 (can also join to instance 2) - - ```bash - toyconsul join --id instance1 --addr 127.0.0.1:7001 --rpc-addr toyconsul.instance3.sock - ``` - -- In the fourth terminal - - - Insert a key - value to the instance1 - - ```bash - toyconsul register --name web --addr 192.0.0.1:8080 --rpc-addr toyconsul.instance1.sock - toyconsul register --name db --addr 192.0.0.1:8081 --rpc-addr toyconsul.instance2.sock - ``` - - - After some seconds, you can get the value from any one of three instances - - ```bash - toyconsul list --rpc-addr toyconsul.instance1.sock - ``` - - ```bash - toyconsul list --rpc-addr toyconsul.instance2.sock - ``` - - ```bash - toyconsul list --rpc-addr toyconsul.instance3.sock - ``` diff --git a/legacy/examples/toyconsul/src/main.rs b/legacy/examples/toyconsul/src/main.rs deleted file mode 100644 index 6aad0a32..00000000 --- a/legacy/examples/toyconsul/src/main.rs +++ /dev/null @@ -1,584 +0,0 @@ -use std::{net::SocketAddr, path::PathBuf, sync::Arc}; - -use clap::Parser; -use crossbeam_skiplist::SkipMap; -use serf::{ - MemberlistOptions, Options, - delegate::CompositeDelegate, - event::{Event as SerfEvent, EventProducer, EventSubscriber}, - net::{NetTransportOptions, NodeId, TokioNetTransport}, - tokio::{TokioSocketAddrResolver, TokioTcp, TokioTcpSerf}, - types::{MaybeResolvedAddress, SmolStr}, -}; - -use tokio::{ - io::{AsyncReadExt, AsyncWriteExt}, - net::{UnixListener, UnixStream}, - sync::{ - mpsc::{UnboundedReceiver, UnboundedSender}, - oneshot, - }, -}; - -type Result = std::result::Result>; - -type ConsulDelegate = CompositeDelegate; - -#[derive(Clone, serde::Serialize, serde::Deserialize)] -struct Service { - name: SmolStr, - addr: SocketAddr, -} - -struct Inner { - serf: TokioTcpSerf, - services: SkipMap, - tx: UnboundedSender, -} - -#[derive(Clone)] -struct ToyConsul { - inner: Arc, -} - -impl ToyConsul { - async fn new( - opts: Options, - net_opts: NetTransportOptions, - ) -> Result { - let (tx, rx) = tokio::sync::mpsc::unbounded_channel(); - let (producer, subscriber) = EventProducer::unbounded(); - let serf = - TokioTcpSerf::with_event_producer(net_opts, opts.with_event_buffer_size(256), producer) - .await?; - - let this = Self { - inner: Inner { - serf, - services: SkipMap::new(), - tx, - } - .into(), - }; - - this.clone().handle_serf_events(subscriber); - this.clone().handle_rpc_events(rx); - - Ok(this) - } - - fn handle_rpc_events(self, mut rx: UnboundedReceiver) { - tokio::spawn(async move { - loop { - tokio::select! { - _ = tokio::signal::ctrl_c() => { - tracing::info!("toyconsul: shutting down rpc listener"); - } - ev = rx.recv() => { - match ev { - Some(Event::Register { name, addr, tx }) => { - let service = Service { name, addr }; - match bincode::serde::encode_to_vec(&service, bincode::config::standard()) { - Ok(data) => { - // broadcast a register event to all members - match self.inner.serf.user_event("register", data, false).await { - Ok(_) => { - let _ = tx.send(Ok(())); - } - Err(e) => { - tracing::error!(err=%e, "toyconsul: fail to send register event"); - let _ = tx.send(Err(e.into())); - } - } - } - Err(e) => { - tracing::error!(err=%e, "toyconsul: fail to encode register response"); - let _ = tx.send(Err(e.into())); - } - } - } - Some(Event::List { tx }) => { - let services = self.inner.services.iter().map(|ent| ent.value().clone()).collect(); - let _ = tx.send(Ok(services)); - } - Some(Event::Join { addr, tx }) => { - let res = self.inner.serf.join(MaybeResolvedAddress::Resolved(addr), false).await; - let _ = tx.send(res.map_err(Into::into).map(|_| ())); - } - None => { - break; - } - } - } - } - } - }); - } - - fn handle_serf_events( - self, - subscriber: EventSubscriber< - TokioNetTransport, - ConsulDelegate, - >, - ) { - tokio::spawn(async move { - loop { - tokio::select! { - _ = tokio::signal::ctrl_c() => { - tracing::info!("toyconsul: shutting down event listener"); - } - ev = subscriber.recv() => { - let Ok(ev) = ev else { - tracing::info!("serf event channel closed, loop quit now"); - break; - }; - - if let SerfEvent::User(ev) = ev { - match ev.name().as_str() { - "register" => { - let payload = ev.payload(); - let service: Service = match bincode::serde::decode_from_slice(payload, bincode::config::standard()) { - Ok((service, _)) => service, - Err(e) => { - tracing::error!(err=%e, "toyconsul: fail to decode register event"); - continue; - } - }; - self.inner.services.insert(service.name.clone(), service); - }, - other => { - tracing::warn!("toyconsul: unknown user event {}", other); - } - } - } - } - } - } - }); - } - - async fn handle_register( - &self, - name: SmolStr, - addr: SocketAddr, - stream: &mut W, - ) -> Result<()> { - let (tx, rx) = oneshot::channel(); - if let Err(e) = self.inner.tx.send(Event::Register { name, addr, tx }) { - tracing::error!(err=%e, "toyconsul: fail to send get event"); - return Ok(()); - } - - let resp = rx.await?; - tracing::info!(value=?resp, "toyconsul: fetch key"); - match bincode::serde::encode_to_vec( - resp.map_err(|e| e.to_string()), - bincode::config::standard(), - ) { - Ok(resp) => { - let mut prefixed_data = vec![0; resp.len() + 4]; - prefixed_data[..4].copy_from_slice(&(resp.len() as u32).to_le_bytes()); - prefixed_data[4..].copy_from_slice(&resp); - if let Err(e) = stream.write_all(&prefixed_data).await { - tracing::error!(err=%e, "toyconsul: fail to write rpc response"); - } else { - tracing::info!(data=?prefixed_data, "toyconsul: send get response"); - } - } - Err(e) => { - tracing::error!(err=%e, "toyconsul: fail to encode rpc response"); - } - } - Ok(()) - } - - async fn handle_join( - &self, - addr: SocketAddr, - stream: &mut W, - ) -> Result<()> { - let (tx, rx) = oneshot::channel(); - self.inner.tx.send(Event::Join { addr, tx })?; - - let resp = rx.await?; - if let Err(e) = resp { - let res = std::result::Result::<(), String>::Err(e.to_string()); - match bincode::serde::encode_to_vec(&res, bincode::config::standard()) { - Ok(resp) => { - let mut prefixed_data = vec![0; resp.len() + 4]; - prefixed_data[..4].copy_from_slice(&(resp.len() as u32).to_le_bytes()); - prefixed_data[4..].copy_from_slice(&resp); - if let Err(e) = stream.write_all(&prefixed_data).await { - tracing::error!(err=%e, "toyconsul: fail to write rpc response"); - } - } - Err(e) => { - tracing::error!(err=%e, "toyconsul: fail to encode rpc response"); - } - } - } else { - let res = std::result::Result::<(), String>::Ok(()); - match bincode::serde::encode_to_vec(&res, bincode::config::standard()) { - Ok(resp) => { - let mut prefixed_data = vec![0; resp.len() + 4]; - prefixed_data[..4].copy_from_slice(&(resp.len() as u32).to_le_bytes()); - prefixed_data[4..].copy_from_slice(&resp); - if let Err(e) = stream.write_all(&prefixed_data).await { - tracing::error!(err=%e, "toyconsul: fail to write rpc response"); - } - } - Err(e) => { - tracing::error!(err=%e, "toyconsul: fail to encode rpc response"); - } - } - } - - Ok(()) - } - - async fn handle_list(&self, stream: &mut W) -> Result<()> { - let (tx, rx) = oneshot::channel(); - self.inner.tx.send(Event::List { tx })?; - - let resp = rx.await?; - - match resp { - Ok(s) => { - let res = std::result::Result::, String>::Ok(s); - match bincode::serde::encode_to_vec(&res, bincode::config::standard()) { - Ok(resp) => { - let mut prefixed_data = vec![0; resp.len() + 4]; - prefixed_data[..4].copy_from_slice(&(resp.len() as u32).to_le_bytes()); - prefixed_data[4..].copy_from_slice(&resp); - if let Err(e) = stream.write_all(&prefixed_data).await { - tracing::error!(err=%e, "toyconsul: fail to write rpc response"); - } - } - Err(e) => { - tracing::error!(err=%e, "toyconsul: fail to encode rpc response"); - } - } - } - Err(e) => { - let res = std::result::Result::<(), String>::Err(e.to_string()); - match bincode::serde::encode_to_vec(&res, bincode::config::standard()) { - Ok(resp) => { - let mut prefixed_data = vec![0; resp.len() + 4]; - prefixed_data[..4].copy_from_slice(&(resp.len() as u32).to_le_bytes()); - prefixed_data[4..].copy_from_slice(&resp); - if let Err(e) = stream.write_all(&prefixed_data).await { - tracing::error!(err=%e, "toyconsul: fail to write rpc response"); - } - } - Err(e) => { - tracing::error!(err=%e, "toyconsul: fail to encode rpc response"); - } - } - } - } - Ok(()) - } -} - -#[derive(clap::Args)] -struct StartArgs { - /// The id of the db instance - #[clap(short, long)] - id: NodeId, - /// The address the memberlist should bind to - #[clap(short, long)] - addr: SocketAddr, - /// The rpc address to listen on commands - #[clap(short, long)] - rpc_addr: std::path::PathBuf, -} - -#[derive(clap::Subcommand)] -enum Commands { - /// Start the toyconsul instance - Start(StartArgs), - /// Join to an existing toyconsul cluster - Join { - #[clap(short, long)] - addr: SocketAddr, - #[clap(short, long)] - rpc_addr: std::path::PathBuf, - }, - /// Register a service to the toyconsul - Register { - #[clap(short, long)] - name: String, - #[clap(short, long)] - addr: SocketAddr, - #[clap(short, long)] - rpc_addr: std::path::PathBuf, - }, - /// List all services in the toyconsul - List { - #[clap(short, long)] - rpc_addr: std::path::PathBuf, - }, -} - -#[derive(clap::Parser)] -#[command(name = "toyconsul")] -#[command(about = "CLI for toyconsul example", long_about = None)] -struct Cli { - #[clap(subcommand)] - command: Commands, -} - -#[derive(serde::Serialize, serde::Deserialize)] -enum Op { - Register { name: SmolStr, addr: SocketAddr }, - List, - Join { addr: SocketAddr }, -} - -enum Event { - Register { - name: SmolStr, - addr: SocketAddr, - tx: oneshot::Sender>, - }, - List { - tx: oneshot::Sender>>, - }, - Join { - addr: SocketAddr, - tx: oneshot::Sender>, - }, -} - -#[tokio::main] -async fn main() -> Result<()> { - let filter = std::env::var("TOY_CONSUL_LOG").unwrap_or_else(|_| "info".to_owned()); - tracing::subscriber::set_global_default( - tracing_subscriber::fmt::fmt() - .without_time() - .with_line_number(true) - .with_env_filter(filter) - .with_file(false) - .with_target(true) - .with_ansi(true) - .finish(), - ) - .unwrap(); - - let cli = Cli::parse(); - match cli.command { - Commands::Join { addr, rpc_addr } => { - handle_join_cmd(addr, rpc_addr).await?; - } - Commands::Register { - name, - addr, - rpc_addr, - } => { - handle_register_cmd(name, addr, rpc_addr).await?; - } - Commands::Start(args) => { - handle_start_cmd(args).await?; - } - Commands::List { rpc_addr } => { - handle_list_cmd(rpc_addr).await?; - } - } - - Ok(()) -} - -async fn handle_join_cmd(addr: SocketAddr, rpc_addr: std::path::PathBuf) -> Result<()> { - let conn = UnixStream::connect(rpc_addr).await?; - let data = bincode::serde::encode_to_vec(&Op::Join { addr }, bincode::config::standard())?; - - let (reader, mut writer) = conn.into_split(); - - let mut prefixed_data = vec![0; data.len() + 4]; - prefixed_data[..4].copy_from_slice(&(data.len() as u32).to_le_bytes()); - prefixed_data[4..].copy_from_slice(&data); - - writer.write_all(&prefixed_data).await?; - writer.shutdown().await?; - - let mut reader = tokio::io::BufReader::new(reader); - let mut len_buf = [0; 4]; - reader.read_exact(&mut len_buf).await?; - let len = u32::from_le_bytes(len_buf) as usize; - - let mut buf = vec![0; len]; - reader.read_exact(&mut buf).await?; - let (res, _) = bincode::serde::decode_from_slice::, _>( - &buf, - bincode::config::standard(), - )?; - match res { - Ok(_) => { - println!("join successfully"); - } - Err(e) => { - println!("fail to join {e}") - } - } - Ok(()) -} - -async fn handle_register_cmd( - name: String, - addr: SocketAddr, - rpc_addr: std::path::PathBuf, -) -> Result<()> { - let conn = UnixStream::connect(rpc_addr).await?; - let data = bincode::serde::encode_to_vec( - &Op::Register { - name: name.clone().into(), - addr, - }, - bincode::config::standard(), - )?; - - let (reader, mut writer) = conn.into_split(); - - let mut prefixed_data = vec![0; data.len() + 4]; - prefixed_data[..4].copy_from_slice(&(data.len() as u32).to_le_bytes()); - prefixed_data[4..].copy_from_slice(&data); - - writer.write_all(&prefixed_data).await?; - writer.shutdown().await?; - - let mut reader = tokio::io::BufReader::new(reader); - let mut len_buf = [0; 4]; - reader.read_exact(&mut len_buf).await?; - let len = u32::from_le_bytes(len_buf) as usize; - - let mut buf = vec![0; len]; - reader.read_exact(&mut buf).await?; - let (res, _) = bincode::serde::decode_from_slice::, _>( - &buf, - bincode::config::standard(), - )?; - match res { - Ok(_) => { - println!("register {}({}) successfully", name, addr); - } - Err(e) => { - println!("fail to register {e}"); - } - } - Ok(()) -} - -async fn handle_start_cmd(args: StartArgs) -> Result<()> { - let opts = Options::new().with_memberlist_options(MemberlistOptions::local()); - let net_opts = - NetTransportOptions::new(args.id).with_bind_addresses([args.addr].into_iter().collect()); - - let consul = ToyConsul::new(opts, net_opts).await?; - - struct Guard { - sock: std::path::PathBuf, - } - - impl Drop for Guard { - fn drop(&mut self) { - if let Err(e) = std::fs::remove_file(&self.sock) { - tracing::error!(err=%e, "toyconsul: fail to remove rpc sock"); - } - } - } - - let _guard = Guard { - sock: args.rpc_addr.clone(), - }; - - let listener = UnixListener::bind(&args.rpc_addr)?; - - tracing::info!("toyconsul: start listening on {}", args.rpc_addr.display()); - - loop { - tokio::select! { - conn = listener.accept() => { - let (stream, _) = conn?; - let mut stream = tokio::io::BufReader::new(stream); - let mut len_buf = [0; 4]; - stream.read_exact(&mut len_buf).await?; - let len = u32::from_le_bytes(len_buf) as usize; - - let mut data = vec![0; len]; - if let Err(e) = stream.read_exact(&mut data).await { - tracing::error!(err=%e, "toyconsul: fail to read from rpc stream"); - continue; - } - - let op: Op = match bincode::serde::decode_from_slice(&data, bincode::config::standard()) { - Ok((op, _)) => op, - Err(e) => { - tracing::error!(err=%e, "toyconsul: fail to decode rpc message"); - continue; - } - }; - - match op { - Op::Join { addr } => { - consul.handle_join(addr, &mut stream).await?; - } - Op::Register { - name, - addr, - } => { - consul.handle_register(name, addr, &mut stream).await?; - }, - Op::List => { - consul.handle_list(&mut stream).await?; - } - } - - if let Err(e) = stream.into_inner().shutdown().await { - tracing::error!(err=%e, "toyconsul: fail to shutdown rpc stream"); - } - } - _ = tokio::signal::ctrl_c() => { - break; - } - } - } - Ok(()) -} - -async fn handle_list_cmd(rpc_addr: PathBuf) -> Result<()> { - let conn = UnixStream::connect(rpc_addr).await?; - let data = bincode::serde::encode_to_vec(&Op::List, bincode::config::standard())?; - - let (reader, mut writer) = conn.into_split(); - - let mut prefixed_data = vec![0; data.len() + 4]; - prefixed_data[..4].copy_from_slice(&(data.len() as u32).to_le_bytes()); - prefixed_data[4..].copy_from_slice(&data); - - writer.write_all(&prefixed_data).await?; - writer.shutdown().await?; - - let mut reader = tokio::io::BufReader::new(reader); - let mut len_buf = [0; 4]; - reader.read_exact(&mut len_buf).await?; - let len = u32::from_le_bytes(len_buf) as usize; - - let mut buf = vec![0; len]; - reader.read_exact(&mut buf).await?; - let (res, _) = bincode::serde::decode_from_slice::, String>, _>( - &buf, - bincode::config::standard(), - )?; - match res { - Ok(services) => { - for service in services { - println!("{}({})", service.name, service.addr); - } - } - Err(e) => { - println!("fail to list {e}") - } - } - Ok(()) -} diff --git a/legacy/fuzz/.gitignore b/legacy/fuzz/.gitignore deleted file mode 100644 index 1a45eee7..00000000 --- a/legacy/fuzz/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -target -corpus -artifacts -coverage diff --git a/legacy/fuzz/Cargo.toml b/legacy/fuzz/Cargo.toml deleted file mode 100644 index 45f4a377..00000000 --- a/legacy/fuzz/Cargo.toml +++ /dev/null @@ -1,20 +0,0 @@ -[package] -name = "serf-types-fuzz" -version = "0.0.0" -publish = false -edition = "2024" - -[package.metadata] -cargo-fuzz = true - -[dependencies] -libfuzzer-sys = "0.4" - -serf-core = { workspace = true, features = ["arbitrary", "default", "encryption", "test"] } - -[[bin]] -name = "messages" -path = "fuzz_targets/messages.rs" -test = false -doc = false -bench = false diff --git a/legacy/fuzz/fuzz_targets/messages.rs b/legacy/fuzz/fuzz_targets/messages.rs deleted file mode 100644 index c5e4dc47..00000000 --- a/legacy/fuzz/fuzz_targets/messages.rs +++ /dev/null @@ -1,15 +0,0 @@ -#![no_main] -#![allow(clippy::type_complexity)] - -use libfuzzer_sys::fuzz_target; - -use serf_core::types::{ - Node, - fuzzy::{Message, encodable_round_trip}, -}; - -fuzz_target!( - |data: (Message>, Option>>)| { - assert!(encodable_round_trip(data.0, data.1)); - } -); diff --git a/legacy/serf-core/Cargo.toml b/legacy/serf-core/Cargo.toml deleted file mode 100644 index 8abb31c6..00000000 --- a/legacy/serf-core/Cargo.toml +++ /dev/null @@ -1,100 +0,0 @@ -[package] -name = "serf-core" -version.workspace = true -rust-version.workspace = true -edition.workspace = true -license.workspace = true -description.workspace = true -repository.workspace = true -readme.workspace = true -homepage.workspace = true -keywords.workspace = true -categories.workspace = true - -[features] -default = ["metrics"] -metrics = ["memberlist-core/metrics", "dep:metrics"] - -encryption = ["memberlist-core/encryption", "serde", "serde_json"] - -crc32 = ["memberlist-core/crc32"] -murmur3 = ["memberlist-core/murmur3"] -xxhash64 = ["memberlist-core/xxhash64"] -xxhash32 = ["memberlist-core/xxhash32"] -xxhash3 = ["memberlist-core/xxhash3"] - -snappy = ["memberlist-core/snappy"] -zstd = ["memberlist-core/zstd"] -lz4 = ["memberlist-core/lz4"] -brotli = ["memberlist-core/brotli"] - -serde = [ - "dep:serde", - "dep:humantime-serde", - "bitflags/serde", - "memberlist-core/serde", - "smol_str/serde", - "smallvec/serde", - "indexmap/serde", -] - -test = ["memberlist-core/test", "paste", "tracing-subscriber", "tempfile"] - -arbitrary = ["dep:arbitrary", "memberlist-core/arbitrary", "smol_str/arbitrary"] -quickcheck = ["dep:quickcheck", "memberlist-core/quickcheck"] - -[dependencies] -auto_impl = "1" -atomic_refcell = "0.1" -arc-swap = "1" -async-lock = "3" -async-channel = "2" -bitflags = "2" -byteorder.workspace = true -crossbeam-queue = "0.3" -derive_more.workspace = true -futures = { workspace = true, features = ["default"] } -either = "1" -indexmap.workspace = true -once_cell = "1" # remove this dependency when [feature(lazy_cell)] is stabilized -parking_lot = { version = "0.12", features = ["send_guard"] } -pin-project = "1" -rand.workspace = true -regex.workspace = true -scopeguard = "1" -smol_str.workspace = true -smallvec.workspace = true -thiserror.workspace = true -viewit.workspace = true -memberlist-core.workspace = true - -metrics = { version = "0.24", optional = true } - -serde = { workspace = true, optional = true } -humantime-serde = { workspace = true, optional = true } -serde_json = { version = "1", optional = true } - -arbitrary = { workspace = true, optional = true, default-features = false, features = ["derive"] } -quickcheck = { workspace = true, optional = true, default-features = false } - -# test features -paste = { version = "1", optional = true } -tracing-subscriber = { version = "0.3", optional = true, features = [ - "env-filter", - "tracing-log", -] } -tempfile = { version = "3", optional = true } - -[dev-dependencies] -agnostic-lite = { version = "0.6", features = ["tokio"] } -tokio = { version = "1", features = ["full"] } -futures = { workspace = true, features = ["executor"] } -tempfile = "3" -memberlist-core = { workspace = true, features = ["quickcheck", "arbitrary", "test"] } -quickcheck_macros = "1" -quickcheck.workspace = true -paste = "1" - -[package.metadata.docs.rs] -all-features = true -rustdoc-args = ["--cfg", "docsrs"] \ No newline at end of file diff --git a/legacy/serf-core/README.md b/legacy/serf-core/README.md deleted file mode 120000 index 32d46ee8..00000000 --- a/legacy/serf-core/README.md +++ /dev/null @@ -1 +0,0 @@ -../README.md \ No newline at end of file diff --git a/legacy/serf-core/src/broadcast.rs b/legacy/serf-core/src/broadcast.rs deleted file mode 100644 index d9a1728e..00000000 --- a/legacy/serf-core/src/broadcast.rs +++ /dev/null @@ -1,79 +0,0 @@ -use async_channel::Sender; -use memberlist_core::{Broadcast, bytes::Bytes}; - -#[derive(Debug, PartialEq, Eq, Hash, Clone)] -pub(crate) struct BroadcastId; - -impl core::fmt::Display for BroadcastId { - fn fmt(&self, _f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - Ok(()) - } -} - -#[viewit::viewit] -#[derive(Debug)] -pub(crate) struct SerfBroadcast { - msg: Bytes, - notify_tx: Option>, -} - -impl Broadcast for SerfBroadcast { - type Id = BroadcastId; - type Message = Bytes; - - fn id(&self) -> Option<&Self::Id> { - None - } - - fn invalidates(&self, _other: &Self) -> bool { - false - } - - fn message(&self) -> &Self::Message { - &self.msg - } - - async fn finished(&self) { - if let Some(ref tx) = self.notify_tx { - let _ = tx.send(()).await; - } - } - - fn encoded_len(msg: &Self::Message) -> usize { - msg.len() - } -} - -#[cfg(test)] -#[tokio::test] -async fn test_broadcast_finished() { - use futures::{self, FutureExt}; - use std::time::Duration; - - let (tx, rx) = async_channel::unbounded(); - - let b = SerfBroadcast { - msg: Bytes::new(), - notify_tx: Some(tx), - }; - - b.finished().await; - - futures::select! { - _ = rx.recv().fuse() => {} - _ = tokio::time::sleep(Duration::from_millis(10)).fuse() => { - panic!("expected broadcast to be finished") - } - } -} - -#[cfg(test)] -#[tokio::test] -async fn test_broadcast_finished_no_sender() { - let b = SerfBroadcast { - msg: Bytes::new(), - notify_tx: None, - }; - - b.finished().await; -} diff --git a/legacy/serf-core/src/coalesce.rs b/legacy/serf-core/src/coalesce.rs deleted file mode 100644 index 64660b7f..00000000 --- a/legacy/serf-core/src/coalesce.rs +++ /dev/null @@ -1,155 +0,0 @@ -mod member; -pub(crate) use member::*; -mod user; -pub(crate) use user::*; - -use std::{future::Future, time::Duration}; - -use async_channel::{Receiver, Sender, bounded}; -use futures::FutureExt; -use memberlist_core::{ - agnostic_lite::RuntimeLite, - tracing, - transport::{AddressResolver, Transport}, -}; - -use crate::delegate::Delegate; - -use super::event::CrateEvent; - -pub(crate) struct ClosedOutChannel; - -pub(crate) trait Coalescer: Send + Sync + 'static { - type Delegate: Delegate< - Id = ::Id, - Address = <::Resolver as AddressResolver>::ResolvedAddress, - >; - type Transport: Transport; - - fn name(&self) -> &'static str; - - fn handle(&self, event: &CrateEvent) -> bool; - - /// Invoked to coalesce the given event - fn coalesce(&mut self, event: CrateEvent); - - /// Invoked to flush the coalesced events - fn flush( - &mut self, - out_tx: &Sender>, - ) -> impl Future> + Send; -} - -/// Returns an event channel where the events are coalesced -/// using the given coalescer. -pub(crate) fn coalesced_event( - out_tx: Sender>, - shutdown_rx: Receiver<()>, - c_period: Duration, - q_period: Duration, - c: C, -) -> Sender> { - let (in_tx, in_rx) = bounded(1024); - <::Runtime as RuntimeLite>::spawn_detach(coalesce_loop::( - in_rx, - out_tx, - shutdown_rx, - c_period, - q_period, - c, - )); - in_tx -} - -/// A simple long-running routine that manages the high-level -/// flow of coalescing based on quiescence and a maximum quantum period. -async fn coalesce_loop( - in_rx: Receiver>, - out_tx: Sender>, - shutdown_rx: Receiver<()>, - coalesce_peirod: Duration, - quiescent_period: Duration, - mut c: C, -) { - let mut quiescent = None; - let mut quantum = None; - let mut shutdown = false; - - loop { - futures::select! { - ev = in_rx.recv().fuse() => { - let Ok(ev) = ev else { - // if we receive an error, it means the channel is closed. We should return - return; - }; - - // Ignore any non handled events - if !c.handle(&ev) { - if let Err(e) = out_tx.send(ev).await { - tracing::error!(err=%e, "serf: fail send event to out channel in {} coalesce thread", c.name()); - return; - } - continue; - } - - // Start a new quantum if we need to - // and restart the quiescent timer - if quantum.is_none() { - quantum = Some(<::Runtime as RuntimeLite>::sleep(coalesce_peirod)); - } - quiescent = Some(<::Runtime as RuntimeLite>::sleep(quiescent_period)); - - // Coalesce the event - c.coalesce(ev); - } - _ = async { - if let Some(quantum) = quantum.take() { - quantum.await; - - } else { - std::future::pending::<()>().await; - } - }.fuse() => { - // Flush the coalesced events - if c.flush(&out_tx).await.is_err() { - tracing::error!(err="closed channel", "serf: fail send event to out channel in {} coalesce thread", c.name()); - return; - } - - // Restart ingestion if we are not done - if !shutdown { - quiescent = None; - quantum = None; - continue; - } - - return; - } - _ = async { - if let Some(quiescent) = quiescent.take() { - quiescent.await; - } else { - std::future::pending::<()>().await; - } - }.fuse() => { - // Flush the coalesced events - if c.flush(&out_tx).await.is_err() { - tracing::error!(err="closed channel", "serf: fail send event to out channel in {} coalesce thread", c.name()); - return; - } - - // Restart ingestion if we are not done - if !shutdown { - quantum = None; - quiescent = None; - continue; - } - - return; - } - _ = shutdown_rx.recv().fuse() => { - shutdown = true; - } - } - } -} diff --git a/legacy/serf-core/src/coalesce/member.rs b/legacy/serf-core/src/coalesce/member.rs deleted file mode 100644 index 495bd790..00000000 --- a/legacy/serf-core/src/coalesce/member.rs +++ /dev/null @@ -1,392 +0,0 @@ -use std::{collections::HashMap, marker::PhantomData}; - -use async_channel::Sender; -use memberlist_core::{ - CheapClone, - proto::TinyVec, - transport::{Node, Transport}, -}; - -use crate::{ - delegate::Delegate, - event::{CrateEvent, MemberEventMut, MemberEventType}, - types::Member, -}; - -use super::Coalescer; - -pub(crate) struct CoalesceEvent { - pub(super) ty: MemberEventType, - member: Member, -} - -#[derive(Default)] -pub(crate) struct MemberEventCoalescer { - last_events: HashMap, MemberEventType>, - latest_events: HashMap, CoalesceEvent>, - _m: PhantomData, -} - -impl MemberEventCoalescer { - pub(crate) fn new() -> Self { - Self { - last_events: HashMap::new(), - latest_events: HashMap::new(), - _m: PhantomData, - } - } -} - -impl Coalescer for MemberEventCoalescer -where - D: Delegate, - T: Transport, -{ - type Delegate = D; - type Transport = T; - - fn name(&self) -> &'static str { - "member_event_coalescer" - } - - fn handle(&self, event: &CrateEvent) -> bool { - matches!(event, CrateEvent::Member(_)) - } - - fn coalesce(&mut self, event: CrateEvent) { - let CrateEvent::Member(event) = event else { - unreachable!(); - }; - - let (ty, members) = event.into(); - for member in members.iter() { - self.latest_events.insert( - member.node().cheap_clone(), - CoalesceEvent { - ty, - member: member.clone(), - }, - ); - } - } - - async fn flush( - &mut self, - out_tx: &Sender>, - ) -> Result<(), super::ClosedOutChannel> { - let mut events: HashMap> = - HashMap::with_capacity(self.latest_events.len()); - // Coalesce the various events we got into a single set of events. - for (id, cev) in self.latest_events.drain() { - match self.last_events.get(&id) { - Some(&previous) if previous == cev.ty && cev.ty != MemberEventType::Update => { - continue; - } - Some(_) | None => { - // Update our last event - self.last_events.insert(id, cev.ty); - - // Add it to our event - match events.entry(cev.ty) { - std::collections::hash_map::Entry::Occupied(mut ent) => { - ent.get_mut().members.push(cev.member); - } - std::collections::hash_map::Entry::Vacant(ent) => { - ent.insert(MemberEventMut { - ty: cev.ty, - members: TinyVec::from(cev.member), - }); - } - } - } - } - } - - // Send out those events - for event in events.into_values() { - if out_tx.send(CrateEvent::from(event.freeze())).await.is_err() { - return Err(super::ClosedOutChannel); - } - } - Ok(()) - } -} - -#[cfg(all(test, feature = "test"))] -#[allow(clippy::collapsible_match)] -mod tests { - use std::time::Duration; - - use crate::types::{MemberStatus, UserEventMessage}; - use futures::FutureExt; - use memberlist_core::{ - agnostic_lite::{RuntimeLite, tokio::TokioRuntime}, - transport::{resolver::socket_addr::SocketAddrResolver, unimplemented::UnimplementedTransport}, - }; - use smol_str::SmolStr; - - use crate::{ - DefaultDelegate, - coalesce::coalesced_event, - event::{CrateEventType, MemberEvent}, - }; - - use super::*; - - type Transport = UnimplementedTransport, TokioRuntime>; - - type Delegate = DefaultDelegate; - - #[tokio::test] - async fn test_member_event_coealesce_basic() { - let (tx, rx) = async_channel::unbounded(); - let (_shutdown_tx, shutdown_rx) = async_channel::bounded(1); - let coalescer = MemberEventCoalescer::::new(); - - let in_ = coalesced_event( - tx, - shutdown_rx, - Duration::from_millis(20), - Duration::from_millis(20), - coalescer, - ); - - let send = vec![ - MemberEvent { - ty: MemberEventType::Join, - members: TinyVec::from(Member::new( - Node::new("foo".into(), "127.0.0.1:8080".parse().unwrap()), - Default::default(), - MemberStatus::None, - )) - .into(), - }, - MemberEvent { - ty: MemberEventType::Leave, - members: TinyVec::from(Member::new( - Node::new("foo".into(), "127.0.0.1:8080".parse().unwrap()), - Default::default(), - MemberStatus::None, - )) - .into(), - }, - MemberEvent { - ty: MemberEventType::Leave, - members: TinyVec::from(Member::new( - Node::new("bar".into(), "127.0.0.1:8080".parse().unwrap()), - Default::default(), - MemberStatus::None, - )) - .into(), - }, - MemberEvent { - ty: MemberEventType::Update, - members: TinyVec::from(Member::new( - Node::new("zip".into(), "127.0.0.1:8080".parse().unwrap()), - [("role", "foo")].into_iter().collect(), - MemberStatus::None, - )) - .into(), - }, - MemberEvent { - ty: MemberEventType::Update, - members: TinyVec::from(Member::new( - Node::new("zip".into(), "127.0.0.1:8080".parse().unwrap()), - [("role", "bar")].into_iter().collect(), - MemberStatus::None, - )) - .into(), - }, - MemberEvent { - ty: MemberEventType::Reap, - members: TinyVec::from(Member::new( - Node::new("dead".into(), "127.0.0.1:8080".parse().unwrap()), - Default::default(), - MemberStatus::None, - )) - .into(), - }, - ]; - - for event in send { - in_.send(CrateEvent::from(event)).await.unwrap(); - } - - let mut events = HashMap::new(); - let timeout = TokioRuntime::sleep(Duration::from_millis(40)); - futures::pin_mut!(timeout); - loop { - futures::select! { - e = rx.recv().fuse() => { - let e = e.unwrap(); - events.insert(e.ty(), e.clone()); - } - _ = (&mut timeout).fuse() => { - break; - }, - } - } - - assert_eq!(events.len(), 3); - - match events.get(&CrateEventType::Member(MemberEventType::Leave)) { - None => panic!(""), - Some(e) => match e { - CrateEvent::Member(MemberEvent { members, .. }) => { - assert_eq!(members.len(), 2); - - let expected = ["bar", "foo"]; - let mut names = [members[0].node.id().clone(), members[1].node.id().clone()]; - names.sort(); - - assert_eq!(names, expected); - } - _ => panic!(""), - }, - } - - match events.get(&CrateEventType::Member(MemberEventType::Update)) { - None => panic!(""), - Some(e) => match e { - CrateEvent::Member(MemberEvent { members, .. }) => { - assert_eq!(members.len(), 1); - assert_eq!(members[0].node.id(), "zip"); - assert_eq!(members[0].tags().get("role").unwrap(), "bar"); - } - _ => panic!(""), - }, - } - - match events.get(&CrateEventType::Member(MemberEventType::Reap)) { - None => panic!(""), - Some(e) => match e { - CrateEvent::Member(MemberEvent { members, .. }) => { - assert_eq!(members.len(), 1); - assert_eq!(members[0].node.id(), "dead"); - } - _ => panic!(""), - }, - } - } - - #[tokio::test] - async fn test_member_event_coalesce_tag_update() { - let (tx, rx) = async_channel::unbounded(); - let (_shutdown_tx, shutdown_rx) = async_channel::bounded(1); - let coalescer = MemberEventCoalescer::::new(); - - let in_ = coalesced_event( - tx, - shutdown_rx, - Duration::from_millis(5), - Duration::from_millis(5), - coalescer, - ); - - in_ - .send(CrateEvent::from(MemberEvent { - ty: MemberEventType::Update, - members: TinyVec::from(Member::new( - Node::new("foo".into(), "127.0.0.1:8080".parse().unwrap()), - [("role", "foo")].into_iter().collect(), - MemberStatus::None, - )) - .into(), - })) - .await - .unwrap(); - - TokioRuntime::sleep(Duration::from_millis(30)).await; - - futures::select! { - e = rx.recv().fuse() => { - let e = e.unwrap(); - - match e { - CrateEvent::Member(MemberEvent { ty, .. }) => { - assert!(matches!(ty, MemberEventType::Update)); - } - _ => panic!("expected update"), - } - } - default => panic!("expected update"), - } - - // Second update should not be suppressed even though - // last event was an update - in_ - .send(CrateEvent::from(MemberEvent { - ty: MemberEventType::Update, - members: TinyVec::from(Member::new( - Node::new("foo".into(), "127.0.0.1:8080".parse().unwrap()), - [("role", "bar")].into_iter().collect(), - MemberStatus::None, - )) - .into(), - })) - .await - .unwrap(); - TokioRuntime::sleep(Duration::from_millis(10)).await; - - futures::select! { - e = rx.recv().fuse() => { - let e = e.unwrap(); - - match e { - CrateEvent::Member(MemberEvent { ty, .. }) => { - assert!(matches!(ty, MemberEventType::Update)); - } - _ => panic!("expected update"), - } - } - default => panic!("expected update"), - } - } - - #[test] - fn test_member_event_coalesce_pass_through() { - let cases = [ - (CrateEvent::from(UserEventMessage::default()), false), - ( - CrateEvent::from(MemberEvent { - ty: MemberEventType::Join, - members: TinyVec::new().into(), - }), - true, - ), - ( - CrateEvent::from(MemberEvent { - ty: MemberEventType::Leave, - members: TinyVec::new().into(), - }), - true, - ), - ( - CrateEvent::from(MemberEvent { - ty: MemberEventType::Failed, - members: TinyVec::new().into(), - }), - true, - ), - ( - CrateEvent::from(MemberEvent { - ty: MemberEventType::Update, - members: TinyVec::new().into(), - }), - true, - ), - ( - CrateEvent::from(MemberEvent { - ty: MemberEventType::Reap, - members: TinyVec::new().into(), - }), - true, - ), - ]; - - for (event, handle) in cases.iter() { - let coalescer = MemberEventCoalescer::::new(); - assert_eq!(coalescer.handle(event), *handle); - } - } -} diff --git a/legacy/serf-core/src/coalesce/user.rs b/legacy/serf-core/src/coalesce/user.rs deleted file mode 100644 index 2d1e9b79..00000000 --- a/legacy/serf-core/src/coalesce/user.rs +++ /dev/null @@ -1,232 +0,0 @@ -use std::marker::PhantomData; - -use crate::types::UserEventMessage; -use indexmap::IndexMap; -use memberlist_core::proto::TinyVec; -use smol_str::SmolStr; - -use crate::types::LamportTime; - -use super::*; - -struct LatestUserEvents { - ltime: LamportTime, - events: TinyVec, -} - -#[derive(Default)] -#[repr(transparent)] -pub(crate) struct UserEventCoalescer { - events: IndexMap, - _m: PhantomData<(D, T)>, -} - -impl UserEventCoalescer { - pub(crate) fn new() -> Self { - Self { - events: IndexMap::new(), - _m: PhantomData, - } - } -} - -impl Coalescer for UserEventCoalescer -where - D: Delegate, - T: Transport, -{ - type Delegate = D; - type Transport = T; - - fn name(&self) -> &'static str { - "user_event_coalescer" - } - - fn handle(&self, event: &CrateEvent) -> bool { - match event { - CrateEvent::User(e) => e.cc(), - _ => false, - } - } - - fn coalesce(&mut self, event: CrateEvent) { - let event = match event { - CrateEvent::User(e) => e.clone(), - _ => unreachable!(), - }; - - let ltime = event.ltime(); - match self.events.get_mut(event.name()) { - None => { - self.events.insert( - event.name().clone(), - LatestUserEvents { - ltime, - events: TinyVec::from(event), - }, - ); - } - Some(latest) => { - if latest.ltime < ltime { - latest.events.clear(); - latest.ltime = ltime; - latest.events.push(event); - return; - } - - // If the the same age, save it - if latest.ltime == ltime { - latest.events.push(event); - } - } - } - } - - async fn flush( - &mut self, - out_tx: &Sender>, - ) -> Result<(), super::ClosedOutChannel> { - for (_, latest) in self.events.drain(..) { - for event in latest.events { - if out_tx.send(CrateEvent::from(event)).await.is_err() { - return Err(super::ClosedOutChannel); - } - } - } - Ok(()) - } -} - -#[cfg(all(test, feature = "test"))] -mod tests { - use agnostic_lite::tokio::TokioRuntime; - use memberlist_core::transport::{ - resolver::socket_addr::SocketAddrResolver, unimplemented::UnimplementedTransport, - }; - - use crate::{ - DefaultDelegate, - event::{MemberEvent, MemberEventType}, - }; - - use super::*; - - type Transport = UnimplementedTransport, TokioRuntime>; - - type Delegate = DefaultDelegate; - - #[tokio::test] - async fn test_user_event_coalesce_basic() { - let (tx, rx) = async_channel::unbounded(); - let (_shutdown_tx, shutdown_rx) = async_channel::bounded(1); - let coalescer = UserEventCoalescer::::new(); - - let in_ = coalesced_event( - tx, - shutdown_rx, - Duration::from_millis(20), - Duration::from_millis(20), - coalescer, - ); - - let send = vec![ - UserEventMessage::default() - .with_name("foo".into()) - .with_cc(true) - .with_ltime(1.into()), - UserEventMessage::default() - .with_name("foo".into()) - .with_cc(true) - .with_ltime(2.into()), - UserEventMessage::default() - .with_name("bar".into()) - .with_cc(true) - .with_ltime(2.into()) - .with_payload("test1".into()), - UserEventMessage::default() - .with_name("bar".into()) - .with_cc(true) - .with_ltime(2.into()) - .with_payload("test2".into()), - ]; - - for event in send { - in_.send(CrateEvent::from(event)).await.unwrap(); - } - - let mut got_foo = false; - let mut got_bar1 = false; - let mut got_bar2 = false; - - loop { - futures::select! { - _ = TokioRuntime::sleep(Duration::from_millis(40)).fuse() => break, - event = rx.recv().fuse() => { - let event = event.unwrap(); - match event { - CrateEvent::User(e) => { - match e.name().as_str() { - "foo" => { - assert_eq!(e.ltime(), 2.into(), "bad ltime for foo"); - got_foo = true; - } - "bar" => { - assert_eq!(e.ltime(), 2.into(), "bad ltime for bar"); - if e.payload().eq("test1".as_bytes()) { - got_bar1 = true; - } - - if e.payload().eq("test2".as_bytes()) { - got_bar2 = true; - } - } - _ => unreachable!(), - } - } - _ => unreachable!(), - } - } - } - } - - assert!(got_foo && got_bar1 && got_bar2, "missing events"); - } - - #[test] - fn test_user_event_coalesce_pass_through() { - let cases = [ - (CrateEvent::from(UserEventMessage::default()), false), - ( - CrateEvent::from(UserEventMessage::default().with_cc(true)), - true, - ), - ( - CrateEvent::from(MemberEvent { - ty: MemberEventType::Join, - members: TinyVec::new().into(), - }), - false, - ), - ( - CrateEvent::from(MemberEvent { - ty: MemberEventType::Leave, - members: TinyVec::new().into(), - }), - false, - ), - ( - CrateEvent::from(MemberEvent { - ty: MemberEventType::Failed, - members: TinyVec::new().into(), - }), - false, - ), - ]; - - let coalescer = UserEventCoalescer::::new(); - - for (idx, (event, should_coalesce)) in cases.iter().enumerate() { - assert_eq!(coalescer.handle(event), *should_coalesce, "bad: {idx}"); - } - } -} diff --git a/legacy/serf-core/src/delegate.rs b/legacy/serf-core/src/delegate.rs deleted file mode 100644 index 563ed2bb..00000000 --- a/legacy/serf-core/src/delegate.rs +++ /dev/null @@ -1,23 +0,0 @@ -use memberlist_core::{CheapClone, transport::Id}; - -mod merge; -pub use merge::*; - -mod reconnect; -pub use reconnect::*; - -mod composite; -pub use composite::*; - -/// [`Delegate`] is the trait that clients must implement if they want to hook -/// into the gossip layer of [`Serf`](crate::Serf). All the methods must be thread-safe, -/// as they can and generally will be called concurrently. -pub trait Delegate: - MergeDelegate::Id, Address = ::Address> - + ReconnectDelegate::Id, Address = ::Address> -{ - /// The id type of the delegate - type Id: Id; - /// The address type of the delegate - type Address: CheapClone + Send + Sync + 'static; -} diff --git a/legacy/serf-core/src/delegate/composite.rs b/legacy/serf-core/src/delegate/composite.rs deleted file mode 100644 index 9d74bc0e..00000000 --- a/legacy/serf-core/src/delegate/composite.rs +++ /dev/null @@ -1,114 +0,0 @@ -use memberlist_core::{CheapClone, transport::Id}; - -use crate::types::Member; - -use super::{ - DefaultMergeDelegate, Delegate, MergeDelegate, NoopReconnectDelegate, ReconnectDelegate, -}; - -use std::sync::Arc; - -/// `CompositeDelegate` is a helpful struct to split the [`Delegate`] into multiple small delegates, -/// so that users do not need to implement full [`Delegate`] when they only want to custom some methods -/// in the [`Delegate`]. -pub struct CompositeDelegate, R = NoopReconnectDelegate> -{ - merge: M, - reconnect: R, - _m: std::marker::PhantomData<(I, A)>, -} - -impl Default for CompositeDelegate { - fn default() -> Self { - Self::new() - } -} - -impl CompositeDelegate { - /// Returns a new `CompositeDelegate`. - pub fn new() -> Self { - Self { - merge: Default::default(), - reconnect: Default::default(), - _m: std::marker::PhantomData, - } - } -} - -impl CompositeDelegate -where - M: MergeDelegate, -{ - /// Set the [`MergeDelegate`] for the `CompositeDelegate`. - pub fn with_merge_delegate(self, merge: NM) -> CompositeDelegate { - CompositeDelegate { - merge, - reconnect: self.reconnect, - _m: std::marker::PhantomData, - } - } -} - -impl CompositeDelegate { - /// Set the [`ReconnectDelegate`] for the `CompositeDelegate`. - pub fn with_reconnect_delegate(self, reconnect: NR) -> CompositeDelegate { - CompositeDelegate { - reconnect, - merge: self.merge, - _m: std::marker::PhantomData, - } - } -} - -impl MergeDelegate for CompositeDelegate -where - I: Id + Send + Sync + 'static, - A: CheapClone + Send + Sync + 'static, - M: MergeDelegate, - R: Send + Sync + 'static, -{ - type Error = M::Error; - - type Id = M::Id; - - type Address = M::Address; - - async fn notify_merge( - &self, - members: Arc<[Member]>, - ) -> Result<(), Self::Error> { - self.merge.notify_merge(members).await - } -} - -impl ReconnectDelegate for CompositeDelegate -where - I: Id + Send + Sync + 'static, - A: CheapClone + Send + Sync + 'static, - M: Send + Sync + 'static, - R: ReconnectDelegate, -{ - type Id = R::Id; - - type Address = R::Address; - - fn reconnect_timeout( - &self, - member: &Member, - timeout: std::time::Duration, - ) -> std::time::Duration { - self.reconnect.reconnect_timeout(member, timeout) - } -} - -impl Delegate for CompositeDelegate -where - I: Id + Send + Sync + 'static, - A: CheapClone + Send + Sync + 'static, - M: MergeDelegate, - R: ReconnectDelegate, -{ - type Id = I; - - type Address = A; -} diff --git a/legacy/serf-core/src/delegate/merge.rs b/legacy/serf-core/src/delegate/merge.rs deleted file mode 100644 index 7190ffa1..00000000 --- a/legacy/serf-core/src/delegate/merge.rs +++ /dev/null @@ -1,55 +0,0 @@ -use memberlist_core::{CheapClone, transport::Id}; -use std::{future::Future, sync::Arc}; - -use crate::types::Member; - -/// Used to involve a client in -/// a potential cluster merge operation. Namely, when -/// a node does a promised push/pull (as part of a join), -/// the delegate is involved and allowed to cancel the join -/// based on custom logic. The merge delegate is NOT invoked -/// as part of the push-pull anti-entropy. -#[auto_impl::auto_impl(Box, Arc)] -pub trait MergeDelegate: Send + Sync + 'static { - /// The error type of the delegate - type Error: std::error::Error + Send + Sync + 'static; - /// The id type of the delegate - type Id: Id; - /// The address type of the delegate - type Address: CheapClone + Send + Sync + 'static; - - /// Invoked when a merge could take place. - /// Provides a list of the nodes known by the peer. If - /// the return value is `Err`, the merge is canceled. - fn notify_merge( - &self, - members: Arc<[Member]>, - ) -> impl Future> + Send; -} - -/// A default implementation of the `MergeDelegate` trait. -#[derive(Debug, Clone, Copy)] -pub struct DefaultMergeDelegate(std::marker::PhantomData<(I, A)>); - -impl Default for DefaultMergeDelegate { - fn default() -> Self { - Self(Default::default()) - } -} - -impl MergeDelegate for DefaultMergeDelegate -where - I: Id + Send + Sync + 'static, - A: CheapClone + Send + Sync + 'static, -{ - type Error = std::convert::Infallible; - type Id = I; - type Address = A; - - async fn notify_merge( - &self, - _members: Arc<[Member]>, - ) -> Result<(), Self::Error> { - Ok(()) - } -} diff --git a/legacy/serf-core/src/delegate/reconnect.rs b/legacy/serf-core/src/delegate/reconnect.rs deleted file mode 100644 index 98b34765..00000000 --- a/legacy/serf-core/src/delegate/reconnect.rs +++ /dev/null @@ -1,56 +0,0 @@ -use std::time::Duration; - -use memberlist_core::{CheapClone, transport::Id}; - -use crate::types::Member; - -/// Implemented to allow overriding the reconnect timeout for individual members. -#[auto_impl::auto_impl(Box, Arc)] -pub trait ReconnectDelegate: Send + Sync + 'static { - /// The id type of the delegate - type Id: Id; - /// The address type of the delegate - type Address: CheapClone + Send + Sync + 'static; - - /// Returns the reconnect timeout for the given member. - fn reconnect_timeout( - &self, - member: &Member, - timeout: Duration, - ) -> Duration; -} - -/// Noop implementation of `ReconnectDelegate`. -#[derive(Debug)] -pub struct NoopReconnectDelegate(std::marker::PhantomData<(I, A)>); - -impl Default for NoopReconnectDelegate { - fn default() -> Self { - Self(Default::default()) - } -} - -impl Clone for NoopReconnectDelegate { - fn clone(&self) -> Self { - *self - } -} - -impl Copy for NoopReconnectDelegate {} - -impl ReconnectDelegate for NoopReconnectDelegate -where - I: Id + Send + Sync + 'static, - A: CheapClone + Send + Sync + 'static, -{ - type Id = I; - type Address = A; - - fn reconnect_timeout( - &self, - _member: &Member, - timeout: Duration, - ) -> Duration { - timeout - } -} diff --git a/legacy/serf-core/src/error.rs b/legacy/serf-core/src/error.rs deleted file mode 100644 index 8de090c2..00000000 --- a/legacy/serf-core/src/error.rs +++ /dev/null @@ -1,390 +0,0 @@ -use std::sync::Arc; - -use memberlist_core::{ - delegate::DelegateError as MemberlistDelegateError, proto::TinyVec, transport::Transport, -}; - -use crate::{ - delegate::{Delegate, MergeDelegate}, - serf::{SerfDelegate, SerfState}, - types::Member, -}; - -pub use crate::snapshot::SnapshotError; - -/// Error trait for [`Delegate`] -#[derive(thiserror::Error)] -pub enum SerfDelegateError { - /// Serf error - #[error(transparent)] - Serf(#[from] SerfError), - /// [`MergeDelegate`] error - #[error(transparent)] - Merge(::Error), -} - -impl core::fmt::Debug for SerfDelegateError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Merge(err) => write!(f, "{err:?}"), - Self::Serf(err) => write!(f, "{err:?}"), - } - } -} - -impl SerfDelegateError { - /// Create a delegate error from a merge delegate error. - #[inline] - pub const fn merge(err: ::Error) -> Self { - Self::Merge(err) - } - - /// Create a delegate error from a serf error. - #[inline] - pub const fn serf(err: crate::error::SerfError) -> Self { - Self::Serf(err) - } -} - -impl From>> for SerfDelegateError -where - D: Delegate, - T: Transport, -{ - fn from(value: MemberlistDelegateError>) -> Self { - match value { - MemberlistDelegateError::AliveDelegate(e) => e, - MemberlistDelegateError::MergeDelegate(e) => e, - } - } -} - -/// Error type for the serf crate. -#[derive(thiserror::Error)] -pub enum Error -where - D: Delegate, - T: Transport, -{ - /// Returned when the underlyhing memberlist error - #[error(transparent)] - Memberlist(#[from] memberlist_core::error::Error>), - /// Returned when the serf error - #[error(transparent)] - Serf(#[from] SerfError), - /// Returned when the relay error - #[error(transparent)] - Relay(#[from] RelayError), - /// Multiple errors - #[error("errors:\n{}", format_multiple_errors(.0))] - Multiple(Arc<[Self]>), -} - -impl core::fmt::Debug for Error -where - D: Delegate, - T: Transport, -{ - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - Self::Memberlist(e) => write!(f, "{e:?}"), - Self::Serf(e) => write!(f, "{e:?}"), - Self::Relay(e) => write!(f, "{e:?}"), - Self::Multiple(e) => write!(f, "{e:?}"), - } - } -} - -impl From for Error -where - D: Delegate, - T: Transport, -{ - fn from(value: SnapshotError) -> Self { - Self::Serf(SerfError::Snapshot(value)) - } -} - -impl From for Error -where - D: Delegate, - T: Transport, -{ - fn from(e: memberlist_core::proto::EncodeError) -> Self { - Self::Serf(e.into()) - } -} - -impl From for Error -where - D: Delegate, - T: Transport, -{ - fn from(e: memberlist_core::proto::DecodeError) -> Self { - Self::Serf(e.into()) - } -} - -impl Error -where - D: Delegate, - T: Transport, -{ - /// Create a query response too large error - #[inline] - pub const fn query_response_too_large(limit: usize, got: usize) -> Self { - Self::Serf(SerfError::QueryResponseTooLarge { limit, got }) - } - - /// Create a query timeout error - #[inline] - pub const fn query_timeout() -> Self { - Self::Serf(SerfError::QueryTimeout) - } - - /// Create a query already response error - #[inline] - pub const fn query_already_responsed() -> Self { - Self::Serf(SerfError::QueryAlreadyResponsed) - } - - /// Create a query response delivery failed error - #[inline] - pub const fn query_response_delivery_failed() -> Self { - Self::Serf(SerfError::QueryResponseDeliveryFailed) - } - - /// Create a relayed response too large error - #[inline] - pub const fn relayed_response_too_large(size: usize) -> Self { - Self::Serf(SerfError::RelayedResponseTooLarge(size)) - } - - /// Create a relay error - #[inline] - pub const fn relay(err: RelayError) -> Self { - Self::Relay(err) - } - - /// Create a fail truncate response error - #[inline] - pub const fn fail_truncate_response() -> Self { - Self::Serf(SerfError::FailTruncateResponse) - } - - /// Create a tags too large error - #[inline] - pub const fn tags_too_large(size: usize) -> Self { - Self::Serf(SerfError::TagsTooLarge(size)) - } - - /// Create a query too large error - #[inline] - pub const fn query_too_large(size: usize) -> Self { - Self::Serf(SerfError::QueryTooLarge(size)) - } - - /// Create a user event limit too large error - #[inline] - pub const fn user_event_limit_too_large(size: usize) -> Self { - Self::Serf(SerfError::UserEventLimitTooLarge(size)) - } - - /// Create a user event limit too large error - #[inline] - pub const fn user_event_too_large(size: usize) -> Self { - Self::Serf(SerfError::UserEventTooLarge(size)) - } - - /// Create a raw user event too large error - #[inline] - pub const fn raw_user_event_too_large(size: usize) -> Self { - Self::Serf(SerfError::RawUserEventTooLarge(size)) - } - - /// Create a broadcast channel closed error - #[inline] - pub const fn broadcast_channel_closed() -> Self { - Self::Serf(SerfError::BroadcastChannelClosed) - } - - /// Create a removal broadcast timeout error - #[inline] - pub const fn removal_broadcast_timeout() -> Self { - Self::Serf(SerfError::RemovalBroadcastTimeout) - } - - /// Create a snapshot error - #[inline] - pub const fn snapshot(err: SnapshotError) -> Self { - Self::Serf(SerfError::Snapshot(err)) - } - - /// Create a bad leave status error - #[inline] - pub const fn bad_leave_status(status: SerfState) -> Self { - Self::Serf(SerfError::BadLeaveStatus(status)) - } - - /// Create a bad join status error - #[inline] - pub const fn bad_join_status(status: SerfState) -> Self { - Self::Serf(SerfError::BadJoinStatus(status)) - } - - /// Create a coordinates disabled error - #[inline] - pub const fn coordinates_disabled() -> Self { - Self::Serf(SerfError::CoordinatesDisabled) - } -} - -/// [`Serf`](crate::Serf) error. -#[derive(Debug, thiserror::Error)] -pub enum SerfError { - /// Returned when the user event exceeds the configured limit. - #[error("user event exceeds configured limit of {0} bytes before encoding")] - UserEventLimitTooLarge(usize), - /// Returned when the user event exceeds the sane limit. - #[error("user event exceeds sane limit of {0} bytes before encoding")] - UserEventTooLarge(usize), - /// Returned when the join status is bad. - #[error("join called on {0} statues")] - BadJoinStatus(SerfState), - /// Returned when the leave status is bad. - #[error("leave called on {0} statues")] - BadLeaveStatus(SerfState), - /// Returned when the encoded user event exceeds the sane limit after encoding. - #[error("user event exceeds sane limit of {0} bytes after encoding")] - RawUserEventTooLarge(usize), - /// Returned when the query size exceeds the configured limit. - #[error("query exceeds limit of {0} bytes")] - QueryTooLarge(usize), - /// Returned when the query is timeout. - #[error("query response is past the deadline")] - QueryTimeout, - /// Returned when the query response is too large. - #[error("query response ({got} bytes) exceeds limit of {limit} bytes")] - QueryResponseTooLarge { - /// The query response size limit. - limit: usize, - /// The query response size. - got: usize, - }, - /// Returned when the query has already been responded. - #[error("query response already sent")] - QueryAlreadyResponsed, - /// Returned when failed to truncate response so that it fits into message. - #[error("failed to truncate response so that it fits into message")] - FailTruncateResponse, - /// Returned when the tags too large. - #[error("encoded length of tags exceeds limit of {0} bytes")] - TagsTooLarge(usize), - /// Returned when the relayed response is too large. - #[error("relayed response exceeds limit of {0} bytes")] - RelayedResponseTooLarge(usize), - /// Returned when failed to deliver query response, dropping. - #[error("failed to deliver query response, dropping")] - QueryResponseDeliveryFailed, - /// Returned when the coordinates are disabled. - #[error("coordinates are disabled")] - CoordinatesDisabled, - /// Returned when snapshot error. - #[error(transparent)] - Snapshot(#[from] SnapshotError), - /// Returned when trying to decode a serf data - #[error(transparent)] - Decode(#[from] memberlist_core::proto::DecodeError), - /// Returned when trying to encode a serf data - #[error(transparent)] - Encode(#[from] memberlist_core::proto::EncodeError), - /// Returned when timed out broadcasting node removal. - #[error("timed out broadcasting node removal")] - RemovalBroadcastTimeout, - /// Returned when the timed out broadcasting channel closed. - #[error("timed out broadcasting channel closed")] - BroadcastChannelClosed, -} - -/// Relay error from remote nodes. -pub struct RelayError( - #[allow(clippy::type_complexity)] - TinyVec<( - Member, - memberlist_core::error::Error>, - )>, -) -where - D: Delegate, - T: Transport; - -impl - From< - TinyVec<( - Member, - memberlist_core::error::Error>, - )>, - > for RelayError -where - D: Delegate, - T: Transport, -{ - fn from( - value: TinyVec<( - Member, - memberlist_core::error::Error>, - )>, - ) -> Self { - Self(value) - } -} - -impl core::fmt::Display for RelayError -where - D: Delegate, - T: Transport, -{ - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - writeln!(f, "relay errors:")?; - - for (member, err) in self.0.iter() { - writeln!( - f, - "\tfailed to send relay response to {}: {}", - member.node().id(), - err - )?; - } - Ok(()) - } -} - -impl core::fmt::Debug for RelayError -where - D: Delegate, - T: Transport, -{ - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - core::fmt::Display::fmt(self, f) - } -} - -impl std::error::Error for RelayError -where - D: Delegate, - T: Transport, -{ -} - -fn format_multiple_errors(errors: &[Error]) -> String -where - D: Delegate, - T: Transport, -{ - errors - .iter() - .enumerate() - .map(|(i, err)| format!(" {}. {}", i + 1, err)) - .collect::>() - .join("\n") -} diff --git a/legacy/serf-core/src/event.rs b/legacy/serf-core/src/event.rs deleted file mode 100644 index fa1ae2de..00000000 --- a/legacy/serf-core/src/event.rs +++ /dev/null @@ -1,512 +0,0 @@ -use std::{pin::Pin, sync::Arc, task::Poll, time::Duration}; - -use self::error::Error; - -use super::{delegate::Delegate, types::Epoch, *}; - -mod crate_event; - -use async_channel::Sender; -pub use async_channel::{RecvError, TryRecvError}; - -use crate::types::{LamportTime, Member, Node, QueryFlag, QueryResponseMessage, UserEventMessage}; -use async_lock::Mutex; -pub(crate) use crate_event::*; -use futures::Stream; -use memberlist_core::{CheapClone, bytes::Bytes, proto::TinyVec, transport::Transport}; -use smol_str::SmolStr; - -pub(crate) struct QueryContext -where - D: Delegate, - T: Transport, -{ - pub(crate) query_timeout: Duration, - pub(crate) span: Mutex>, - pub(crate) this: Serf, -} - -impl QueryContext -where - D: Delegate, - T: Transport, -{ - fn check_response_size(&self, size: usize) -> Result<(), Error> { - if size > self.this.inner.opts.query_response_size_limit { - Err(Error::query_response_too_large( - self.this.inner.opts.query_response_size_limit, - size, - )) - } else { - Ok(()) - } - } - - async fn respond_with_message_and_response( - &self, - respond_to: &T::ResolvedAddress, - relay_factor: u8, - raw: Bytes, - resp: QueryResponseMessage, - ) -> Result<(), Error> { - self.check_response_size(raw.len())?; - - let mut mu = self.span.lock().await; - - if let Some(span) = *mu { - // Ensure we aren't past our response deadline - if span.elapsed() > self.query_timeout { - return Err(Error::query_timeout()); - } - - // Send the response directly to the originator - self.this.inner.memberlist.send(respond_to, raw).await?; - - // Relay the response through up to relayFactor other nodes - self - .this - .relay_response(relay_factor, resp.from.cheap_clone(), resp) - .await?; - - // Clear the deadline, responses sent - *mu = None; - Ok(()) - } else { - Err(Error::query_already_responsed()) - } - } - - async fn respond( - &self, - respond_to: &T::ResolvedAddress, - id: u32, - ltime: LamportTime, - relay_factor: u8, - msg: Bytes, - ) -> Result<(), Error> { - let resp = QueryResponseMessage { - ltime, - id, - from: self.this.advertise_node(), - flags: QueryFlag::empty(), - payload: msg, - }; - let buf = crate::types::encode_message_to_bytes(&resp)?; - self - .respond_with_message_and_response(respond_to, relay_factor, buf, resp) - .await - } -} - -/// Query event -pub struct QueryEvent -where - D: Delegate, - T: Transport, -{ - pub(crate) ltime: LamportTime, - pub(crate) name: SmolStr, - pub(crate) payload: Bytes, - - pub(crate) ctx: Arc>, - pub(crate) id: u32, - /// source node - pub(crate) from: Node, - /// Number of duplicate responses to relay back to sender - pub(crate) relay_factor: u8, -} - -impl QueryEvent -where - D: Delegate, - T: Transport, -{ - /// Returns the lamport time of the query - #[inline] - pub const fn lamport_time(&self) -> LamportTime { - self.ltime - } - - /// Returns the name of the query - #[inline] - pub const fn name(&self) -> &SmolStr { - &self.name - } - - /// Returns the payload of the query - #[inline] - pub const fn payload(&self) -> &Bytes { - &self.payload - } - - /// Returns the id of the query - #[inline] - pub const fn id(&self) -> u32 { - self.id - } - - /// Returns the source node of the query - #[inline] - pub const fn from(&self) -> &Node { - &self.from - } -} - -impl PartialEq for QueryEvent -where - D: Delegate, - T: Transport, -{ - fn eq(&self, other: &Self) -> bool { - self.id == other.id - && self.from == other.from - && self.relay_factor == other.relay_factor - && self.ltime == other.ltime - && self.name == other.name - && self.payload == other.payload - } -} - -impl AsRef> for QueryEvent -where - D: Delegate, - T: Transport, -{ - fn as_ref(&self) -> &QueryEvent { - self - } -} - -impl Clone for QueryEvent -where - D: Delegate, - T: Transport, -{ - fn clone(&self) -> Self { - Self { - ltime: self.ltime, - name: self.name.clone(), - payload: self.payload.clone(), - ctx: self.ctx.clone(), - id: self.id, - from: self.from.clone(), - relay_factor: self.relay_factor, - } - } -} - -impl core::fmt::Display for QueryEvent -where - D: Delegate, - T: Transport, -{ - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "query") - } -} - -impl QueryEvent -where - D: Delegate, - T: Transport, -{ - #[cfg(feature = "encryption")] - pub(crate) fn create_response( - &self, - buf: Bytes, - ) -> QueryResponseMessage { - QueryResponseMessage { - ltime: self.ltime, - id: self.id, - from: self.ctx.this.inner.memberlist.advertise_node(), - flags: QueryFlag::empty(), - payload: buf, - } - } - - #[cfg(feature = "encryption")] - pub(crate) fn check_response_size(&self, size: usize) -> Result<(), Error> { - self.ctx.check_response_size(size) - } - - #[cfg(feature = "encryption")] - pub(crate) async fn respond_with_message_and_response( - &self, - raw: Bytes, - resp: QueryResponseMessage, - ) -> Result<(), Error> { - self - .ctx - .respond_with_message_and_response(self.from.address(), self.relay_factor, raw, resp) - .await - } - - /// Used to send a response to the user query - pub async fn respond(&self, msg: Bytes) -> Result<(), Error> { - self - .ctx - .respond( - self.from().address(), - self.id, - self.ltime, - self.relay_factor, - msg, - ) - .await - } -} - -/// The event type for member event -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "serde", serde(rename_all = "kebab-case", untagged))] -pub enum MemberEventType { - /// Join event - #[cfg_attr(feature = "serde", serde(rename = "member-join"))] - Join, - /// Leave event - #[cfg_attr(feature = "serde", serde(rename = "member-leave"))] - Leave, - /// Failed event - #[cfg_attr(feature = "serde", serde(rename = "member-failed"))] - Failed, - /// Update event - #[cfg_attr(feature = "serde", serde(rename = "member-update"))] - Update, - /// Reap event - #[cfg_attr(feature = "serde", serde(rename = "member-reap"))] - Reap, -} - -impl MemberEventType { - /// Returns the string representation of the event type. - #[inline] - pub const fn as_str(&self) -> &'static str { - match self { - Self::Join => "member-join", - Self::Leave => "member-leave", - Self::Failed => "member-failed", - Self::Update => "member-update", - Self::Reap => "member-reap", - } - } -} - -impl core::fmt::Display for MemberEventType { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - match self { - Self::Join => write!(f, "member-join"), - Self::Leave => write!(f, "member-leave"), - Self::Failed => write!(f, "member-failed"), - Self::Update => write!(f, "member-update"), - Self::Reap => write!(f, "member-reap"), - } - } -} - -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct MemberEventMut { - pub(crate) ty: MemberEventType, - pub(crate) members: TinyVec>, -} - -impl MemberEventMut { - pub(crate) fn freeze(self) -> MemberEvent { - MemberEvent { - ty: self.ty, - members: Arc::new(self.members), - } - } -} - -/// MemberEvent is the struct used for member related events -/// Because Serf coalesces events, an event may contain multiple members. -#[derive(Debug, PartialEq)] -pub struct MemberEvent { - pub(crate) ty: MemberEventType, - pub(crate) members: Arc>>, -} - -impl Clone for MemberEvent { - fn clone(&self) -> Self { - Self { - ty: self.ty, - members: self.members.clone(), - } - } -} - -impl CheapClone for MemberEvent {} - -impl core::fmt::Display for MemberEvent { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "{}", self.ty) - } -} - -impl MemberEvent { - /// Returns the event type of this member event - pub fn ty(&self) -> MemberEventType { - self.ty - } - - /// Returns the members of this event - pub fn members(&self) -> &[Member] { - &self.members - } -} - -impl From> for (MemberEventType, Arc>>) { - fn from(event: MemberEvent) -> Self { - (event.ty, event.members) - } -} - -/// The event produced by the Serf instance. -#[derive(derive_more::From)] -pub enum Event -where - D: Delegate, - T: Transport, -{ - /// Member related events - Member(MemberEvent), - /// User events - User(UserEventMessage), - /// Query events - Query(QueryEvent), -} - -impl Clone for Event -where - D: Delegate, - T: Transport, -{ - fn clone(&self) -> Self { - match self { - Self::Member(e) => Self::Member(e.cheap_clone()), - Self::User(e) => Self::User(e.cheap_clone()), - Self::Query(e) => Self::Query(e.clone()), - } - } -} - -/// The producer of the Serf events. -#[derive(Debug)] -pub struct EventProducer -where - D: Delegate, - T: Transport, -{ - pub(crate) tx: Sender>, -} - -impl EventProducer -where - D: Delegate, - T: Transport, -{ - /// Creates a bounded producer and subscriber. - /// - /// The created subscriber has space to hold at most cap events at a time. - /// Users must actively consume the events from the subscriber to prevent the producer from blocking. - pub fn bounded(size: usize) -> (Self, EventSubscriber) { - let (tx, rx) = async_channel::bounded(size); - (Self { tx }, EventSubscriber { rx }) - } - - /// Creates an unbounded producer and subscriber. - /// - /// The created subscriber has no limit on the number of events it can hold. - pub fn unbounded() -> (Self, EventSubscriber) { - let (tx, rx) = async_channel::unbounded(); - (Self { tx }, EventSubscriber { rx }) - } -} - -/// Subscribe the events from the Serf instance. -#[pin_project::pin_project] -#[derive(Debug)] -pub struct EventSubscriber -where - D: Delegate, - T: Transport, -{ - #[pin] - pub(crate) rx: async_channel::Receiver>, -} - -impl EventSubscriber -where - D: Delegate, - T: Transport, -{ - /// Receives a event from the subscriber. - /// - /// If the subscriber is empty, this method waits until there is a event. - /// - /// If the subscriber is closed, this method receives a event or returns an error if there are no more events - pub async fn recv(&self) -> Result, RecvError> { - loop { - match self.rx.recv().await { - Ok(CrateEvent::InternalQuery { .. }) => continue, - Ok(CrateEvent::Member(e)) => return Ok(Event::Member(e)), - Ok(CrateEvent::User(e)) => return Ok(Event::User(e)), - Ok(CrateEvent::Query(e)) => return Ok(Event::Query(e)), - Err(e) => return Err(e), - } - } - } - - /// Tries to receive a event from the subscriber. - /// - /// If the subscriber is empty, this method returns an error. - /// If the subscriber is closed, this method receives a event or returns an error if there are no more events - pub fn try_recv(&self) -> Result, TryRecvError> { - loop { - match self.rx.try_recv() { - Ok(CrateEvent::InternalQuery { .. }) => continue, - Ok(CrateEvent::Member(e)) => return Ok(Event::Member(e)), - Ok(CrateEvent::User(e)) => return Ok(Event::User(e)), - Ok(CrateEvent::Query(e)) => return Ok(Event::Query(e)), - Err(e) => return Err(e), - } - } - } - - /// Returns `true` if the subscriber is empty. - pub fn is_empty(&self) -> bool { - self.rx.is_empty() - } - - /// Returns `true` if the channel is closed. - pub fn is_closed(&self) -> bool { - self.rx.is_closed() - } - - /// Returns the number of events in the subscriber. - pub fn len(&self) -> usize { - self.rx.len() - } -} - -impl Stream for EventSubscriber -where - D: Delegate, - T: Transport, -{ - type Item = Event; - - fn poll_next(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll> { - match > as Stream>::poll_next(self.project().rx, cx) { - Poll::Ready(Some(event)) => match event { - CrateEvent::Member(e) => Poll::Ready(Some(Event::Member(e))), - CrateEvent::User(e) => Poll::Ready(Some(Event::User(e))), - CrateEvent::Query(e) => Poll::Ready(Some(Event::Query(e))), - CrateEvent::InternalQuery { .. } => Poll::Pending, - }, - Poll::Ready(None) => Poll::Ready(None), - Poll::Pending => Poll::Pending, - } - } -} diff --git a/legacy/serf-core/src/event/crate_event.rs b/legacy/serf-core/src/event/crate_event.rs deleted file mode 100644 index df450495..00000000 --- a/legacy/serf-core/src/event/crate_event.rs +++ /dev/null @@ -1,225 +0,0 @@ -use crate::types::{QueryMessage, QueryMessageRef}; -use memberlist_core::proto::{Data, DecodeError}; - -use super::*; - -pub(crate) trait QueryMessageExt { - fn decode_internal_query(&self) -> Option, DecodeError>>; -} - -impl QueryMessageExt for QueryMessage -where - I: Data, -{ - fn decode_internal_query(&self) -> Option, DecodeError>> { - Some(Ok(match self.name().as_str() { - INTERNAL_PING => InternalQueryEvent::Ping, - INTERNAL_CONFLICT => { - return Some( - ::decode(&self.payload).map(|(_, id)| InternalQueryEvent::Conflict(id)), - ); - } - #[cfg(feature = "encryption")] - INTERNAL_INSTALL_KEY => InternalQueryEvent::InstallKey, - #[cfg(feature = "encryption")] - INTERNAL_USE_KEY => InternalQueryEvent::UseKey, - #[cfg(feature = "encryption")] - INTERNAL_REMOVE_KEY => InternalQueryEvent::RemoveKey, - #[cfg(feature = "encryption")] - INTERNAL_LIST_KEYS => InternalQueryEvent::ListKey, - _ => return None, - })) - } -} - -impl<'a, I, A> QueryMessageExt for QueryMessageRef<'a, I::Ref<'a>, A> -where - I: Data, -{ - fn decode_internal_query(&self) -> Option, DecodeError>> { - Some(Ok(match self.name() { - INTERNAL_PING => InternalQueryEvent::Ping, - INTERNAL_CONFLICT => { - return Some( - ::decode(self.payload()).map(|(_, id)| InternalQueryEvent::Conflict(id)), - ); - } - #[cfg(feature = "encryption")] - INTERNAL_INSTALL_KEY => InternalQueryEvent::InstallKey, - #[cfg(feature = "encryption")] - INTERNAL_USE_KEY => InternalQueryEvent::UseKey, - #[cfg(feature = "encryption")] - INTERNAL_REMOVE_KEY => InternalQueryEvent::RemoveKey, - #[cfg(feature = "encryption")] - INTERNAL_LIST_KEYS => InternalQueryEvent::ListKey, - _ => return None, - })) - } -} - -const INTERNAL_PING: &str = "_serf_ping"; -const INTERNAL_CONFLICT: &str = "_serf_conflict"; -#[cfg(feature = "encryption")] -pub(crate) const INTERNAL_INSTALL_KEY: &str = "_serf_install_key"; -#[cfg(feature = "encryption")] -pub(crate) const INTERNAL_USE_KEY: &str = "_serf_use_key"; -#[cfg(feature = "encryption")] -pub(crate) const INTERNAL_REMOVE_KEY: &str = "_serf_remove_key"; -#[cfg(feature = "encryption")] -pub(crate) const INTERNAL_LIST_KEYS: &str = "_serf_list_keys"; - -#[cfg(feature = "test")] -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "serde", serde(rename_all = "kebab-case", untagged))] -pub enum CrateEventType { - Member(MemberEventType), - User, - Query, - InternalQuery, -} - -pub(crate) enum CrateEvent -where - D: Delegate, - T: Transport, -{ - Member(MemberEvent), - User(UserEventMessage), - Query(QueryEvent), - InternalQuery { - kind: InternalQueryEvent, - query: QueryEvent, - }, -} - -impl Clone for CrateEvent -where - D: Delegate, - T: Transport, -{ - fn clone(&self) -> Self { - match self { - Self::Member(e) => Self::Member(e.clone()), - Self::User(e) => Self::User(e.clone()), - Self::Query(e) => Self::Query(e.clone()), - Self::InternalQuery { kind, query } => Self::InternalQuery { - kind: kind.clone(), - query: query.clone(), - }, - } - } -} - -impl CrateEvent -where - D: Delegate, - T: Transport, -{ - /// Returns the type of the event - #[cfg(feature = "test")] - #[inline] - pub(crate) fn ty(&self) -> CrateEventType { - match self { - Self::Member(e) => CrateEventType::Member(e.ty), - Self::User(_) => CrateEventType::User, - Self::Query(_) => CrateEventType::Query, - Self::InternalQuery { .. } => CrateEventType::InternalQuery, - } - } - - pub(crate) fn is_internal_query(&self) -> bool { - matches!(self, Self::InternalQuery { .. }) - } -} - -impl From> for CrateEvent -where - D: Delegate, - T: Transport, -{ - fn from(value: MemberEvent) -> Self { - Self::Member(value) - } -} - -impl From for CrateEvent -where - D: Delegate, - T: Transport, -{ - fn from(value: UserEventMessage) -> Self { - Self::User(value) - } -} - -impl From> for CrateEvent -where - D: Delegate, - T: Transport, -{ - fn from(value: QueryEvent) -> Self { - Self::Query(value) - } -} - -impl From<(InternalQueryEvent, QueryEvent)> for CrateEvent -where - D: Delegate, - T: Transport, -{ - fn from(value: (InternalQueryEvent, QueryEvent)) -> Self { - Self::InternalQuery { - kind: value.0, - query: value.1, - } - } -} - -pub enum InternalQueryEvent { - Ping, - Conflict(I), - #[cfg(feature = "encryption")] - InstallKey, - #[cfg(feature = "encryption")] - UseKey, - #[cfg(feature = "encryption")] - RemoveKey, - #[cfg(feature = "encryption")] - ListKey, -} - -impl Clone for InternalQueryEvent { - fn clone(&self) -> Self { - match self { - Self::Ping => Self::Ping, - Self::Conflict(e) => Self::Conflict(e.clone()), - #[cfg(feature = "encryption")] - Self::InstallKey => Self::InstallKey, - #[cfg(feature = "encryption")] - Self::UseKey => Self::UseKey, - #[cfg(feature = "encryption")] - Self::RemoveKey => Self::RemoveKey, - #[cfg(feature = "encryption")] - Self::ListKey => Self::ListKey, - } - } -} - -impl InternalQueryEvent { - #[inline] - pub(crate) const fn as_str(&self) -> &'static str { - match self { - Self::Ping => INTERNAL_PING, - Self::Conflict(_) => INTERNAL_CONFLICT, - #[cfg(feature = "encryption")] - Self::InstallKey => INTERNAL_INSTALL_KEY, - #[cfg(feature = "encryption")] - Self::UseKey => INTERNAL_USE_KEY, - #[cfg(feature = "encryption")] - Self::RemoveKey => INTERNAL_REMOVE_KEY, - #[cfg(feature = "encryption")] - Self::ListKey => INTERNAL_LIST_KEYS, - } - } -} diff --git a/legacy/serf-core/src/key_manager.rs b/legacy/serf-core/src/key_manager.rs deleted file mode 100644 index f53183c8..00000000 --- a/legacy/serf-core/src/key_manager.rs +++ /dev/null @@ -1,348 +0,0 @@ -use std::{collections::HashMap, sync::OnceLock}; - -use crate::types::MessageRef; -use async_channel::Receiver; -use async_lock::RwLock; -use futures::StreamExt; -use memberlist_core::{CheapClone, proto::SecretKey, tracing, transport::Transport}; -use smol_str::{SmolStr, format_smolstr}; - -use crate::event::{ - INTERNAL_INSTALL_KEY, INTERNAL_LIST_KEYS, INTERNAL_REMOVE_KEY, INTERNAL_USE_KEY, - InternalQueryEvent, -}; - -use super::{ - Serf, SerfWeakRef, - delegate::Delegate, - error::Error, - serf::{NodeResponse, QueryResponse}, - types::KeyRequestMessage, -}; - -/// KeyResponse is used to relay a query for a list of all keys in use. -#[viewit::viewit( - vis_all = "pub(crate)", - getters(style = "move", vis_all = "pub"), - setters(skip) -)] -#[derive(Default, Debug)] -pub struct KeyResponse { - /// Map of node id to response message - #[viewit(getter( - const, - style = "ref", - attrs(doc = "Returns a map of node id to response message.") - ))] - messages: HashMap, - /// Total nodes memberlist knows of - #[viewit(getter(const, attrs(doc = "Returns the total nodes memberlist knows of.")))] - num_nodes: usize, - /// Total responses received - #[viewit(getter(const, attrs(doc = "Returns the total responses received.")))] - num_resp: usize, - /// Total errors from request - #[viewit(getter(const, attrs(doc = "Returns the total errors from request.")))] - num_err: usize, - - /// A mapping of the value of the key bytes to the - /// number of nodes that have the key installed. - #[viewit(getter( - const, - style = "ref", - attrs( - doc = "Returns a mapping of the value of the key bytes to the number of nodes that have the key installed.." - ) - ))] - keys: HashMap, - - /// A mapping of the value of the primary - /// key bytes to the number of nodes that have the key installed. - #[viewit(getter( - const, - style = "ref", - attrs( - doc = "Returns a mapping of the value of the primary key bytes to the number of nodes that have the key installed." - ) - ))] - primary_keys: HashMap, -} - -/// KeyRequestOptions is used to contain optional parameters for a keyring operation -pub struct KeyRequestOptions { - /// The number of duplicate query responses to send by relaying through - /// other nodes, for redundancy - pub relay_factor: u8, -} - -/// `KeyManager` encapsulates all functionality within Serf for handling -/// encryption keyring changes across a cluster. -pub struct KeyManager -where - D: Delegate, - T: Transport, -{ - serf: OnceLock>, - /// The lock is used to serialize keys related handlers - l: RwLock<()>, -} - -impl KeyManager -where - D: Delegate, - T: Transport, -{ - pub(crate) fn new() -> Self { - Self { - serf: OnceLock::new(), - l: RwLock::new(()), - } - } - - pub(crate) fn store(&self, serf: SerfWeakRef) { - // No error handling here, because we never call this in parallel - let _ = self.serf.set(serf); - } - - fn this(&self) -> Option> { - self.serf.get().and_then(|weak_ref| weak_ref.upgrade()) - } - - /// Handles broadcasting a query to all members and gathering - /// responses from each of them, returning a list of messages from each node - /// and any applicable error conditions. - pub async fn install_key( - &self, - key: SecretKey, - opts: Option, - ) -> Result, Error> { - let _mu = self.l.write().await; - self - .handle_key_request( - Some(key), - INTERNAL_INSTALL_KEY, - opts, - InternalQueryEvent::InstallKey, - ) - .await - } - - /// Handles broadcasting a primary key change to all members in the - /// cluster, and gathering any response messages. If successful, there should - /// be an empty KeyResponse returned. - pub async fn use_key( - &self, - key: SecretKey, - opts: Option, - ) -> Result, Error> { - let _mu = self.l.write().await; - self - .handle_key_request( - Some(key), - INTERNAL_USE_KEY, - opts, - InternalQueryEvent::UseKey, - ) - .await - } - - /// Handles broadcasting a key to the cluster for removal. Each member - /// will receive this event, and if they have the key in their keyring, remove - /// it. If any errors are encountered, RemoveKey will collect and relay them. - pub async fn remove_key( - &self, - key: SecretKey, - opts: Option, - ) -> Result, Error> { - let _mu = self.l.write().await; - self - .handle_key_request( - Some(key), - INTERNAL_REMOVE_KEY, - opts, - InternalQueryEvent::RemoveKey, - ) - .await - } - - /// Used to collect installed keys from members in a Serf cluster - /// and return an aggregated list of all installed keys. This is useful to - /// operators to ensure that there are no lingering keys installed on any agents. - /// Since having multiple keys installed can cause performance penalties in some - /// cases, it's important to verify this information and remove unneeded keys. - pub async fn list_keys(&self) -> Result, Error> { - let _mu = self.l.read().await; - self - .handle_key_request(None, INTERNAL_LIST_KEYS, None, InternalQueryEvent::ListKey) - .await - } - - pub(crate) async fn handle_key_request( - &self, - key: Option, - ty: &str, - opts: Option, - event: InternalQueryEvent, - ) -> Result, Error> { - let kr = KeyRequestMessage { key }; - let buf = crate::types::encode_message_to_bytes(&kr)?; - - let Some(this) = self.this() else { - return Ok(KeyResponse { - num_nodes: 0, - messages: HashMap::new(), - num_resp: 0, - num_err: 0, - keys: HashMap::new(), - primary_keys: HashMap::new(), - }); - }; - - let mut q_param = this.default_query_param().await; - if let Some(opts) = opts { - q_param.relay_factor = opts.relay_factor; - } - let qresp: QueryResponse = this - .internal_query(SmolStr::new(ty), buf, Some(q_param), event) - .await?; - - // Handle the response stream and populate the KeyResponse - let resp = Self::stream_key_response(&this, qresp.response_rx()).await; - - // Check the response for any reported failure conditions - if resp.num_err > 0 { - tracing::error!( - "serf: {}/{} nodes reported failure", - resp.num_err, - resp.num_nodes - ); - } - - if resp.num_resp != resp.num_nodes { - tracing::error!( - "serf: {}/{} nodes responded success", - resp.num_resp, - resp.num_nodes - ); - } - - Ok(resp) - } - - async fn stream_key_response( - this: &Serf, - ch: Receiver>, - ) -> KeyResponse { - let mut resp = KeyResponse { - num_nodes: this.num_members().await, - messages: HashMap::new(), - num_resp: 0, - num_err: 0, - keys: HashMap::new(), - primary_keys: HashMap::new(), - }; - futures::pin_mut!(ch); - while let Some(r) = ch.next().await { - resp.num_resp += 1; - - // Decode the response - if r.payload.is_empty() { - resp - .messages - .insert(r.from.id().cheap_clone(), SmolStr::new("empty payload")); - resp.num_err += 1; - - if resp.num_resp == resp.num_nodes { - return resp; - } - continue; - } - - let node_response = - match crate::types::decode_message::(&r.payload) { - Ok(msg) => match msg { - MessageRef::KeyResponse(kr) => kr, - msg => { - tracing::error!(type=%msg.ty(), "serf: invalid key query response type"); - - resp.messages.insert( - r.from.id().cheap_clone(), - format_smolstr!("invalid key query response: {}", msg.ty()), - ); - resp.num_err += 1; - - if resp.num_resp == resp.num_nodes { - return resp; - } - continue; - } - }, - Err(e) => { - tracing::error!(err=%e, "serf: failed to decode key query response"); - resp - .messages - .insert(r.from.id().cheap_clone(), format_smolstr!("{e}")); - resp.num_err += 1; - - if resp.num_resp == resp.num_nodes { - return resp; - } - continue; - } - }; - - if !node_response.result() { - resp.messages.insert( - r.from.id().cheap_clone(), - SmolStr::new(node_response.message()), - ); - resp.num_err += 1; - } - - if node_response.result() && !node_response.message().is_empty() { - tracing::warn!("serf: {}", node_response.message()); - resp.messages.insert( - r.from.id().cheap_clone(), - SmolStr::new(node_response.message()), - ); - } - - // Currently only used for key list queries, this adds keys to a counter - // and increments them for each node response which contains them. - let res = node_response - .keys() - .iter::() - .try_for_each(|res| { - res.map(|k| { - let count = resp.keys.entry(k).or_insert(0); - *count += 1; - }) - }); - - if let Err(e) = res { - resp.messages.insert( - r.from.id().cheap_clone(), - SmolStr::new(format!("Failed to decode key query response: {:?}", e)), - ); - resp.num_err += 1; - - if resp.num_resp == resp.num_nodes { - return resp; - } - continue; - } - - if let Some(pk) = node_response.primary_key() { - let ctr = resp.primary_keys.entry(*pk).or_insert(0); - *ctr += 1; - } - - // Return early if all nodes have responded. This allows us to avoid - // waiting for the full timeout when there is nothing left to do. - if resp.num_resp == resp.num_nodes { - return resp; - } - } - resp - } -} diff --git a/legacy/serf-core/src/lib.rs b/legacy/serf-core/src/lib.rs deleted file mode 100644 index 7a51d92c..00000000 --- a/legacy/serf-core/src/lib.rs +++ /dev/null @@ -1,125 +0,0 @@ -#![doc = include_str!("../README.md")] -#![doc(html_logo_url = "https://raw.githubusercontent.com/al8n/serf/main/art/logo_72x72.png")] -#![forbid(unsafe_code)] -// #![deny(warnings, missing_docs)] -#![allow(clippy::type_complexity)] -#![cfg_attr(docsrs, feature(doc_cfg))] -#![cfg_attr(docsrs, allow(unused_attributes))] - -pub(crate) mod broadcast; - -mod coalesce; - -/// Events for [`Serf`] -pub mod event; - -/// Errors for `serf`. -pub mod error; - -/// Delegate traits and its implementations. -pub mod delegate; - -mod options; -pub use options::*; - -/// The types used in `serf`. -pub mod types; -pub use types::coordinate; - -/// Secret key management. -#[cfg(feature = "encryption")] -#[cfg_attr(docsrs, doc(cfg(feature = "encryption")))] -pub mod key_manager; - -mod serf; -pub use serf::*; - -mod snapshot; -pub use snapshot::*; - -fn invalid_data_io_error(e: E) -> std::io::Error { - std::io::Error::new(std::io::ErrorKind::InvalidData, e) -} - -/// All unit test fns are exported in the `tests` module. -/// This module is used for users want to use other async runtime, -/// and want to use the test if memberlist also works with their runtime. -#[cfg(feature = "test")] -#[cfg_attr(docsrs, doc(cfg(feature = "test")))] -pub mod tests { - pub use memberlist_core::tests::{AnyError, next_socket_addr_v4, next_socket_addr_v6}; - pub use paste; - - pub use super::serf::base::tests::{serf::*, *}; - - /// Add `test` prefix to the predefined unit test fn with a given [`Runtime`](memberlist_core::agnostic_lite::RuntimeLite) - #[cfg(any(feature = "test", test))] - #[cfg_attr(docsrs, doc(cfg(any(feature = "test", test))))] - #[macro_export] - macro_rules! unit_tests { - ($runtime:ty => $run:ident($($fn:ident), +$(,)?)) => { - $( - ::serf_core::tests::paste::paste! { - #[test] - fn [< test_ $fn >] () { - $run($fn::<$runtime>()); - } - } - )* - }; - } - - /// Add `test` prefix to the predefined unit test fn with a given [`Runtime`](memberlist_core::agnostic_lite::RuntimeLite) - #[cfg(any(feature = "test", test))] - #[cfg_attr(docsrs, doc(cfg(any(feature = "test", test))))] - #[macro_export] - macro_rules! unit_tests_with_expr { - ($run:ident($( - $(#[$outer:meta])* - $fn:ident( $expr:expr ) - ), +$(,)?)) => { - $( - ::serf_core::tests::paste::paste! { - #[test] - $(#[$outer])* - fn [< test_ $fn >] () { - $run(async move { - $expr - }); - } - } - )* - }; - } - - /// Initialize the tracing for the unit tests. - pub fn initialize_tests_tracing() { - use std::sync::Once; - static TRACE: Once = Once::new(); - TRACE.call_once(|| { - let filter = std::env::var("SERF_TESTING_LOG") - .unwrap_or_else(|_| "serf_core=debug,memberlist_core=info".to_owned()); - memberlist_core::tracing::subscriber::set_global_default( - tracing_subscriber::fmt::fmt() - .without_time() - .with_line_number(true) - .with_env_filter(filter) - .with_file(false) - .with_target(true) - .with_ansi(true) - .finish(), - ) - .unwrap(); - }); - } - - /// Run the unit test with a given async runtime sequentially. - pub fn run(block_on: B, fut: F) - where - B: FnOnce(F) -> F::Output, - F: std::future::Future, - { - // initialize_tests_tracing(); - block_on(fut); - } -} diff --git a/legacy/serf-core/src/options.rs b/legacy/serf-core/src/options.rs deleted file mode 100644 index 167bf18b..00000000 --- a/legacy/serf-core/src/options.rs +++ /dev/null @@ -1,590 +0,0 @@ -use std::{path::PathBuf, sync::Arc, time::Duration}; - -use arc_swap::ArcSwap; -pub use memberlist_core::Options as MemberlistOptions; -use smol_str::SmolStr; - -use super::types::{DelegateVersion, ProtocolVersion, Tags}; - -fn tags(tags: &Arc>) -> Arc { - tags.load().clone() -} - -/// The configuration for creating a Serf instance. -#[viewit::viewit(getters(vis_all = "pub"), setters(vis_all = "pub", prefix = "with"))] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub struct Options { - /// The tags for this role, if any. This is used to provide arbitrary - /// key/value metadata per-node. For example, a "role" tag may be used to - /// differentiate "load-balancer" from a "web" role as parts of the same cluster. - /// Tags are deprecating 'Role', and instead it acts as a special key in this - /// map. - #[viewit( - vis = "pub(crate)", - getter( - vis = "pub", - style = "ref", - result(converter(style = "ref", fn = "tags",), type = "Arc",), - attrs( - doc = "Returns the tags for this role, if any. This is used to provide arbitrary key/value metadata per-node. For example, a \"role\" tag may be used to differentiate \"load-balancer\" from a \"web\" role as parts of the same cluster." - ) - ), - setter(skip) - )] - #[cfg_attr(feature = "serde", serde(with = "tags_serde"))] - tags: Arc>, - - /// The protocol version to speak - #[viewit( - getter(const, attrs(doc = "Returns the protocol version to speak")), - setter(attrs(doc = "Sets the protocol version to speak")) - )] - protocol_version: ProtocolVersion, - - /// The delegate version to speak - #[viewit( - getter(const, attrs(doc = "Returns the delegate version to speak")), - setter(attrs(doc = "Sets the delegate version to speak")) - )] - delegate_version: DelegateVersion, - - /// The amount of time to wait for a broadcast - /// message to be sent to the cluster. Broadcast messages are used for - /// things like leave messages and force remove messages. If this is not - /// set, a timeout of 5 seconds will be set. - #[cfg_attr(feature = "serde", serde(with = "humantime_serde"))] - #[viewit( - getter( - const, - attrs( - doc = "Returns the amount of time to wait for a broadcast message to be sent to the cluster." - ) - ), - setter(attrs( - doc = "Sets the amount of time to wait for a broadcast message to be sent to the cluster." - )) - )] - broadcast_timeout: Duration, - - /// For our leave (node dead) message to propagate - /// through the cluster. In particular, we want to stay up long enough to - /// service any probes from other nodes before they learn about us - /// leaving and stop probing. Otherwise, we risk getting node failures as - /// we leave. - #[cfg_attr(feature = "serde", serde(with = "humantime_serde"))] - #[viewit( - getter(const, attrs(doc = "Returns the leave propagate delay.")), - setter(attrs(doc = "Sets the leave propagate delay.")) - )] - leave_propagate_delay: Duration, - - /// The settings below relate to Serf's event coalescence feature. Serf - /// is able to coalesce multiple events into single events in order to - /// reduce the amount of noise that is sent along the event channel. For example - /// if five nodes quickly join, the event channel will be sent one EventMemberJoin - /// containing the five nodes rather than five individual EventMemberJoin - /// events. Coalescence can mitigate potential flapping behavior. - /// - /// Coalescence is disabled by default and can be enabled by setting - /// `coalesce_period`. - /// - /// `coalesce_period` specifies the time duration to coalesce events. - /// For example, if this is set to 5 seconds, then all events received - /// within 5 seconds that can be coalesced will be. - /// - #[cfg_attr(feature = "serde", serde(with = "humantime_serde"))] - #[viewit( - getter(const, attrs(doc = "Returns the coalesce period.")), - setter(attrs(doc = "Sets the coalesce period.")) - )] - coalesce_period: Duration, - - /// specifies the duration of time where if no events - /// are received, coalescence immediately happens. For example, if - /// `coalesce_period` is set to 10 seconds but `quiescent_period` is set to 2 - /// seconds, then the events will be coalesced and dispatched if no - /// new events are received within 2 seconds of the last event. Otherwise, - /// every event will always be delayed by at least 10 seconds. - #[cfg_attr(feature = "serde", serde(with = "humantime_serde"))] - #[viewit( - getter( - const, - attrs( - doc = "Returns the specifies the duration of time where if no events are received, coalescence immediately happens." - ) - ), - setter(attrs( - doc = "Sets specifies the duration of time where if no events are received, coalescence immediately happens." - )) - )] - quiescent_period: Duration, - - /// The settings below relate to Serf's user event coalescing feature. - /// The settings operate like above but only affect user messages and - /// not the Member* messages that Serf generates. - #[cfg_attr(feature = "serde", serde(with = "humantime_serde"))] - #[viewit( - getter(const, attrs(doc = "Returns the user event coalesce period.")), - setter(attrs(doc = "Sets the user event coalesce period.")) - )] - user_coalesce_period: Duration, - - /// The settings below relate to Serf's user event coalescing feature. - /// The settings operate like above but only affect user messages and - /// not the Member* messages that Serf generates. - #[cfg_attr(feature = "serde", serde(with = "humantime_serde"))] - #[viewit( - getter(const, attrs(doc = "Returns the user quiescent period.")), - setter(attrs(doc = "Sets the user quiescent period.")) - )] - user_quiescent_period: Duration, - - /// The interval when the reaper runs. If this is not - /// set (it is zero), it will be set to a reasonable default. - #[cfg_attr(feature = "serde", serde(with = "humantime_serde"))] - #[viewit( - getter(const, attrs(doc = "Returns the interval when the reaper runs.")), - setter(attrs(doc = "Sets the interval when the reaper runs.")) - )] - reap_interval: Duration, - - /// The interval when we attempt to reconnect - /// to failed nodes. If this is not set (it is zero), it will be set - /// to a reasonable default. - #[cfg_attr(feature = "serde", serde(with = "humantime_serde"))] - #[viewit( - getter( - const, - attrs(doc = "Returns the interval when we attempt to reconnect to failed nodes.") - ), - setter(attrs(doc = "Sets the interval when we attempt to reconnect to failed nodes.")) - )] - reconnect_interval: Duration, - - /// The amount of time to attempt to reconnect to - /// a failed node before giving up and considering it completely gone. - #[cfg_attr(feature = "serde", serde(with = "humantime_serde"))] - #[viewit( - getter( - const, - attrs( - doc = "Returns the amount of time to attempt to reconnect to a failed node before giving up and considering it completely gone." - ) - ), - setter(attrs( - doc = "Sets the amount of time to attempt to reconnect to a failed node before giving up and considering it completely gone." - )) - )] - reconnect_timeout: Duration, - - /// The amount of time to keep around nodes - /// that gracefully left as tombstones for syncing state with other - /// Serf nodes. - #[cfg_attr(feature = "serde", serde(with = "humantime_serde"))] - #[viewit( - getter( - const, - attrs( - doc = "Returns the amount of time to keep around nodes that gracefully left as tombstones for syncing state with other Serf nodes." - ) - ), - setter(attrs( - doc = "Sets the amount of time to keep around nodes that gracefully left as tombstones for syncing state with other Serf nodes." - )) - )] - tombstone_timeout: Duration, - - /// The amount of time less than which we consider a node - /// being failed and rejoining looks like a flap for telemetry purposes. - /// This should be set less than a typical reboot time, but large enough - /// to see actual events, given our expected detection times for a failed - /// node. - #[cfg_attr(feature = "serde", serde(with = "humantime_serde"))] - #[viewit( - getter( - const, - attrs( - doc = "Returns the amount of time less than which we consider a node being failed and rejoining looks like a flap for telemetry purposes." - ) - ), - setter(attrs( - doc = "Sets the amount of time less than which we consider a node being failed and rejoining looks like a flap for telemetry purposes." - )) - )] - flap_timeout: Duration, - - /// The interval at which we check the message - /// queue to apply the warning and max depth. - #[cfg_attr(feature = "serde", serde(with = "humantime_serde"))] - #[viewit( - getter( - const, - attrs( - doc = "Returns the interval at which we check the message queue to apply the warning and max depth." - ) - ), - setter(attrs( - doc = "Sets the interval at which we check the message queue to apply the warning and max depth." - )) - )] - queue_check_interval: Duration, - - /// Used to generate warning message if the - /// number of queued messages to broadcast exceeds this number. This - /// is to provide the user feedback if events are being triggered - /// faster than they can be disseminated - #[viewit( - getter(const, attrs(doc = "Returns the queue depth warning.")), - setter(attrs(doc = "Sets the queue depth warning.")) - )] - queue_depth_warning: usize, - - /// Used to start dropping messages if the number - /// of queued messages to broadcast exceeds this number. This is to - /// prevent an unbounded growth of memory utilization - #[viewit( - getter(const, attrs(doc = "Returns the max queue depth.")), - setter(attrs(doc = "Sets the max queue depth.")) - )] - max_queue_depth: usize, - - /// if >0 will enforce a lower limit for dropping messages - /// and then the max will be max(MinQueueDepth, 2*SizeOfCluster). This - /// defaults to 0 which disables this dynamic sizing feature. If this is - /// >0 then `max_queue_depth` will be ignored. - #[viewit( - getter( - const, - attrs( - doc = "Returns if `>0` will enforce a lower limit for dropping messages and then the max will be `max(min_queue_depth, 2 * size_of_cluster)`. This defaults to 0 which disables this dynamic sizing feature. If this is `>0` then `max_queue_depth` will be ignored." - ) - ), - setter(attrs( - doc = "Sets if `>0` will enforce a lower limit for dropping messages and then the max will be `max(min_queue_depth, 2 * size_of_cluster)`. This defaults to 0 which disables this dynamic sizing feature. If this is `>0` then `max_queue_depth` will be ignored." - )) - )] - min_queue_depth: usize, - - /// Used to determine how long we store recent - /// join and leave intents. This is used to guard against the case where - /// Serf broadcasts an intent that arrives before the Memberlist event. - /// It is important that this not be too short to avoid continuous - /// rebroadcasting of dead events. - #[cfg_attr(feature = "serde", serde(with = "humantime_serde"))] - #[viewit( - getter( - const, - attrs(doc = "Returns how long we store recent join and leave intents.") - ), - setter(attrs(doc = "Sets how long we store recent join and leave intents.")) - )] - recent_intent_timeout: Duration, - - /// Used to control how many events are buffered. - /// This is used to prevent re-delivery of events to a client. The buffer - /// must be large enough to handle all "recent" events, since Serf will - /// not deliver messages that are older than the oldest entry in the buffer. - /// Thus if a client is generating too many events, it's possible that the - /// buffer gets overrun and messages are not delivered. - #[viewit( - getter(const, attrs(doc = "Returns how many events are buffered.")), - setter(attrs(doc = "Sets how many events are buffered.")) - )] - event_buffer_size: usize, - - /// used to control how many queries are buffered. - /// This is used to prevent re-delivery of queries to a client. The buffer - /// must be large enough to handle all "recent" events, since Serf will not - /// deliver queries older than the oldest entry in the buffer. - /// Thus if a client is generating too many queries, it's possible that the - /// buffer gets overrun and messages are not delivered. - #[viewit( - getter(const, attrs(doc = "Returns how many queries are buffered.")), - setter(attrs(doc = "Sets how many queries are buffered.")) - )] - query_buffer_size: usize, - - /// Configures the default timeout multipler for a query to run if no - /// specific value is provided. Queries are real-time by nature, where the - /// reply is time sensitive. As a result, results are collected in an async - /// fashion, however the query must have a bounded duration. We want the timeout - /// to be long enough that all nodes have time to receive the message, run a handler, - /// and generate a reply. Once the timeout is exceeded, any further replies are ignored. - /// The default value is - /// - /// ```text - /// timeout = gossip_interval * query_timeout_mult * log(N+1) - /// ``` - #[viewit( - getter( - const, - attrs( - doc = "Returns the default timeout multipler for a query to run if no specific value is provided." - ) - ), - setter(attrs( - doc = "Sets the default timeout multipler for a query to run if no specific value is provided." - )) - )] - query_timeout_mult: usize, - - /// Limit the outbound payload sizes for queries, respectively. These must fit - /// in a UDP packet with some additional overhead, so tuning these - /// past the default values of 1024 will depend on your network - /// configuration. - #[viewit( - getter( - const, - attrs(doc = "Returns the limit of the outbound payload sizes for queries.") - ), - setter(attrs(doc = "Sets the limit of the outbound payload sizes for queries.")) - )] - query_response_size_limit: usize, - - /// Limit the inbound payload sizes for queries, respectively. These must fit - /// in a UDP packet with some additional overhead, so tuning these - /// past the default values of 1024 will depend on your network - /// configuration. - #[viewit( - getter( - const, - attrs(doc = "Returns the limit of the inbound payload sizes for queries.") - ), - setter(attrs(doc = "Sets the limit of the inbound payload sizes for queries.")) - )] - query_size_limit: usize, - - /// The memberlist configuration that Serf will - /// use to do the underlying membership management and gossip. - #[viewit( - getter( - const, - style = "ref", - attrs( - doc = "Returns the memberlist configuration that Serf will use to do the underlying membership management and gossip." - ) - ), - setter(attrs( - doc = "Sets the memberlist configuration that Serf will use to do the underlying membership management and gossip." - )) - )] - memberlist_options: MemberlistOptions, - - /// If provided is used to snapshot live nodes as well - /// as lamport clock values. When Serf is started with a snapshot, - /// it will attempt to join all the previously known nodes until one - /// succeeds and will also avoid replaying old user events. - #[viewit( - getter( - const, - style = "ref", - result(converter(fn = "Option::as_ref"), type = "Option<&PathBuf>"), - attrs(doc = "Returns the path to the snapshot file.") - ), - setter(attrs(doc = "Sets the path to the snapshot file.")) - )] - snapshot_path: Option, - - /// Controls our interaction with the snapshot file. - /// When set to false (default), a leave causes a Serf to not rejoin - /// the cluster until an explicit join is received. If this is set to - /// true, we ignore the leave, and rejoin the cluster on start. - #[viewit( - getter( - const, - attrs(doc = "Returns if Serf will rejoin the cluster after a leave.") - ), - setter(attrs(doc = "Sets if Serf will rejoin the cluster after a leave.")) - )] - rejoin_after_leave: bool, - - /// Controls if Serf will actively attempt - /// to resolve a name conflict. Since each Serf member must have a unique - /// name, a cluster can run into issues if multiple nodes claim the same - /// name. Without automatic resolution, Serf merely logs some warnings, but - /// otherwise does not take any action. Automatic resolution detects the - /// conflict and issues a special query which asks the cluster for the - /// Name -> IP:Port mapping. If there is a simple majority of votes, that - /// node stays while the other node will leave the cluster and exit. - #[viewit( - getter( - const, - attrs(doc = "Returns if Serf will attempt to resolve a name conflict.") - ), - setter(attrs(doc = "Sets if Serf will attempt to resolve a name conflict.")) - )] - enable_id_conflict_resolution: bool, - - /// Controls if Serf will maintain an estimate of this - /// node's network coordinate internally. A network coordinate is useful - /// for estimating the network distance (i.e. round trip time) between - /// two nodes. Enabling this option adds some overhead to ping messages. - #[viewit( - getter( - const, - attrs( - doc = "Returns if Serf will maintain an estimate of this node's network coordinate internally." - ) - ), - setter(attrs( - doc = "Sets if Serf will maintain an estimate of this node's network coordinate internally." - )) - )] - disable_coordinates: bool, - - /// Provides the location of a writable file where Serf can - /// persist changes to the encryption keyring. - #[cfg(feature = "encryption")] - #[viewit( - getter( - const, - style = "ref", - result(converter(fn = "Option::as_ref"), type = "Option<&PathBuf>"), - attrs( - doc = "Returns the location of a writable file where Serf can persist changes to the encryption keyring.", - cfg(feature = "encryption") - ) - ), - setter(attrs( - doc = "Sets the location of a writable file where Serf can persist changes to the encryption keyring.", - cfg(feature = "encryption") - )) - )] - keyring_file: Option, - - /// Maximum byte size limit of user event `name` + `payload` in bytes. - /// It's optimal to be relatively small, since it's going to be gossiped through the cluster. - #[viewit( - getter( - const, - attrs( - doc = "Returns the maximum byte size limit of user event `name` + `payload` in bytes." - ) - ), - setter(attrs( - doc = "Sets the maximum byte size limit of user event `name` + `payload` in bytes." - )) - )] - max_user_event_size: usize, -} - -impl Default for Options { - #[inline] - fn default() -> Self { - Self::new() - } -} - -impl Clone for Options { - #[inline] - fn clone(&self) -> Self { - Self { - memberlist_options: self.memberlist_options.clone(), - #[cfg(feature = "encryption")] - keyring_file: self.keyring_file.clone(), - snapshot_path: self.snapshot_path.clone(), - tags: self.tags.clone(), - ..*self - } - } -} - -impl Options { - /// Returns a new instance of `Options` with default configurations. - #[inline] - pub fn new() -> Self { - Self { - tags: Arc::new(ArcSwap::from_pointee(Tags::default())), - protocol_version: ProtocolVersion::V1, - delegate_version: DelegateVersion::V1, - broadcast_timeout: Duration::from_secs(5), - leave_propagate_delay: Duration::from_secs(1), - coalesce_period: Duration::ZERO, - quiescent_period: Duration::ZERO, - user_coalesce_period: Duration::ZERO, - user_quiescent_period: Duration::ZERO, - reap_interval: Duration::from_secs(15), - reconnect_interval: Duration::from_secs(30), - reconnect_timeout: Duration::from_secs(3600 * 24), - tombstone_timeout: Duration::from_secs(3600 * 24), - flap_timeout: Duration::from_secs(60), - queue_check_interval: Duration::from_secs(30), - queue_depth_warning: 128, - max_queue_depth: 4096, - min_queue_depth: 0, - recent_intent_timeout: Duration::from_secs(60 * 5), - event_buffer_size: 512, - query_buffer_size: 512, - query_timeout_mult: 16, - query_response_size_limit: 1024, - query_size_limit: 1024, - memberlist_options: MemberlistOptions::lan(), - snapshot_path: None, - rejoin_after_leave: false, - enable_id_conflict_resolution: true, - disable_coordinates: false, - #[cfg(feature = "encryption")] - keyring_file: None, - max_user_event_size: 512, - } - } - - /// Sets the tags for this node. - #[inline] - pub fn with_tags, V: Into>( - self, - tags: impl Iterator, - ) -> Self { - self - .tags - .store(Arc::new(tags.map(|(k, v)| (k.into(), v.into())).collect())); - self - } - - #[inline] - pub(crate) fn queue_opts(&self) -> QueueOptions { - QueueOptions { - max_queue_depth: self.max_queue_depth, - min_queue_depth: self.min_queue_depth, - check_interval: self.queue_check_interval, - depth_warning: self.queue_depth_warning, - #[cfg(feature = "metrics")] - metric_labels: self.memberlist_options.metric_labels().clone(), - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub(crate) struct QueueOptions { - pub(crate) max_queue_depth: usize, - pub(crate) min_queue_depth: usize, - pub(crate) check_interval: Duration, - pub(crate) depth_warning: usize, - #[cfg(feature = "metrics")] - pub(crate) metric_labels: Arc, -} - -#[cfg(feature = "serde")] -mod tags_serde { - use std::sync::Arc; - - use arc_swap::ArcSwap; - use serde::{Deserialize, Deserializer, Serialize, Serializer}; - - use crate::types::Tags; - - pub fn serialize(tags: &Arc>, serializer: S) -> Result - where - S: Serializer, - { - let tags = tags.load(); - Tags::serialize(&**tags, serializer) - } - - pub fn deserialize<'de, D>(deserializer: D) -> Result>, D::Error> - where - D: Deserializer<'de>, - { - Tags::deserialize(deserializer).map(|map| Arc::new(ArcSwap::from_pointee(map))) - } -} diff --git a/legacy/serf-core/src/serf.rs b/legacy/serf-core/src/serf.rs deleted file mode 100644 index 0a842bc8..00000000 --- a/legacy/serf-core/src/serf.rs +++ /dev/null @@ -1,255 +0,0 @@ -use std::{ - collections::HashMap, - sync::{Arc, Weak, atomic::AtomicBool}, -}; - -use async_lock::{Mutex, RwLock}; -use atomic_refcell::AtomicRefCell; -use futures::stream::FuturesUnordered; -use memberlist_core::{ - Memberlist, - agnostic_lite::{AsyncSpawner, RuntimeLite}, - proto::MediumVec, - queue::TransmitLimitedQueue, - transport::{AddressResolver, Transport}, -}; - -use super::{ - Options, - broadcast::SerfBroadcast, - delegate::{CompositeDelegate, Delegate}, - event::CrateEvent, - snapshot::SnapshotHandle, - types::{ - LamportClock, LamportTime, Members, UserEvents, - coordinate::{Coordinate, CoordinateClient}, - }, -}; - -mod api; -pub(crate) mod base; - -mod delegate; -pub(crate) use delegate::*; - -mod query; -pub use query::*; - -mod internal_query; - -/// Maximum 128 KB snapshot -pub(crate) const SNAPSHOT_SIZE_LIMIT: u64 = 128 * 1024; - -/// Maximum 9KB for event name and payload -const USER_EVENT_SIZE_LIMIT: usize = 9 * 1024; - -/// Exports the default delegate type -pub type DefaultDelegate = CompositeDelegate< - ::Id, - <::Resolver as AddressResolver>::ResolvedAddress, ->; - -pub(crate) struct CoordCore { - pub(crate) client: CoordinateClient, - pub(crate) cache: parking_lot::RwLock>, -} - -/// Stores all the query ids at a specific time -#[derive(Debug, Clone, Eq, PartialEq)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub(crate) struct Queries { - ltime: LamportTime, - query_ids: MediumVec, -} - -#[derive(Default)] -pub(crate) struct QueryCore { - responses: HashMap>, - min_time: LamportTime, - buffer: Vec>, -} - -#[viewit::viewit] -pub(crate) struct EventCore { - min_time: LamportTime, - buffer: Vec>, -} - -/// The state of the Serf instance. -#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] -pub enum SerfState { - /// Alive state - Alive, - /// Leaving state - Leaving, - /// Left state - Left, - /// Shutdown state - Shutdown, -} - -impl SerfState { - /// Returns the string representation of the state. - pub const fn as_str(&self) -> &'static str { - match self { - Self::Alive => "alive", - Self::Leaving => "leaving", - Self::Left => "left", - Self::Shutdown => "shutdown", - } - } -} - -impl core::fmt::Display for SerfState { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.as_str()) - } -} - -struct NumMembers(Arc>>); - -impl Clone for NumMembers { - fn clone(&self) -> Self { - Self(self.0.clone()) - } -} - -impl From>>> for NumMembers { - fn from(value: Arc>>) -> Self { - Self(value) - } -} - -impl memberlist_core::queue::NodeCalculator for NumMembers -where - I: Send + Sync + 'static, - A: Send + Sync + 'static, -{ - async fn num_nodes(&self) -> usize { - self.0.read().await.states.len() - } -} - -pub(crate) struct SerfCore> -where - D: Delegate, - T: Transport, -{ - pub(crate) clock: LamportClock, - pub(crate) event_clock: LamportClock, - pub(crate) query_clock: LamportClock, - - broadcasts: Arc>>, - event_broadcasts: Arc>>, - query_broadcasts: Arc>>, - - pub(crate) memberlist: Memberlist>, - pub(crate) members: Arc>>, - event_tx: async_channel::Sender>, - pub(crate) event_join_ignore: AtomicBool, - - pub(crate) event_core: RwLock, - query_core: Arc>>, - handles: AtomicRefCell< - FuturesUnordered<<::Spawner as AsyncSpawner>::JoinHandle<()>>, - >, - pub(crate) opts: Options, - - state: parking_lot::Mutex, - - join_lock: Mutex<()>, - - snapshot: Option, - #[cfg(feature = "encryption")] - key_manager: crate::key_manager::KeyManager, - shutdown_tx: async_channel::Sender<()>, - shutdown_rx: async_channel::Receiver<()>, - - pub(crate) coord_core: Option>>, -} - -/// Serf is a single node that is part of a single cluster that gets -/// events about joins/leaves/failures/etc. It is created with the Create -/// method. -/// -/// All functions on the Serf structure are safe to call concurrently. -#[repr(transparent)] -pub struct Serf> -where - D: Delegate, - T: Transport, -{ - pub(crate) inner: Arc>, -} - -impl Clone for Serf -where - D: Delegate, - T: Transport, -{ - fn clone(&self) -> Self { - Self { - inner: self.inner.clone(), - } - } -} - -impl Serf -where - D: Delegate, - T: Transport, -{ - /// Creates a weak reference to this `Serf` instance. - /// - /// This is used to break reference cycles between `Serf` and components that hold a back-reference, - /// such as [`SerfDelegate`] or [`KeyManager`]. Since Rust uses reference counting for ownership, - /// strong references would prevent the `Serf` instance from being dropped even when no longer needed. - /// - /// A weak reference does not keep the inner `SerfCore` alive. It must be upgraded to a strong - /// reference using [`SerfWeakRef::upgrade()`] before use, which returns `None` if the `Serf` - /// has already been destroyed. - /// - /// # Returns - /// - /// A [`SerfWeakRef`] that may be safely stored without preventing cleanup. - fn downgrade(&self) -> SerfWeakRef { - SerfWeakRef { - inner: Arc::downgrade(&self.inner), - } - } -} - -/// A weak reference to a [`Serf`] instance. -/// -/// This type allows holding a non-owning reference to a `Serf` without extending its lifetime. -/// It is primarily used to break reference cycles. -/// -/// To access the inner `Serf`, call [`upgrade()`](Self::upgrade), which returns `Some(Serf)` if the -/// original instance is still alive, or `None` if it has been dropped. -/// -/// This type is analogous to [`std::sync::Weak`] in relation to [`std::sync::Arc`]. -#[repr(transparent)] -pub(crate) struct SerfWeakRef> -where - D: Delegate, - T: Transport, -{ - inner: Weak>, -} - -impl SerfWeakRef -where - D: Delegate, - T: Transport, -{ - /// Attempts to upgrade this weak reference to a strong one. - /// - /// Returns `Some(Serf)` if the referenced `Serf` is still alive, otherwise `None`. - /// - /// This is typically called within delegate methods or background tasks that need temporary - /// access to the `Serf` instance. Always handle the `None` case gracefully, as it indicates - /// the `Serf` has already begun shutting down. - pub(crate) fn upgrade(&self) -> Option> { - self.inner.upgrade().map(|inner| Serf { inner }) - } -} diff --git a/legacy/serf-core/src/serf/api.rs b/legacy/serf-core/src/serf/api.rs deleted file mode 100644 index 33f2f352..00000000 --- a/legacy/serf-core/src/serf/api.rs +++ /dev/null @@ -1,602 +0,0 @@ -use std::sync::atomic::Ordering; - -use futures::{FutureExt, StreamExt}; -use memberlist_core::{ - CheapClone, - bytes::Bytes, - proto::{Data, MaybeResolvedAddress, Meta, Node, OneOrMore, SmallVec}, - tracing, -}; -use smol_str::SmolStr; - -use crate::{ - error::Error, - event::EventProducer, - types::{LeaveMessage, Member, Tags, UserEventMessage}, -}; - -use super::*; - -impl Serf -where - T: Transport, -{ - /// Creates a new Serf instance with the given transport and options. - pub async fn new( - transport: T::Options, - opts: Options, - ) -> Result>> { - Self::new_in( - None, - None, - transport, - opts, - #[cfg(any(test, feature = "test"))] - None, - ) - .await - } - - /// Creates a new Serf instance with the given transport and options. - pub async fn with_event_producer( - transport: T::Options, - opts: Options, - ev: EventProducer>, - ) -> Result>> { - Self::new_in( - Some(ev.tx), - None, - transport, - opts, - #[cfg(any(test, feature = "test"))] - None, - ) - .await - } -} - -impl Serf -where - D: Delegate, - T: Transport, -{ - /// Creates a new Serf instance with the given transport and options. - pub async fn with_delegate( - transport: T::Options, - opts: Options, - delegate: D, - ) -> Result> { - Self::new_in( - None, - Some(delegate), - transport, - opts, - #[cfg(any(test, feature = "test"))] - None, - ) - .await - } - - /// Creates a new Serf instance with the given transport, options, event sender, and delegate. - pub async fn with_event_producer_and_delegate( - transport: T::Options, - opts: Options, - ev: EventProducer, - delegate: D, - ) -> Result> { - Self::new_in( - Some(ev.tx), - Some(delegate), - transport, - opts, - #[cfg(any(test, feature = "test"))] - None, - ) - .await - } - - /// Returns the local node's ID - #[inline] - pub fn local_id(&self) -> &T::Id { - self.inner.memberlist.local_id() - } - - /// Returns the local node's ID and the advertised address - #[inline] - pub fn advertise_node(&self) -> Node { - self.inner.memberlist.advertise_node() - } - - /// A predicate that determines whether or not encryption - /// is enabled, which can be possible in one of 2 cases: - /// - Single encryption key passed at agent start (no persistence) - /// - Keyring file provided at agent start - #[inline] - #[cfg(feature = "encryption")] - #[cfg_attr(docsrs, doc(cfg(feature = "encryption")))] - pub fn encryption_enabled(&self) -> bool { - self.inner.memberlist.encryption_enabled() - } - - /// Returns a receiver that can be used to wait for - /// Serf to shutdown. - #[inline] - pub fn shutdown_rx(&self) -> async_channel::Receiver<()> { - self.inner.shutdown_rx.clone() - } - - /// The current state of this Serf instance. - #[inline] - pub fn state(&self) -> SerfState { - *self.inner.state.lock() - } - - /// Returns a point-in-time snapshot of the members of this cluster. - #[inline] - pub async fn members(&self) -> OneOrMore> { - self - .inner - .members - .read() - .await - .states - .values() - .map(|s| s.member.cheap_clone()) - .collect() - } - - /// Used to provide operator debugging information - #[inline] - pub async fn stats(&self) -> Stats { - let (num_members, num_failed, num_left, health_score) = { - let members = self.inner.members.read().await; - let num_members = members.states.len(); - let num_failed = members.failed_members.len(); - let num_left = members.left_members.len(); - let health_score = self.inner.memberlist.health_score(); - (num_members, num_failed, num_left, health_score) - }; - - #[cfg(not(feature = "encryption"))] - let encrypted = false; - #[cfg(feature = "encryption")] - let encrypted = self.inner.memberlist.encryption_enabled(); - - Stats { - members: num_members, - failed: num_failed, - left: num_left, - health_score, - member_time: self.inner.clock.time().into(), - event_time: self.inner.event_clock.time().into(), - query_time: self.inner.query_clock.time().into(), - intent_queue: self.inner.broadcasts.num_queued().await, - event_queue: self.inner.event_broadcasts.num_queued().await, - query_queue: self.inner.query_broadcasts.num_queued().await, - encrypted, - coordinate_resets: self - .inner - .coord_core - .as_ref() - .map(|coord| coord.client.stats().resets), - } - } - - /// Returns the number of nodes in the serf cluster, regardless of - /// their health or status. - #[inline] - pub async fn num_members(&self) -> usize { - self.inner.members.read().await.states.len() - } - - /// Returns the key manager for the current serf instance - #[cfg(feature = "encryption")] - #[cfg_attr(docsrs, doc(cfg(feature = "encryption")))] - #[inline] - pub fn key_manager(&self) -> &crate::key_manager::KeyManager { - &self.inner.key_manager - } - - /// Returns the Member information for the local node - #[inline] - pub async fn local_member(&self) -> Member { - self - .inner - .members - .read() - .await - .states - .get(self.inner.memberlist.local_id()) - .unwrap() - .member - .cheap_clone() - } - - /// Used to dynamically update the tags associated with - /// the local node. This will propagate the change to the rest of - /// the cluster. Blocks until a the message is broadcast out. - #[inline] - pub async fn set_tags(&self, tags: Tags) -> Result<(), Error> { - // Check that the meta data length is okay - let tags_encoded_len = tags.encoded_len_with_length_delimited(); - if tags_encoded_len > Meta::MAX_SIZE { - return Err(Error::tags_too_large(tags_encoded_len)); - } - // update the config - self.inner.opts.tags.store(Arc::new(tags)); - - // trigger a memberlist update - self - .inner - .memberlist - .update_node(self.inner.opts.broadcast_timeout) - .await - .map_err(From::from) - } - - /// Used to broadcast a custom user event with a given - /// name and payload. If the configured size limit is exceeded and error will be returned. - /// If coalesce is enabled, nodes are allowed to coalesce this event. - #[inline] - pub async fn user_event( - &self, - name: impl Into, - payload: impl Into, - coalesce: bool, - ) -> Result<(), Error> { - let name: SmolStr = name.into(); - let payload: Bytes = payload.into(); - let payload_size_before_encoding = name.len() + payload.len(); - - // Check size before encoding to prevent needless encoding and return early if it's over the specified limit. - if payload_size_before_encoding > self.inner.opts.max_user_event_size { - return Err(Error::user_event_limit_too_large( - self.inner.opts.max_user_event_size, - )); - } - - if payload_size_before_encoding > USER_EVENT_SIZE_LIMIT { - return Err(Error::user_event_too_large(USER_EVENT_SIZE_LIMIT)); - } - - // Create a message - let msg = UserEventMessage { - ltime: self.inner.event_clock.time(), - name: name.clone(), - payload, - cc: coalesce, - }; - - // Start broadcasting the event - let len = crate::types::encoded_message_len(&msg); - - // Check the size after encoding to be sure again that - // we're not attempting to send over the specified size limit. - if len > self.inner.opts.max_user_event_size { - return Err(Error::raw_user_event_too_large(len)); - } - - if len > USER_EVENT_SIZE_LIMIT { - return Err(Error::raw_user_event_too_large(len)); - } - - let raw = crate::types::encode_message_to_bytes(&msg)?; - - self.inner.event_clock.increment(); - - // Process update locally - self.handle_user_event(either::Either::Right(msg)).await; - - self - .inner - .event_broadcasts - .queue_broadcast(SerfBroadcast { - msg: raw, - notify_tx: None, - }) - .await; - Ok(()) - } - - /// Used to broadcast a new query. The query must be fairly small, - /// and an error will be returned if the size limit is exceeded. Query parameters are optional, - /// and if not provided, a sane set of defaults will be used. - pub async fn query( - &self, - name: impl Into, - payload: impl Into, - params: Option>, - ) -> Result, Error> { - self - .query_in(name.into(), payload.into(), params, None) - .await - } - - /// Joins an existing Serf cluster. Returns the resolved address of node - /// successfully contacted. If `ignore_old` is true, then any - /// user messages sent prior to the join will be ignored. - pub async fn join( - &self, - node: MaybeResolvedAddress, - ignore_old: bool, - ) -> Result> { - // Do a quick state check - let current_state = self.state(); - if current_state != SerfState::Alive { - return Err(Error::bad_join_status(current_state)); - } - - // Hold the joinLock, this is to make eventJoinIgnore safe - let _join_lock = self.inner.join_lock.lock().await; - - // Ignore any events from a potential join. This is safe since we hold - // the joinLock and nobody else can be doing a Join - if ignore_old { - self.inner.event_join_ignore.store(true, Ordering::SeqCst); - } - - // Have memberlist attempt to join - match self.inner.memberlist.join(node).await { - Ok(node) => { - // Start broadcasting the update - if let Err(e) = self.broadcast_join(self.inner.clock.time()).await { - if ignore_old { - self.inner.event_join_ignore.store(false, Ordering::SeqCst); - } - return Err(e); - } - if ignore_old { - self.inner.event_join_ignore.store(false, Ordering::SeqCst); - } - - Ok(node) - } - Err(e) => { - if ignore_old { - self.inner.event_join_ignore.store(false, Ordering::SeqCst); - } - Err(Error::from(e)) - } - } - } - - /// Joins an existing Serf cluster. Returns the resolved address of nodes - /// successfully contacted. If `ignore_old` is true, then any - /// user messages sent prior to the join will be ignored. - pub async fn join_many( - &self, - existing: impl Iterator>, - ignore_old: bool, - ) -> Result, (SmallVec, Error)> { - // Do a quick state check - let current_state = self.state(); - if current_state != SerfState::Alive { - return Err((SmallVec::new(), Error::bad_join_status(current_state))); - } - - // Hold the joinLock, this is to make eventJoinIgnore safe - let _join_lock = self.inner.join_lock.lock().await; - - // Ignore any events from a potential join. This is safe since we hold - // the joinLock and nobody else can be doing a Join - if ignore_old { - self.inner.event_join_ignore.store(true, Ordering::SeqCst); - } - - // Have memberlist attempt to join - match self.inner.memberlist.join_many(existing).await { - Ok(joined) => { - // Start broadcasting the update - if let Err(e) = self.broadcast_join(self.inner.clock.time()).await { - self.inner.event_join_ignore.store(false, Ordering::SeqCst); - return Err((joined, e)); - } - self.inner.event_join_ignore.store(false, Ordering::SeqCst); - Ok(joined) - } - Err((joined, err)) => { - // If we joined any nodes, broadcast the join message - if !joined.is_empty() { - // Start broadcasting the update - if let Err(e) = self.broadcast_join(self.inner.clock.time()).await { - self.inner.event_join_ignore.store(false, Ordering::SeqCst); - return Err(( - joined, - Error::Multiple(std::sync::Arc::from_iter([err.into(), e])), - )); - } - - self.inner.event_join_ignore.store(false, Ordering::SeqCst); - Err((joined, Error::from(err))) - } else { - self.inner.event_join_ignore.store(false, Ordering::SeqCst); - Err((joined, Error::from(err))) - } - } - } - } - - /// Gracefully exits the cluster. It is safe to call this multiple - /// times. - /// If the Leave broadcast timeout, Leave() will try to finish the sequence as best effort. - pub async fn leave(&self) -> Result<(), Error> { - // Check the current state - { - let mut s = self.inner.state.lock(); - match *s { - SerfState::Left => return Ok(()), - SerfState::Leaving => return Err(Error::bad_leave_status(*s)), - SerfState::Shutdown => return Err(Error::bad_leave_status(*s)), - _ => { - // Set the state to leaving - *s = SerfState::Leaving; - } - } - } - - // If we have a snapshot, mark we are leaving - if let Some(ref snap) = self.inner.snapshot { - snap.leave().await; - } - - // Construct the message for the graceful leave - let msg = LeaveMessage { - ltime: self.inner.clock.time(), - id: self.inner.memberlist.local_id().cheap_clone(), - prune: false, - }; - - self.inner.clock.increment(); - - // Process the leave locally - self.handle_node_leave_intent(&msg).await; - - // Only broadcast the leave message if there is at least one - // other node alive. - if self.has_alive_members().await { - let (notify_tx, notify_rx) = async_channel::bounded(1); - let msg = crate::types::encode_message_to_bytes(&msg)?; - self.broadcast(msg, Some(notify_tx)).await?; - - futures::select! { - _ = notify_rx.recv().fuse() => { - // We got a response, so we are done - } - _ = ::sleep(self.inner.opts.broadcast_timeout).fuse() => { - tracing::warn!("serf: timeout while waiting for graceful leave"); - } - } - } - - // Attempt the memberlist leave - if let Err(e) = self - .inner - .memberlist - .leave(self.inner.opts.broadcast_timeout) - .await - { - tracing::warn!("serf: timeout waiting for leave broadcast: {}", e); - } - - // Wait for the leave to propagate through the cluster. The broadcast - // timeout is how long we wait for the message to go out from our own - // queue, but this wait is for that message to propagate through the - // cluster. In particular, we want to stay up long enough to service - // any probes from other nodes before they learn about us leaving. - ::sleep(self.inner.opts.leave_propagate_delay).await; - - // Transition to Left only if we not already shutdown - { - let mut s = self.inner.state.lock(); - match *s { - SerfState::Shutdown => {} - _ => { - *s = SerfState::Left; - } - } - } - Ok(()) - } - - /// Forcibly removes a failed node from the cluster - /// immediately, instead of waiting for the reaper to eventually reclaim it. - /// This also has the effect that Serf will no longer attempt to reconnect - /// to this node. - pub async fn remove_failed_node(&self, id: T::Id) -> Result<(), Error> { - self.force_leave(id, false).await - } - - /// Forcibly removes a failed node from the cluster - /// immediately, instead of waiting for the reaper to eventually reclaim it. - /// This also has the effect that Serf will no longer attempt to reconnect - /// to this node. - pub async fn remove_failed_node_prune(&self, id: T::Id) -> Result<(), Error> { - self.force_leave(id, true).await - } - - /// Forcefully shuts down the Serf instance, stopping all network - /// activity and background maintenance associated with the instance. - /// - /// This is not a graceful shutdown, and should be preceded by a call - /// to Leave. Otherwise, other nodes in the cluster will detect this node's - /// exit as a node failure. - /// - /// It is safe to call this method multiple times. - pub async fn shutdown(&self) -> Result<(), Error> { - { - let mut s = self.inner.state.lock(); - match *s { - SerfState::Shutdown => return Ok(()), - SerfState::Left => {} - _ => { - tracing::warn!("serf: shutdown without a leave"); - } - } - - // Wait to close the shutdown channel until after we've shut down the - // memberlist and its associated network resources, since the shutdown - // channel signals that we are cleaned up outside of Serf. - *s = SerfState::Shutdown; - } - self.inner.memberlist.shutdown().await?; - self.inner.shutdown_tx.close(); - - // Wait for the snapshoter to finish if we have one - if let Some(ref snap) = self.inner.snapshot { - snap.wait().await; - } - - loop { - if let Ok(mut handles) = self.inner.handles.try_borrow_mut() { - let mut futs = core::mem::take(&mut *handles); - while futs.next().await.is_some() {} - break; - } - } - - Ok(()) - } - - /// Returns the network coordinate of the local node. - pub fn cooridate(&self) -> Result> { - if let Some(ref coord) = self.inner.coord_core { - return Ok(coord.client.get_coordinate()); - } - - Err(Error::coordinates_disabled()) - } - - /// Returns the network coordinate for the node with the given - /// name. This will only be valid if `disable_coordinates` is set to `false`. - pub fn cached_coordinate(&self, id: &T::Id) -> Result, Error> { - if let Some(ref coord) = self.inner.coord_core { - return Ok(coord.cache.read().get(id).cloned()); - } - - Err(Error::coordinates_disabled()) - } - - /// Returns the underlying [`Memberlist`] instance - #[inline] - pub fn memberlist(&self) -> &Memberlist> { - &self.inner.memberlist - } -} - -#[viewit::viewit(vis_all = "", getters(vis_all = "pub", prefix = "get"), setters(skip))] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub struct Stats { - members: usize, - failed: usize, - left: usize, - health_score: usize, - member_time: u64, - event_time: u64, - query_time: u64, - intent_queue: usize, - event_queue: usize, - query_queue: usize, - encrypted: bool, - #[cfg_attr(feature = "serde", serde(skip_serializing_if = "Option::is_none"))] - coordinate_resets: Option, -} diff --git a/legacy/serf-core/src/serf/base.rs b/legacy/serf-core/src/serf/base.rs deleted file mode 100644 index 61f57ee1..00000000 --- a/legacy/serf-core/src/serf/base.rs +++ /dev/null @@ -1,1873 +0,0 @@ -use std::time::Duration; - -use crate::types::{MessageRef, Tags, UserEventMessageRef}; -use either::Either; -use futures::{FutureExt, StreamExt}; -use memberlist_core::{ - CheapClone, - agnostic_lite::AfterHandle, - bytes::Bytes, - delegate::EventDelegate, - proto::{Data, MaybeResolvedAddress, Meta, Node, NodeState, OneOrMore, TinyVec}, - tracing, -}; -use rand::{RngExt, SeedableRng}; -use smol_str::SmolStr; - -use crate::{ - QueueOptions, - coalesce::{MemberEventCoalescer, UserEventCoalescer, coalesced_event}, - error::Error, - event::{InternalQueryEvent, MemberEvent, MemberEventType, QueryContext, QueryEvent}, - snapshot::{Snapshot, open_and_replay_snapshot}, - types::{ - DelegateVersion, Epoch, JoinMessage, LeaveMessage, Member, MemberState, MemberStatus, - MemberlistDelegateVersion, MemberlistProtocolVersion, MessageType, NodeIntent, ProtocolVersion, - QueryFlag, QueryMessage, QueryResponseMessage, UserEvent, UserEventMessage, - coordinate::CoordinateOptions, - }, -}; - -use self::internal_query::SerfQueries; - -use super::*; - -/// Re-export the unit tests -#[cfg(feature = "test")] -#[cfg_attr(docsrs, doc(cfg(feature = "test")))] -pub mod tests; - -impl Serf -where - D: Delegate, - T: Transport, -{ - #[cfg(any(feature = "test", test))] - pub(crate) async fn with_message_dropper( - transport: T::Options, - opts: Options, - message_dropper: Box, - ) -> Result> { - Self::new_in( - None, - None, - transport, - opts, - #[cfg(any(feature = "test", test))] - Some(message_dropper), - ) - .await - } - - pub(crate) async fn new_in( - ev: Option>>, - delegate: Option, - transport: T::Options, - opts: Options, - #[cfg(any(test, feature = "test"))] message_dropper: Option>, - ) -> Result> { - if opts.max_user_event_size > USER_EVENT_SIZE_LIMIT { - return Err(Error::user_event_limit_too_large(USER_EVENT_SIZE_LIMIT)); - } - - // Check that the meta data length is okay - { - let tags = opts.tags.load(); - if !tags.as_ref().is_empty() { - let len = tags.encoded_len_with_length_delimited(); - let meta_encoded_len = 1 + (len as u32).encoded_len() + len; - if meta_encoded_len > Meta::MAX_SIZE { - return Err(Error::tags_too_large(len)); - } - } - } - - let (shutdown_tx, shutdown_rx) = async_channel::bounded(1); - - let handles = FuturesUnordered::new(); - let event_tx = ev.map(|mut event_tx| { - // Check if serf member event coalescing is enabled - if opts.coalesce_period > Duration::ZERO && opts.quiescent_period > Duration::ZERO { - let c = MemberEventCoalescer::new(); - - event_tx = coalesced_event( - event_tx, - shutdown_rx.clone(), - opts.coalesce_period, - opts.quiescent_period, - c, - ); - } - - // Check if user event coalescing is enabled - if opts.user_coalesce_period > Duration::ZERO && opts.user_quiescent_period > Duration::ZERO { - let c = UserEventCoalescer::new(); - event_tx = coalesced_event( - event_tx, - shutdown_rx.clone(), - opts.user_coalesce_period, - opts.user_quiescent_period, - c, - ); - } - - event_tx - }); - - // Listen for internal Serf queries. This is setup before the snapshotter, since - // we want to capture the query-time, but the internal listener does not passthrough - // the queries - let (event_tx, handle) = SerfQueries::new(event_tx.clone(), shutdown_rx.clone()); - handles.push(handle); - - let clock = LamportClock::new(); - let event_clock = LamportClock::new(); - let query_clock = LamportClock::new(); - let mut event_min_time = LamportTime::ZERO; - let mut query_min_time = LamportTime::ZERO; - - // Try access the snapshot - let (old_clock, old_event_clock, old_query_clock, event_tx, alive_nodes, handle) = - if let Some(sp) = opts.snapshot_path.as_ref() { - let rs = open_and_replay_snapshot(sp, opts.rejoin_after_leave)?; - let old_clock = rs.last_clock; - let old_event_clock = rs.last_event_clock; - let old_query_clock = rs.last_query_clock; - let (event_tx, alive_nodes, handle) = Snapshot::from_replay_result( - rs, - SNAPSHOT_SIZE_LIMIT, - opts.rejoin_after_leave, - clock.clone(), - event_tx, - shutdown_rx.clone(), - #[cfg(feature = "metrics")] - opts.memberlist_options.metric_labels().clone(), - )?; - event_min_time = old_event_clock + LamportTime::new(1); - query_min_time = old_query_clock + LamportTime::new(1); - ( - old_clock, - old_event_clock, - old_query_clock, - event_tx, - alive_nodes, - Some(handle), - ) - } else { - ( - LamportTime::new(0), - LamportTime::new(0), - LamportTime::new(0), - event_tx, - TinyVec::new(), - None, - ) - }; - - // Set up network coordinate client. - let coord = (!opts.disable_coordinates).then_some({ - CoordinateClient::with_options(CoordinateOptions { - #[cfg(feature = "metrics")] - metric_labels: opts.memberlist_options.metric_labels().clone(), - ..Default::default() - }) - }); - let members = Arc::new(RwLock::new(Members::default())); - let num_members = NumMembers::from(members.clone()); - // Setup the various broadcast queues, which we use to send our own - // custom broadcasts along the gossip channel. - let broadcasts = Arc::new(TransmitLimitedQueue::::new( - opts.memberlist_options.retransmit_mult(), - num_members.clone(), - )); - let event_broadcasts = Arc::new(TransmitLimitedQueue::::new( - opts.memberlist_options.retransmit_mult(), - num_members.clone(), - )); - let query_broadcasts = Arc::new(TransmitLimitedQueue::::new( - opts.memberlist_options.retransmit_mult(), - num_members.clone(), - )); - - // Create a buffer for events and queries - let event_buffer = vec![None; opts.event_buffer_size]; - let query_buffer = vec![None; opts.query_buffer_size]; - - // Ensure our lamport clock is at least 1, so that the default - // join LTime of 0 does not cause issues - clock.increment(); - event_clock.increment(); - query_clock.increment(); - - // Restore the clock from snap if we have one - clock.witness(old_clock); - event_clock.witness(old_event_clock); - query_clock.witness(old_query_clock); - - // Create the underlying memberlist that will manage membership - // and failure detection for the Serf instance. - let memberlist = Memberlist::with_delegate( - { - #[cfg(any(test, feature = "test"))] - { - match message_dropper { - Some(dropper) => SerfDelegate::with_dropper(delegate, dropper, opts.tags.clone()), - None => SerfDelegate::new(delegate, opts.tags.clone()), - } - } - #[cfg(not(any(test, feature = "test")))] - { - SerfDelegate::new(delegate, opts.tags.clone()) - } - }, - transport, - opts.memberlist_options.clone(), - ) - .await?; - - let c = SerfCore { - clock, - event_clock, - query_clock, - broadcasts, - memberlist, - members, - event_broadcasts, - event_join_ignore: AtomicBool::new(false), - event_core: RwLock::new(EventCore { - min_time: event_min_time, - buffer: event_buffer, - }), - query_broadcasts, - query_core: Arc::new(RwLock::new(QueryCore { - min_time: query_min_time, - responses: HashMap::new(), - buffer: query_buffer, - })), - opts, - handles: AtomicRefCell::new(handles), - state: parking_lot::Mutex::new(SerfState::Alive), - join_lock: Mutex::new(()), - snapshot: handle, - #[cfg(feature = "encryption")] - key_manager: crate::key_manager::KeyManager::new(), - shutdown_tx, - shutdown_rx: shutdown_rx.clone(), - coord_core: coord.map(|cc| { - Arc::new(CoordCore { - client: cc, - cache: parking_lot::RwLock::new(HashMap::new()), - }) - }), - event_tx, - }; - let this = Serf { inner: Arc::new(c) }; - // update delegate - let that = this.downgrade(); - let memberlist_delegate = this.inner.memberlist.delegate().unwrap(); - memberlist_delegate.store(that); - let local_node = this.inner.memberlist.local_state().await; - if let Some(local_node) = local_node { - memberlist_delegate.notify_join(local_node).await; - } - - // update key manager - #[cfg(feature = "encryption")] - { - let that = this.downgrade(); - this.inner.key_manager.store(that); - } - - let handles = this.inner.handles.borrow(); - // Start the background tasks. See the documentation above each method - // for more information on their role. - let h = Reaper { - coord_core: this.inner.coord_core.clone(), - memberlist: this.inner.memberlist.clone(), - members: this.inner.members.clone(), - event_tx: this.inner.event_tx.clone(), - shutdown_rx: shutdown_rx.clone(), - reap_interval: this.inner.opts.reap_interval, - reconnect_timeout: this.inner.opts.reconnect_timeout, - recent_intent_timeout: this.inner.opts.recent_intent_timeout, - tombstone_timeout: this.inner.opts.tombstone_timeout, - } - .spawn(); - handles.push(h); - - let h = Reconnector { - members: this.inner.members.clone(), - memberlist: this.inner.memberlist.clone(), - shutdown_rx: shutdown_rx.clone(), - reconnect_interval: this.inner.opts.reconnect_interval, - } - .spawn(); - handles.push(h); - - let h = QueueChecker { - name: "serf.queue.intent", - queue: this.inner.broadcasts.clone(), - members: this.inner.members.clone(), - opts: this.inner.opts.queue_opts(), - shutdown_rx: shutdown_rx.clone(), - } - .spawn::(); - handles.push(h); - - let h = QueueChecker { - name: "serf.queue.event", - queue: this.inner.event_broadcasts.clone(), - members: this.inner.members.clone(), - opts: this.inner.opts.queue_opts(), - shutdown_rx: shutdown_rx.clone(), - } - .spawn::(); - handles.push(h); - - let h = QueueChecker { - name: "serf.queue.query", - queue: this.inner.query_broadcasts.clone(), - members: this.inner.members.clone(), - opts: this.inner.opts.queue_opts(), - shutdown_rx: shutdown_rx.clone(), - } - .spawn::(); - handles.push(h); - - // Attempt to re-join the cluster if we have known nodes - if !alive_nodes.is_empty() { - let memberlist = this.inner.memberlist.clone(); - Self::handle_rejoin(memberlist, alive_nodes); - } - drop(handles); - Ok(this) - } - - pub(crate) async fn has_alive_members(&self) -> bool { - let members = self.inner.members.read().await; - for member in members.states.values() { - if member.member.node.id() == self.inner.memberlist.local_id() { - continue; - } - - if member.member.status == MemberStatus::Alive { - return true; - } - } - - false - } - - /// Takes a Serf message type, encodes it for the wire, and queues - /// the broadcast. If a notify channel is given, this channel will be closed - /// when the broadcast is sent. - pub(crate) async fn broadcast( - &self, - msg: Bytes, - notify_tx: Option>, - ) -> Result<(), Error> { - self - .inner - .broadcasts - .queue_broadcast(SerfBroadcast { msg, notify_tx }) - .await; - Ok(()) - } - - /// Broadcasts a new join intent with a - /// given clock value. It is used on either join, or if - /// we need to refute an older leave intent. Cannot be called - /// with the memberLock held. - pub(crate) async fn broadcast_join(&self, ltime: LamportTime) -> Result<(), Error> { - // Construct message to update our lamport clock - let msg = JoinMessage::new(ltime, self.inner.memberlist.local_id().cheap_clone()); - self.inner.clock.witness(ltime); - - // Process update locally - self.handle_node_join_intent(&msg).await; - - let msg = crate::types::encode_message_to_bytes(&msg)?; - // Start broadcasting the update - if let Err(e) = self.broadcast(msg, None).await { - tracing::warn!(err=%e, "serf: failed to broadcast join intent"); - return Err(e); - } - - Ok(()) - } - - /// Serialize the current keyring and save it to a file. - #[cfg(feature = "encryption")] - pub(crate) async fn write_keyring_file(&self) -> std::io::Result<()> { - let Some(path) = self.inner.opts.keyring_file() else { - return Ok(()); - }; - - if let Some(keyring) = self.inner.memberlist.keyring() { - let encoded_keys = keyring - .keys() - .map(|k| k.to_base64()) - .collect::>(); - - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - let mut opts = std::fs::OpenOptions::new(); - opts.truncate(true).write(true).create(true).mode(0o600); - return opts.open(path).and_then(|file| { - serde_json::to_writer_pretty(file, &encoded_keys).map_err(std::io::Error::other) - }); - } - // TODO: I don't know how to set permissions on windows - // need helps :) - #[cfg(windows)] - { - let mut opts = std::fs::OpenOptions::new(); - opts.truncate(true).write(true).create(true); - return opts.open(path).and_then(|file| { - serde_json::to_writer_pretty(file, &encoded_keys).map_err(std::io::Error::other) - }); - } - } - - Ok(()) - } - - #[cfg(feature = "test")] - pub(crate) async fn get_queue_max(&self) -> usize { - let mut max = self.inner.opts.max_queue_depth; - if self.inner.opts.min_queue_depth > 0 { - let num_members = self.inner.members.read().await.states.len(); - max = num_members * 2; - - if max < self.inner.opts.min_queue_depth { - max = self.inner.opts.min_queue_depth; - } - } - max - } - - /// Forcibly removes a failed node from the cluster - /// immediately, instead of waiting for the reaper to eventually reclaim it. - /// This also has the effect that Serf will no longer attempt to reconnect - /// to this node. - pub(crate) async fn force_leave(&self, id: T::Id, prune: bool) -> Result<(), Error> { - // Construct the message to broadcast - let msg = LeaveMessage { - ltime: self.inner.clock.time(), - id, - prune, - }; - - // Process our own event - self.handle_node_leave_intent(&msg).await; - - // If we have no members, then we don't need to broadcast - if !self.has_alive_members().await { - return Ok(()); - } - - let msg = crate::types::encode_message_to_bytes(&msg)?; - // Broadcast the remove - let (ntx, nrx) = async_channel::bounded(1); - self.broadcast(msg, Some(ntx)).await?; - - // Wait for the broadcast - ::timeout(self.inner.opts.broadcast_timeout, nrx.recv()) - .await - .map_err(|_| Error::removal_broadcast_timeout())? - .map_err(|_| Error::broadcast_channel_closed()) - } -} - -struct Reaper -where - D: Delegate, - T: Transport, -{ - coord_core: Option>>, - memberlist: Memberlist>, - members: Arc>>, - event_tx: async_channel::Sender>, - shutdown_rx: async_channel::Receiver<()>, - reap_interval: Duration, - reconnect_timeout: Duration, - recent_intent_timeout: Duration, - tombstone_timeout: Duration, -} - -macro_rules! erase_node { - ($tx:ident <- $coord:ident($members:ident[$id:ident].$m:ident)) => {{ - // takes a node completely out of the member list - $members.states.remove($id); - - // Tell the coordinate client the node has gone away and delete - // its cached coordinates. - if let Some(cc) = $coord { - cc.client.forget_node($id); - cc.cache.write().remove($id); - } - - // Send out event - let _ = $tx - .send(CrateEvent::from(MemberEvent { - ty: MemberEventType::Reap, - members: Arc::new(TinyVec::from($m.member.clone())), - })) - .await; - }}; -} - -macro_rules! reap { - ( - $tx:ident <- $local_id:ident.$reconnector:ident($timeout: ident($members: ident.$ty: ident, $coord:ident)) - ) => {{ - let mut n = $members.$ty.len(); - let mut i = 0; - while i < n { - let m = $members.$ty[i].clone(); - let mut member_timeout = $timeout; - if let Some(r) = $reconnector { - member_timeout = r.reconnect_timeout(&m.member, member_timeout); - } - - // Skip if the timeout is not yet reached - if let Some(leave_time) = m.leave_time { - if leave_time.elapsed() <= member_timeout { - i += 1; - continue; - } - } - - // Delete from the list - $members.$ty.swap_remove(i); - n -= 1; - - // Delete from members and send out event - let id = m.member.node.id(); - tracing::info!("serf: event member reap: {} reaps {}", $local_id, id); - - erase_node!($tx <- $coord($members[id].m)); - } - }}; -} - -impl Reaper -where - D: Delegate, - T: Transport, -{ - async fn run(self) { - let tick = ::interval(self.reap_interval); - futures::pin_mut!(tick); - loop { - futures::select! { - _ = tick.next().fuse() => { - let mut ms = self.members.write().await; - let local_id = self.memberlist.local_id(); - Self::reap_failed(local_id, &mut ms, &self.event_tx, self.memberlist.delegate().and_then(|d| d.delegate()), self.coord_core.as_deref(), self.reconnect_timeout).await; - Self::reap_left(local_id, &mut ms, &self.event_tx, self.memberlist.delegate().and_then(|d| d.delegate()), self.coord_core.as_deref(), self.tombstone_timeout).await; - reap_intents(&mut ms.recent_intents, Epoch::now(), self.recent_intent_timeout); - if self.shutdown_rx.is_closed() { - break; - } - } - _ = self.shutdown_rx.recv().fuse() => { - break; - } - } - } - tracing::debug!("serf: reaper exits"); - } - - fn spawn(self) -> <::Spawner as AsyncSpawner>::JoinHandle<()> { - ::spawn(async move { - self.run().await; - }) - } - - async fn reap_failed( - local_id: &T::Id, - old: &mut Members, - event_tx: &async_channel::Sender>, - reconnector: Option<&D>, - coord: Option<&CoordCore>, - timeout: Duration, - ) { - reap!(event_tx <- local_id.reconnector(timeout(old.failed_members, coord))) - } - - async fn reap_left( - local_id: &T::Id, - old: &mut Members, - event_tx: &async_channel::Sender>, - reconnector: Option<&D>, - coord: Option<&CoordCore>, - timeout: Duration, - ) { - reap!(event_tx <- local_id.reconnector(timeout(old.left_members, coord))) - } -} - -struct Reconnector -where - T: Transport, - D: Delegate, -{ - members: Arc>>, - memberlist: Memberlist>, - shutdown_rx: async_channel::Receiver<()>, - reconnect_interval: Duration, -} - -impl Reconnector -where - D: Delegate, - T: Transport, -{ - fn spawn(self) -> <::Spawner as AsyncSpawner>::JoinHandle<()> { - let mut rng = rand::rngs::StdRng::from_rng(&mut rand::rng()); - - ::spawn(async move { - let tick = ::interval(self.reconnect_interval); - futures::pin_mut!(tick); - loop { - futures::select! { - _ = tick.next().fuse() => { - let mu = self.members.read().await; - let num_failed = mu.failed_members.len(); - // Nothing to do if there are no failed members - if num_failed == 0 { - continue; - } - - // Probability we should attempt to reconect is given - // by num failed / (num members - num failed - num left) - // This means that we probabilistically expect the cluster - // to attempt to connect to each failed member once per - // reconnect interval - - let num_alive = (mu.states.len() - num_failed - mu.left_members.len()).max(1); - let prob = num_failed as f32 / num_alive as f32; - let r: f32 = rng.random(); - if r > prob { - tracing::debug!("serf: forgoing reconnect for random throttling"); - continue; - } - - // Select a random member to try and join - let idx: usize = rng.random_range(0..num_failed); - let member = &mu.failed_members[idx]; - - let (id, address) = member.member.node().cheap_clone().into_components(); - drop(mu); // release read lock - tracing::info!("serf: attempting to reconnect to {}", id); - // Attempt to join at the memberlist level - if let Err(e) = self.memberlist.join(MaybeResolvedAddress::resolved(address)).await { - tracing::warn!("serf: failed to reconnect {}: {}", id, e); - } else { - tracing::info!("serf: successfully reconnected to {}", id); - } - } - _ = self.shutdown_rx.recv().fuse() => { - break; - } - } - } - - tracing::debug!("serf: reconnector exits"); - }) - } -} - -struct QueueChecker { - name: &'static str, - queue: Arc>>, - members: Arc>>, - opts: QueueOptions, - shutdown_rx: async_channel::Receiver<()>, -} - -impl QueueChecker -where - I: Send + Sync + 'static, - A: Send + Sync + 'static, -{ - fn spawn(self) -> <::Spawner as AsyncSpawner>::JoinHandle<()> { - R::spawn(async move { - let tick = R::interval(self.opts.check_interval); - futures::pin_mut!(tick); - loop { - futures::select! { - _ = tick.next().fuse() => { - let numq = self.queue.num_queued().await; - #[cfg(feature = "metrics")] - { - metrics::gauge!(self.name, self.opts.metric_labels.iter()).set(numq as f64); - } - if numq >= self.opts.depth_warning { - tracing::warn!("serf: queue {} depth: {}", self.name, numq); - } - - let max = self.get_queue_max().await; - if numq >= max { - tracing::warn!("serf: {} queue depth ({}) exceeds limit ({}), dropping messages!", self.name, numq, max); - self.queue.prune(max).await; - } - } - _ = self.shutdown_rx.recv().fuse() => { - break; - } - } - } - - tracing::debug!("serf: {} queue checker exits", self.name); - }) - } - - async fn get_queue_max(&self) -> usize { - let mut max = self.opts.max_queue_depth; - if self.opts.min_queue_depth > 0 { - let num_members = self.members.read().await.states.len(); - max = num_members * 2; - - if max < self.opts.min_queue_depth { - max = self.opts.min_queue_depth; - } - } - max - } -} - -// ---------------------------------Hanlders Methods------------------------------- -impl Serf -where - D: Delegate, - T: Transport, -{ - /// Called when a user event broadcast is - /// received. Returns if the message should be rebroadcast. - pub(crate) async fn handle_user_event( - &self, - msg: Either, UserEventMessage>, - ) -> bool { - let (ltime, name) = match &msg { - Either::Left(msg) => (msg.ltime(), msg.name()), - Either::Right(msg) => (msg.ltime, msg.name.as_str()), - }; - - // Witness a potentially newer time - self.inner.event_clock.witness(ltime); - - let mut el = self.inner.event_core.write().await; - - // Ignore if it is before our minimum event time - if ltime < el.min_time { - return false; - } - - // Check if this message is too old - let bltime = LamportTime::new(el.buffer.len() as u64); - let cur_time = self.inner.event_clock.time(); - if cur_time > bltime && ltime < cur_time - bltime { - tracing::warn!( - "serf: received old event {} from time {} (current: {})", - name, - ltime, - cur_time - ); - return false; - } - - // Check if we've already seen this - let idx = u64::from(ltime % bltime) as usize; - let seen: Option<&mut UserEvents> = el.buffer[idx].as_mut(); - - let msg = match msg { - Either::Left(msg) => match UserEventMessage::from_ref(msg) { - Ok(msg) => msg, - Err(e) => { - tracing::warn!("serf: failed to decode user event message: {}", e); - return false; - } - }, - Either::Right(msg) => msg, - }; - - let user_event = UserEvent { - name: msg.name.clone(), - payload: msg.payload.clone(), - }; - if let Some(seen) = seen { - for prev in seen.events.iter() { - if user_event.eq(prev) { - return false; - } - } - seen.events.push(user_event); - } else { - el.buffer[idx] = Some(UserEvents { - ltime, - events: OneOrMore::from(user_event), - }); - } - - #[cfg(feature = "metrics")] - { - metrics::counter!( - "serf.events", - self.inner.opts.memberlist_options.metric_labels().iter() - ) - .increment(1); - - // TODO: how to avoid allocating here? - let named = format!("serf.events.{}", msg.name); - metrics::counter!( - named, - self.inner.opts.memberlist_options.metric_labels().iter() - ) - .increment(1); - } - - if let Err(e) = self.inner.event_tx.send(msg.into()).await { - tracing::error!("serf: failed to send user event: {}", e); - } - - true - } - - #[allow(clippy::too_many_arguments)] - pub(crate) fn query_event( - &self, - ltime: LamportTime, - name: SmolStr, - payload: Bytes, - timeout: Duration, - id: u32, - from: Node, - relay_factor: u8, - ) -> QueryEvent { - QueryEvent { - ltime, - name, - payload, - ctx: Arc::new(QueryContext { - query_timeout: timeout, - span: Mutex::new(Some(Epoch::now())), - this: self.clone(), - }), - id, - from, - relay_factor, - } - } - - pub(crate) async fn internal_query( - &self, - name: SmolStr, - payload: Bytes, - params: Option>, - ty: InternalQueryEvent, - ) -> Result, Error> { - self.query_in(name, payload, params, Some(ty)).await - } - - pub(crate) async fn query_in( - &self, - name: SmolStr, - payload: Bytes, - params: Option>, - ty: Option>, - ) -> Result, Error> { - // Provide default parameters if none given. - let params = match params { - Some(params) if params.timeout != Duration::ZERO => params, - Some(mut params) => { - params.timeout = self.default_query_timeout().await; - params - } - None => self.default_query_param().await, - }; - - // Get the local node - let local = self.inner.memberlist.advertise_node(); - - // Setup the flags - let flags = if params.request_ack { - QueryFlag::ACK - } else { - QueryFlag::empty() - }; - - // Create the message - let q = QueryMessage { - ltime: self.inner.query_clock.time(), - id: rand::random(), - from: local.cheap_clone(), - filters: params.filters, - flags, - relay_factor: params.relay_factor, - timeout: params.timeout, - name: name.clone(), - payload, - }; - - // Encode the query - let len = crate::types::encoded_message_len(&q); - - // Check the size - if len > self.inner.opts.query_size_limit { - return Err(Error::query_too_large(len)); - } - - let raw = crate::types::encode_message_to_bytes(&q)?; - - // Register QueryResponse to track acks and responses - let resp = QueryResponse::from_query(&q, self.inner.memberlist.num_online_members().await); - self - .register_query_response(params.timeout, resp.clone()) - .await; - - // Process query locally - self.handle_query(Either::Right(q), ty).await?; - - // Start broadcasting the event - self - .inner - .query_broadcasts - .queue_broadcast(SerfBroadcast { - msg: raw, - notify_tx: None, - }) - .await; - Ok(resp) - } - - /// Used to setup the listeners for the query, - /// and to schedule closing the query after the timeout. - pub(crate) async fn register_query_response( - &self, - timeout: Duration, - resp: QueryResponse, - ) { - let tresps = self.inner.query_core.clone(); - let mut resps = self.inner.query_core.write().await; - // Map the LTime to the QueryResponse. This is necessarily 1-to-1, - // since we increment the time for each new query. - let ltime = resp.ltime; - resps.responses.insert(ltime, resp); - - // Setup a timer to close the response and deregister after the timeout - ::spawn_after(timeout, async move { - let mut resps = tresps.write().await; - if let Some(resp) = resps.responses.remove(<ime) { - resp.close().await; - } - }) - .detach(); - } - - /// Called when a query broadcast is - /// received. Returns if the message should be rebroadcast. - pub(crate) async fn handle_query( - &self, - q: Either< - as Data>::Ref<'_>, - QueryMessage, - >, - ty: Option>, - ) -> Result { - let (qm_ltime, qm_id, qm_timeout, no_broadcast, ack, name, filters) = match q.as_ref() { - Either::Left(q) => ( - q.ltime(), - q.id(), - q.timeout(), - q.no_broadcast(), - q.ack(), - q.name(), - Either::Left(*q.filters()), - ), - Either::Right(q) => ( - q.ltime, - q.id(), - q.timeout(), - q.no_broadcast(), - q.ack(), - q.name.as_str(), - Either::Right(q.filters.as_slice()), - ), - }; - - // Witness a potentially newer time - self.inner.query_clock.witness(qm_ltime); - - let mut query = self.inner.query_core.write().await; - - // Ignore if it is before our minimum query time - if qm_ltime < query.min_time { - return Ok(false); - } - - // Check if this message is too old - let cur_time = self.inner.query_clock.time(); - let q_time = LamportTime::new(query.buffer.len() as u64); - if cur_time > q_time && qm_ltime < cur_time - q_time { - tracing::warn!( - "serf: received old query {} from time {} (current: {})", - name, - qm_ltime, - cur_time - ); - return Ok(false); - } - - // Check if we've already seen this - let idx = u64::from(qm_ltime % q_time) as usize; - let fresh = match query.buffer[idx].as_mut() { - // Slot holds this ltime: dedup by query id, then record this id. - Some(seen) if seen.ltime == qm_ltime => { - for &prev in seen.query_ids.iter() { - if qm_id == prev { - // Seen this ID already - return Ok(false); - } - } - seen.query_ids.push(qm_id); - false - } - // Empty slot, or a stale entry from a different ltime that wrapped onto - // this ring index: start a fresh record. Appending onto the stale entry - // (leaving its `ltime` unchanged) would break dedup for the new ltime and - // let a re-gossiped query be reprocessed. Mirrors serf's `handleQuery`. - _ => true, - }; - if fresh { - query.buffer[idx] = Some(Queries { - ltime: qm_ltime, - query_ids: MediumVec::from(qm_id), - }); - } - - // update some metrics - #[cfg(feature = "metrics")] - { - metrics::counter!( - "serf.queries", - self.inner.opts.memberlist_options.metric_labels().iter() - ) - .increment(1); - - // TODO: how to avoid allocating here? - let named = format!("serf.queries.{}", name); - metrics::counter!( - named, - self.inner.opts.memberlist_options.metric_labels().iter() - ) - .increment(1); - } - - // Check if we should rebroadcast, this may be disabled by a flag - let mut rebroadcast = true; - if no_broadcast { - rebroadcast = false; - } - - // Filter the query - if !self.should_process_query(filters)? { - // Even if we don't process it further, we should rebroadcast, - // since it is the first time we've seen this. - return Ok(rebroadcast); - } - - // Send ack if requested, without waiting for client to respond() - let (name, payload, from, relay_factor) = if ack { - let ack = QueryResponseMessage { - ltime: qm_ltime, - id: qm_id, - from: self.inner.memberlist.advertise_node(), - flags: QueryFlag::ACK, - payload: Bytes::new(), - }; - - match crate::types::encode_message_to_bytes(&ack) { - Ok(raw) => { - let (name, payload, from, relay_factor) = match q { - Either::Left(q) => ( - SmolStr::new(q.name()), - Bytes::copy_from_slice(q.payload()), - Node::from_ref(*q.from())?, - q.relay_factor(), - ), - Either::Right(q) => (q.name, q.payload, q.from, q.relay_factor), - }; - - if let Err(e) = self.inner.memberlist.send(from.address(), raw).await { - tracing::error!(err=%e, "serf: failed to send ack"); - } - - if let Err(e) = self.relay_response(relay_factor, from.clone(), ack).await { - tracing::error!(err=%e, "serf: failed to relay ack"); - } - (name, payload, from, relay_factor) - } - Err(e) => { - tracing::error!(err=%e, "serf: failed to format ack"); - match q { - Either::Left(q) => ( - SmolStr::new(q.name()), - Bytes::copy_from_slice(q.payload()), - Node::from_ref(*q.from())?, - q.relay_factor(), - ), - Either::Right(q) => (q.name, q.payload, q.from, q.relay_factor), - } - } - } - } else { - match q { - Either::Left(q) => ( - SmolStr::new(q.name()), - Bytes::copy_from_slice(q.payload()), - Node::from_ref(*q.from())?, - q.relay_factor(), - ), - Either::Right(q) => (q.name, q.payload, q.from, q.relay_factor), - } - }; - - let ev = self.query_event( - qm_ltime, - name, - payload, - qm_timeout, - qm_id, - from, - relay_factor, - ); - - if let Err(e) = self - .inner - .event_tx - .send(match ty { - Some(ty) => (ty, ev).into(), - None => ev.into(), - }) - .await - { - tracing::error!(err=%e, "serf: failed to send query"); - } - - Ok(rebroadcast) - } - - /// Called when a query response is - /// received. - pub(crate) async fn handle_query_response( - &self, - resp: as Data>::Ref<'_>, - ) -> Result<(), memberlist_core::proto::DecodeError> { - // Look for a corresponding QueryResponse - let qc = self - .inner - .query_core - .read() - .await - .responses - .get(&resp.ltime()) - .cloned(); - if let Some(query) = qc { - // Verify the ID matches - if query.id != resp.id() { - tracing::warn!( - "serf: query reply ID mismatch (local: {}, response: {})", - query.id, - resp.id() - ); - return Ok(()); - } - - let resp = QueryResponseMessage::::from_ref(resp)?; - - query - .handle_query_response::( - resp, - self.local_id(), - #[cfg(feature = "metrics")] - self.inner.opts.memberlist_options.metric_labels(), - ) - .await; - } else { - tracing::warn!( - "serf: reply for non-running query (LTime: {}, ID: {}) From: {:?}", - resp.ltime(), - resp.id(), - resp.from() - ); - } - - Ok(()) - } - - /// Called when a node join event is received - /// from memberlist. - pub(crate) async fn handle_node_join(&self, n: Arc>) { - let mut members = self.inner.members.write().await; - - #[cfg(any(test, feature = "test"))] - { - if let Some(ref dropper) = self.inner.memberlist.delegate().unwrap().message_dropper { - if dropper.should_drop(MessageType::Join) { - return; - } - } - } - - let node = n.node(); - let tags = if !n.meta().is_empty() { - match ::decode(n.meta()) { - Ok((readed, tags)) => { - tracing::trace!(read = %readed, tags=?tags, "serf: decode tags successfully"); - tags - } - Err(e) => { - tracing::error!(err=%e, "serf: failed to decode tags"); - return; - } - } - } else { - Default::default() - }; - - let (old_status, fut) = if let Some(member) = members.states.get_mut(node.id()) { - let old_status = member.member.status; - #[cfg(feature = "metrics")] - let dead_time = member.leave_time.map(|t| t.elapsed()); - #[cfg(feature = "metrics")] - if old_status == MemberStatus::Failed { - if let Some(dead_time) = dead_time { - if dead_time < self.inner.opts.flap_timeout { - metrics::counter!( - "serf.member.flap", - self.inner.opts.memberlist_options.metric_labels().iter() - ) - .increment(1); - } - } - } - - *member = MemberState { - member: Member { - node: node.cheap_clone(), - tags: Arc::new(tags), - status: MemberStatus::Alive, - protocol_version: member.member.protocol_version, - delegate_version: member.member.delegate_version, - memberlist_delegate_version: member.member.memberlist_delegate_version, - memberlist_protocol_version: member.member.memberlist_protocol_version, - }, - status_time: member.status_time, - leave_time: None, - }; - - ( - old_status, - self.inner.event_tx.send( - MemberEvent { - ty: MemberEventType::Join, - members: Arc::new(TinyVec::from(member.member.clone())), - } - .into(), - ), - ) - } else { - // Check if we have a join or leave intent. The intent buffer - // will only hold one event for this node, so the more recent - // one will take effect. - let mut status = MemberStatus::Alive; - let mut status_ltime = LamportTime::new(0); - if let Some(t) = recent_intent(&members.recent_intents, n.id(), MessageType::Join) { - status_ltime = t; - } - - if let Some(t) = recent_intent(&members.recent_intents, n.id(), MessageType::Leave) { - status_ltime = t; - status = MemberStatus::Leaving; - } - - let ms = MemberState { - member: Member { - node: node.cheap_clone(), - tags: Arc::new(tags), - status, - protocol_version: self.inner.opts.protocol_version, - delegate_version: self.inner.opts.delegate_version, - memberlist_delegate_version: self.inner.opts.memberlist_options.delegate_version(), - memberlist_protocol_version: self.inner.opts.memberlist_options.protocol_version(), - }, - status_time: status_ltime, - leave_time: None, - }; - let member = ms.member.clone(); - members.states.insert(node.id().cheap_clone(), ms); - ( - MemberStatus::None, - self.inner.event_tx.send( - MemberEvent { - ty: MemberEventType::Join, - members: Arc::new(TinyVec::from(member)), - } - .into(), - ), - ) - }; - - if matches!(old_status, MemberStatus::Failed | MemberStatus::Left) { - remove_old_member(&mut members.failed_members, node.id()); - remove_old_member(&mut members.left_members, node.id()); - } - - // update some metrics - #[cfg(feature = "metrics")] - metrics::counter!( - "serf.member.join", - self.inner.opts.memberlist_options.metric_labels().iter() - ) - .increment(1); - - tracing::info!("serf: member join: {}", node); - if let Err(e) = fut.await { - tracing::error!(err=%e, "serf: failed to send member event"); - } - } - - /// Called when a node broadcasts a - /// join message to set the lamport time of its join - pub(crate) async fn handle_node_join_intent(&self, join_msg: &JoinMessage) -> bool { - // Witness a potentially newer time - self.inner.clock.witness(join_msg.ltime); - - let mut members = self.inner.members.write().await; - match members.states.get_mut(join_msg.id()) { - Some(member) => { - // Check if this time is newer than what we have - if join_msg.ltime <= member.status_time { - return false; - } - - // Update the LTime - member.status_time = join_msg.ltime; - - // If we are in the leaving state, we should go back to alive, - // since the leaving message must have been for an older time - - if member.member.status == MemberStatus::Leaving { - member.member.status = MemberStatus::Alive; - } - - true - } - None => { - // Rebroadcast only if this was an update we hadn't seen before. - upsert_intent( - &mut members.recent_intents, - join_msg.id(), - MessageType::Join, - join_msg.ltime, - Epoch::now, - ) - } - } - } - - pub(crate) async fn handle_node_leave(&self, n: Arc>) { - let mut members = self.inner.members.write().await; - - let Some(member_state) = members.states.get_mut(n.id()) else { - return; - }; - - let mut ms = member_state.member.status; - let member = match ms { - MemberStatus::Leaving => { - member_state.member.status = MemberStatus::Left; - - ms = MemberStatus::Left; - member_state.leave_time = Some(Epoch::now()); - let member_state = member_state.clone(); - let member = member_state.member.clone(); - members.left_members.push(member_state); - member - } - MemberStatus::Alive => { - member_state.member.status = MemberStatus::Failed; - ms = MemberStatus::Failed; - member_state.leave_time = Some(Epoch::now()); - let member_state = member_state.clone(); - let member = member_state.member.clone(); - members.failed_members.push(member_state); - member - } - _ => { - tracing::warn!("serf: bad state when leave: {}", ms); - return; - } - }; - - // Send an event along - let ty = if ms != MemberStatus::Left { - MemberEventType::Failed - } else { - MemberEventType::Leave - }; - - // Update some metrics - #[cfg(feature = "metrics")] - metrics::counter!( - "serf.member.leave", - self.inner.opts.memberlist_options.metric_labels().iter() - ) - .increment(1); - - tracing::info!("serf: {}: {}", ty.as_str(), member.node()); - - if let Err(e) = self - .inner - .event_tx - .send( - MemberEvent { - ty, - members: Arc::new(TinyVec::from(member)), - } - .into(), - ) - .await - { - tracing::error!(err=%e, "serf: failed to send member event: {}", e); - } - } - - pub(crate) async fn handle_node_leave_intent(&self, msg: &LeaveMessage) -> bool { - let state = self.state(); - - // Witness a potentially newer time - self.inner.clock.witness(msg.ltime); - - let mut members = self.inner.members.write().await; - - if !members.states.contains_key(msg.id()) { - return upsert_intent( - &mut members.recent_intents, - msg.id(), - MessageType::Leave, - msg.ltime, - Epoch::now, - ); - } - - let members = atomic_refcell::AtomicRefCell::new(&mut *members); - let mut members_mut = members.borrow_mut(); - let member = members_mut.states.get_mut(msg.id()).unwrap(); - // If the message is old, then it is irrelevant and we can skip it - if msg.ltime <= member.status_time { - return false; - } - - // Refute us leaving if we are in the alive state - // Must be done in another goroutine since we have the memberLock - if msg.id().eq(self.inner.memberlist.local_id()) && state == SerfState::Alive { - tracing::debug!("serf: refuting an older leave intent"); - let this = self.clone(); - let ltime = self.inner.clock.time(); - ::spawn_detach(async move { - if let Err(e) = this.broadcast_join(ltime).await { - tracing::error!(err=%e, "serf: failed to broadcast join"); - } - }); - return false; - } - - // Always update the lamport time even when the status does not change - // (despite the variable naming implying otherwise). - // - // By updating this statusLTime here we ensure that the earlier conditional - // on "leaveMsg.LTime <= member.statusLTime" will prevent an infinite - // rebroadcast when seeing two successive leave message for the same - // member. Without this fix a leave message that arrives after a member is - // already marked as leaving/left will cause it to be rebroadcast without - // marking it locally as witnessed. If more than one serf instance in the - // cluster experiences this series of events then they will rebroadcast - // each other's messages about the affected node indefinitely. - // - // This eventually leads to overflowing serf intent queues - // - https://github.com/hashicorp/consul/issues/8179 - // - https://github.com/hashicorp/consul/issues/7960 - member.status_time = msg.ltime; - - // State transition depends on current state - match member.member.status { - MemberStatus::None => false, - MemberStatus::Alive => { - member.member.status = MemberStatus::Leaving; - - if msg.prune { - let owned = member.clone(); - drop(members_mut); - self.handle_prune(&owned, *members.borrow_mut()).await; - } - true - } - MemberStatus::Leaving | MemberStatus::Left => { - if msg.prune { - let owned = member.clone(); - drop(members_mut); - self.handle_prune(&owned, *members.borrow_mut()).await; - } - true - } - MemberStatus::Failed => { - member.member.status = MemberStatus::Left; - let owned = member.clone(); - drop(members_mut); - - let mut members_mut = members.borrow_mut(); - // Remove from the failed list and add to the left list. We add - // to the left list so that when we do a sync, other nodes will - // remove it from their failed list. - members_mut - .failed_members - .retain(|m| m.member.node.id().ne(owned.member.node.id())); - members_mut.left_members.push(owned.clone()); - - // We must push a message indicating the node has now - // left to allow higher-level applications to handle the - // graceful leave. - tracing::info!("serf: EventMemberLeave (forced): {}", owned.member.node); - - if let Err(e) = self - .inner - .event_tx - .send( - MemberEvent { - ty: MemberEventType::Leave, - members: Arc::new(TinyVec::from(owned.member.clone())), - } - .into(), - ) - .await - { - tracing::error!(err=%e, "serf: failed to send member event"); - } - - if msg.prune { - self.handle_prune(&owned, *members_mut).await; - } - - true - } - status => { - tracing::warn!(status=%status, "serf: received leave intent for unknown member status"); - member.member.status = MemberStatus::Leaving; - - if msg.prune { - let owned = member.clone(); - drop(members_mut); - self.handle_prune(&owned, *members.borrow_mut()).await; - } - true - } - } - } - - /// Called when a node meta data update - /// has taken place - pub(crate) async fn handle_node_update(&self, n: Arc>) { - let tags = match ::decode(n.meta()) { - Ok((readed, tags)) => { - tracing::trace!(read = %readed, tags=?tags, "serf: decode tags successfully"); - tags - } - Err(e) => { - tracing::error!(err=%e, "serf: failed to decode tags"); - return; - } - }; - let mut members = self.inner.members.write().await; - let id = n.id(); - if let Some(ms) = members.states.get_mut(id) { - // Update the member attributes - ms.member = Member { - node: n.node(), - tags: Arc::new(tags), - status: ms.member.status, - protocol_version: ProtocolVersion::V1, - delegate_version: DelegateVersion::V1, - memberlist_delegate_version: MemberlistDelegateVersion::V1, - memberlist_protocol_version: MemberlistProtocolVersion::V1, - }; - - #[cfg(feature = "metrics")] - metrics::counter!( - "serf.member.update", - self.inner.opts.memberlist_options.metric_labels().iter() - ) - .increment(1); - - tracing::info!("serf: member update: {}", id); - if let Err(e) = self - .inner - .event_tx - .send( - MemberEvent { - ty: MemberEventType::Update, - members: Arc::new(TinyVec::from(ms.member.clone())), - } - .into(), - ) - .await - { - tracing::error!(err=%e, "serf: failed to send member event"); - } - } - } - - /// Waits for nodes that are leaving and then forcibly - /// erases a member from the list of members - pub(crate) async fn handle_prune( - &self, - member: &MemberState, - members: &mut Members, - ) { - let ms = member.member.status; - if ms == MemberStatus::Leaving { - ::sleep( - self.inner.opts.broadcast_timeout + self.inner.opts.leave_propagate_delay, - ) - .await; - } - - let node = member.member.node(); - let id = node.id(); - tracing::info!("serf: EventMemberReap (forced): {}", node); - - // If we are leaving or left we may be in that list of members - if matches!(ms, MemberStatus::Leaving | MemberStatus::Left) { - remove_old_member(&mut members.left_members, id); - } - - let tx = &self.inner.event_tx; - let coord = self.inner.coord_core.as_deref(); - erase_node!(tx <- coord(members[id].member)) - } - - /// Invoked when a join detects a conflict over a name. - /// This means two different nodes (IP/Port) are claiming the same name. Memberlist - /// will reject the "new" node mapping, but we can still be notified. - pub(crate) async fn handle_node_conflict( - &self, - existing: Arc>, - other: Arc>, - ) { - // Log a basic warning if the node is not us... - if existing.id() != self.inner.memberlist.local_id() { - tracing::warn!( - "serf: node conflict detected between {}({}) and {}({})", - existing.id(), - existing.address(), - other.id(), - other.address(), - ); - return; - } - - // The current node is conflicting! This is an error - tracing::error!( - "serf: node id conflicts with another node at {}. node id must be unique! (resolution enabled: {})", - other.id(), - self.inner.opts.enable_id_conflict_resolution - ); - - // If automatic resolution is enabled, kick off the resolution - if self.inner.opts.enable_id_conflict_resolution { - let this = self.clone(); - ::spawn_detach(async move { this.resolve_node_conflict().await }); - } - } - - /// Used to determine which node should remain during - /// a name conflict. This is done by running an internal query. - async fn resolve_node_conflict(&self) { - // Get the local node - let local_id = self.inner.memberlist.local_id(); - let local_advertise_addr = self.inner.memberlist.advertise_address(); - - let payload = match local_id.encode_to_bytes() { - Ok(id) => id, - Err(e) => { - tracing::error!(err=%e, "serf: failed to encode local id"); - return; - } - }; - - // Start an id resolution query - let ty = InternalQueryEvent::Conflict(local_id.clone()); - let resp = match self - .internal_query(SmolStr::new(ty.as_str()), payload, None, ty) - .await - { - Ok(resp) => resp, - Err(e) => { - tracing::error!(err=%e, "serf: failed to start node id resolution query"); - return; - } - }; - - // Counter to determine winner - let mut responses = 0usize; - let mut matching = 0usize; - - // Gather responses - let resp_rx = resp.response_rx(); - while let Ok(r) = resp_rx.recv().await { - let res = crate::types::decode_message::(&r.payload); - match res { - Ok(msg) => { - match msg { - MessageRef::ConflictResponse(resp) => { - // Update the counters - responses += 1; - match ::from_ref(*resp.member().node().address()) { - Ok(addr) => { - if addr.eq(local_advertise_addr) { - matching += 1; - } - } - Err(e) => { - tracing::error!(err=%e, "serf: failed to decode conflict query response"); - continue; - } - } - } - msg => { - tracing::warn!( - type = %msg.ty(), - "serf: invalid conflict query response type", - ); - continue; - } - } - } - Err(e) => { - tracing::error!(err=%e, "serf: failed to decode conflict query response"); - continue; - } - } - } - - // Query over, determine if we should live - let majority = (responses / 2) + 1; - if matching >= majority { - tracing::info!( - "serf: majority in node id conflict resolution [{} / {}]", - matching, - responses - ); - return; - } - - // Since we lost the vote, we need to exit - tracing::warn!( - "serf: minority in name conflict resolution, quiting [{} / {}]", - matching, - responses - ); - - if let Err(e) = self.shutdown().await { - tracing::error!(err=%e, "serf: failed to shutdown"); - } - } - - pub(crate) fn handle_rejoin( - memberlist: Memberlist>, - alive_nodes: TinyVec>>, - ) { - ::spawn_detach(async move { - for prev in alive_nodes { - // Do not attempt to join ourself - if prev.id().eq(memberlist.local_id()) { - continue; - } - - tracing::info!("serf: attempting re-join to previously known node {}", prev); - if let Err(e) = memberlist.join(prev.address().cheap_clone()).await { - tracing::warn!( - "serf: failed to re-join to previously known node {}: {}", - prev, - e - ); - } else { - tracing::info!("serf: re-joined to previously known node: {}", prev); - return; - } - } - - tracing::warn!("serf: failed to re-join to any previously known node"); - }); - } -} - -/// Used to remove an old member from a list of old -/// members. -fn remove_old_member(old: &mut OneOrMore>, id: &I) { - old.retain(|m| m.member.node.id() != id); -} - -/// Clears out any intents that are older than the timeout. Make sure -/// the memberLock is held when passing in the Serf instance's recentIntents -/// member. -fn reap_intents(intents: &mut HashMap, now: Epoch, timeout: Duration) { - intents.retain(|_, intent| (now - intent.wall_time) <= timeout); -} - -fn recent_intent( - intents: &HashMap, - id: &I, - ty: MessageType, -) -> Option { - match intents.get(id) { - Some(intent) if intent.ty == ty => Some(intent.ltime), - _ => None, - } -} - -fn upsert_intent( - intents: &mut HashMap, - node: &I, - t: MessageType, - ltime: LamportTime, - stamper: impl FnOnce() -> Epoch, -) -> bool -where - I: CheapClone + Eq + core::hash::Hash, -{ - match intents.entry(node.cheap_clone()) { - std::collections::hash_map::Entry::Occupied(mut ent) => { - let intent = ent.get_mut(); - if ltime > intent.ltime { - intent.ty = t; - intent.ltime = ltime; - intent.wall_time = stamper(); - true - } else { - false - } - } - std::collections::hash_map::Entry::Vacant(ent) => { - ent.insert(NodeIntent { - ty: t, - wall_time: stamper(), - ltime, - }); - true - } - } -} diff --git a/legacy/serf-core/src/serf/base/tests.rs b/legacy/serf-core/src/serf/base/tests.rs deleted file mode 100644 index b66ff279..00000000 --- a/legacy/serf-core/src/serf/base/tests.rs +++ /dev/null @@ -1,459 +0,0 @@ -use std::time::Duration; - -use crate::types::{ - MessageRef, MessageType, Node, PushPullMessage, QueryFlag, QueryMessage, UserEvent, - UserEventMessage, -}; -use async_channel::Receiver; -use memberlist_core::{ - agnostic_lite::RuntimeLite, - bytes::Bytes, - delegate::NodeDelegate, - proto::{MaybeResolvedAddress, OneOrMore, TinyVec}, -}; -use smol_str::SmolStr; - -use crate::{ - event::{CrateEvent, CrateEventType, MemberEvent, MemberEventType}, - types::Epoch, -}; - -use super::*; - -pub(crate) mod serf; - -fn test_config() -> Options { - let mut opts = Options::new(); - opts.memberlist_options = opts - .memberlist_options - .with_gossip_interval(Duration::from_millis(5)) - .with_probe_interval(Duration::from_millis(50)) - .with_probe_timeout(Duration::from_millis(25)) - .with_timeout(Duration::from_millis(100)) - .with_suspicion_mult(1); - opts - .with_reap_interval(Duration::from_secs(1)) - .with_reconnect_interval(Duration::from_millis(100)) - .with_reconnect_timeout(Duration::from_micros(1)) - .with_tombstone_timeout(Duration::from_micros(1)) -} - -async fn wait_until_num_nodes(desired_nodes: usize, serfs: &[Serf]) -where - D: Delegate, - T: Transport, -{ - let start = Epoch::now(); - loop { - ::sleep(Duration::from_millis(25)).await; - let mut conds = Vec::with_capacity(serfs.len()); - for (idx, s) in serfs.iter().enumerate() { - let n = s.num_members().await; - if n == desired_nodes { - conds.push(true); - continue; - } - - if start.elapsed() > Duration::from_secs(7) { - panic!("s{} got {} expected {}", idx + 1, n, desired_nodes); - } - } - if conds.len() == serfs.len() { - break; - } - } -} - -async fn wait_until_intent_queue_len(desired_len: usize, serfs: &[Serf]) -where - D: Delegate, - T: Transport, -{ - let start = Epoch::now(); - loop { - ::sleep(Duration::from_millis(25)).await; - let mut conds = Vec::with_capacity(serfs.len()); - for (idx, s) in serfs.iter().enumerate() { - let stats = s.stats().await; - if stats.get_intent_queue() == desired_len { - conds.push(true); - continue; - } - - if start.elapsed() > Duration::from_secs(7) { - panic!( - "s{} got {} expected {}", - idx + 1, - stats.get_intent_queue(), - desired_len - ); - } - } - if conds.len() == serfs.len() { - break; - } - } -} - -/// tests that the given node had the given sequence of events -/// on the event channel. -async fn test_events( - rx: Receiver>, - node: T::Id, - expected: Vec, -) where - D: Delegate, - T: Transport, -{ - let mut actual = Vec::with_capacity(expected.len()); - - loop { - futures::select! { - event = rx.recv().fuse() => { - let event = event.unwrap(); - match event { - CrateEvent::Member(MemberEvent { ty, members }) => { - let mut found = false; - - for m in members.iter() { - if node.eq(m.node.id()) { - found = true; - break; - } - } - - if found { - actual.push(CrateEventType::Member(ty)); - } - } - _ => continue, - } - } - _ = ::sleep(Duration::from_millis(10)).fuse() => { - break; - } - } - } - - assert_eq!(actual, expected, "bad events for node {:?}", node); -} - -/// tests that the given sequence of usr events -/// on the event channel took place. -async fn test_user_events( - rx: Receiver>, - expected_name: Vec, - expected_payload: Vec, -) where - D: Delegate, - T: Transport, -{ - let mut actual_name = Vec::with_capacity(expected_name.len()); - let mut actual_payload = Vec::with_capacity(expected_payload.len()); - - loop { - futures::select! { - event = rx.recv().fuse() => { - let Ok(event) = event else { break }; - match event { - CrateEvent::User(e) => { - actual_name.push(e.name.clone()); - actual_payload.push(e.payload.clone()); - } - _ => continue, - } - } - _ = ::sleep(Duration::from_millis(10)).fuse() => { - break; - } - } - } - - assert_eq!(actual_name, expected_name); - assert_eq!(actual_payload, expected_payload); -} - -/// tests that the given sequence of query events -/// on the event channel took place. -async fn test_query_events( - rx: Receiver>, - expected_name: Vec, - expected_payload: Vec, -) where - D: Delegate, - T: Transport, -{ - let mut actual_name = Vec::with_capacity(expected_name.len()); - let mut actual_payload = Vec::with_capacity(expected_payload.len()); - - loop { - futures::select! { - event = rx.recv().fuse() => { - let Ok(event) = event else { break }; - match event { - CrateEvent::Query(e) => { - actual_name.push(e.name.clone()); - actual_payload.push(e.payload.clone()); - } - CrateEvent::InternalQuery { query, .. } => { - actual_name.push(query.name.clone()); - actual_payload.push(query.payload.clone()); - } - _ => continue, - } - } - _ = ::sleep(Duration::from_millis(10)).fuse() => { - break; - } - } - } - - assert_eq!(actual_name, expected_name); - assert_eq!(actual_payload, expected_payload); -} - -/// Unit test for queries pass through functionality -pub async fn queries_pass_through(s: Serf) -where - T: Transport, -{ - let (tx, rx) = async_channel::bounded(4); - let (_shutdown_tx, shutdown_rx) = async_channel::bounded(1); - let (event_tx, _handle) = SerfQueries::>::new(Some(tx), shutdown_rx); - - // Push a user event - let event = CrateEvent::from( - UserEventMessage::default() - .with_name("foo".into()) - .with_ltime(42.into()), - ); - event_tx.send(event.clone()).await.unwrap(); - - // Push a query - let query = s.query_event( - 42.into(), - "foo".into(), - Bytes::new(), - Default::default(), - 1, - s.memberlist().advertise_node(), - 0, - ); - event_tx.send(CrateEvent::from(query)).await.unwrap(); - - // Push a member event - let event = CrateEvent::from(MemberEvent { - ty: MemberEventType::Join, - members: TinyVec::new().into(), - }); - event_tx.send(event).await.unwrap(); - - // Should get passed through - for _ in 0..3 { - let sleep = ::sleep(Duration::from_millis(100)); - futures::select! { - _ = rx.recv().fuse() => {}, - _ = sleep.fuse() => panic!("timeout"), - } - } -} - -/// Unit test for queries ping functionality -pub async fn queries_ping(s: Serf) -where - T: Transport, -{ - let (tx, rx) = async_channel::bounded(4); - let (_shutdown_tx, shutdown_rx) = async_channel::bounded(1); - let (event_tx, _handle) = SerfQueries::>::new(Some(tx), shutdown_rx); - - // Push a query - let query = s.query_event( - 42.into(), - "ping".into(), - Bytes::new(), - Default::default(), - 1, - s.memberlist().advertise_node(), - 0, - ); - event_tx - .send(CrateEvent::from((InternalQueryEvent::Ping, query))) - .await - .unwrap(); - - let sleep = ::sleep(Duration::from_millis(50)); - futures::select! { - _ = rx.recv().fuse() => panic!("should not passthrough query!"), - _ = sleep.fuse() => {}, - } -} - -/// Unit test for queries conflict functionality -pub async fn queries_conflict_same_name(s: Serf) -where - T: Transport, -{ - let (tx, rx) = async_channel::bounded(4); - let (_shutdown_tx, shutdown_rx) = async_channel::bounded(1); - let (event_tx, _handle) = SerfQueries::>::new(Some(tx), shutdown_rx); - - // Push a query - let query = s.query_event( - 42.into(), - "conflict".into(), - Bytes::new(), - Default::default(), - 1, - s.memberlist().advertise_node(), - 0, - ); - let id = s.memberlist().local_id().clone(); - event_tx - .send(CrateEvent::from((InternalQueryEvent::Conflict(id), query))) - .await - .unwrap(); - - let sleep = ::sleep(Duration::from_millis(50)); - futures::select! { - _ = rx.recv().fuse() => panic!("should not passthrough query!"), - _ = sleep.fuse() => {}, - } -} - -/// Unit test for queries list key response functionality. -/// -/// This test requires the transport to support encryption. -#[cfg(feature = "encryption")] -pub async fn estimate_max_keys_in_list_key_response_factor( - transport_opts: T::Options, - opts: Options, -) where - T: Transport, -{ - use crate::types::KeyResponseMessage; - use memberlist_core::proto::SecretKey; - - let size_limit = opts.query_response_size_limit() * 10; - let opts = opts.with_query_response_size_limit(size_limit); - let s = Serf::::new(transport_opts, opts).await.unwrap(); - let query = s.query_event( - 0.into(), - Default::default(), - Default::default(), - Default::default(), - 0, - s.memberlist().advertise_node(), - 0, - ); - - let mut resp = KeyResponseMessage::default(); - for _ in 0..=(size_limit / 25) { - resp.keys.push(SecretKey::from([1; 16])); - } - - let mut found = 0; - for i in (0..=resp.keys.len()).rev() { - let dst = crate::types::encode_message_to_bytes(&resp).unwrap(); - - let qresp = query.create_response(dst); - let dst = crate::types::encode_message_to_bytes(&qresp).unwrap(); - if query.check_response_size(dst.len()).is_err() { - resp.keys.truncate(i); - continue; - } - found = i; - break; - } - - assert_ne!(found, 0, "Do not find anything!"); - - println!( - "max keys in response with {} bytes: {}", - size_limit, - resp.keys.len() - ); - println!("factor: {}", size_limit / resp.keys.len()); -} - -/// Unit test for queries list key response functionality. -/// -/// This test requires the transport to support encryption. -#[cfg(feature = "encryption")] -pub async fn key_list_key_response_with_correct_size(transport_opts: T::Options, opts: Options) -where - T: Transport, -{ - use crate::types::KeyResponseMessage; - use memberlist_core::proto::SecretKey; - - let opts = opts.with_query_response_size_limit(1024); - let s = Serf::::new(transport_opts, opts).await.unwrap(); - let query = s.query_event( - 0.into(), - Default::default(), - Default::default(), - Default::default(), - 0, - s.memberlist().advertise_node(), - 0, - ); - - let k = [0; 16]; - let encoded_len = SecretKey::from(k).encoded_len(); - let cases = [ - (0, false, KeyResponseMessage::default()), - (1, false, { - let mut msg = KeyResponseMessage::default(); - msg.add_key(SecretKey::from(k)); - msg - }), - // has 50 keys which makes the response bigger than 1024 bytes. - (50, true, { - let mut msg = KeyResponseMessage::default(); - for _ in 0..50 { - msg.add_key(SecretKey::from(k)); - } - msg - }), - // this test when the list of keys length is less than the max allowed, in this test case 1024/encoded_len - (encoded_len, true, { - let mut msg = KeyResponseMessage::default(); - for _ in 0..encoded_len - 2 { - msg.add_key(SecretKey::from(k)); - } - msg - }), - // this test when the list of keys length is equal the max allowed, in this test case 1024/25 = 40 - (encoded_len, true, { - let mut msg = KeyResponseMessage::default(); - for _ in 0..encoded_len { - msg.add_key(SecretKey::from(k)); - } - msg - }), - // this test when the list of keys length is equal the max allowed, in this test case 1024/25 = 40 - (18, true, { - let mut msg = KeyResponseMessage::default(); - for _ in 0..18 { - msg.add_key(SecretKey::from(k)); - } - msg - }), - ]; - - for (expected, has_msg, mut resp) in cases { - if let Err(e) = SerfQueries::key_list_response_with_correct_size(&query, &mut resp) { - println!("error: {:?}", e); - continue; - } - - if resp.keys.len() != expected { - println!("expected: {}, got: {}", expected, resp.keys.len()); - } - - if has_msg && !resp.message.contains("truncated") { - println!("truncation message should be set"); - } - } -} diff --git a/legacy/serf-core/src/serf/base/tests/serf.rs b/legacy/serf-core/src/serf/base/tests/serf.rs deleted file mode 100644 index 1a9ff9b7..00000000 --- a/legacy/serf-core/src/serf/base/tests/serf.rs +++ /dev/null @@ -1,950 +0,0 @@ -use std::sync::atomic::{AtomicUsize, Ordering}; - -use memberlist_core::{tests::AnyError, transport::Id}; - -use crate::{ - event::EventProducer, - options::MemberlistOptions, - types::{Member, MemberState, MemberStatus, Tags}, -}; - -use super::*; - -/// Unit tests for the serf events related functionalities -pub mod event; - -/// Unit tests for the serf leave related functionalities -pub mod leave; - -/// Unit tests for the serf join related functionalities -pub mod join; - -/// Unit tests for the serf ping delegate related functionalities -pub mod delegate; - -/// Unit tests for the serf reconnect related functionalities -pub mod reconnect; - -/// Unit tests for the serf remove related functionalities -pub mod remove; - -/// Unit tests for serf reap related functionalities -pub mod reap; - -/// Unit tests for the serf snapshot related functionalities -pub mod snapshot; - -fn test_member_status( - members: &HashMap>, - id: I, - status: MemberStatus, -) -> Result<(), AnyError> { - for member in members.values() { - if id.eq(member.member.node.id()) { - if member.member.status != status { - return Err(AnyError::from(format!( - "expected member {} to have status {:?}, got {:?}", - id, status, member.member.status - ))); - } - return Ok(()); - } - } - Err(AnyError::from(format!("member {} not found", id))) -} - -/// Unit tests for the get queue max -pub async fn serf_get_queue_max( - transport_opts: T::Options, - mut get_addr: impl FnMut(usize) -> T::ResolvedAddress, -) where - T: Transport, - T::Options: Clone, -{ - let s = Serf::::new(transport_opts.clone(), test_config()) - .await - .unwrap(); - - // We don't need a running Serf so fake it out with the required - // state. - { - let mut members = s.inner.members.write().await; - members.states.clear(); - for i in 0..100 { - let name: SmolStr = format!("Member{i}").into(); - members.states.insert( - name.clone(), - MemberState { - member: Member::new( - Node::new(name.clone(), get_addr(i)), - Default::default(), - MemberStatus::Alive, - ), - status_time: 0.into(), - leave_time: None, - }, - ); - } - } - - // Default mode just uses the max depth. - let got = s.get_queue_max().await; - let want = 4096; - assert_eq!(got, want); - - // Now configure a min which should take precedence. - s.shutdown().await.unwrap(); - ::sleep(Duration::from_secs(2)).await; - - let sn = Serf::::new( - transport_opts.clone(), - test_config().with_min_queue_depth(1024), - ) - .await - .unwrap(); - - { - let mut members = sn.inner.members.write().await; - members.states.clear(); - let old_members = s.inner.members.read().await; - members.states.clone_from(&old_members.states); - } - - let got = sn.get_queue_max().await; - let want = 1024; - assert_eq!(got, want); - - sn.shutdown().await.unwrap(); - ::sleep(Duration::from_secs(2)).await; - - // Bring it under the number of nodes, so the calculation based on - // the number of nodes takes precedence. - let snn = Serf::::new(transport_opts, test_config().with_min_queue_depth(16)) - .await - .unwrap(); - - { - let mut members = snn.inner.members.write().await; - members.states.clear(); - let old_members = sn.inner.members.read().await; - members.states.clone_from(&old_members.states); - } - - let got = snn.get_queue_max().await; - let want = 200; - assert_eq!(got, want); - - // Try adjusting the node count. - { - let mut members = snn.inner.members.write().await; - let name = SmolStr::new("another"); - members.states.insert( - name.clone(), - MemberState { - member: Member::new( - Node::new(name.clone(), get_addr(10000)), - Default::default(), - MemberStatus::Alive, - ), - status_time: 0.into(), - leave_time: None, - }, - ); - } - - let got = snn.get_queue_max().await; - let want = 202; - assert_eq!(got, want); - snn.shutdown().await.unwrap(); - drop(snn); -} - -/// Unit tests for the update -pub async fn serf_update( - transport_opts1: T::Options, - transport_opts2: T::Options, - get_transport: impl FnOnce(T::Id, T::ResolvedAddress) -> F + Copy, -) where - T: Transport, - T::Options: Clone, - F: core::future::Future, -{ - let (event_tx, event_rx) = EventProducer::bounded(64); - let s1 = Serf::::with_event_producer(transport_opts1, test_config(), event_tx) - .await - .unwrap(); - let s2 = Serf::::new(transport_opts2.clone(), test_config()) - .await - .unwrap(); - let (s2id, s2addr) = s2.advertise_node().into_components(); - - let mut serfs = vec![s1, s2]; - wait_until_num_nodes(1, &serfs).await; - - let node = serfs[1] - .inner - .memberlist - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - wait_until_num_nodes(2, &serfs).await; - // Now force the shutdown of s2 so it appears to fail. - serfs[1].shutdown().await.unwrap(); - drop(serfs.pop().unwrap()); - - // Don't wait for a failure to be detected. Bring back s2 immediately - let start = Epoch::now(); - let s2 = loop { - match Serf::::new( - get_transport(s2id.clone(), s2addr.clone()).await, - test_config().with_tags([("foo", "bar")].into_iter()), - ) - .await - { - Ok(s) => break s, - Err(e) => { - ::sleep(Duration::from_secs(1)).await; - if start.elapsed() > Duration::from_secs(20) { - panic!("timed out: {}", e); - } - } - } - }; - - let s1node = serfs[0].advertise_node(); - s2.join( - s1node - .map_address(MaybeResolvedAddress::resolved) - .address() - .clone(), - false, - ) - .await - .unwrap(); - serfs.push(s2); - wait_until_num_nodes(2, &serfs).await; - - test_events( - event_rx.rx, - node.id().clone(), - [ - CrateEventType::Member(MemberEventType::Join), - CrateEventType::Member(MemberEventType::Update), - ] - .into_iter() - .collect(), - ) - .await; - - // Verify that the member data got updated. - let mut found = false; - let members = serfs[0].inner.members.read().await; - - for member in members.states.values() { - if member.member.node.id().eq(node.id()) - && member.member.tags().get("foo").map(|v| v.as_str()) == Some("bar") - { - found = true; - break; - } - } - assert!(found, "did not found s2 in members"); - - for s in serfs.iter() { - s.shutdown().await.unwrap(); - } -} - -/// Unit tests for the role -pub async fn serf_role(transport_opts1: T::Options, transport_opts2: T::Options) -where - T: Transport, -{ - let s1 = Serf::::new( - transport_opts1, - test_config().with_tags([("role", "web")].into_iter()), - ) - .await - .unwrap(); - let s2 = Serf::::new( - transport_opts2, - test_config().with_tags([("role", "lb")].into_iter()), - ) - .await - .unwrap(); - - let serfs = [s1, s2]; - wait_until_num_nodes(1, &serfs).await; - - let node = serfs[1] - .inner - .memberlist - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - wait_until_num_nodes(2, &serfs).await; - - let mut roles = HashMap::new(); - - let start = Epoch::now(); - let mut cond1 = false; - let mut cond2 = false; - loop { - ::sleep(Duration::from_millis(25)).await; - - let members = serfs[0].inner.members.read().await; - for m in members.states.values() { - roles.insert( - m.member.node.id().clone(), - m.member.tags().get("role").cloned().unwrap(), - ); - } - - if let Some(role) = roles.get(node.id()) { - if role == "lb" { - cond1 = true; - } - } - - if let Some(role) = roles.get(serfs[0].local_id()) { - if role == "web" { - cond2 = true; - } - } - - if cond1 && cond2 { - break; - } - - if start.elapsed() > Duration::from_secs(7) { - panic!("timed out"); - } - } -} - -/// Unit test for serf state -pub async fn serf_state(transport_opts1: T::Options) -where - T: Transport, -{ - let s1 = Serf::::new(transport_opts1, test_config()) - .await - .unwrap(); - - assert_eq!(s1.state(), SerfState::Alive); - - s1.leave().await.unwrap(); - - assert_eq!(s1.state(), SerfState::Left); - - s1.shutdown().await.unwrap(); - - assert_eq!(s1.state(), SerfState::Shutdown); -} - -/// Unit tests for serf set tags -pub async fn serf_set_tags(transport_opts1: T::Options, transport_opts2: T::Options) -where - T: Transport, -{ - let (event_tx, event_rx) = EventProducer::bounded(4); - let s1 = Serf::::with_event_producer(transport_opts1, test_config(), event_tx) - .await - .unwrap(); - let s2 = Serf::::new(transport_opts2, test_config()) - .await - .unwrap(); - - let serfs = [s1, s2]; - - wait_until_num_nodes(1, &serfs).await; - - let node = serfs[1] - .inner - .memberlist - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - wait_until_num_nodes(2, &serfs).await; - - // Update the tags - serfs[0] - .set_tags([("port", "8080")].into_iter().collect()) - .await - .unwrap(); - - serfs[1] - .set_tags([("datacenter", "east-aws")].into_iter().collect()) - .await - .unwrap(); - - let start = Epoch::now(); - let mut cond1 = false; - let mut cond2 = false; - let mut cond3 = false; - let mut cond4 = false; - - loop { - ::sleep(Duration::from_millis(25)).await; - - let m1m = serfs[0].members().await; - let mut m1m_tags = HashMap::with_capacity(2); - for m in m1m { - m1m_tags.insert(m.node.id().clone(), m.tags.clone()); - } - - if m1m_tags.get(serfs[0].local_id()).map(|t| t.get("port")) == Some(Some(&"8080".into())) { - cond1 = true; - } - - if m1m_tags - .get(serfs[1].local_id()) - .map(|t| t.get("datacenter")) - == Some(Some(&"east-aws".into())) - { - cond2 = true; - } - - let m2m = serfs[1].members().await; - let mut m2m_tags = HashMap::with_capacity(2); - for m in m2m { - m2m_tags.insert(m.node.id().clone(), m.tags.clone()); - } - - if m2m_tags.get(serfs[0].local_id()).map(|t| t.get("port")) == Some(Some(&"8080".into())) { - cond3 = true; - } - - if m2m_tags - .get(serfs[1].local_id()) - .map(|t| t.get("datacenter")) - == Some(Some(&"east-aws".into())) - { - cond4 = true; - } - - if cond1 && cond2 && cond3 && cond4 { - break; - } - - if start.elapsed() > Duration::from_secs(7) { - panic!("timed out"); - } - } - - // we check the events to make sure we got failures. - test_events( - event_rx.rx, - node.id().clone(), - [ - CrateEventType::Member(MemberEventType::Join), - CrateEventType::Member(MemberEventType::Update), - ] - .into_iter() - .collect(), - ) - .await; - - for s in serfs.iter() { - s.shutdown().await.unwrap(); - } -} - -/// Unit tests for serf num nodes -pub async fn serf_num_nodes(transport_opts1: T::Options, transport_opts2: T::Options) -where - T: Transport, -{ - let s1 = Serf::::new(transport_opts1, test_config()) - .await - .unwrap(); - let s2 = Serf::::new(transport_opts2, test_config()) - .await - .unwrap(); - - assert_eq!(s1.num_members().await, 1); - - let serfs = [s1, s2]; - wait_until_num_nodes(1, &serfs).await; - - let node = serfs[1] - .inner - .memberlist - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - wait_until_num_nodes(2, &serfs).await; -} - -/// Unit tests for serf coordinates -pub async fn serf_coordinates( - transport_opts1: T::Options, - transport_opts2: T::Options, - transport_opts3: T::Options, -) where - T: Transport, -{ - const PROBE_INTERVAL: Duration = Duration::from_millis(2); - - let opts = test_config() - .with_disable_coordinates(false) - .with_memberlist_options(memberlist_core::Options::lan().with_probe_interval(PROBE_INTERVAL)); - let s1 = Serf::::new(transport_opts1, opts.clone()).await.unwrap(); - let s2 = Serf::::new(transport_opts2, opts).await.unwrap(); - - let mut serfs = vec![s1, s2]; - wait_until_num_nodes(1, &serfs).await; - - // Make sure both nodes start out the origin so we can prove they did - // an update later. - let c1 = serfs[0].cooridate().unwrap(); - let c2 = serfs[1].cooridate().unwrap(); - - const ZERO_THRESHOLD: f64 = 20.0e-6; - - assert!( - c1.distance_to(&c2).as_secs_f64() <= ZERO_THRESHOLD, - "coordinates didn't start at the origin" - ); - - // Join the two nodes together and give them time to probe each other. - let node = serfs[1] - .inner - .memberlist - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - wait_until_num_nodes(2, &serfs).await; - - let start = Epoch::now(); - let mut cond1 = false; - let mut cond2 = false; - let mut cond3 = false; - let mut cond4 = false; - let s2id = serfs[1].local_id().clone(); - let s1id = serfs[0].local_id().clone(); - loop { - ::sleep(Duration::from_millis(25)).await; - - // See if they know about each other. - - if serfs[0].cached_coordinate(&s2id.clone()).is_ok() { - cond1 = true; - } else if start.elapsed() > Duration::from_secs(7) { - panic!("s1 didn't get a coordinate for s2"); - } - - if serfs[1].cached_coordinate(&s1id.clone()).is_ok() { - cond2 = true; - } else if start.elapsed() > Duration::from_secs(7) { - panic!("s2 didn't get a coordinate for s1"); - } - - // With only one ping they won't have a good estimate of the other node's - // coordinate, but they should both have updated their own coordinate. - let c1 = serfs[0].cooridate().unwrap(); - let c2 = serfs[1].cooridate().unwrap(); - - if c1.distance_to(&c2).as_secs_f64() >= ZERO_THRESHOLD { - cond3 = true; - } else if start.elapsed() > Duration::from_secs(7) { - panic!("coordinates didn't update after probes"); - } - - // Make sure they cached their own current coordinate after the update. - let c1c = serfs[0].cached_coordinate(&s1id.clone()).unwrap(); - match c1c { - None => { - if start.elapsed() > Duration::from_secs(7) { - panic!("s1 didn't cache its own coordinate"); - } - } - Some(c1c) => { - if c1 == c1c { - cond4 = true; - } else if start.elapsed() > Duration::from_secs(7) { - panic!("s1 coordinates are not equal"); - } - } - } - - if cond1 && cond2 && cond3 && cond4 { - break; - } - - if start.elapsed() > Duration::from_secs(7) { - panic!( - "timed out cond1 {} cond2 {} cond3 {} cond4 {}", - cond1, cond2, cond3, cond4 - ); - } - } - - // Break up the cluster and make sure the coordinates get removed by - // the reaper. - serfs[1].shutdown().await.unwrap(); - let t = serfs[1].inner.opts.reap_interval * 4; - drop(serfs.pop().unwrap()); - - ::sleep(t).await; - - wait_until_num_nodes(1, &serfs).await; - - let start = Epoch::now(); - loop { - ::sleep(Duration::from_millis(25)).await; - - if serfs[0].cached_coordinate(&s2id.clone()).unwrap().is_none() { - break; - } - - if start.elapsed() > Duration::from_secs(7) { - panic!("s1 should have removed s2's cached coordinate"); - } - } - - // Try a setup with coordinates disabled. - let s3 = Serf::::new( - transport_opts3, - test_config() - .with_disable_coordinates(true) - .with_memberlist_options(memberlist_core::Options::lan().with_probe_interval(PROBE_INTERVAL)), - ) - .await - .unwrap(); - - serfs.push(s3); - wait_until_num_nodes(1, &serfs).await; - - let node = serfs[0] - .inner - .memberlist - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[1].join(node.address().clone(), false).await.unwrap(); - - wait_until_num_nodes(2, &serfs).await; - - let start = Epoch::now(); - let mut cond1 = false; - let mut cond2 = false; - loop { - ::sleep(Duration::from_millis(25)).await; - - // See if they know about each other. - - if let Err(e) = serfs[1].cooridate() { - if e.to_string().contains("coordinates are disabled") { - cond1 = true; - } - } - - if serfs[1].cached_coordinate(&s1id.clone()).is_err() { - cond2 = true; - } - - if cond1 && cond2 { - break; - } - - if start.elapsed() > Duration::from_secs(14) { - panic!("timed out: cond1 {} cond2 {}", cond1, cond2); - } - } - - for s in serfs.iter() { - s.shutdown().await.unwrap(); - } -} - -/// Unit tests for serf name resolution -/// -/// set_id is a function that takes the transport options and the id of the node, and returns the -/// transport options with the id set to the given id. -pub async fn serf_name_resolution( - transport_opts1: T::Options, - transport_opts2: T::Options, - transport_opts3: T::Options, - set_id: impl FnOnce(T::Options, T::Id) -> T::Options, -) where - T: Transport, -{ - let s1 = Serf::::new(transport_opts1, test_config()) - .await - .unwrap(); - let s2 = Serf::::new(transport_opts2, test_config()) - .await - .unwrap(); - let s3 = Serf::::new( - set_id(transport_opts3, s1.local_id().clone()), - test_config(), - ) - .await - .unwrap(); - - let serfs = [s1, s2, s3]; - wait_until_num_nodes(1, &serfs).await; - - // Join s1 to s2 first. s2 should vote for s1 in conflict - let node = serfs[1] - .inner - .memberlist - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - wait_until_num_nodes(2, &serfs[..2]).await; - wait_until_num_nodes(1, &serfs[2..]).await; - - let node = serfs[2] - .inner - .memberlist - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - // Wait for the query period to end - ::sleep(serfs[0].default_query_timeout().await * 30).await; - - let start = Epoch::now(); - let mut cond1 = false; - let mut cond2 = false; - let mut cond3 = false; - loop { - // s3 should have shutdown, while s1 is running - ::sleep(Duration::from_millis(25)).await; - - if serfs[0].state() == SerfState::Alive { - cond1 = true; - } - - if serfs[1].state() == SerfState::Alive { - cond2 = true; - } - - if serfs[2].state() == SerfState::Shutdown { - cond3 = true; - } - - if cond1 && cond2 && cond3 { - break; - } - - if start.elapsed() > Duration::from_secs(14) { - println!("cond1 {cond1} cond2 {cond2} cond3 {cond3}"); - panic!("timed out"); - } - } - - for s in serfs.iter() { - s.shutdown().await.unwrap(); - } -} - -/// Unit test for serf local member -pub async fn serf_local_member(opts: T::Options) -where - T: Transport, -{ - let s = Serf::::new(opts, test_config()).await.unwrap(); - - let local = s.local_member().await; - assert_eq!(local.node.id(), s.local_id()); - - assert_eq!(local.tags, s.inner.opts.tags()); - assert_eq!(local.status, MemberStatus::Alive); - - let new_tags = [("foo", "bar"), ("test", "ing")] - .into_iter() - .collect::(); - s.set_tags(new_tags.clone()).await.unwrap(); - - let local = s.local_member().await; - assert_eq!(&*local.tags, &new_tags); -} - -/// Unit test for serf stats -pub async fn serf_stats(opts: T::Options) -where - T: Transport, -{ - let s = Serf::::new(opts, test_config()).await.unwrap(); - - let stats = s.stats().await; - assert_eq!(stats.get_event_queue(), 0); - assert_eq!(stats.get_event_time(), 1); - assert_eq!(stats.get_failed(), 0); - assert_eq!(stats.get_intent_queue(), 0); - assert_eq!(stats.get_left(), 0); - assert_eq!(stats.get_health_score(), 0); - assert_eq!(stats.get_member_time(), 1); - assert_eq!(stats.get_members(), 1); - assert!(!stats.get_encrypted()); -} - -/// Unit test for serf write keying file -#[cfg(feature = "encryption")] -pub async fn serf_write_keyring_file( - get_transport_opts: impl FnOnce(memberlist_core::proto::SecretKey) -> (T::Options, MemberlistOptions), -) where - T: Transport, -{ - use std::io::Read; - - const EXISTING: &str = "T9jncgl9mbLus+baTTa7q7nPSUrXwbDi2dhbtqir37s="; - const NEW_KEY: &str = "HvY8ubRZMgafUOWvrOadwOckVa1wN3QWAo46FVKbVN8="; - - let td = tempfile::tempdir().unwrap(); - let mut p = td.path().join("serf_write_keying_file"); - p.set_extension("json"); - - let sk = crate::types::SecretKey::try_from(EXISTING).unwrap(); - - let (topts, mopts) = get_transport_opts(sk); - let serf = Serf::::new( - topts, - test_config() - .with_keyring_file(Some(p.clone())) - .with_memberlist_options(mopts), - ) - .await - .unwrap(); - assert!( - serf.encryption_enabled(), - "write keyring file test only works on encrypted serf" - ); - - let manager = serf.key_manager(); - let new_sk = crate::types::SecretKey::try_from(NEW_KEY).unwrap(); - manager.install_key(new_sk, None).await.unwrap(); - - let mut keyring_file = std::fs::File::open(&p).unwrap(); - let mut s = String::new(); - keyring_file.read_to_string(&mut s).unwrap(); - - let lines = s.split('\n').collect::>(); - assert_eq!(lines.len(), 4); - - // Ensure both the original key and the new key are present in the file - assert!(s.contains(EXISTING)); - assert!(s.contains(NEW_KEY)); - - // Ensure the existing key remains primary. This is in position 1 because - // the file writer will use json.MarshalIndent(), leaving the first line as - // the opening bracket. - assert!(lines[1].contains(EXISTING)); - - // Swap primary keys - manager.use_key(new_sk, None).await.unwrap(); - - let mut keyring_file = std::fs::File::open(&p).unwrap(); - let mut s = String::new(); - keyring_file.read_to_string(&mut s).unwrap(); - - let lines = s.split('\n').collect::>(); - assert_eq!(lines.len(), 4); - - // Key order should have changed in keyring file - assert!(lines[1].contains(NEW_KEY)); - - // Remove the old key - manager.remove_key(sk, None).await.unwrap(); - - let mut keyring_file = std::fs::File::open(&p).unwrap(); - let mut s = String::new(); - keyring_file.read_to_string(&mut s).unwrap(); - - let lines = s.split('\n').collect::>(); - // Only the new key should now be present in the keyring file - assert_eq!(lines.len(), 3); - - assert!(lines[1].contains(NEW_KEY)); - - let resp = manager.list_keys().await.unwrap(); - assert_eq!(resp.primary_keys().len(), 1); - assert_eq!(resp.keys().len(), 1); -} - -#[test] -fn test_recent_intent() { - assert!(recent_intent::(&HashMap::new(), &"foo".into(), MessageType::Join).is_none()); - - let now = Epoch::now(); - let expire = || now - Duration::from_secs(2); - let save = || now; - - let mut intents = HashMap::::new(); - assert!(recent_intent(&intents, &"foo".into(), MessageType::Join).is_none()); - - assert!(upsert_intent( - &mut intents, - &"foo".into(), - MessageType::Join, - 1.into(), - expire - )); - assert!(upsert_intent( - &mut intents, - &"bar".into(), - MessageType::Leave, - 2.into(), - expire - )); - assert!(upsert_intent( - &mut intents, - &"baz".into(), - MessageType::Join, - 3.into(), - save - )); - assert!(upsert_intent( - &mut intents, - &"bar".into(), - MessageType::Join, - 4.into(), - expire - )); - assert!(!upsert_intent( - &mut intents, - &"bar".into(), - MessageType::Join, - 0.into(), - expire - )); - assert!(upsert_intent( - &mut intents, - &"bar".into(), - MessageType::Join, - 5.into(), - expire - )); - - let ltime = recent_intent(&intents, &"foo".into(), MessageType::Join).unwrap(); - assert_eq!(ltime, 1.into()); - - let ltime = recent_intent(&intents, &"bar".into(), MessageType::Join).unwrap(); - assert_eq!(ltime, 5.into()); - - let ltime = recent_intent(&intents, &"baz".into(), MessageType::Join).unwrap(); - assert_eq!(ltime, 3.into()); - - assert!(recent_intent(&intents, &"tubez".into(), MessageType::Join).is_none()); - - reap_intents(&mut intents, Epoch::now(), Duration::from_secs(1)); - assert!(recent_intent(&intents, &"foo".into(), MessageType::Join).is_none()); - assert!(recent_intent(&intents, &"bar".into(), MessageType::Join).is_none()); - let ltime = recent_intent(&intents, &"baz".into(), MessageType::Join).unwrap(); - assert_eq!(ltime, 3.into()); - assert!(recent_intent(&intents, &"tubez".into(), MessageType::Join).is_none()); - reap_intents( - &mut intents, - Epoch::now() + Duration::from_secs(2), - Duration::from_secs(1), - ); - assert!(recent_intent(&intents, &"baz".into(), MessageType::Join).is_none()); -} diff --git a/legacy/serf-core/src/serf/base/tests/serf/delegate.rs b/legacy/serf-core/src/serf/base/tests/serf/delegate.rs deleted file mode 100644 index 3e037577..00000000 --- a/legacy/serf-core/src/serf/base/tests/serf/delegate.rs +++ /dev/null @@ -1,350 +0,0 @@ -use super::*; - -/// Unit test for delegate node meta -pub async fn delegate_nodemeta(transport_opts: T::Options) -where - T: Transport, -{ - let opts = test_config(); - let s = Serf::::new( - transport_opts, - opts.with_tags([("role", "test")].into_iter()), - ) - .await - .unwrap(); - let meta = s.inner.memberlist.delegate().unwrap().node_meta(32).await; - - let (_, tags) = Tags::decode(&meta).unwrap(); - assert_eq!(tags.get("role"), Some(&SmolStr::new("test"))); - - s.shutdown().await.unwrap(); -} - -/// Unit test for delegate node meta panic -pub async fn delegate_nodemeta_panic(transport_opts: T::Options) -where - T: Transport, -{ - let opts = test_config(); - let s = Serf::::new( - transport_opts, - opts.with_tags([("role", "test")].into_iter()), - ) - .await - .unwrap(); - s.inner.memberlist.delegate().unwrap().node_meta(1).await; - s.shutdown().await.unwrap(); -} - -/// Unit test for delegate local state -pub async fn delegate_local_state(transport_opts1: T::Options, transport_opts2: T::Options) -where - T: Transport, -{ - let opts = test_config().with_event_buffer_size(0); - let s1 = Serf::::new(transport_opts1, opts).await.unwrap(); - - let opts = test_config().with_event_buffer_size(0); - let s2 = Serf::::new(transport_opts2, opts).await.unwrap(); - - let serfs = [s1, s2]; - wait_until_num_nodes(1, &serfs).await; - - let (id, addr) = serfs[1].memberlist().advertise_node().into_components(); - - serfs[0] - .join(MaybeResolvedAddress::resolved(addr), false) - .await - .unwrap(); - - wait_until_num_nodes(2, &serfs).await; - - serfs[0] - .user_event("test", Bytes::from_static(b"test"), false) - .await - .unwrap(); - - serfs[0].query("foo", Bytes::new(), None).await.unwrap(); - - // s2 can leave now - serfs[1].leave().await.unwrap(); - - // Do a state dump - let buf = serfs[0] - .memberlist() - .delegate() - .unwrap() - .local_state(false) - .await; - - // Attempt a decode - let pp = crate::types::decode_message::(&buf).unwrap(); - - let MessageRef::PushPull(pp) = pp else { - panic!("bad message") - }; - let pp = as Data>::from_ref(pp).unwrap(); - - // Verify lamport clock - assert_eq!(pp.ltime(), serfs[0].inner.clock.time(), "bad lamport clock"); - - // Verify the status - // Leave waits until propagation so this should only have one member - assert_eq!(pp.status_ltimes().len(), 1, "missing ltimes"); - assert_eq!(pp.left_members().len(), 0, "should have no left memebers"); - assert_eq!( - pp.event_ltime(), - serfs[0].inner.event_clock.time(), - "bad event clock" - ); - assert_eq!( - pp.events().len(), - serfs[0].inner.event_core.read().await.buffer.len(), - "should send full event buffer" - ); - assert_eq!( - pp.query_ltime(), - serfs[0].inner.query_clock.time(), - "bad query clock" - ); - - for s in serfs { - s.shutdown().await.unwrap(); - } -} - -/// Unit test for delegate merge remote state -pub async fn delegate_merge_remote_state(transport_opts: T::Options) -where - T: Transport, -{ - let opts = test_config(); - let s = Serf::::new(transport_opts, opts).await.unwrap(); - let d = s.memberlist().delegate().unwrap(); - - // Make a fake push pull - let pp = PushPullMessage { - ltime: 42.into(), - status_ltimes: [ - (SmolStr::new("test"), 20.into()), - (SmolStr::new("foo"), 15.into()), - ] - .into_iter() - .collect(), - left_members: ["foo".into()].into_iter().collect(), - event_ltime: 50.into(), - events: TinyVec::from(UserEvents { - ltime: 45.into(), - events: OneOrMore::from(UserEvent { - name: "test".into(), - payload: Bytes::new(), - }), - }), - query_ltime: 100.into(), - }; - - let buf = crate::types::encode_message_to_bytes(&pp).unwrap(); - - // Merge in fake state - d.merge_remote_state(&buf, false).await; - - // Verify lamport - assert_eq!(s.inner.clock.time(), 42.into(), "bad lamport clock"); - - let members = s.inner.members.read().await; - // Verify pending join for test - let ltime = recent_intent( - &members.recent_intents, - &SmolStr::new("test"), - MessageType::Join, - ) - .unwrap(); - assert_eq!(ltime, 20.into(), "bad join ltime"); - // Verify pending leave for foo - let ltime = recent_intent( - &members.recent_intents, - &SmolStr::new("foo"), - MessageType::Leave, - ) - .unwrap(); - assert_eq!(ltime, 16.into(), "bad leave ltime"); - - // Verify event clock - assert_eq!(s.inner.event_clock.time(), 50.into(), "bad event clock"); - let buf = s.inner.event_core.read().await; - assert!(buf.buffer[45].is_some(), "missing event buffer for time"); - assert_eq!(buf.buffer[45].as_ref().unwrap().events[0].name, "test"); - assert_eq!(s.inner.query_clock.time(), 100.into(), "bad query clock"); - - s.shutdown().await.unwrap(); -} - -/// Unit test for serf ping delegate versioning -pub async fn serf_ping_delegate_versioning( - transport_opts1: T::Options, - transport_opts2: T::Options, -) where - T: Transport, -{ - const PROBE_INTERVAL: Duration = Duration::from_millis(2); - - let s1 = Serf::::new( - transport_opts1, - test_config() - .with_disable_coordinates(false) - .with_memberlist_options(memberlist_core::Options::lan().with_probe_interval(PROBE_INTERVAL)), - ) - .await - .unwrap(); - - let s2 = Serf::::new( - transport_opts2, - test_config() - .with_disable_coordinates(false) - .with_memberlist_options(memberlist_core::Options::lan().with_probe_interval(PROBE_INTERVAL)), - ) - .await - .unwrap(); - - // Monkey patch s1 to send weird versions of the ping messages. - s1.memberlist() - .delegate() - .unwrap() - .ping_versioning_test - .store(true, Ordering::SeqCst); - - let serfs = [s1, s2]; - wait_until_num_nodes(1, &serfs).await; - - let node = serfs[1] - .memberlist() - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - // They both should show 2 members, but only s1 should know about s2 - // in the cache, since s1 spoke an alien ping protocol. - wait_until_num_nodes(2, &serfs).await; - - let start = Epoch::now(); - let mut cond1 = false; - let mut cond2 = false; - let s1id = serfs[0].local_id().clone(); - let s2id = serfs[1].local_id().clone(); - - loop { - ::sleep(Duration::from_millis(25)).await; - - if serfs[0].cached_coordinate(&s2id).unwrap().is_some() { - cond1 = true; - } else if start.elapsed() > Duration::from_secs(7) { - panic!("s1 didn't get a coordinate for s2"); - } - - if serfs[1].cached_coordinate(&s1id).unwrap().is_none() { - cond2 = true; - } else if start.elapsed() > Duration::from_secs(7) { - panic!("s2 got an unexpected coordinate for s1"); - } - - if cond1 && cond2 { - break; - } - - if start.elapsed() > Duration::from_secs(7) { - panic!("s1: {} s2: {}", cond1, cond2); - } - } - - for s in serfs.iter() { - s.shutdown().await.unwrap(); - } -} - -/// Unit test for serf ping delegate rogue coordinate -pub async fn serf_ping_delegate_rogue_coordinate( - transport_opts1: T::Options, - transport_opts2: T::Options, -) where - T: Transport, -{ - const PROBE_INTERVAL: Duration = Duration::from_millis(2); - - let opts = test_config().with_disable_coordinates(false); - let s1 = Serf::::new( - transport_opts1, - opts - .with_memberlist_options(memberlist_core::Options::lan().with_probe_interval(PROBE_INTERVAL)), - ) - .await - .unwrap(); - - let opts = test_config().with_disable_coordinates(false); - let s2 = Serf::::new( - transport_opts2, - opts - .with_memberlist_options(memberlist_core::Options::lan().with_probe_interval(PROBE_INTERVAL)), - ) - .await - .unwrap(); - - // Monkey patch s1 to send ping messages with bad coordinates. - s1.memberlist() - .delegate() - .unwrap() - .ping_dimension_test - .store(true, Ordering::SeqCst); - - let serfs = [s1, s2]; - wait_until_num_nodes(1, &serfs).await; - - let node = serfs[1] - .memberlist() - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - // They both should show 2 members, but only s1 should know about s2 - // in the cache, since s1 spoke an alien ping protocol. - wait_until_num_nodes(2, &serfs).await; - - let start = Epoch::now(); - let mut cond1 = false; - let mut cond2 = false; - let s1id = serfs[0].local_id().clone(); - let s2id = serfs[1].local_id().clone(); - - // They both should show 2 members, but only s1 should know about s2 - // in the cache, since s1 sent a bad coordinate. - loop { - ::sleep(Duration::from_millis(25)).await; - - let s1c = serfs[0].cached_coordinate(&s2id).unwrap(); - // println!("s1c {:?}", s1c); - if s1c.is_some() { - cond1 = true; - } else if start.elapsed() > Duration::from_secs(7) { - panic!("s1 didn't get a coordinate for s2"); - } - - let s2c = serfs[1].cached_coordinate(&s1id).unwrap(); - // println!("s2c {:?}", s2c); - if s2c.is_none() { - cond2 = true; - } else if start.elapsed() > Duration::from_secs(7) { - panic!("s2 got an unexpected coordinate for s1"); - } - - if cond1 && cond2 { - break; - } - - if start.elapsed() > Duration::from_secs(7) { - panic!("s1: {} s2: {}", cond1, cond2); - } - } - - for s in serfs.iter() { - s.shutdown().await.unwrap(); - } -} diff --git a/legacy/serf-core/src/serf/base/tests/serf/event.rs b/legacy/serf-core/src/serf/base/tests/serf/event.rs deleted file mode 100644 index cc8216fa..00000000 --- a/legacy/serf-core/src/serf/base/tests/serf/event.rs +++ /dev/null @@ -1,1106 +0,0 @@ -use memberlist_core::proto::DataRef; - -use crate::types::{Filter, QueryResponseMessageRef, TagFilter}; - -use super::*; - -/// Unit tests for the user event old message -pub async fn user_event_old_message(transport_opts: T::Options) -where - T: Transport, -{ - let opts = test_config(); - let event_buffer = opts.event_buffer_size; - let s1 = Serf::::new(transport_opts, opts).await.unwrap(); - - // increase the ltime artificially - s1.inner - .event_clock - .witness(((event_buffer + 1000) as u64).into()); - assert!( - !s1 - .handle_user_event(Either::Right( - UserEventMessage::default() - .with_ltime(1.into()) - .with_name("old".into()) - )) - .await, - "should not rebroadcast" - ); - s1.shutdown().await.unwrap(); -} - -/// Unit tests for the user event smae clock -pub async fn user_event_same_clock(transport_opts: T::Options) -where - T: Transport, -{ - let opts = test_config(); - let (event_tx, event_rx) = EventProducer::bounded(4); - let s1 = Serf::::with_event_producer(transport_opts, opts, event_tx) - .await - .unwrap(); - - let msg = UserEventMessage::default() - .with_ltime(1.into()) - .with_name("first".into()) - .with_payload(Bytes::from_static(b"test")); - assert!( - s1.handle_user_event(Either::Right(msg)).await, - "should rebroadcast" - ); - - let msg = UserEventMessage::default() - .with_ltime(1.into()) - .with_name("first".into()) - .with_payload(Bytes::from_static(b"newpayload")); - assert!( - s1.handle_user_event(Either::Right(msg)).await, - "should rebroadcast" - ); - - let msg = UserEventMessage::default() - .with_ltime(1.into()) - .with_name("second".into()) - .with_payload(Bytes::from_static(b"other")); - assert!( - s1.handle_user_event(Either::Right(msg)).await, - "should rebroadcast" - ); - - test_user_events( - event_rx.rx, - ["first", "first", "second"] - .into_iter() - .map(Into::into) - .collect(), - ["test", "newpayload", "other"] - .into_iter() - .map(Into::into) - .collect(), - ) - .await; - - s1.shutdown().await.unwrap(); -} - -/// Unit tests for the events failed -pub async fn serf_events_failed(transport_opts1: T::Options, transport_opts2: T::Options) -where - T: Transport, -{ - let (event_tx, event_rx) = EventProducer::bounded(4); - let s1 = Serf::::with_event_producer(transport_opts1, test_config(), event_tx) - .await - .unwrap(); - let s2 = Serf::::new(transport_opts2, test_config()) - .await - .unwrap(); - - let serfs = [s1, s2]; - wait_until_num_nodes(1, &serfs).await; - - let node = serfs[1] - .inner - .memberlist - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - wait_until_num_nodes(2, &serfs).await; - - serfs[1].shutdown().await.unwrap(); - - wait_until_num_nodes(1, &serfs[..1]).await; - - // Since s2 shutdown, we check the events to make sure we got failures. - let node = serfs[1].inner.memberlist.local_id().clone(); - test_events( - event_rx.rx, - node, - [ - CrateEventType::Member(MemberEventType::Join), - CrateEventType::Member(MemberEventType::Failed), - CrateEventType::Member(MemberEventType::Reap), - ] - .into_iter() - .collect(), - ) - .await; -} - -/// Unit tests for the events join -pub async fn serf_events_join(transport_opts1: T::Options, transport_opts2: T::Options) -where - T: Transport, -{ - let (event_tx, event_rx) = EventProducer::bounded(4); - let s1 = Serf::::with_event_producer(transport_opts1, test_config(), event_tx) - .await - .unwrap(); - let s2 = Serf::::new(transport_opts2, test_config()) - .await - .unwrap(); - - let serfs = [s1, s2]; - wait_until_num_nodes(1, &serfs).await; - - let node = serfs[1] - .inner - .memberlist - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - wait_until_num_nodes(2, &serfs).await; - - let node = serfs[1].inner.memberlist.local_id().clone(); - test_events( - event_rx.rx, - node, - [CrateEventType::Member(MemberEventType::Join)] - .into_iter() - .collect(), - ) - .await; - - for s in serfs { - let _ = s.shutdown().await; - } -} - -/// Unit tests for the events leave -/// Unit tests for the events failed -pub async fn serf_events_leave(transport_opts1: T::Options, transport_opts2: T::Options) -where - T: Transport, -{ - let (event_tx, event_rx) = EventProducer::bounded(4); - let s1 = Serf::::with_event_producer( - transport_opts1, - test_config().with_reap_interval(Duration::from_secs(30)), - event_tx, - ) - .await - .unwrap(); - let s2 = Serf::::new(transport_opts2, test_config()) - .await - .unwrap(); - - let serfs = [s1, s2]; - wait_until_num_nodes(1, &serfs).await; - - let node = serfs[1] - .inner - .memberlist - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - wait_until_num_nodes(2, &serfs).await; - - serfs[1].leave().await.unwrap(); - - let start = Epoch::now(); - loop { - ::sleep(Duration::from_millis(25)).await; - - let members = serfs[0].inner.members.read().await; - if test_member_status(&members.states, node.id().clone(), MemberStatus::Left).is_ok() { - break; - } - - if start.elapsed() > Duration::from_secs(7) { - panic!("timed out"); - } - } - - // Now that s2 has left, we check the events to make sure we got - // a leave event in s1 about the leave. - let node = serfs[1].inner.memberlist.local_id().clone(); - test_events( - event_rx.rx, - node, - [ - CrateEventType::Member(MemberEventType::Join), - CrateEventType::Member(MemberEventType::Leave), - ] - .into_iter() - .collect(), - ) - .await; -} - -#[derive(Debug, Clone)] -struct DropJoins { - drop: Arc, -} - -impl MessageDropper for DropJoins { - fn should_drop(&self, ty: MessageType) -> bool { - match ty { - MessageType::Join | MessageType::PushPull => self.drop.load(Ordering::SeqCst) == 1, - _ => false, - } - } -} - -impl DropJoins { - fn new() -> Self { - Self { - drop: Arc::new(AtomicUsize::new(0)), - } - } -} - -/// Unit tests for the events leave avoid infinite leave rebroadcast -pub async fn serf_events_leave_avoid_infinite_rebroadcast( - transport_opts1: T::Options, - transport_opts2: T::Options, - transport_opts3: T::Options, - transport_opts4: T::Options, - get_transport_opts: impl FnOnce(T::Id, T::ResolvedAddress) -> F + Copy, -) where - T: Transport, - F: core::future::Future, -{ - // This test is a variation of the normal leave test that is crafted - // specifically to handle a situation where two unique leave events for the - // same node reach two other nodes in the wrong order which causes them to - // infinitely rebroadcast the leave event without updating their own - // lamport clock for that node. - let config_local = |opts: Options| opts.with_reap_interval(Duration::from_secs(30)); - - let (event_tx1, event_rx1) = EventProducer::bounded(4); - let s1 = Serf::::with_event_producer(transport_opts1, config_local(test_config()), event_tx1) - .await - .unwrap(); - let s2 = Serf::::new(transport_opts2, config_local(test_config())) - .await - .unwrap(); - - // Allow s3 and s4 to drop joins in the future. - let d = DropJoins::new(); - let s3 = Serf::::with_message_dropper( - transport_opts3, - config_local(test_config()), - Box::new(d.clone()), - ) - .await - .unwrap(); - let s4 = Serf::::with_message_dropper( - transport_opts4, - config_local(test_config()), - Box::new(d.clone()), - ) - .await - .unwrap(); - - let mut serfs = vec![s1, s2]; - wait_until_num_nodes(1, &serfs).await; - - let node = serfs[1] - .inner - .memberlist - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - - serfs[0].join(node.address().clone(), false).await.unwrap(); - s3.join(node.address().clone(), false).await.unwrap(); - s4.join(node.address().clone(), false).await.unwrap(); - - // S2 leaves gracefully - serfs[1].leave().await.unwrap(); - serfs[1].shutdown().await.unwrap(); - - // Make s3 and s4 drop inbound join messages and push-pulls for a bit so it won't see - // s2 rejoin - d.drop.store(1, Ordering::SeqCst); - let s2node = serfs[1].advertise_node(); - let (s2id, s2addr) = s2node.clone().into_components(); - let _ = serfs.pop().unwrap(); - - // Bring back s2 by mimicking its name and address - ::sleep(Duration::from_secs(2)).await; - - let s2 = Serf::::new( - get_transport_opts(s2id, s2addr).await, - config_local(test_config().with_rejoin_after_leave(true)), - ) - .await - .unwrap(); - - let s1node = serfs[0] - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - s2.join(s1node.address().clone(), false).await.unwrap(); - - serfs.push(s2); - serfs.push(s3); - serfs.push(s4); - - wait_until_num_nodes(4, &serfs).await; - - // Now leave a second time but before s3 and s4 see the rejoin (due to the gate) - serfs[1].leave().await.unwrap(); - - let s2 = serfs.remove(1); - wait_until_intent_queue_len(0, &serfs).await; - - let start = Epoch::now(); - let mut cond1 = false; - let mut cond2 = false; - let mut cond3 = false; - loop { - ::sleep(Duration::from_millis(25)).await; - - if !cond1 { - let members = serfs[0].inner.members.read().await; - if test_member_status(&members.states, node.id().clone(), MemberStatus::Left).is_ok() { - cond1 = true; - } - } - - if !cond2 { - let members = serfs[1].inner.members.read().await; - if test_member_status(&members.states, node.id().clone(), MemberStatus::Left).is_ok() { - cond2 = true; - } - } - - if !cond3 { - let members = serfs[2].inner.members.read().await; - if test_member_status(&members.states, node.id().clone(), MemberStatus::Left).is_ok() { - cond3 = true; - } - } - - if cond1 && cond2 && cond3 { - break; - } - - if start.elapsed() > Duration::from_secs(7) { - panic!("timed out"); - } - } - - // Now that s2 has left, we check the events to make sure we got - // a leave event in s1 about the leave. - test_events( - event_rx1.rx, - s2.inner.memberlist.local_id().clone(), - [ - CrateEventType::Member(MemberEventType::Join), - CrateEventType::Member(MemberEventType::Leave), - CrateEventType::Member(MemberEventType::Join), - CrateEventType::Member(MemberEventType::Leave), - ] - .into_iter() - .collect(), - ) - .await; -} - -/// Unit tests for the remove failed events leave -pub async fn serf_remove_failed_events_leave( - transport_opts1: T::Options, - transport_opts2: T::Options, -) where - T: Transport, -{ - let (event_tx, event_rx) = EventProducer::bounded(4); - let s1 = Serf::::with_event_producer(transport_opts1, test_config(), event_tx) - .await - .unwrap(); - let s2 = Serf::::new(transport_opts2, test_config()) - .await - .unwrap(); - - let serfs = [s1, s2]; - wait_until_num_nodes(1, &serfs).await; - - let node = serfs[1] - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - wait_until_num_nodes(2, &serfs).await; - - serfs[1].shutdown().await.unwrap(); - - let t = serfs[1].inner.opts.memberlist_options.probe_interval(); - ::sleep(t * 15).await; - - serfs[0] - .remove_failed_node(node.id().clone()) - .await - .unwrap(); - - let start = Epoch::now(); - loop { - ::sleep(Duration::from_millis(25)).await; - - let members = serfs[0].inner.members.read().await; - if test_member_status(&members.states, node.id().clone(), MemberStatus::Left).is_ok() { - break; - } - - if start.elapsed() > Duration::from_secs(7) { - panic!("timed out"); - } - } - - // Now that s2 has failed and been marked as left, we check the - // events to make sure we got a leave event in s1 about the leave. - test_events( - event_rx.rx, - serfs[1].inner.memberlist.local_id().clone(), - [ - CrateEventType::Member(MemberEventType::Join), - CrateEventType::Member(MemberEventType::Failed), - CrateEventType::Member(MemberEventType::Leave), - ] - .into_iter() - .collect(), - ) - .await; -} - -/// Unit tests for the events user -pub async fn serf_event_user(transport_opts1: T::Options, transport_opts2: T::Options) -where - T: Transport, -{ - let (event_tx, event_rx) = EventProducer::bounded(4); - let s1 = Serf::::new(transport_opts1, test_config()) - .await - .unwrap(); - let s2 = Serf::::with_event_producer(transport_opts2, test_config(), event_tx) - .await - .unwrap(); - - let serfs = [s1, s2]; - wait_until_num_nodes(1, &serfs).await; - - let node = serfs[1] - .inner - .memberlist - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - wait_until_num_nodes(2, &serfs).await; - - // Fire a user event - serfs[0] - .user_event("event!", Bytes::from_static(b"test"), false) - .await - .unwrap(); - - // Fire a user event - serfs[0] - .user_event("second", Bytes::from_static(b"foobar"), false) - .await - .unwrap(); - - // check the events to make sure we got - // a leave event in s1 about the leave. - test_user_events( - event_rx.rx, - ["event!", "second"].into_iter().map(Into::into).collect(), - vec![Bytes::from_static(b"test"), Bytes::from_static(b"foobar")], - ) - .await; -} - -/// Unit tests for the events user size limit -pub async fn serf_event_user_size_limit(transport_opts1: T::Options) -where - T: Transport, -{ - let (event_tx, _event_rx) = EventProducer::bounded(4); - let s1 = Serf::::with_event_producer(transport_opts1, test_config(), event_tx) - .await - .unwrap(); - - let serfs = [s1]; - wait_until_num_nodes(1, &serfs).await; - - let p = Bytes::from(vec![0; serfs[0].inner.opts.max_user_event_size]); - let err = serfs[0] - .user_event("this is too large an event", p, false) - .await - .unwrap_err() - .to_string(); - assert!(err.contains("user event exceeds")); -} - -/// Unit test for default query -pub async fn default_query(transport_opts: T::Options) -where - T: Transport, -{ - let opts = test_config(); - let s = [Serf::::new(transport_opts, opts).await.unwrap()]; - - wait_until_num_nodes(1, &s).await; - let timeout = s[0].default_query_timeout().await; - assert_eq!( - timeout, - s[0].inner.opts.memberlist_options.gossip_interval() - * (s[0].inner.opts.query_timeout_mult as u32) - ); - - let params = s[0].default_query_param().await; - - assert!(params.filters.is_empty()); - assert!(!params.request_ack); - assert_eq!(params.timeout, timeout); -} - -/// Unit test for should process functionallity -pub async fn should_process(transport_opts: T::Options) -where - T: Transport, -{ - let opts = test_config(); - let s = Serf::::new( - transport_opts, - opts.with_tags([("role", "webserver"), ("datacenter", "east-aws")].into_iter()), - ) - .await - .unwrap(); - let mut params = s.default_query_param().await; - params.filters.push(Filter::Id( - [ - "foo".into(), - "bar".into(), - s.memberlist().local_id().clone(), - ] - .into_iter() - .collect(), - )); - params.filters.push(Filter::Tag( - TagFilter::new() - .with_tag("role".into()) - .with_expr("^web".try_into().unwrap()), - )); - params.filters.push(Filter::Tag( - TagFilter::new() - .with_tag("datacenter".into()) - .with_expr("aws$".try_into().unwrap()), - )); - - assert!( - s.should_process_query(Either::Right(¶ms.filters)) - .unwrap() - ); - - // Omit node - let mut params = s.default_query_param().await; - params - .filters - .push(Filter::Id(["foo".into(), "bar".into()].into())); - - assert!( - !s.should_process_query(Either::Right(¶ms.filters)) - .unwrap() - ); - - // Filter on missing tag - let mut params = s.default_query_param().await; - params.filters.push(Filter::Tag( - TagFilter::new() - .with_tag("other".into()) - .with_expr("cool".try_into().unwrap()), - )); - - assert!( - !s.should_process_query(Either::Right(¶ms.filters)) - .unwrap() - ); - - // Bad tag - let mut params = s.default_query_param().await; - params.filters.push(Filter::Tag( - TagFilter::new() - .with_tag("role".into()) - .with_expr("db".try_into().unwrap()), - )); - - assert!( - !s.should_process_query(Either::Right(¶ms.filters)) - .unwrap() - ); -} - -/// Unit tests for the query old message -pub async fn query_old_message(transport_opts: T::Options, from: Node) -where - T: Transport, -{ - let opts = test_config(); - let event_buffer = opts.query_buffer_size; - let s1 = Serf::::new(transport_opts, opts).await.unwrap(); - // increase the ltime artificially - s1.inner - .query_clock - .witness(((event_buffer + 1000) as u64).into()); - assert!( - !s1 - .handle_query( - Either::Right(QueryMessage { - ltime: 1.into(), - id: 0, - from, - filters: Default::default(), - flags: QueryFlag::empty(), - relay_factor: 0, - timeout: Default::default(), - name: "old".into(), - payload: Bytes::new(), - }), - None - ) - .await - .unwrap(), - "should not rebroadcast" - ); - - s1.shutdown().await.unwrap(); -} - -/// Unit tests for the query same clock -pub async fn query_same_clock(transport_opts: T::Options, from: Node) -where - T: Transport, -{ - let opts = test_config(); - let (event_tx, event_rx) = EventProducer::bounded(4); - let s1 = Serf::::with_event_producer(transport_opts, opts, event_tx) - .await - .unwrap(); - - let msg = QueryMessage { - ltime: 1.into(), - id: 1, - from: from.clone(), - filters: Default::default(), - flags: QueryFlag::empty(), - relay_factor: 0, - timeout: Default::default(), - name: "foo".into(), - payload: Bytes::from_static(b"test"), - }; - - assert!( - s1.handle_query(Either::Right(msg.clone()), None) - .await - .unwrap(), - "should rebroadcast" - ); - assert!( - !s1 - .handle_query(Either::Right(msg.clone()), None) - .await - .unwrap(), - "should not rebroadcast" - ); - - let msg = QueryMessage { - ltime: 1.into(), - id: 2, - from: from.clone(), - filters: Default::default(), - flags: QueryFlag::empty(), - relay_factor: 0, - timeout: Default::default(), - name: "bar".into(), - payload: Bytes::from_static(b"newpayload"), - }; - - assert!( - s1.handle_query(Either::Right(msg.clone()), None) - .await - .unwrap(), - "should rebroadcast" - ); - assert!( - !s1 - .handle_query(Either::Right(msg.clone()), None) - .await - .unwrap(), - "should not rebroadcast" - ); - - let msg = QueryMessage { - ltime: 1.into(), - id: 3, - from: from.clone(), - filters: Default::default(), - flags: QueryFlag::empty(), - relay_factor: 0, - timeout: Default::default(), - name: "baz".into(), - payload: Bytes::from_static(b"other"), - }; - assert!( - s1.handle_query(Either::Right(msg.clone()), None) - .await - .unwrap(), - "should rebroadcast" - ); - assert!( - !s1 - .handle_query(Either::Right(msg.clone()), None) - .await - .unwrap(), - "should not rebroadcast" - ); - - test_query_events( - event_rx.rx, - ["foo", "bar", "baz"].into_iter().map(Into::into).collect(), - ["test", "newpayload", "other"] - .into_iter() - .map(Into::into) - .collect(), - ) - .await; - - s1.shutdown().await.unwrap(); -} - -/// Unit test for serf query -pub async fn serf_query(transport_opts1: T::Options, transport_opts2: T::Options) -where - T: Transport, -{ - let (event_tx, event_rx) = EventProducer::bounded(4); - - let s1 = Serf::::with_event_producer(transport_opts1, test_config(), event_tx) - .await - .unwrap(); - - let wg = Arc::new(()); - let wg1 = wg.clone(); - let (ctx_tx, ctx_rx) = async_channel::bounded::<()>(1); - scopeguard::defer!(ctx_tx.close();); - - ::spawn_detach(async move { - let _wg = wg1; - loop { - futures::select! { - _ = ctx_rx.recv().fuse() => { - break; - }, - e = event_rx.rx.recv().fuse() => { - let e = e.unwrap(); - match e.ty() { - CrateEventType::Query => { - match e { - CrateEvent::Query(q) => { - q.respond(Bytes::from_static(b"test")).await.unwrap(); - break; - } - _ => unreachable!(), - } - }, - _ => continue, - } - }, - _ = ::sleep(Duration::from_secs(1)).fuse() => { - tracing::error!("timeout"); - break; - }, - } - } - }); - - let s2 = Serf::::new(transport_opts2, test_config()) - .await - .unwrap(); - - let serfs = [s1, s2]; - wait_until_num_nodes(1, &serfs).await; - - let node = serfs[1] - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - wait_until_num_nodes(2, &serfs).await; - - // Start a query from s2; - let mut params = serfs[1].default_query_param().await; - params.request_ack = true; - let resp = serfs[1] - .query("load", Bytes::from_static(b"sup girl"), Some(params)) - .await - .unwrap(); - - let mut acks = vec![]; - let mut responses = vec![]; - - let ack_rx = resp.ack_rx().unwrap(); - let resp_rx = resp.response_rx(); - - for _ in 0..3 { - futures::select! { - a = ack_rx.recv().fuse() => { - let a = a.unwrap(); - acks.push(a); - }, - r = resp_rx.recv().fuse() => { - let r = r.unwrap(); - assert_eq!(r.from, serfs[0].advertise_node()); - assert_eq!(r.payload, Bytes::from_static(b"test")); - responses.push(r); - }, - _ = ::sleep(Duration::from_secs(1)).fuse() => { - panic!("timeout"); - }, - } - } - assert_eq!(acks.len(), 2, "missing acks {acks:?}"); - assert_eq!(responses.len(), 1, "missing responses {responses:?}"); -} - -/// Unit test for serf query filter -pub async fn serf_query_filter( - transport_opts1: T::Options, - transport_opts2: T::Options, - transport_opts3: T::Options, -) where - T: Transport, -{ - let (event_tx, event_rx) = EventProducer::bounded(4); - - let s1 = Serf::::with_event_producer(transport_opts1, test_config(), event_tx) - .await - .unwrap(); - - let wg = Arc::new(()); - let wg1 = wg.clone(); - let (ctx_tx, ctx_rx) = async_channel::bounded::<()>(1); - scopeguard::defer!(ctx_tx.close();); - - ::spawn_detach(async move { - let _wg = wg1; - loop { - futures::select! { - _ = ctx_rx.recv().fuse() => { - break; - }, - e = event_rx.rx.recv().fuse() => { - let e = e.unwrap(); - match e.ty() { - CrateEventType::Query => { - match e { - CrateEvent::Query(q) => { - q.respond(Bytes::from_static(b"test")).await.unwrap(); - break; - } - _ => unreachable!(), - } - }, - _ => continue, - } - }, - _ = ::sleep(Duration::from_secs(1)).fuse() => { - tracing::error!("timeout"); - break; - }, - } - } - }); - - let s2 = Serf::::new(transport_opts2, test_config()) - .await - .unwrap(); - - let mut serfs = vec![s1, s2]; - wait_until_num_nodes(1, &serfs).await; - - let node = serfs[1] - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - wait_until_num_nodes(2, &serfs).await; - - let s3 = Serf::::new(transport_opts3, test_config()) - .await - .unwrap(); - serfs.push(s3); - - wait_until_num_nodes(1, &serfs[2..]).await; - - let node = serfs[2] - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - wait_until_num_nodes(3, &serfs).await; - - // Filter to only s1! - let mut params = serfs[1].default_query_param().await; - params.filters.push(Filter::Id( - [serfs[0].memberlist().local_id().clone()] - .into_iter() - .collect(), - )); - params.request_ack = true; - params.relay_factor = 1; - - // Start a query from s2; - let resp = serfs[1] - .query("load", Bytes::from_static(b"sup girl"), Some(params)) - .await - .unwrap(); - - let mut acks = vec![]; - let mut responses = vec![]; - - let ack_rx = resp.ack_rx().unwrap(); - let resp_rx = resp.response_rx(); - - for _ in 0..2 { - futures::select! { - a = ack_rx.recv().fuse() => { - let a = a.unwrap(); - acks.push(a); - }, - r = resp_rx.recv().fuse() => { - let r = r.unwrap(); - assert_eq!(r.from, serfs[0].advertise_node()); - assert_eq!(r.payload, Bytes::from_static(b"test")); - responses.push(r); - }, - _ = ::sleep(Duration::from_secs(1)).fuse() => { - panic!("timeout"); - }, - } - } - - assert_eq!(acks.len(), 1, "missing acks {acks:?}"); - assert_eq!(responses.len(), 1, "missing responses {responses:?}"); - - for s in serfs.iter() { - s.shutdown().await.unwrap(); - } -} - -/// Unit test for serf query deduplicate -pub async fn serf_query_deduplicate(transport_opts: T::Options) -where - T: Transport, -{ - let opts = test_config(); - let s = Serf::::new(transport_opts, opts).await.unwrap(); - - // Set up a dummy query and response - let mq = QueryMessage { - ltime: 123.into(), - id: 123, - from: s.advertise_node(), - filters: Default::default(), - flags: QueryFlag::ACK, - relay_factor: 0, - timeout: Duration::from_secs(1), - name: Default::default(), - payload: Default::default(), - }; - let query = QueryResponse::from_query(&mq, 3); - let response = QueryResponseMessage { - ltime: mq.ltime, - id: mq.id, - from: s.advertise_node(), - flags: QueryFlag::empty(), - payload: Default::default(), - }; - { - let mut qc = s.inner.query_core.write().await; - qc.responses.insert(mq.ltime, query.clone()); - } - - let buf = response.encode_to_bytes().unwrap(); - let (_, resp_ref) = as DataRef< - '_, - QueryResponseMessage, - >>::decode(&buf) - .unwrap(); - - // Send a few duplicate responses - s.handle_query_response(resp_ref).await.unwrap(); - s.handle_query_response(resp_ref).await.unwrap(); - - let mut resp_ref2 = resp_ref; - resp_ref2.flags |= QueryFlag::ACK; - s.handle_query_response(resp_ref2).await.unwrap(); - s.handle_query_response(resp_ref2).await.unwrap(); - - // Ensure we only get one NodeResponse off the channel - let resp_rx = query.response_rx(); - futures::select! { - _ = resp_rx.recv().fuse() => {}, - default => { - panic!("should have a response") - } - } - - let ack_rx = query.ack_rx().unwrap(); - futures::select! { - _ = ack_rx.recv().fuse() => {}, - default => { - panic!("should have an ack") - } - } - - futures::select! { - _ = resp_rx.recv().fuse() => { - panic!("should not have a second response") - }, - default => {} - } - - futures::select! { - _ = ack_rx.recv().fuse() => { - panic!("should not have a second ack") - }, - default => {} - } -} - -/// Unit test for serf query size limit -pub async fn serf_query_size_limit(transport_opts: T::Options) -where - T: Transport, -{ - let opts = test_config(); - let size_limit = opts.query_size_limit; - let s = Serf::::new(transport_opts, opts).await.unwrap(); - - let name = "this is too large a query"; - let payload = vec![0; size_limit]; - let Err(err) = s.query(name, payload, None).await else { - panic!("expected error"); - }; - assert!(err.to_string().contains("query exceeds limit of")); -} - -/// Unit test for serf query size limit increased -pub async fn serf_query_size_limit_increased(transport_opts: T::Options) -where - T: Transport, -{ - let opts = test_config(); - let size_limit = opts.query_size_limit; - let s = Serf::::new(transport_opts, opts.with_query_size_limit(size_limit * 2)) - .await - .unwrap(); - - let name = "this is too large a query"; - let payload = vec![0; size_limit]; - s.query(name, payload, None).await.unwrap(); -} diff --git a/legacy/serf-core/src/serf/base/tests/serf/join.rs b/legacy/serf-core/src/serf/base/tests/serf/join.rs deleted file mode 100644 index 7588f727..00000000 --- a/legacy/serf-core/src/serf/base/tests/serf/join.rs +++ /dev/null @@ -1,613 +0,0 @@ -use std::marker::PhantomData; - -use crate::delegate::MergeDelegate; - -use super::*; - -/// Unit tests for the join intent buffer early -pub async fn join_intent_buffer_early(transport_opts: T::Options) -where - T: Transport, -{ - let opts = test_config(); - let s1 = Serf::::new(transport_opts, opts).await.unwrap(); - - // Deliver a join intent message early - let j = JoinMessage { - ltime: 10.into(), - id: "test".into(), - }; - - assert!(s1.handle_node_join_intent(&j).await, "should rebroadcast"); - assert!( - !s1.handle_node_join_intent(&j).await, - "should not rebroadcast" - ); - - // Check that we buffered - { - let members = s1.inner.members.read().await; - let ltime = recent_intent(&members.recent_intents, &"test".into(), MessageType::Join).unwrap(); - assert_eq!(ltime, 10.into(), "bad buffer"); - } - - s1.shutdown().await.unwrap(); -} - -/// Unit tests for the join intent old message -pub async fn join_intent_old_message(transport_opts: T::Options, addr: T::ResolvedAddress) -where - T: Transport, -{ - let opts = test_config(); - let s1 = Serf::::new(transport_opts, opts).await.unwrap(); - - { - let mut members = s1.inner.members.write().await; - members.states.insert( - "test".into(), - MemberState { - member: Member { - node: Node::new("test".into(), addr), - tags: Arc::new(Default::default()), - status: MemberStatus::Alive, - memberlist_protocol_version: crate::types::MemberlistProtocolVersion::V1, - memberlist_delegate_version: crate::types::MemberlistDelegateVersion::V1, - protocol_version: crate::types::ProtocolVersion::V1, - delegate_version: crate::types::DelegateVersion::V1, - }, - status_time: 12.into(), - leave_time: None, - }, - ); - } - - let j = JoinMessage { - ltime: 10.into(), - id: "test".into(), - }; - - assert!( - !s1.handle_node_join_intent(&j).await, - "should not rebroadcast" - ); - - // Check that we didn't buffer anything - { - let members = s1.inner.members.read().await; - assert!( - recent_intent(&members.recent_intents, &"test".into(), MessageType::Join).is_none(), - "should not have buffered intent" - ); - } - - s1.shutdown().await.unwrap(); -} - -/// Unit tests for the join intent newer -pub async fn join_intent_newer(transport_opts: T::Options, addr: T::ResolvedAddress) -where - T: Transport, -{ - let opts = test_config(); - let s1 = Serf::::new(transport_opts, opts).await.unwrap(); - { - let mut members = s1.inner.members.write().await; - members.states.insert( - "test".into(), - MemberState { - member: Member { - node: Node::new("test".into(), addr), - tags: Arc::new(Default::default()), - status: MemberStatus::Alive, - memberlist_protocol_version: crate::types::MemberlistProtocolVersion::V1, - memberlist_delegate_version: crate::types::MemberlistDelegateVersion::V1, - protocol_version: crate::types::ProtocolVersion::V1, - delegate_version: crate::types::DelegateVersion::V1, - }, - status_time: 12.into(), - leave_time: None, - }, - ); - } - - let j = JoinMessage { - ltime: 14.into(), - id: "test".into(), - }; - - assert!(s1.handle_node_join_intent(&j).await, "should rebroadcast"); - - { - let members = s1.inner.members.read().await; - assert!( - recent_intent(&members.recent_intents, &"test".into(), MessageType::Join).is_none(), - "should not have buffered intent" - ); - - let m = members.states.get("test").unwrap(); - assert_eq!(m.status_time, 14.into(), "should update join time"); - assert_eq!(s1.inner.clock.time(), 15.into(), "should update clock"); - } - - s1.shutdown().await.unwrap(); -} - -/// Unit tests for the join intent reset leaving -pub async fn join_intent_reset_leaving(transport_opts: T::Options, addr: T::ResolvedAddress) -where - T: Transport, -{ - let opts = test_config(); - let s1 = Serf::::new(transport_opts, opts).await.unwrap(); - - { - let mut members = s1.inner.members.write().await; - members.states.insert( - "test".into(), - MemberState { - member: Member { - node: Node::new("test".into(), addr), - tags: Arc::new(Default::default()), - status: MemberStatus::Leaving, - memberlist_protocol_version: crate::types::MemberlistProtocolVersion::V1, - memberlist_delegate_version: crate::types::MemberlistDelegateVersion::V1, - protocol_version: crate::types::ProtocolVersion::V1, - delegate_version: crate::types::DelegateVersion::V1, - }, - status_time: 12.into(), - leave_time: None, - }, - ); - } - - let j = JoinMessage { - ltime: 14.into(), - id: "test".into(), - }; - - assert!(s1.handle_node_join_intent(&j).await, "should rebroadcast"); - - { - let members = s1.inner.members.read().await; - assert!( - recent_intent(&members.recent_intents, &"test".into(), MessageType::Join).is_none(), - "should not have buffered intent" - ); - - let m = members.states.get("test").unwrap(); - assert_eq!(m.status_time, 14.into(), "should update join time"); - assert_eq!(m.member.status, MemberStatus::Alive, "should update status"); - assert_eq!(s1.inner.clock.time(), 15.into(), "should update clock"); - } - - s1.shutdown().await.unwrap(); -} - -/// Unit tests for the join leave ltime logic -pub async fn join_leave_ltime(transport_opts1: T::Options, transport_opts2: T::Options) -where - T: Transport, -{ - let opts = test_config(); - let s1 = Serf::::new(transport_opts1, opts).await.unwrap(); - let opts = test_config(); - let s2 = Serf::::new(transport_opts2, opts).await.unwrap(); - - let serfs = [s1, s2]; - wait_until_num_nodes(1, &serfs).await; - - let (_, addr) = serfs[1].inner.memberlist.advertise_node().into_components(); - serfs[0] - .join(MaybeResolvedAddress::resolved(addr), false) - .await - .unwrap(); - - wait_until_num_nodes(2, &serfs).await; - - let now = Epoch::now(); - - loop { - let members = serfs[1].inner.members.read().await; - let mut cond1 = false; - let mut cond2 = false; - if let Some(m) = members.states.get(serfs[0].inner.memberlist.local_id()) { - if m.status_time == 1.into() { - cond1 = true; - } - - if serfs[1].inner.clock.time() > m.status_time { - cond2 = true; - } - } - - if cond1 && cond2 { - break; - } - - if now.elapsed() > Duration::from_secs(7) { - panic!("timed out waiting for status time to be updated"); - } - - ::sleep(Duration::from_millis(25)).await; - } - - let old_clock = serfs[1].inner.clock.time(); - - serfs[0].leave().await.unwrap(); - - loop { - let mut cond1 = false; - - if serfs[1].inner.clock.time() > old_clock { - cond1 = true; - } - - if cond1 { - break; - } - - if now.elapsed() > Duration::from_secs(7) { - panic!( - "leave should increment ({} / {})", - serfs[1].inner.clock.time(), - old_clock - ); - } - - ::sleep(Duration::from_millis(25)).await; - } - - for s in serfs.iter() { - s.shutdown().await.unwrap(); - } -} - -/// Unit tests for the join pending intent logic -pub async fn join_pending_intent(transport_opts: T::Options, addr: T::ResolvedAddress) -where - T: Transport, -{ - let opts = test_config(); - let s1 = Serf::::new(transport_opts, opts).await.unwrap(); - { - let mut members = s1.inner.members.write().await; - upsert_intent::( - &mut members.recent_intents, - &"test".into(), - MessageType::Join, - 5.into(), - Epoch::now, - ); - } - - s1.handle_node_join(Arc::new(NodeState { - id: "test".into(), - addr, - meta: Meta::empty(), - state: memberlist_core::proto::State::Alive, - protocol_version: crate::types::MemberlistProtocolVersion::V1, - delegate_version: crate::types::MemberlistDelegateVersion::V1, - })) - .await; - - { - let members = s1.inner.members.read().await; - let m = members.states.get("test").unwrap(); - assert_eq!(m.status_time, 5.into()); - assert_eq!(m.member.status, MemberStatus::Alive); - } - - s1.shutdown().await.unwrap(); -} - -/// Unit tests for the join pending intent logic -pub async fn join_pending_intents(transport_opts: T::Options, addr: T::ResolvedAddress) -where - T: Transport, -{ - let opts = test_config(); - let s1 = Serf::::new(transport_opts, opts).await.unwrap(); - { - let mut members = s1.inner.members.write().await; - upsert_intent::( - &mut members.recent_intents, - &"test".into(), - MessageType::Join, - 5.into(), - Epoch::now, - ); - upsert_intent::( - &mut members.recent_intents, - &"test".into(), - MessageType::Leave, - 6.into(), - Epoch::now, - ); - } - - s1.handle_node_join(Arc::new(NodeState { - id: "test".into(), - addr, - meta: Meta::empty(), - state: memberlist_core::proto::State::Alive, - protocol_version: crate::types::MemberlistProtocolVersion::V1, - delegate_version: crate::types::MemberlistDelegateVersion::V1, - })) - .await; - - { - let members = s1.inner.members.read().await; - let m = members.states.get("test").unwrap(); - assert_eq!(m.status_time, 6.into()); - assert_eq!(m.member.status, MemberStatus::Leaving); - } - - s1.shutdown().await.unwrap(); -} - -/// Unit tests for the join leave -pub async fn serf_join_leave(transport_opts1: T::Options, transport_opts2: T::Options) -where - T: Transport, -{ - let s1 = Serf::::new(transport_opts1, test_config()) - .await - .unwrap(); - let reap_interval = s1.inner.opts.reap_interval(); - let s2 = Serf::::new(transport_opts2, test_config()) - .await - .unwrap(); - - let serfs = [s1, s2]; - wait_until_num_nodes(1, &serfs).await; - - let node = serfs[1] - .inner - .memberlist - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - wait_until_num_nodes(2, &serfs).await; - - serfs[1].leave().await.unwrap(); - - ::sleep(reap_interval * 2).await; - - wait_until_num_nodes(1, &serfs).await; -} - -/// Unit test for serf join leave join -pub async fn serf_join_leave_join(transport_opts1: T::Options, transport_opts2: T::Options) -where - T: Transport, - T::Options: Clone, -{ - let s1 = Serf::::new( - transport_opts1, - test_config().with_reap_interval(Duration::from_secs(10)), - ) - .await - .unwrap(); - let s2 = Serf::::new( - transport_opts2.clone(), - test_config().with_reap_interval(Duration::from_secs(10)), - ) - .await - .unwrap(); - - let mut serfs = [s1, s2]; - wait_until_num_nodes(1, &serfs).await; - - let node = serfs[1] - .inner - .memberlist - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - wait_until_num_nodes(2, &serfs).await; - - serfs[1].leave().await.unwrap(); - serfs[1].shutdown().await.unwrap(); - - let t = serfs[1].inner.opts.memberlist_options.probe_interval() * 5; - // Give the reaper time to reap nodes - ::sleep(t).await; - - // s1 should see the node as having left - let start = Epoch::now(); - loop { - ::sleep(Duration::from_millis(25)).await; - - let members = serfs[0].inner.members.read().await; - let mut any_left = false; - for member in members.states.values() { - if member.member.status == MemberStatus::Left { - any_left = true; - break; - } - } - - if any_left { - break; - } - - if !any_left && start.elapsed() > Duration::from_secs(7) { - panic!("Node should have left"); - } - } - - // Bring the node back - let s2 = Serf::::new( - transport_opts2, - test_config().with_reap_interval(Duration::from_secs(10)), - ) - .await - .unwrap(); - - serfs[1] = s2; - wait_until_num_nodes(1, &serfs[1..]).await; - - // Re-attempt the join - let node = serfs[1] - .inner - .memberlist - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - let start = Epoch::now(); - loop { - ::sleep(Duration::from_millis(25)).await; - - let members = serfs[0].inner.members.read().await; - let mut any_left = false; - for member in members.states.values() { - if member.member.status == MemberStatus::Left { - any_left = true; - break; - } - } - - if !any_left { - break; - } - - if start.elapsed() > Duration::from_secs(7) { - panic!("all nodes should be alive!"); - } - } - - for s in serfs.iter() { - s.shutdown().await.unwrap(); - } -} - -/// Unit test for serf join ignore old -pub async fn serf_join_ignore_old(transport_opts1: T::Options, transport_opts2: T::Options) -where - T: Transport, -{ - let s1 = Serf::::new(transport_opts1, test_config()) - .await - .unwrap(); - - let (event_tx, event_rx) = EventProducer::bounded(4); - let s2 = Serf::::with_event_producer(transport_opts2, test_config(), event_tx) - .await - .unwrap(); - - let serfs = [s1, s2]; - wait_until_num_nodes(1, &serfs).await; - - // Fire a user event - serfs[0] - .user_event("event!", Bytes::from_static(b"test"), false) - .await - .unwrap(); - ::sleep(Duration::from_secs(10)).await; - - serfs[0] - .user_event("second", Bytes::from_static(b"foobar"), false) - .await - .unwrap(); - ::sleep(Duration::from_secs(10)).await; - - // join with ignoreOld set to true! should not get events - let node = serfs[0] - .inner - .memberlist - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[1].join(node.address().clone(), true).await.unwrap(); - - wait_until_num_nodes(2, &serfs).await; - - // check the events to make sure we got nothing - test_user_events(event_rx.rx, vec![], vec![]).await; - - for s in serfs.iter() { - s.shutdown().await.unwrap(); - } -} - -#[derive(Debug, thiserror::Error)] -#[error("merge canceled")] -struct CancelMergeError; - -#[derive(Clone)] -struct CancelMergeDelegate { - invoked: Arc, - _phantom: PhantomData, -} - -impl MergeDelegate for CancelMergeDelegate { - type Error = CancelMergeError; - - type Id = SmolStr; - - type Address = A; - - async fn notify_merge( - &self, - _members: Arc<[Member]>, - ) -> Result<(), Self::Error> { - self.invoked.store(true, Ordering::SeqCst); - Err(CancelMergeError) - } -} - -/// Unit test for serf join cancel -pub async fn serf_join_cancel(transport_opts1: T::Options, transport_opts2: T::Options) -where - T: Transport, -{ - let cmd1 = CancelMergeDelegate { - invoked: Arc::new(AtomicBool::new(false)), - _phantom: PhantomData, - }; - let s1 = Serf::::with_delegate( - transport_opts1, - test_config(), - DefaultDelegate::::new().with_merge_delegate(cmd1.clone()), - ) - .await - .unwrap(); - let cmd2 = CancelMergeDelegate { - invoked: Arc::new(AtomicBool::new(false)), - _phantom: PhantomData, - }; - let s2 = Serf::::with_delegate( - transport_opts2, - test_config(), - DefaultDelegate::::new().with_merge_delegate(cmd2.clone()), - ) - .await - .unwrap(); - - let serfs = [s1, s2]; - wait_until_num_nodes(0, &serfs).await; - - let node = serfs[1] - .inner - .memberlist - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - - let err = serfs[0] - .join(node.address().clone(), false) - .await - .unwrap_err(); - assert!(err.to_string().contains("merge canceled")); - - wait_until_num_nodes(0, &serfs).await; - - assert!(cmd1.invoked.load(Ordering::SeqCst)); - assert!(cmd2.invoked.load(Ordering::SeqCst)); - - for s in serfs.iter() { - s.shutdown().await.unwrap(); - } -} diff --git a/legacy/serf-core/src/serf/base/tests/serf/leave.rs b/legacy/serf-core/src/serf/base/tests/serf/leave.rs deleted file mode 100644 index 8b313c61..00000000 --- a/legacy/serf-core/src/serf/base/tests/serf/leave.rs +++ /dev/null @@ -1,516 +0,0 @@ -use super::*; - -/// Unit tests for the leave intent buffer early -pub async fn leave_intent_buffer_early(transport_opts: T::Options) -where - T: Transport, -{ - let opts = test_config(); - let s1 = Serf::::new(transport_opts, opts).await.unwrap(); - - // Deliver a leave intent message early - let j = LeaveMessage { - ltime: 10.into(), - id: "test".into(), - prune: false, - }; - - assert!(s1.handle_node_leave_intent(&j).await, "should rebroadcast"); - assert!( - !s1.handle_node_leave_intent(&j).await, - "should not rebroadcast" - ); - - // Check that we buffered - { - let members = s1.inner.members.read().await; - let ltime = recent_intent(&members.recent_intents, &"test".into(), MessageType::Leave).unwrap(); - assert_eq!(ltime, 10.into(), "bad buffer"); - } - - s1.shutdown().await.unwrap(); -} - -/// Unit tests for the leave intent old message -pub async fn leave_intent_old_message(transport_opts: T::Options, addr: T::ResolvedAddress) -where - T: Transport, -{ - let opts = test_config(); - let s1 = Serf::::new(transport_opts, opts).await.unwrap(); - - { - let mut members = s1.inner.members.write().await; - members.states.insert( - "test".into(), - MemberState { - member: Member { - node: Node::new("test".into(), addr), - tags: Arc::new(Default::default()), - status: MemberStatus::Alive, - memberlist_protocol_version: crate::types::MemberlistProtocolVersion::V1, - memberlist_delegate_version: crate::types::MemberlistDelegateVersion::V1, - protocol_version: crate::types::ProtocolVersion::V1, - delegate_version: crate::types::DelegateVersion::V1, - }, - status_time: 12.into(), - leave_time: None, - }, - ); - } - - let j = LeaveMessage { - ltime: 10.into(), - id: "test".into(), - prune: false, - }; - - assert!( - !s1.handle_node_leave_intent(&j).await, - "should not rebroadcast" - ); - - { - let members = s1.inner.members.read().await; - assert!( - recent_intent(&members.recent_intents, &"test".into(), MessageType::Leave).is_none(), - "should not have buffered intent" - ); - } - - s1.shutdown().await.unwrap(); -} - -/// Unit tests for the leave intent newer -pub async fn leave_intent_newer(transport_opts: T::Options, addr: T::ResolvedAddress) -where - T: Transport, -{ - let opts = test_config(); - let s1 = Serf::::new(transport_opts, opts).await.unwrap(); - { - let mut members = s1.inner.members.write().await; - members.states.insert( - "test".into(), - MemberState { - member: Member { - node: Node::new("test".into(), addr), - tags: Arc::new(Default::default()), - status: MemberStatus::Alive, - memberlist_protocol_version: crate::types::MemberlistProtocolVersion::V1, - memberlist_delegate_version: crate::types::MemberlistDelegateVersion::V1, - protocol_version: crate::types::ProtocolVersion::V1, - delegate_version: crate::types::DelegateVersion::V1, - }, - status_time: 12.into(), - leave_time: None, - }, - ); - } - - let j = LeaveMessage { - ltime: 14.into(), - id: "test".into(), - prune: false, - }; - - assert!(s1.handle_node_leave_intent(&j).await, "should rebroadcast"); - - { - let members = s1.inner.members.read().await; - assert!( - recent_intent(&members.recent_intents, &"test".into(), MessageType::Leave).is_none(), - "should not have buffered intent" - ); - - let m = members.states.get("test").unwrap(); - assert_eq!( - m.member.status, - MemberStatus::Leaving, - "should update status" - ); - assert_eq!(s1.inner.clock.time(), 15.into(), "should update clock"); - } - - s1.shutdown().await.unwrap(); -} - -/// Unit tests for the force leave failed -pub async fn serf_force_leave_failed( - transport_opts1: T::Options, - transport_opts2: T::Options, - transport_opts3: T::Options, -) where - T: Transport, -{ - let s1 = Serf::::new(transport_opts1, test_config()) - .await - .unwrap(); - let s2 = Serf::::new(transport_opts2, test_config()) - .await - .unwrap(); - let s3 = Serf::::new(transport_opts3, test_config()) - .await - .unwrap(); - - let mut serfs = [s1, s2, s3]; - wait_until_num_nodes(1, &serfs).await; - - let node = serfs[1] - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - let node = serfs[2] - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - wait_until_num_nodes(3, &serfs).await; - - serfs[1].shutdown().await.unwrap(); - - let s2id = serfs[1].local_id().clone(); - - let start = Epoch::now(); - loop { - ::sleep(Duration::from_millis(25)).await; - - let members = serfs[0].inner.members.read().await; - if test_member_status(&members.states, s2id.clone(), MemberStatus::Failed).is_ok() { - break; - } - - if start.elapsed() > Duration::from_secs(7) { - panic!("timed out"); - } - } - serfs[0].force_leave(s2id, true).await.unwrap(); - serfs.swap(1, 2); - wait_until_num_nodes(2, &serfs[..2]).await; -} - -/// Unit tests for the force leave leaving -pub async fn serf_force_leave_leaving( - transport_opts1: T::Options, - transport_opts2: T::Options, - transport_opts3: T::Options, -) where - T: Transport, -{ - const TOMBSTONE_TIMEOUT: Duration = Duration::from_secs(3600); - const LEAVE_PROPAGATE_DELAY: Duration = Duration::from_secs(5); - - let s1 = Serf::::new( - transport_opts1, - test_config() - .with_tombstone_timeout(TOMBSTONE_TIMEOUT) - .with_leave_propagate_delay(LEAVE_PROPAGATE_DELAY), - ) - .await - .unwrap(); - let s2 = Serf::::new( - transport_opts2, - test_config() - .with_tombstone_timeout(TOMBSTONE_TIMEOUT) - .with_leave_propagate_delay(LEAVE_PROPAGATE_DELAY), - ) - .await - .unwrap(); - let s3 = Serf::::new( - transport_opts3, - test_config() - .with_tombstone_timeout(TOMBSTONE_TIMEOUT) - .with_leave_propagate_delay(LEAVE_PROPAGATE_DELAY), - ) - .await - .unwrap(); - - let mut serfs = [s1, s2, s3]; - wait_until_num_nodes(1, &serfs).await; - - let node = serfs[1] - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - let node = serfs[2] - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - wait_until_num_nodes(3, &serfs).await; - - //Put s2 in left state - serfs[1].leave().await.unwrap(); - - let s2id = serfs[1].local_id().clone(); - - let start = Epoch::now(); - loop { - ::sleep(Duration::from_millis(25)).await; - - let members = serfs[0].inner.members.read().await; - if test_member_status(&members.states, s2id.clone(), MemberStatus::Left).is_ok() { - break; - } - - if start.elapsed() > Duration::from_secs(7) { - panic!("timed out"); - } - } - - serfs[0].force_leave(s2id, true).await.unwrap(); - serfs.swap(1, 2); - wait_until_num_nodes(2, &serfs[..2]).await; -} - -/// Unit tests for the force leave left -pub async fn serf_force_leave_left( - transport_opts1: T::Options, - transport_opts2: T::Options, - transport_opts3: T::Options, -) where - T: Transport, -{ - const TOMBSTONE_TIMEOUT: Duration = Duration::from_secs(3600); - - let s1 = Serf::::new( - transport_opts1, - test_config().with_tombstone_timeout(TOMBSTONE_TIMEOUT), - ) - .await - .unwrap(); - let s2 = Serf::::new( - transport_opts2, - test_config().with_tombstone_timeout(TOMBSTONE_TIMEOUT), - ) - .await - .unwrap(); - let s3 = Serf::::new( - transport_opts3, - test_config().with_tombstone_timeout(TOMBSTONE_TIMEOUT), - ) - .await - .unwrap(); - - let mut serfs = [s1, s2, s3]; - wait_until_num_nodes(1, &serfs).await; - - let node = serfs[1] - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - let node = serfs[2] - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - wait_until_num_nodes(3, &serfs).await; - - //Put s2 in left state - serfs[1].leave().await.unwrap(); - - let s2id = serfs[1].local_id().clone(); - - let start = Epoch::now(); - loop { - ::sleep(Duration::from_millis(25)).await; - - let members = serfs[0].inner.members.read().await; - if test_member_status(&members.states, s2id.clone(), MemberStatus::Left).is_ok() { - break; - } - - if start.elapsed() > Duration::from_secs(7) { - panic!("timed out"); - } - } - - serfs[0].force_leave(s2id, true).await.unwrap(); - serfs.swap(1, 2); - wait_until_num_nodes(2, &serfs[..2]).await; -} - -/// Unit tests for the leave rejoin different role -pub async fn serf_leave_rejoin_different_role( - transport_opts1: T::Options, - transport_opts2: T::Options, -) where - T: Transport, - T::Options: Clone, -{ - let s1 = Serf::::new(transport_opts1, test_config()) - .await - .unwrap(); - let s2 = Serf::::new(transport_opts2.clone(), test_config()) - .await - .unwrap(); - - let mut serfs = [s1, s2]; - wait_until_num_nodes(1, &serfs).await; - - let node = serfs[1] - .inner - .memberlist - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - wait_until_num_nodes(2, &serfs).await; - - serfs[1].leave().await.unwrap(); - serfs[1].shutdown().await.unwrap(); - - ::sleep(Duration::from_secs(10)).await; - - // Make s3 look just like s2, but create a new node with a new role - let s3 = Serf::::new( - transport_opts2, - test_config().with_tags([("role", "bar")].into_iter()), - ) - .await - .unwrap(); - - let node = serfs[0] - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - - serfs[1] = s3; - - serfs[1].join(node.address().clone(), false).await.unwrap(); - - wait_until_num_nodes(2, &serfs).await; - - let start = Epoch::now(); - let s3id = serfs[1].local_id().clone(); - loop { - ::sleep(Duration::from_millis(25)).await; - - let members = serfs[0].inner.members.read().await; - let mut find = None; - for (id, member) in members.states.iter() { - if s3id.eq(id) { - find = Some(member); - break; - } - } - - if let Some(member) = find { - let role = member.member.tags.get("role"); - assert_eq!(role, Some(&"bar".into()), "bad role: {:?}", role); - return; - } - - if start.elapsed() > Duration::from_secs(7) { - panic!("timed out"); - } - } -} - -/// Unit tests for the serf leave snapshot recovery -pub async fn serf_leave_snapshot_recovery( - transport_opts1: T::Options, - transport_opts2: T::Options, - get_transport: impl FnOnce(T::Id, T::ResolvedAddress) -> F + Copy, -) where - T: Transport, - F: core::future::Future, -{ - let td = tempfile::tempdir().unwrap(); - let snap_path = td.path().join("serf_leave_snapshot_recovery"); - - let s1 = Serf::::new( - transport_opts1, - test_config().with_reap_interval(Duration::from_secs(30)), - ) - .await - .unwrap(); - let s2 = Serf::::new( - transport_opts2, - test_config() - .with_snapshot_path(Some(snap_path.clone())) - .with_reap_interval(Duration::from_secs(30)), - ) - .await - .unwrap(); - - let mut serfs = vec![s1, s2]; - wait_until_num_nodes(1, &serfs).await; - - let node = serfs[1] - .inner - .memberlist - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - wait_until_num_nodes(2, &serfs).await; - - // Put s2 in left state - serfs[1].leave().await.unwrap(); - serfs[1].shutdown().await.unwrap(); - let t = serfs[1].inner.opts.memberlist_options.probe_interval * 5; - let (s2id, s2addr) = serfs[1].advertise_node().into_components(); - let _ = serfs.pop().unwrap(); - - ::sleep(t).await; - - let start = Epoch::now(); - loop { - ::sleep(Duration::from_millis(25)).await; - - let members = serfs[0].inner.members.read().await; - if test_member_status(&members.states, s2id.clone(), MemberStatus::Left).is_ok() { - break; - } - - if start.elapsed() > Duration::from_secs(7) { - panic!("timed out"); - } - } - - // Restart s2 from the snapshot now! - let s2 = Serf::::new( - get_transport(s2id.clone(), s2addr).await, - test_config() - .with_snapshot_path(Some(snap_path.clone())) - .with_reap_interval(Duration::from_secs(30)), - ) - .await - .unwrap(); - serfs.push(s2); - - // Wait for the node to auto rejoin - - // Verify that s2 did not join - let start = Epoch::now(); - let mut cond1 = false; - let mut cond2 = false; - loop { - if !cond1 { - let num = serfs[1].num_members().await; - if num == 1 { - cond1 = true; - } - } - - if !cond2 { - let members = serfs[0].inner.members.read().await; - if test_member_status(&members.states, s2id.clone(), MemberStatus::Left).is_ok() { - cond2 = true; - } - } - - if cond1 && cond2 { - break; - } - - if start.elapsed() > Duration::from_secs(7) { - panic!("time out"); - } - - ::sleep(Duration::from_millis(25)).await; - } -} diff --git a/legacy/serf-core/src/serf/base/tests/serf/reap.rs b/legacy/serf-core/src/serf/base/tests/serf/reap.rs deleted file mode 100644 index c5759713..00000000 --- a/legacy/serf-core/src/serf/base/tests/serf/reap.rs +++ /dev/null @@ -1,171 +0,0 @@ -use super::*; - -/// Unit test for reap handler shutdown -pub async fn serf_reap_handler_shutdown(opts: T::Options) -where - T: Transport, -{ - let s = Serf::::new(opts, test_config()).await.unwrap(); - - // Make sure the reap handler exits on shutdown. - let (tx, rx) = async_channel::bounded::<()>(1); - - let s1 = s.clone(); - let reap = Reaper { - coord_core: s1.inner.coord_core.clone(), - memberlist: s1.inner.memberlist.clone(), - members: s1.inner.members.clone(), - event_tx: s1.inner.event_tx.clone(), - shutdown_rx: s1.inner.shutdown_rx.clone(), - reap_interval: s1.inner.opts.reap_interval, - reconnect_timeout: s1.inner.opts.reconnect_timeout, - recent_intent_timeout: s1.inner.opts.recent_intent_timeout, - tombstone_timeout: s1.inner.opts.tombstone_timeout, - }; - ::spawn_detach(async move { - reap.run().await; - tx.close(); - }); - - s.shutdown().await.unwrap(); - - futures::select! { - _ = rx.recv().fuse() => {}, - _ = ::sleep(Duration::from_secs(1)).fuse() => { - panic!("reap handler did not exit"); - } - } -} - -/// Unit test for reap handler -pub async fn serf_reap_handler(opts: T::Options, addr: T::ResolvedAddress) -where - T: Transport, -{ - let s = Serf::::new( - opts, - test_config() - .with_reap_interval(Duration::from_nanos(1)) - .with_tombstone_timeout(Duration::from_secs(6)) - .with_recent_intent_timeout(Duration::from_secs(7)), - ) - .await - .unwrap(); - - { - let mut members = s.inner.members.write().await; - let n = Node::new("foo".into(), addr.clone()); - members.left_members.push(MemberState { - member: Member::new(n.clone(), Default::default(), MemberStatus::None), - status_time: 0.into(), - leave_time: Some(Epoch::now()), - }); - members.left_members.push(MemberState { - member: Member::new(n.clone(), Default::default(), MemberStatus::None), - status_time: 0.into(), - leave_time: Some(Epoch::now() - Duration::from_secs(5)), - }); - members.left_members.push(MemberState { - member: Member::new(n.clone(), Default::default(), MemberStatus::None), - status_time: 0.into(), - leave_time: Some(Epoch::now() - Duration::from_secs(10)), - }); - upsert_intent::( - &mut members.recent_intents, - &"alice".into(), - MessageType::Join, - 1.into(), - Epoch::now, - ); - upsert_intent::( - &mut members.recent_intents, - &"bob".into(), - MessageType::Join, - 2.into(), - || Epoch::now() - Duration::from_secs(10), - ); - upsert_intent::( - &mut members.recent_intents, - &"carol".into(), - MessageType::Leave, - 1.into(), - Epoch::now, - ); - upsert_intent::( - &mut members.recent_intents, - &"doug".into(), - MessageType::Leave, - 2.into(), - || Epoch::now() - Duration::from_secs(10), - ); - } - - let s1 = s.clone(); - ::spawn_detach(async move { - ::sleep(Duration::from_millis(1)).await; - s1.shutdown().await.unwrap(); - }); - - let reap = Reaper { - coord_core: s.inner.coord_core.clone(), - memberlist: s.inner.memberlist.clone(), - members: s.inner.members.clone(), - event_tx: s.inner.event_tx.clone(), - shutdown_rx: s.inner.shutdown_rx.clone(), - reap_interval: s.inner.opts.reap_interval, - reconnect_timeout: s.inner.opts.reconnect_timeout, - recent_intent_timeout: s.inner.opts.recent_intent_timeout, - tombstone_timeout: s.inner.opts.tombstone_timeout, - }; - reap.run().await; - - let members = s.inner.members.read().await; - assert_eq!(members.left_members.len(), 2); - - recent_intent(&members.recent_intents, &"alice".into(), MessageType::Join).unwrap(); - assert!(recent_intent(&members.recent_intents, &"bob".into(), MessageType::Join).is_none()); - recent_intent(&members.recent_intents, &"carol".into(), MessageType::Leave).unwrap(); - assert!(recent_intent(&members.recent_intents, &"doug".into(), MessageType::Leave).is_none()); -} - -/// Unit test for reap -pub async fn serf_reap(opts: T::Options, addr: T::ResolvedAddress) -where - T: Transport, -{ - let s = Serf::::new(opts, test_config()).await.unwrap(); - - { - let mut members = s.inner.members.write().await; - let n = Node::new("foo".into(), addr.clone()); - members.left_members.push(MemberState { - member: Member::new(n.clone(), Default::default(), MemberStatus::None), - status_time: 0.into(), - leave_time: Some(Epoch::now()), - }); - members.left_members.push(MemberState { - member: Member::new(n.clone(), Default::default(), MemberStatus::None), - status_time: 0.into(), - leave_time: Some(Epoch::now() - Duration::from_secs(5)), - }); - members.left_members.push(MemberState { - member: Member::new(n.clone(), Default::default(), MemberStatus::None), - status_time: 0.into(), - leave_time: Some(Epoch::now() - Duration::from_secs(10)), - }); - - let (tx, _rx) = async_channel::bounded(64); - - Reaper::>::reap_left( - s.local_id(), - &mut members, - &tx, - None, - None, - Duration::from_secs(6), - ) - .await; - } - - s.shutdown().await.unwrap(); -} diff --git a/legacy/serf-core/src/serf/base/tests/serf/reconnect.rs b/legacy/serf-core/src/serf/base/tests/serf/reconnect.rs deleted file mode 100644 index 41fc210e..00000000 --- a/legacy/serf-core/src/serf/base/tests/serf/reconnect.rs +++ /dev/null @@ -1,282 +0,0 @@ -use core::{marker::PhantomData, net::SocketAddr}; - -use memberlist_core::transport::resolver::socket_addr::SocketAddrResolver; - -use crate::delegate::ReconnectDelegate; - -use super::*; - -/// Unit test for reconnect -pub async fn serf_reconnect( - transport_opts1: T::Options, - transport_opts2: T::Options, - get_transport: impl FnOnce(T::Id, T::ResolvedAddress) -> F + Copy, -) where - T: Transport, - F: core::future::Future, -{ - let (event_tx, event_rx) = EventProducer::bounded(64); - - let s1 = Serf::::with_event_producer(transport_opts1, test_config(), event_tx) - .await - .unwrap(); - let s2 = Serf::::new(transport_opts2, test_config()) - .await - .unwrap(); - - let mut serfs = vec![s1, s2]; - wait_until_num_nodes(1, &serfs).await; - - let node = serfs[1] - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - wait_until_num_nodes(2, &serfs).await; - - serfs[1].shutdown().await.unwrap(); - - let t = serfs[1].inner.opts.memberlist_options.probe_interval(); - let (s2id, s2addr) = serfs[1].advertise_node().into_components(); - let _ = serfs.pop().unwrap(); - ::sleep(t * 15).await; - - // Bring back s2 - let s2 = Serf::::new(get_transport(s2id, s2addr).await, test_config()) - .await - .unwrap(); - serfs.push(s2); - - wait_until_num_nodes(2, &serfs).await; - - let node = serfs[1].local_id().clone(); - test_events( - event_rx.rx, - node, - [ - CrateEventType::Member(MemberEventType::Join), - CrateEventType::Member(MemberEventType::Failed), - CrateEventType::Member(MemberEventType::Join), - ] - .into_iter() - .collect(), - ) - .await; - - for s in serfs.iter() { - println!("start shutdown {}", s.local_id()); - s.shutdown().await.unwrap(); - println!("finish shutdown {}", s.local_id()); - } -} - -/// Unit test for reconnect -pub async fn serf_reconnect_same_ip( - transport_opts1: T::Options, - transport2_id: T::Id, - get_transport: impl FnOnce(T::Id, T::ResolvedAddress) -> F + Copy, -) where - T: Transport>, - T::Options: Clone, - R: RuntimeLite, - F: core::future::Future, -{ - let (event_tx, event_rx) = EventProducer::bounded(64); - - let s1 = Serf::::with_event_producer(transport_opts1, test_config(), event_tx) - .await - .unwrap(); - let s2addr = { - let mut addr = *s1.advertise_node().address(); - let port = addr.port() + 1; - addr.set_port(port); - addr - }; - let s2 = Serf::::new( - get_transport(transport2_id.clone(), s2addr).await, - test_config(), - ) - .await - .unwrap(); - - let ip1 = s1.inner.memberlist.advertise_address().ip(); - let ip2 = s2.inner.memberlist.advertise_address().ip(); - assert_eq!(ip1, ip2, "require same ip address 1: {ip1} 2: {ip2}"); - - let mut serfs = vec![s1, s2]; - wait_until_num_nodes(1, &serfs).await; - - let node = serfs[1] - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - wait_until_num_nodes(2, &serfs).await; - - serfs[1].shutdown().await.unwrap(); - - let t = serfs[1].inner.opts.memberlist_options.probe_interval(); - drop(serfs.pop().unwrap()); - ::sleep(t * 10).await; - - // Bring back s2 - let s2 = Serf::::new( - get_transport(transport2_id.clone(), s2addr).await, - test_config(), - ) - .await - .unwrap(); - serfs.push(s2); - - wait_until_num_nodes(2, &serfs).await; - - let node = serfs[1].local_id().clone(); - test_events( - event_rx.rx, - node, - [ - CrateEventType::Member(MemberEventType::Join), - CrateEventType::Member(MemberEventType::Failed), - CrateEventType::Member(MemberEventType::Join), - ] - .into_iter() - .collect(), - ) - .await; -} - -#[derive(Clone)] -struct ReconnectOverride { - timeout: Duration, - called: Arc, - _marker: PhantomData, -} - -impl ReconnectDelegate for ReconnectOverride -where - A: CheapClone + Send + Sync + 'static, -{ - type Id = SmolStr; - - type Address = A; - - fn reconnect_timeout( - &self, - _member: &Member, - _timeout: Duration, - ) -> Duration { - self.called.store(true, Ordering::SeqCst); - self.timeout - } -} - -/// Unit test for serf per node reconnect timeout -pub async fn serf_per_node_reconnect_timeout( - transport_opts1: T::Options, - transport_opts2: T::Options, -) where - T: Transport, -{ - let (event_tx, event_rx) = EventProducer::bounded(4); - - let ro1 = ReconnectOverride { - timeout: Duration::from_micros(1), - called: Arc::new(AtomicBool::new(false)), - _marker: PhantomData, - }; - - let s1 = Serf::::with_event_producer_and_delegate( - transport_opts1, - test_config().with_reconnect_timeout(Duration::from_secs(30)), - event_tx, - DefaultDelegate::::new().with_reconnect_delegate(ro1.clone()), - ) - .await - .unwrap(); - let s2 = Serf::::new(transport_opts2, test_config()) - .await - .unwrap(); - - let start = Epoch::now(); - let mut cond1 = false; - let mut cond2 = false; - loop { - ::sleep(Duration::from_millis(25)).await; - let n = s1.num_members().await; - if n == 1 { - cond1 = true; - } - if start.elapsed() > Duration::from_secs(7) { - panic!("s1 got {} expected {}", n, 1); - } - - let n = s2.num_members().await; - if n == 1 { - cond2 = true; - } - if start.elapsed() > Duration::from_secs(7) { - panic!("s2 got {} expected {}", n, 1); - } - - if cond1 && cond2 { - break; - } - } - - let node = s2 - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - s1.join(node.address().clone(), false).await.unwrap(); - - let start = Epoch::now(); - let mut cond1 = false; - let mut cond2 = false; - loop { - ::sleep(Duration::from_millis(25)).await; - let n = s1.num_members().await; - if n == 2 { - cond1 = true; - } - if start.elapsed() > Duration::from_secs(7) { - panic!("s1 got {} expected {}", n, 2); - } - - let n = s2.num_members().await; - if n == 2 { - cond2 = true; - } - if start.elapsed() > Duration::from_secs(7) { - panic!("s2 got {} expected {}", n, 2); - } - - if cond1 && cond2 { - break; - } - } - - s2.shutdown().await.unwrap(); - - let serfs = [s1]; - - wait_until_num_nodes(1, &serfs).await; - - // Since s2 shutdown, we check the events to make sure we got failures. - test_events( - event_rx.rx, - s2.local_id().clone(), - [ - CrateEventType::Member(MemberEventType::Join), - CrateEventType::Member(MemberEventType::Failed), - CrateEventType::Member(MemberEventType::Reap), - ] - .into_iter() - .collect(), - ) - .await; - - assert!( - ro1.called.load(Ordering::SeqCst), - "reconnect override was not used" - ); - serfs[0].shutdown().await.unwrap(); -} diff --git a/legacy/serf-core/src/serf/base/tests/serf/remove.rs b/legacy/serf-core/src/serf/base/tests/serf/remove.rs deleted file mode 100644 index c5e8a812..00000000 --- a/legacy/serf-core/src/serf/base/tests/serf/remove.rs +++ /dev/null @@ -1,222 +0,0 @@ -use super::*; - -/// Unit test for remove failed node -pub async fn serf_remove_failed_node( - transport_opts1: T::Options, - transport_opts2: T::Options, - transport_opts3: T::Options, -) where - T: Transport, -{ - let s1 = Serf::::new(transport_opts1, test_config()) - .await - .unwrap(); - let s2 = Serf::::new(transport_opts2, test_config()) - .await - .unwrap(); - let s3 = Serf::::new(transport_opts3, test_config()) - .await - .unwrap(); - - let mut serfs = [s1, s2, s3]; - wait_until_num_nodes(1, &serfs).await; - - let node = serfs[1] - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - let node = serfs[2] - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - wait_until_num_nodes(3, &serfs).await; - - // Now force the shutdown of s2 so it appears to fail. - serfs[1].shutdown().await.unwrap(); - - let t = serfs[1].inner.opts.memberlist_options.probe_interval(); - - ::sleep(t * 5).await; - - let s2id = serfs[1].local_id().clone(); - - let start = Epoch::now(); - loop { - ::sleep(Duration::from_millis(25)).await; - - let members = serfs[0].inner.members.read().await; - if test_member_status(&members.states, s2id.clone(), MemberStatus::Failed).is_ok() { - break; - } - - if start.elapsed() > Duration::from_secs(10) { - println!("{:?}", members.states); - panic!("Failed to mark node as failed"); - } - } - - // Now remove the failed node - serfs[0].remove_failed_node(s2id.clone()).await.unwrap(); - - serfs.swap(1, 2); - - let start = Epoch::now(); - let mut cond1 = false; - let mut cond2 = false; - loop { - ::sleep(Duration::from_millis(25)).await; - - let members = serfs[0].inner.members.read().await; - if test_member_status(&members.states, s2id.clone(), MemberStatus::Left).is_ok() { - cond1 = true; - } - - let members = serfs[1].inner.members.read().await; - if test_member_status(&members.states, s2id.clone(), MemberStatus::Left).is_ok() { - cond2 = true; - } - - if cond1 && cond2 { - break; - } - - if start.elapsed() > Duration::from_secs(7) { - panic!("Failed to remove failed node"); - } - } - - for s in serfs.iter() { - s.shutdown().await.unwrap(); - } -} - -/// Unit test for remove failed node prune -pub async fn serf_remove_failed_node_prune( - transport_opts1: T::Options, - transport_opts2: T::Options, - transport_opts3: T::Options, -) where - T: Transport, -{ - let s1 = Serf::::new(transport_opts1, test_config()) - .await - .unwrap(); - let s2 = Serf::::new(transport_opts2, test_config()) - .await - .unwrap(); - let s3 = Serf::::new(transport_opts3, test_config()) - .await - .unwrap(); - - let mut serfs = [s1, s2, s3]; - wait_until_num_nodes(1, &serfs).await; - - let node = serfs[1] - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - let node = serfs[2] - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - wait_until_num_nodes(3, &serfs).await; - - // Now force the shutdown of s2 so it appears to fail. - serfs[1].shutdown().await.unwrap(); - - let t = serfs[1].inner.opts.memberlist_options.probe_interval(); - - ::sleep(t * 5).await; - - let s2id = serfs[1].local_id().clone(); - - let start = Epoch::now(); - loop { - ::sleep(Duration::from_millis(25)).await; - - let members = serfs[0].inner.members.read().await; - if test_member_status(&members.states, s2id.clone(), MemberStatus::Failed).is_ok() { - break; - } - - if start.elapsed() > Duration::from_secs(7) { - panic!("Failed to mark node as failed"); - } - } - - // Now remove the failed node - serfs[0] - .remove_failed_node_prune(s2id.clone()) - .await - .unwrap(); - - serfs.swap(1, 2); - - wait_until_num_nodes(2, &serfs[..2]).await; - - for s in serfs.iter() { - s.shutdown().await.unwrap(); - } -} - -/// Unit test for remove failed node ourself -pub async fn serf_remove_failed_node_ourself(transport_opts1: T::Options) -where - T: Transport, -{ - let s1 = Serf::::new(transport_opts1, test_config()) - .await - .unwrap(); - - let serfs = [s1]; - wait_until_num_nodes(1, &serfs).await; - - serfs[0] - .remove_failed_node("somebody".into()) - .await - .unwrap(); - - serfs[0].shutdown().await.unwrap(); -} - -/// Unit test for remove old member -#[test] -fn test_remove_old_member() { - let mut members = [ - MemberState { - member: Member::new( - Node::new("foo".into(), 100), - Default::default(), - MemberStatus::None, - ), - status_time: 0.into(), - leave_time: None, - }, - MemberState { - member: Member::new( - Node::new("bar".into(), 100), - Default::default(), - MemberStatus::None, - ), - status_time: 0.into(), - leave_time: Some(Epoch::now() - Duration::from_secs(5)), - }, - MemberState { - member: Member::new( - Node::new("baz".into(), 100), - Default::default(), - MemberStatus::None, - ), - status_time: 0.into(), - leave_time: Some(Epoch::now() - Duration::from_secs(5)), - }, - ] - .into_iter() - .collect(); - remove_old_member::(&mut members, &"bar".into()); - assert_eq!(members.len(), 2); -} diff --git a/legacy/serf-core/src/serf/base/tests/serf/snapshot.rs b/legacy/serf-core/src/serf/base/tests/serf/snapshot.rs deleted file mode 100644 index 4a4d2bb7..00000000 --- a/legacy/serf-core/src/serf/base/tests/serf/snapshot.rs +++ /dev/null @@ -1,788 +0,0 @@ -use std::io::Read; - -use super::*; - -/// Unit test for the snapshoter. -pub async fn snapshoter(transport_opts: T::Options, addr: T::ResolvedAddress) -where - T: Transport, -{ - let dir = tempfile::tempdir().unwrap(); - let p = dir.path().join("snapshoter"); - let s = Serf::::new(transport_opts, test_config()).await.unwrap(); - - let clock = LamportClock::new(); - let (out_tx, out_rx) = async_channel::bounded(64); - let (shutdown_tx, shutdown_rx) = async_channel::bounded(1); - let res = open_and_replay_snapshot::<_, _, _>(&p, false).unwrap(); - let (event_tx, _, handle) = Snapshot::>::from_replay_result( - res, - SNAPSHOT_SIZE_LIMIT, - false, - clock.clone(), - out_tx, - shutdown_rx.clone(), - #[cfg(feature = "metrics")] - Default::default(), - ) - .unwrap(); - - // Write some user events - let ue = UserEventMessage::default() - .with_ltime(42.into()) - .with_name("bar".into()); - event_tx.send(ue.clone().into()).await.unwrap(); - - // Write some queries - let qe = QueryEvent { - ltime: 50.into(), - name: "bar".into(), - payload: Default::default(), - ctx: Arc::new(QueryContext { - query_timeout: Duration::default(), - span: Mutex::new(None), - this: s, - }), - id: 0, - from: Node::new("baz".into(), addr.clone()), - relay_factor: 0, - }; - event_tx.send(qe.clone().into()).await.unwrap(); - - // Write some membership events - clock.witness(100.into()); - - let mejoin = MemberEvent { - ty: MemberEventType::Join, - members: TinyVec::from(Member::new( - Node::new("foo".into(), addr.clone()), - Default::default(), - MemberStatus::None, - )) - .into(), - }; - - let mefail = MemberEvent { - ty: MemberEventType::Failed, - members: TinyVec::from(Member::new( - Node::new("foo".into(), addr.clone()), - Default::default(), - MemberStatus::None, - )) - .into(), - }; - - event_tx.send(mejoin.clone().into()).await.unwrap(); - event_tx.send(mefail.clone().into()).await.unwrap(); - event_tx.send(mejoin.clone().into()).await.unwrap(); - - // Check these get passed through - futures::select! { - e = out_rx.recv().fuse() => { - let e = e.unwrap(); - match e { - CrateEvent::User(e) => { - assert_eq!(e, ue); - }, - _ => panic!("expected user event"), - } - }, - _ = ::sleep(Duration::from_millis(200)).fuse() => { - panic!("timeout"); - } - } - - futures::select! { - e = out_rx.recv().fuse() => { - let e = e.unwrap(); - match e { - CrateEvent::Query(e) => { - if qe.ne(&e) { - panic!("expected query event mismatch"); - } - }, - _ => panic!("expected query event"), - } - }, - _ = ::sleep(Duration::from_millis(200)).fuse() => { - panic!("timeout"); - } - } - - futures::select! { - e = out_rx.recv().fuse() => { - let e = e.unwrap(); - match e { - CrateEvent::Member(e) => { - assert_eq!(e, mejoin); - }, - _ => panic!("expected member event"), - } - }, - _ = ::sleep(Duration::from_millis(200)).fuse() => { - panic!("timeout"); - } - } - - futures::select! { - e = out_rx.recv().fuse() => { - let e = e.unwrap(); - match e { - CrateEvent::Member(e) => { - assert_eq!(e, mefail); - }, - _ => panic!("expected member event"), - } - }, - _ = ::sleep(Duration::from_millis(200)).fuse() => { - panic!("timeout"); - } - } - - futures::select! { - e = out_rx.recv().fuse() => { - let e = e.unwrap(); - match e { - CrateEvent::Member(e) => { - assert_eq!(e, mejoin); - }, - _ => panic!("expected member event"), - } - }, - _ = ::sleep(Duration::from_millis(200)).fuse() => { - panic!("timeout"); - } - } - - // Close the snapshoter - shutdown_tx.close(); - handle.wait().await; - - // Open the snapshoter - let (shutdown_tx, shutdown_rx) = async_channel::bounded(1); - let res = open_and_replay_snapshot::<_, _, _>(&p, false).unwrap(); - - assert_eq!(res.last_clock, 100.into()); - assert_eq!(res.last_event_clock, 42.into()); - assert_eq!(res.last_query_clock, 50.into()); - - let (out_tx, _out_rx) = async_channel::bounded(64); - let (_event_tx, alive_nodes, handle) = Snapshot::>::from_replay_result( - res, - SNAPSHOT_SIZE_LIMIT, - false, - clock.clone(), - out_tx, - shutdown_rx.clone(), - #[cfg(feature = "metrics")] - Default::default(), - ) - .unwrap(); - - assert_eq!(alive_nodes.len(), 1); - let n = &alive_nodes[0]; - assert_eq!(n.id(), "foo"); - assert_eq!(n.address().clone().unwrap_resolved(), addr); - - // Close the snapshoter - shutdown_tx.close(); - handle.wait().await; - - // Open the snapshoter, make sure nothing dies reading with coordinates - // disabled. - let (shutdown_tx, shutdown_rx) = async_channel::bounded(1); - let res = open_and_replay_snapshot::<_, _, _>(&p, false).unwrap(); - - let (out_tx, _out_rx) = async_channel::bounded(64); - let (_event_tx, _, handle) = Snapshot::>::from_replay_result( - res, - SNAPSHOT_SIZE_LIMIT, - false, - clock.clone(), - out_tx, - shutdown_rx.clone(), - #[cfg(feature = "metrics")] - Default::default(), - ) - .unwrap(); - shutdown_tx.close(); - handle.wait().await; -} - -/// Unit test for the snapshoter force compact. -pub async fn snapshoter_force_compact(transport_opts: T::Options, addr: T::ResolvedAddress) -where - T: Transport, -{ - let dir = tempfile::tempdir().unwrap(); - let p = dir.path().join("snapshoter_force_compact"); - let s = Serf::::new(transport_opts, test_config()).await.unwrap(); - - let clock = LamportClock::new(); - let (shutdown_tx, shutdown_rx) = async_channel::bounded(1); - - // Create a very low limit - let res = open_and_replay_snapshot::<_, _, _>(&p, false).unwrap(); - let (out_tx, _out_rx) = async_channel::unbounded(); - let (event_tx, _, handle) = Snapshot::>::from_replay_result( - res, - 1024, - false, - clock.clone(), - out_tx, - shutdown_rx.clone(), - #[cfg(feature = "metrics")] - Default::default(), - ) - .unwrap(); - - // Write lots of user events - for i in 0..1024 { - let ue = UserEventMessage::default().with_ltime(i.into()); - event_tx.send(ue.clone().into()).await.unwrap(); - } - - // Write lots of queries - for i in 0..1024 { - let qe = QueryEvent { - ltime: i.into(), - name: "bar".into(), - payload: Default::default(), - ctx: Arc::new(QueryContext { - query_timeout: Duration::default(), - span: Mutex::new(None), - this: s.clone(), - }), - id: 0, - from: Node::new("baz".into(), addr.clone()), - relay_factor: 0, - }; - event_tx.send(qe.clone().into()).await.unwrap(); - } - - // Wait for drain - while !event_tx.is_empty() { - ::sleep(Duration::from_millis(20)).await; - } - - // Close the snapshoter - shutdown_tx.close(); - handle.wait().await; - - // Open the snapshoter - let res = open_and_replay_snapshot::(&p, false).unwrap(); - - assert_eq!(res.last_event_clock, 1023.into()); - assert_eq!(res.last_query_clock, 1023.into()); -} - -/// Unit test for the snapshoter leave -pub async fn snapshoter_leave(transport_opts: T::Options, addr: T::ResolvedAddress) -where - T: Transport, -{ - let dir = tempfile::tempdir().unwrap(); - let p = dir.path().join("snapshoter_leave"); - let s = Serf::::new(transport_opts, test_config()).await.unwrap(); - - let clock = LamportClock::new(); - let (shutdown_tx, shutdown_rx) = async_channel::bounded(1); - let res = open_and_replay_snapshot::<_, _, _>(&p, false).unwrap(); - let (out_tx, _out_rx) = async_channel::unbounded(); - let (event_tx, _, handle) = Snapshot::>::from_replay_result( - res, - SNAPSHOT_SIZE_LIMIT, - false, - clock.clone(), - out_tx, - shutdown_rx.clone(), - #[cfg(feature = "metrics")] - Default::default(), - ) - .unwrap(); - - // Write a user event - let ue = UserEventMessage::default() - .with_ltime(42.into()) - .with_name("bar".into()); - event_tx.send(ue.clone().into()).await.unwrap(); - - // Write a query - let qe = QueryEvent { - ltime: 50.into(), - name: "uptime".into(), - payload: Default::default(), - ctx: Arc::new(QueryContext { - query_timeout: Duration::default(), - span: Mutex::new(None), - this: s.clone(), - }), - id: 0, - from: Node::new("baz".into(), addr.clone()), - relay_factor: 0, - }; - event_tx.send(qe.clone().into()).await.unwrap(); - - // Write some member events - clock.witness(100.into()); - - let mejoin = MemberEvent { - ty: MemberEventType::Join, - members: TinyVec::from(Member::new( - Node::new("foo".into(), addr.clone()), - Default::default(), - MemberStatus::None, - )) - .into(), - }; - event_tx.send(mejoin.clone().into()).await.unwrap(); - - // wait for drain - while !event_tx.is_empty() { - ::sleep(Duration::from_millis(20)).await; - } - - // Leave the cluster! - handle.leave().await; - - // Close the snapshoter - shutdown_tx.close(); - handle.wait().await; - s.shutdown().await.unwrap(); - drop(s); - - // Open the snapshoter - let (shutdown_tx, shutdown_rx) = async_channel::bounded(1); - let res = open_and_replay_snapshot::<_, _, _>(&p, false).unwrap(); - assert!(res.last_clock == 0.into(), "last_clock: {}", res.last_clock); - assert!( - res.last_event_clock == 0.into(), - "last_event_clock: {}", - res.last_event_clock - ); - assert!( - res.last_query_clock == 0.into(), - "last_query_clock: {}", - res.last_query_clock - ); - let (out_tx, _out_rx) = async_channel::unbounded(); - let (_, alive_nodes, _) = Snapshot::>::from_replay_result( - res, - SNAPSHOT_SIZE_LIMIT, - false, - clock.clone(), - out_tx, - shutdown_rx.clone(), - #[cfg(feature = "metrics")] - Default::default(), - ) - .unwrap(); - - assert!(alive_nodes.is_empty()); - - // Close the snapshoter - shutdown_tx.close(); - handle.wait().await; -} - -/// Unit test for the snapshoter leave rejoin -pub async fn snapshoter_leave_rejoin(transport_opts: T::Options, addr: T::ResolvedAddress) -where - T: Transport, -{ - let dir = tempfile::tempdir().unwrap(); - let p = dir.path().join("snapshoter_leave_rejoin"); - let s = Serf::::new(transport_opts, test_config()).await.unwrap(); - - let clock = LamportClock::new(); - let (shutdown_tx, shutdown_rx) = async_channel::bounded(1); - let res = open_and_replay_snapshot::<_, _, _>(&p, true).unwrap(); - let (out_tx, _out_rx) = async_channel::unbounded(); - let (event_tx, _, handle) = Snapshot::>::from_replay_result( - res, - SNAPSHOT_SIZE_LIMIT, - true, - clock.clone(), - out_tx, - shutdown_rx.clone(), - #[cfg(feature = "metrics")] - Default::default(), - ) - .unwrap(); - - // Write a user event - let ue = UserEventMessage::default() - .with_ltime(42.into()) - .with_name("bar".into()); - event_tx.send(ue.clone().into()).await.unwrap(); - - // Write a query - let qe = QueryEvent { - ltime: 50.into(), - name: "uptime".into(), - payload: Default::default(), - ctx: Arc::new(QueryContext { - query_timeout: Duration::default(), - span: Mutex::new(None), - this: s.clone(), - }), - id: 0, - from: Node::new("baz".into(), addr.clone()), - relay_factor: 0, - }; - event_tx.send(qe.clone().into()).await.unwrap(); - - // Write some member events - clock.witness(100.into()); - - let mejoin = MemberEvent { - ty: MemberEventType::Join, - members: TinyVec::from(Member::new( - Node::new("foo".into(), addr.clone()), - Default::default(), - MemberStatus::None, - )) - .into(), - }; - event_tx.send(mejoin.clone().into()).await.unwrap(); - - // wait for drain - while !event_tx.is_empty() { - ::sleep(Duration::from_millis(20)).await; - } - - // Leave the cluster! - handle.leave().await; - // Close the snapshoter - shutdown_tx.close(); - handle.wait().await; - - let mut f = std::fs::File::open(&p).unwrap(); - let mut d = vec![]; - f.read_to_end(&mut d).unwrap(); - - s.shutdown().await.unwrap(); - drop(s); - - // Open the snapshoter - let (shutdown_tx, shutdown_rx) = async_channel::bounded(1); - let res = open_and_replay_snapshot::<_, _, _>(&p, true).unwrap(); - assert!(res.last_clock == 100.into()); - assert!(res.last_event_clock == 42.into()); - assert!(res.last_query_clock == 50.into()); - let (out_tx, _out_rx) = async_channel::unbounded(); - let (_, alive_nodes, handle) = Snapshot::>::from_replay_result( - res, - SNAPSHOT_SIZE_LIMIT, - false, - clock.clone(), - out_tx, - shutdown_rx.clone(), - #[cfg(feature = "metrics")] - Default::default(), - ) - .unwrap(); - - assert!(!alive_nodes.is_empty()); - - // Close the snapshoter - shutdown_tx.close(); - handle.wait().await; -} - -/// Unit tests for the serf snapshot recovery -pub async fn serf_snapshot_recovery( - transport_opts1: T::Options, - transport_opts2: T::Options, - get_transport: impl FnOnce(T::Id, T::ResolvedAddress) -> F + Copy, -) where - T: Transport, - T::Options: Clone, - F: core::future::Future, -{ - let td = tempfile::tempdir().unwrap(); - let snap_path = td.path().join("serf_snapshot_recovery"); - let s1 = Serf::::new(transport_opts1, test_config()) - .await - .unwrap(); - let s2 = Serf::::new( - transport_opts2.clone(), - test_config().with_snapshot_path(Some(snap_path.clone())), - ) - .await - .unwrap(); - - let mut serfs = vec![s1, s2]; - wait_until_num_nodes(1, &serfs).await; - - let node = serfs[1] - .inner - .memberlist - .advertise_node() - .map_address(MaybeResolvedAddress::resolved); - serfs[0].join(node.address().clone(), false).await.unwrap(); - - wait_until_num_nodes(2, &serfs).await; - - // Fire a user event - serfs[0] - .user_event("event!", Bytes::from_static(b"test"), false) - .await - .unwrap(); - ::sleep(Duration::from_secs(10)).await; - - // Now force the shutdown of s2 so it appears to fail. - serfs[1].shutdown().await.unwrap(); - let (s2id, s2addr) = serfs[1].advertise_node().into_components(); - let t = serfs[1].inner.opts.memberlist_options.probe_interval * 10; - let _ = serfs.pop().unwrap(); - - ::sleep(t).await; - - // Verify that s2 is "failed" - { - let members = serfs[0].inner.members.read().await; - test_member_status(&members.states, node.id().clone(), MemberStatus::Failed).unwrap(); - } - - // Now remove the failed node - serfs[0] - .remove_failed_node(node.id().clone()) - .await - .unwrap(); - - // Verify that s2 is gone - { - let members = serfs[0].inner.members.read().await; - test_member_status(&members.states, node.id().clone(), MemberStatus::Left).unwrap(); - } - - // Listen for events - let (event_tx, event_rx) = EventProducer::bounded(4); - let s2 = Serf::::with_event_producer( - get_transport(s2id, s2addr).await, - test_config().with_snapshot_path(Some(snap_path.clone())), - event_tx, - ) - .await - .unwrap(); - - // Wait for the node to auto rejoin - let start = Epoch::now(); - while start.elapsed() < Duration::from_secs(1) { - let members = serfs[0].members().await; - if members.len() == 2 - && members[0].status == MemberStatus::Alive - && members[1].status == MemberStatus::Alive - { - break; - } - ::sleep(Duration::from_millis(10)).await; - } - - serfs.push(s2); - - // Verify that s2 is "alive" - { - let node = serfs[1].local_id().clone(); - let members = serfs[0].inner.members.read().await; - test_member_status(&members.states, node, MemberStatus::Alive).unwrap(); - } - { - let node = serfs[0].local_id().clone(); - let members = serfs[1].inner.members.read().await; - test_member_status(&members.states, node, MemberStatus::Alive).unwrap(); - } - - // Check the events to make sure we got nothing - test_user_events(event_rx.rx, vec![], vec![]).await; - - for s in serfs.iter() { - s.shutdown().await.unwrap(); - } -} - -#[cfg(test)] -#[tokio::test(flavor = "multi_thread", worker_threads = 16)] -async fn test_snapshoter_slow_disk_not_blocking_event_tx() { - use memberlist_core::{ - agnostic_lite::tokio::TokioRuntime, - transport::{resolver::socket_addr::SocketAddrResolver, unimplemented::UnimplementedTransport}, - }; - - crate::tests::initialize_tests_tracing(); - - type Transport = UnimplementedTransport, TokioRuntime>; - - type Delegate = DefaultDelegate; - - let dir = tempfile::tempdir().unwrap(); - let p = dir - .path() - .join("snapshoter_slow_disk_not_blocking_event_tx"); - - let clock = LamportClock::new(); - let (shutdown_tx, shutdown_rx) = async_channel::bounded(1); - let (out_tx, out_rx) = async_channel::bounded(1024); - let res = open_and_replay_snapshot::<_, _, _>(&p, true).unwrap(); - let (event_tx, _, handle) = Snapshot::::from_replay_result( - res, - SNAPSHOT_SIZE_LIMIT, - true, - clock.clone(), - out_tx, - shutdown_rx.clone(), - #[cfg(feature = "metrics")] - Default::default(), - ) - .unwrap(); - - // We need enough events to be much more than the buffers used which are size - // 1024. This number processes easily within the 500ms we allow below on my - // host provided there is no disk IO on the path (I verified that by just - // returning early in tryAppend using the old blocking code). The new async - // method should pass without disabling disk writes too! - let num_events = 10000; - - // Write lots of member updates (way bigger than our chan buffers) - let (start_tx, start_rx) = async_channel::bounded::<()>(1); - - TokioRuntime::spawn_detach(async move { - let _ = start_rx.recv().await; - - for i in 0..num_events { - let mut e = MemberEvent { - ty: MemberEventType::Join, - members: TinyVec::from(Member::new( - Node::new( - format!("foo{i}").into(), - format!("127.0.{}.{}:5000", (i / 256) % 256, i % 256) - .parse() - .unwrap(), - ), - Default::default(), - MemberStatus::None, - )) - .into(), - }; - - if i % 10 == 0 { - e.ty = MemberEventType::Leave; - } - event_tx.send(e.into()).await.unwrap(); - } - // Pace ourselves - if we just throw these out as fast as possible the - // read loop below can't keep up and we end up dropping messages due to - // backpressure. But we need to still send them all in well less than the - // timeout, 10k messages at 1 microsecond should take 10 ms minimum. In - // practice it's quite a bit more to actually process and because the - // buffer here blocks. - TokioRuntime::sleep(Duration::from_micros(1)).await; - }); - - // Wait for them all to process through and it should be in a lot less time - // than if the disk IO was in serial. This was verified by running this test - // against the old serial implementation and seeing it never come close to - // passing on my laptop with an SSD. It's not the most robust thing ever but - // it's at least a sanity check that we are non-blocking now, and it passes - // reliably at least on my machine. I typically see this complete in around - // 115ms on my machine so this should give plenty of headroom for slower CI - // environments while still being low enough that actual disk IO would - // reliably blow it. - let deadline = TokioRuntime::sleep_until(TokioRuntime::now() + Duration::from_millis(500)); - futures::pin_mut!(deadline); - let mut num_recvd = 0; - let start = Epoch::now(); - - while num_recvd < num_events { - futures::select! { - _ = start_tx.send(()).fuse() => { - continue; - }, - _ = out_rx.recv().fuse() => { - num_recvd += 1; - }, - _ = (&mut deadline).fuse() => { - panic!("timed out after {:?} waiting for messages blocked on fake disk IO? got {} of {}", start.elapsed(), num_recvd, num_events); - } - } - } - - // Close the snapshoter - shutdown_tx.close(); - handle.wait().await; -} - -#[cfg(test)] -#[tokio::test(flavor = "multi_thread", worker_threads = 16)] -async fn test_snapshoter_slow_disk_not_blocking_memberlist() { - use memberlist_core::{ - agnostic_lite::tokio::TokioRuntime, - transport::{resolver::socket_addr::SocketAddrResolver, unimplemented::UnimplementedTransport}, - }; - - type Transport = UnimplementedTransport, TokioRuntime>; - - type Delegate = DefaultDelegate; - - let dir = tempfile::tempdir().unwrap(); - let p = dir - .path() - .join("snapshoter_slow_disk_not_blocking_memberlist"); - - let clock = LamportClock::new(); - let (shutdown_tx, shutdown_rx) = async_channel::bounded(1); - let (out_tx, _out_rx) = async_channel::bounded(1); - let res = open_and_replay_snapshot::<_, _, _>(&p, true).unwrap(); - let (event_tx, _, handle) = Snapshot::::from_replay_result( - res, - SNAPSHOT_SIZE_LIMIT, - true, - clock.clone(), - out_tx, - shutdown_rx.clone(), - #[cfg(feature = "metrics")] - Default::default(), - ) - .unwrap(); - - // We need enough events to be more than the internal buffer sizes - let num_events = 3000; - - for i in 0..num_events { - let mut e = MemberEvent { - ty: MemberEventType::Join, - members: TinyVec::from(Member::new( - Node::new( - format!("foo{i}").into(), - format!("127.0.{}.{}:5000", (i / 256) % 256, i % 256) - .parse() - .unwrap(), - ), - Default::default(), - MemberStatus::None, - )) - .into(), - }; - - if i % 10 == 0 { - e.ty = MemberEventType::Leave; - } - - futures::select! { - _ = event_tx.send(e.into()).fuse() => {}, - default => { - panic!("event_tx should never block"); - } - } - - // Allow just the tiniest time so that the runtime can schedule the - // task that's reading this even if they are both on the same physical - // core (like in CI). - TokioRuntime::sleep(Duration::from_micros(1)).await; - } - - // Close the snapshoter - shutdown_tx.close(); - handle.wait().await; -} diff --git a/legacy/serf-core/src/serf/delegate.rs b/legacy/serf-core/src/serf/delegate.rs deleted file mode 100644 index 297eec74..00000000 --- a/legacy/serf-core/src/serf/delegate.rs +++ /dev/null @@ -1,843 +0,0 @@ -use crate::{ - Serf, SerfWeakRef, - broadcast::SerfBroadcast, - delegate::Delegate, - error::{SerfDelegateError, SerfError}, - event::QueryMessageExt, - types::{ - DelegateVersion, JoinMessage, LamportTime, LeaveMessage, Member, MemberStatus, MessageRef, - ProtocolVersion, PushPullMessageBorrow, RelayMessageRef, UserEventMessage, - }, -}; - -use std::{ - borrow::Cow, - sync::{Arc, OnceLock, atomic::Ordering}, -}; - -use crate::types::{PushPullMessage, Tags}; -use arc_swap::ArcSwap; -use either::Either; -use indexmap::IndexSet; -use memberlist_core::{ - CheapClone, META_MAX_SIZE, - bytes::{Buf, BufMut, Bytes, BytesMut}, - delegate::{ - AliveDelegate, ConflictDelegate, Delegate as MemberlistDelegate, EventDelegate, - MergeDelegate as MemberlistMergeDelegate, NodeDelegate, PingDelegate, - }, - proto::{Data, Meta, NodeState, State, TinyVec}, - tracing, - transport::Transport, -}; - -#[cfg(any(test, feature = "test"))] -use crate::types::MessageType; - -// PingVersion is an internal version for the ping message, above the normal -// versioning we get from the protocol version. This enables small updates -// to the ping message without a full protocol bump. -const PING_VERSION: u8 = 1; - -#[cfg(any(test, feature = "test"))] -pub(crate) trait MessageDropper: Send + Sync + 'static { - fn should_drop(&self, ty: MessageType) -> bool; -} - -/// The memberlist delegate for Serf. -pub struct SerfDelegate -where - D: Delegate, - T: Transport, -{ - serf: OnceLock>, - delegate: Option, - tags: Arc>, - #[cfg(any(test, feature = "test"))] - pub(crate) message_dropper: Option>, - /// Only used for testing purposes - #[cfg(any(test, feature = "test"))] - pub(crate) ping_versioning_test: core::sync::atomic::AtomicBool, - #[cfg(any(test, feature = "test"))] - pub(crate) ping_dimension_test: core::sync::atomic::AtomicBool, -} - -impl SerfDelegate -where - D: Delegate, - T: Transport, -{ - pub(crate) fn new(d: Option, tags: Arc>) -> Self { - Self { - serf: OnceLock::new(), - delegate: d, - tags, - #[cfg(any(test, feature = "test"))] - message_dropper: None, - #[cfg(any(test, feature = "test"))] - ping_versioning_test: core::sync::atomic::AtomicBool::new(false), - #[cfg(any(test, feature = "test"))] - ping_dimension_test: core::sync::atomic::AtomicBool::new(false), - } - } - - #[cfg(any(test, feature = "test"))] - pub(crate) fn with_dropper( - d: Option, - dropper: Box, - tags: Arc>, - ) -> Self { - Self { - serf: OnceLock::new(), - delegate: d, - tags, - #[cfg(any(test, feature = "test"))] - message_dropper: Some(dropper), - #[cfg(any(test, feature = "test"))] - ping_versioning_test: core::sync::atomic::AtomicBool::new(false), - #[cfg(any(test, feature = "test"))] - ping_dimension_test: core::sync::atomic::AtomicBool::new(false), - } - } - - pub(crate) fn delegate(&self) -> Option<&D> { - self.delegate.as_ref() - } - - pub(crate) fn store(&self, s: SerfWeakRef) { - // No error, we never call this in parallel - let _ = self.serf.set(s); - } - - fn this(&self) -> Option> { - self.serf.get().and_then(|weak_ref| weak_ref.upgrade()) - } -} - -impl NodeDelegate for SerfDelegate -where - D: Delegate, - T: Transport, -{ - async fn node_meta(&self, limit: usize) -> Meta { - let tags = self.tags.load(); - match tags.is_empty() { - false => { - let encoded_len = tags.encoded_len(); - let limit = limit.min(Meta::MAX_SIZE); - if encoded_len > limit { - panic!( - "node tags {:?} exceeds length limit of {} bytes", - tags, limit - ); - } - - let mut role_bytes = vec![0; encoded_len]; - match tags.encode(&mut role_bytes) { - Ok(len) => { - if len > limit { - panic!( - "node tags {:?} exceeds length limit of {} bytes", - tags, limit - ); - } - - role_bytes.try_into().unwrap() - } - Err(e) => { - tracing::error!(err=%e, "serf: failed to encode tags"); - Meta::empty() - } - } - } - true => Meta::empty(), - } - } - - async fn notify_message(&self, buf: Cow<'_, [u8]>) { - fn to_owned(buf: Cow<'_, [u8]>) -> Bytes { - match buf { - Cow::Borrowed(buf) => Bytes::copy_from_slice(buf), - Cow::Owned(buf) => Bytes::from(buf), - } - } - - // If we didn't actually receive any data, then ignore it. - if buf.is_empty() { - return; - } - - let Some(this) = self.this() else { - return; - }; - - #[cfg(feature = "metrics")] - { - metrics::histogram!( - "serf.messages.received", - this.inner.opts.memberlist_options.metric_labels.iter() - ) - .record(buf.len() as f64); - } - - let mut rebroadcast = false; - let mut rebroadcast_queue = &this.inner.broadcasts; - let mut relay = None; - match crate::types::decode_message::(buf.as_ref()) { - Ok(msg) => { - #[cfg(any(test, feature = "test"))] - { - if let Some(ref dropper) = this.inner.memberlist.delegate().unwrap().message_dropper { - if dropper.should_drop(MessageType::Join) { - return; - } - } - } - - match msg { - MessageRef::Leave(l) => { - tracing::debug!("serf: leave message: {:?}", l.id()); - match as Data>::from_ref(l) { - Err(e) => { - tracing::error!(err=%e, "serf: failed to decode leave message"); - } - Ok(l) => { - rebroadcast = this.handle_node_leave_intent(&l).await; - } - }; - } - MessageRef::Join(j) => { - tracing::debug!("serf: join message: {:?}", j.id()); - match as Data>::from_ref(j) { - Err(e) => { - tracing::error!(err=%e, "serf: failed to decode join message"); - } - Ok(j) => { - rebroadcast = this.handle_node_join_intent(&j).await; - } - }; - } - MessageRef::UserEvent(ue) => { - rebroadcast = this.handle_user_event(either::Either::Left(ue)).await; - rebroadcast_queue = &this.inner.event_broadcasts; - let rebroadcast_queue_num_queued = rebroadcast_queue.num_queued().await; - tracing::debug!( - name = ue.name(), - payload_len = ue.payload().len(), - rebroadcast, - rebroadcast_queue_num_queued, - "serf: user event message", - ); - } - MessageRef::Query(q) => { - tracing::debug!("serf: query message: {}", q.name()); - match q.decode_internal_query() { - Some(Err(e)) => { - tracing::warn!(err=%e, "serf: failed to decode message"); - } - Some(Ok(res)) => match this.handle_query(either::Either::Left(q), Some(res)).await { - Ok(val) => { - rebroadcast = val; - rebroadcast_queue = &this.inner.query_broadcasts; - } - Err(e) => { - tracing::warn!(err=%e, "serf: failed to decode query message"); - } - }, - None => match this.handle_query(either::Either::Left(q), None).await { - Ok(val) => { - rebroadcast = val; - rebroadcast_queue = &this.inner.query_broadcasts; - } - Err(e) => { - tracing::warn!(err=%e, "serf: failed to decode query message"); - } - }, - } - } - MessageRef::QueryResponse(qr) => { - tracing::debug!("serf: query response message"); - if let Err(e) = this.handle_query_response(qr).await { - tracing::warn!(err=%e, "serf: failed to decode query response message"); - } - } - MessageRef::Relay(RelayMessageRef { - node, - payload, - payload_offset, - }) => { - tracing::debug!("serf: relaying response to node: {:?}", node); - match Data::from_ref(*node.address()) { - Err(e) => { - tracing::error!(err=%e, "serf: failed to encode address"); - } - Ok(addr) => match buf { - Cow::Borrowed(_) => { - relay = Some((addr, Either::Left(Bytes::copy_from_slice(payload)))); - } - Cow::Owned(_) => { - relay = Some((addr, Either::Right((payload_offset, payload.len())))); - } - }, - } - } - msg => { - tracing::warn!("serf: receive unexpected message type: {}", msg.ty()); - } - } - } - Err(e) => { - tracing::warn!(err=%e, "serf: failed to decode message"); - } - } - - if rebroadcast { - rebroadcast_queue - .queue_broadcast(SerfBroadcast { - msg: to_owned(buf), - notify_tx: None, - }) - .await; - } else if let Some((addr, payload)) = relay { - let msg = match payload { - Either::Left(p) => p, - Either::Right((offset, len)) => { - let mut buf = to_owned(buf); - buf.advance(offset); - buf.split_to(len) - } - }; - - if let Err(e) = this.inner.memberlist.send(&addr, msg).await { - tracing::error!(err=%e, "serf: failed to forwarding message to {}", addr); - } - } - } - - async fn broadcast_messages( - &self, - limit: usize, - encoded_len: F, - ) -> impl Iterator + Send - where - F: Fn(Bytes) -> (usize, Bytes) + Send + Sync + 'static, - { - let Some(this) = self.this() else { - return TinyVec::with_capacity(0).into_iter(); - }; - let mut msgs = this.inner.broadcasts.get_broadcasts(limit).await; - - // Determine the bytes used already - let mut bytes_used = 0; - for msg in msgs.iter() { - let (encoded_len, _) = encoded_len(msg.clone()); - bytes_used += encoded_len; - #[cfg(feature = "metrics")] - { - metrics::histogram!( - "serf.messages.sent", - this.inner.opts.memberlist_options.metric_labels.iter() - ) - .record(encoded_len as f64); - } - } - - // Get any additional query broadcasts - let query_msgs = this - .inner - .query_broadcasts - .get_broadcasts(limit - bytes_used) - .await; - for msg in query_msgs.iter() { - let (encoded_len, _) = encoded_len(msg.clone()); - bytes_used += encoded_len; - #[cfg(feature = "metrics")] - { - metrics::histogram!( - "serf.messages.sent", - this.inner.opts.memberlist_options.metric_labels.iter() - ) - .record(encoded_len as f64); - } - } - - // Get any additional event broadcasts - let event_msgs = this - .inner - .event_broadcasts - .get_broadcasts(limit - bytes_used) - .await; - #[cfg(feature = "metrics")] - { - for msg in event_msgs.iter() { - let (encoded_len, _) = encoded_len(msg.clone()); - metrics::histogram!( - "serf.messages.sent", - this.inner.opts.memberlist_options.metric_labels.iter() - ) - .record(encoded_len as f64); - } - } - msgs.extend(query_msgs); - msgs.extend(event_msgs); - msgs.into_iter() - } - - async fn local_state(&self, _join: bool) -> Bytes { - let Some(this) = self.this() else { - return Bytes::new(); - }; - - let members = this.inner.members.read().await; - let events = this.inner.event_core.read().await; - - // Create the message to send - let status_ltimes = members - .states - .values() - .map(|v| (v.member.node.id().cheap_clone(), v.status_time)) - .collect(); - let left_members = members - .left_members - .iter() - .map(|v| v.member.node().id().cheap_clone()) - .collect::>(); - let pp = PushPullMessageBorrow { - ltime: this.inner.clock.time(), - status_ltimes: &status_ltimes, - left_members: &left_members, - event_ltime: this.inner.event_clock.time(), - events: events.buffer.as_slice(), - query_ltime: this.inner.query_clock.time(), - }; - drop(members); - - match crate::types::encode_message_to_bytes(&pp) { - Ok(buf) => { - tracing::trace!(data=?buf.as_ref(), "serf: local state"); - buf - } - Err(e) => { - tracing::error!(err=%e, "serf: failed to encode local state"); - Bytes::new() - } - } - } - - async fn merge_remote_state(&self, buf: &[u8], is_join: bool) { - if buf.is_empty() { - tracing::error!("serf: remote state is zero bytes"); - return; - } - - tracing::trace!(data=?buf, "serf: merge remote state"); - - // Check the message type - let msg = match crate::types::decode_message::(buf) { - Ok(msg) => msg, - Err(e) => { - tracing::error!(err=%e, "serf: fail to decode remote state"); - return; - } - }; - - #[cfg(any(test, feature = "test"))] - { - if let Some(this) = self.this() { - if let Some(ref dropper) = this.inner.memberlist.delegate().unwrap().message_dropper { - if dropper.should_drop(MessageType::PushPull) { - return; - } - } - } - } - - match msg { - MessageRef::PushPull(pp) => { - let ltime = pp.ltime(); - let event_ltime = pp.event_ltime(); - let query_ltime = pp.query_ltime(); - let Some(this) = self.this() else { - return; - }; - - // Witness the Lamport clocks first. - // We subtract 1 since no message with that clock has been sent yet - if ltime > LamportTime::ZERO { - this.inner.clock.witness(ltime - LamportTime::new(1)); - } - if event_ltime > LamportTime::ZERO { - this - .inner - .event_clock - .witness(event_ltime - LamportTime::new(1)); - } - if query_ltime > LamportTime::ZERO { - this - .inner - .query_clock - .witness(query_ltime - LamportTime::new(1)); - } - - let pp = match as Data>::from_ref(pp) { - Ok(pp) => pp, - Err(e) => { - tracing::error!(err=%e, "serf: failed to decode push pull message"); - return; - } - }; - - // Process the left nodes first to avoid the LTimes from incrementing - // in the wrong order. Note that we don't have the actual Lamport time - // for the leave message, so we go one past the join time, since the - // leave must have been accepted after that to get onto the left members - // list. If we didn't do this then the message would not get processed. - for node in &pp.left_members { - if let Some(<ime) = pp.status_ltimes.get(node) { - this - .handle_node_leave_intent(&LeaveMessage { - ltime: ltime + LamportTime::new(1), - id: node.cheap_clone(), - prune: false, - }) - .await; - } else { - tracing::error!( - "serf: {} is in left members, but cannot find the lamport time for it in status", - node - ); - } - } - - // Update any other LTimes - for (node, ltime) in pp.status_ltimes { - // Skip the left nodes - if pp.left_members.contains(&node) { - continue; - } - - // Create an artificial join message - this - .handle_node_join_intent(&JoinMessage { ltime, id: node }) - .await; - } - - // If we are doing a join, and eventJoinIgnore is set - // then we set the eventMinTime to the EventLTime. This - // prevents any of the incoming events from being processed - let event_join_ignore = this.inner.event_join_ignore.load(Ordering::Acquire); - if is_join && event_join_ignore { - let mut ec = this.inner.event_core.write().await; - if event_ltime > ec.min_time { - ec.min_time = event_ltime; - } - } - - // Process all the events - for events in pp.events { - for e in events.events { - this - .handle_user_event(either::Either::Right(UserEventMessage { - ltime: events.ltime, - name: e.name, - payload: e.payload, - cc: false, - })) - .await; - } - } - } - msg => { - tracing::error!("serf: remote state has bad message type {}", msg.ty()); - } - } - } -} - -impl EventDelegate for SerfDelegate -where - D: Delegate, - T: Transport, -{ - type Id = T::Id; - type Address = T::ResolvedAddress; - - async fn notify_join(&self, node: Arc>) { - if let Some(this) = self.this() { - this.handle_node_join(node).await; - } - } - - async fn notify_leave(&self, node: Arc>) { - if let Some(this) = self.this() { - this.handle_node_leave(node).await; - } - } - - async fn notify_update(&self, node: Arc>) { - if let Some(this) = self.this() { - this.handle_node_update(node).await; - } - } -} - -impl AliveDelegate for SerfDelegate -where - D: Delegate, - T: Transport, -{ - type Id = T::Id; - type Address = T::ResolvedAddress; - type Error = SerfDelegateError; - - async fn notify_alive( - &self, - node: Arc>, - ) -> Result<(), Self::Error> { - if let Some(ref d) = self.delegate { - let member = node_to_member::(&node)?; - return d - .notify_merge(Arc::from_iter([member])) - .await - .map_err(SerfDelegateError::Merge); - } - - Ok(()) - } -} - -impl MemberlistMergeDelegate for SerfDelegate -where - D: Delegate, - T: Transport, -{ - type Id = T::Id; - type Address = T::ResolvedAddress; - type Error = SerfDelegateError; - - async fn notify_merge( - &self, - peers: Arc<[NodeState]>, - ) -> Result<(), Self::Error> { - if let Some(ref d) = self.delegate { - let peers = peers - .iter() - .map(node_to_member::) - .collect::, _>>()?; - return d - .notify_merge(peers) - .await - .map_err(SerfDelegateError::Merge); - } - Ok(()) - } -} - -impl ConflictDelegate for SerfDelegate -where - D: Delegate, - T: Transport, -{ - type Id = T::Id; - - type Address = T::ResolvedAddress; - - async fn notify_conflict( - &self, - existing: Arc>, - other: Arc>, - ) { - if let Some(this) = self.this() { - this.handle_node_conflict(existing, other).await; - } - } -} - -impl PingDelegate for SerfDelegate -where - D: Delegate, - T: Transport, -{ - type Id = T::Id; - - type Address = T::ResolvedAddress; - - async fn ack_payload(&self) -> Bytes { - #[cfg(any(feature = "test", test))] - if self.ping_versioning_test.load(Ordering::SeqCst) { - // Send back the next ping version, which is bad by default. - let mut buf = BytesMut::new(); - buf.put_u8(PING_VERSION + 1); - buf.put_slice(b"this is bad and not a real message"); - return buf.freeze(); - } - - #[cfg(any(feature = "test", test))] - if self.ping_dimension_test.load(Ordering::SeqCst) { - let mut buf = BytesMut::new(); - buf.put_u8(PING_VERSION); - - // Make a bad coordinate with the wrong number of dimensions. - let mut coord = crate::coordinate::Coordinate::new(); - let len = coord.portion.len(); - coord.portion.resize(len * 2, 0.0); - - // The rest of the message is the serialized coordinate. - let len = coord.encoded_len(); - buf.resize(len + 1, 0); - if let Err(e) = coord.encode(&mut buf[1..]) { - panic!("failed to encode coordinate: {}", e); - } - return buf.freeze(); - } - - if let Some(c) = self - .this() - .as_ref() - .and_then(|this| this.inner.coord_core.as_ref()) - { - let coord = c.client.get_coordinate(); - let encoded_len = coord.encoded_len() + 1; - let mut buf = BytesMut::with_capacity(encoded_len); - buf.put_u8(PING_VERSION); - buf.resize(encoded_len, 0); - - if let Err(e) = coord.encode(&mut buf[1..]) { - tracing::error!(err=%e, "serf: failed to encode coordinate"); - } - tracing::trace!(coordinate=?coord, data=?buf.as_ref(), "serf: ack payload"); - buf.into() - } else { - Bytes::new() - } - } - - async fn notify_ping_complete( - &self, - node: Arc>, - rtt: std::time::Duration, - payload: Bytes, - ) { - if payload.is_empty() { - return; - } - - let Some(this) = self.this() else { - return; - }; - - tracing::trace!(data=?payload.as_ref(), "serf: receive payload"); - - if let Some(ref c) = this.inner.coord_core { - // Verify ping version in the header. - if payload[0] != PING_VERSION { - tracing::error!(version = %payload[0], "serf: unsupported ping version"); - return; - } - - // Process the remainder of the message as a coordinate. - let coord = match ::decode(&payload[1..]) { - Ok((readed, c)) => { - tracing::trace!(read=%readed, coordinate=?c, "serf: decode coordinate successfully"); - c - } - Err(e) => { - tracing::error!(err=%e, "serf: failed to decode coordinate from ping"); - return; - } - }; - - // Apply the update. - #[cfg(feature = "metrics")] - let before = c.client.get_coordinate(); - match c.client.update(node.id(), &coord, rtt) { - Ok(_after) => { - #[cfg(feature = "metrics")] - { - // Publish some metrics to give us an idea of how much we are - // adjusting each time we update. - let d = before.distance_to(&_after).as_secs_f64() * 1.0e3; - metrics::histogram!( - "serf.coordinate.adjustment-ms", - this.inner.opts.memberlist_options.metric_labels.iter() - ) - .record(d); - } - - // Cache the coordinate for the other node, and add our own - // to the cache as well since it just got updated. This lets - // users call GetCachedCoordinate with our node name, which is - // more friendly. - let mut cache = c.cache.write(); - cache.insert(node.id().cheap_clone(), coord); - cache - .entry(this.inner.memberlist.local_id().cheap_clone()) - .and_modify(|x| { - *x = c.client.get_coordinate(); - }) - .or_insert_with(|| c.client.get_coordinate()); - } - Err(e) => { - #[cfg(feature = "metrics")] - { - metrics::counter!( - "serf.coordinate.rejected", - this.inner.opts.memberlist_options.metric_labels.iter() - ) - .increment(1); - } - - tracing::error!(err=%e, "serf: rejected coordinate from {}", node.id()); - } - } - } - } -} - -impl MemberlistDelegate for SerfDelegate -where - D: Delegate, - T: Transport, -{ - type Id = T::Id; - - type Address = T::ResolvedAddress; -} - -fn node_to_member( - node: &NodeState, -) -> Result, SerfDelegateError> -where - D: Delegate, - T: Transport, -{ - let status = if node.state() == State::Left { - MemberStatus::Left - } else { - MemberStatus::None - }; - - let meta = node.meta(); - if meta.len() > META_MAX_SIZE { - return Err(SerfDelegateError::Serf(SerfError::TagsTooLarge(meta.len()))); - } - - Ok(Member { - node: node.node(), - tags: if !node.meta().is_empty() { - ::decode(node.meta()) - .map(|(read, tags)| { - tracing::trace!(read=%read, tags=?tags, "serf: decode tags successfully"); - Arc::new(tags) - }) - .map_err(|e| SerfDelegateError::Serf(SerfError::from(e)))? - } else { - Default::default() - }, - status, - protocol_version: ProtocolVersion::V1, - delegate_version: DelegateVersion::V1, - memberlist_delegate_version: node.delegate_version(), - memberlist_protocol_version: node.protocol_version(), - }) -} diff --git a/legacy/serf-core/src/serf/internal_query.rs b/legacy/serf-core/src/serf/internal_query.rs deleted file mode 100644 index d55711be..00000000 --- a/legacy/serf-core/src/serf/internal_query.rs +++ /dev/null @@ -1,486 +0,0 @@ -use async_channel::{Receiver, Sender, bounded}; -use futures::FutureExt; -use memberlist_core::{ - agnostic_lite::{AsyncSpawner, RuntimeLite}, - bytes::Bytes, - tracing, - transport::Transport, -}; - -use crate::{ - delegate::Delegate, - event::{CrateEvent, InternalQueryEvent, QueryEvent}, -}; - -#[cfg(feature = "encryption")] -use crate::{ - error::Error, - types::{KeyResponseMessage, MessageRef}, -}; - -#[cfg(feature = "encryption")] -use smol_str::SmolStr; - -/// Used to compute the max number of keys in a list key -/// response. eg 1024/25 = 40. a message with max size of 1024 bytes cannot -/// contain more than 40 keys. There is a test -/// (TestSerfQueries_estimateMaxKeysInListKeyResponse) which does the -/// computation and in case of changes, the value can be adjusted. -#[cfg(feature = "encryption")] -const MIN_ENCODED_KEY_LENGTH: usize = 25; - -pub(crate) struct SerfQueries -where - D: Delegate, - T: Transport, -{ - in_rx: Receiver>, - out_tx: Option>>, - shutdown_rx: Receiver<()>, -} - -impl SerfQueries -where - D: Delegate, - T: Transport, -{ - #[allow(clippy::new_ret_no_self)] - pub(crate) fn new( - out_tx: Option>>, - shutdown_rx: Receiver<()>, - ) -> ( - Sender>, - <::Spawner as AsyncSpawner>::JoinHandle<()>, - ) { - let (in_tx, in_rx) = bounded(1024); - let this = Self { - in_rx, - out_tx, - shutdown_rx, - }; - (in_tx, this.stream()) - } - - /// A long running routine to ingest the event stream - fn stream(self) -> <::Spawner as AsyncSpawner>::JoinHandle<()> { - ::spawn(async move { - loop { - futures::select! { - ev = self.in_rx.recv().fuse() => { - match ev { - Ok(ev) => { - // Check if this is a query we should process - if ev.is_internal_query() { - ::spawn_detach(async move { - Self::handle_query(ev).await; - }); - } else if let Some(ref tx) = self.out_tx { - if let Err(e) = tx.send(ev).await { - tracing::error!(target="serf", err=%e, "failed to send event back in serf query thread"); - } - } - }, - Err(err) => { - tracing::error!(target="serf", err=%err, "failed to receive event in serf query thread"); - return; - } - } - } - _ = self.shutdown_rx.recv().fuse() => { - return; - } - } - } - }) - } - - async fn handle_query(ev: CrateEvent) { - match ev { - CrateEvent::InternalQuery { kind, query } => match kind { - InternalQueryEvent::Ping => {} - InternalQueryEvent::Conflict(conflict) => { - Self::handle_conflict(&conflict, &query).await; - } - #[cfg(feature = "encryption")] - InternalQueryEvent::InstallKey => { - Self::handle_install_key(&query).await; - } - #[cfg(feature = "encryption")] - InternalQueryEvent::UseKey => { - Self::handle_use_key(&query).await; - } - #[cfg(feature = "encryption")] - InternalQueryEvent::RemoveKey => { - Self::handle_remove_key(&query).await; - } - #[cfg(feature = "encryption")] - InternalQueryEvent::ListKey => { - Self::handle_list_keys(&query).await; - } - }, - _ => unreachable!(), - } - } - - /// invoked when we get a query that is attempting to - /// disambiguate a name conflict. They payload is a node name, and the response - /// should the address we believe that node is at, if any. - async fn handle_conflict(conflict: &T::Id, ev: &QueryEvent) { - // The target node id is the payload - - // Do not respond to the query if it is about us - if conflict.eq(ev.ctx.this.inner.memberlist.local_id()) { - return; - } - - tracing::error!( - "serf: local {} got conflict resolution query for '{}'", - ev.ctx.this.inner.memberlist.local_id(), - conflict - ); - - // tracing::debug!("serf: got conflict resolution query for '{}'", conflict); - - // Look for the member info - let out = { - let members = ev.ctx.this.inner.members.read().await; - members.states.get(conflict).cloned() - }; - - // Encode the response - match out { - Some(state) => { - let resp = crate::types::ConflictResponseMessageBorrow::from(state.member()); - match crate::types::encode_message_to_bytes(&resp) { - Ok(raw) => { - if let Err(e) = ev.respond(raw).await { - tracing::error!(target="serf", err=%e, "failed to respond to conflict query"); - } - } - Err(e) => { - tracing::error!(target="serf", err=%e, "failed to encode conflict query response"); - } - } - } - None => { - tracing::warn!("serf: no member status found for '{}'", conflict); - // TODO: consider send something back? - if let Err(e) = ev.respond(Bytes::new()).await { - tracing::error!(target="serf", err=%e, "failed to respond to conflict query"); - } - } - } - } - - /// Invoked whenever a new encryption key is received from - /// another member in the cluster, and handles the process of installing it onto - /// the memberlist keyring. This type of query may fail if the provided key does - /// not fit the constraints that memberlist enforces. If the query fails, the - /// response will contain the error message so that it may be relayed. - #[cfg(feature = "encryption")] - async fn handle_install_key(ev: impl AsRef> + Send) { - let q = ev.as_ref(); - let mut response = KeyResponseMessage::default(); - let req = match crate::types::decode_message::(&q.payload) { - Ok(msg) => match msg { - MessageRef::KeyRequest(req) => req, - msg => { - tracing::error!( - err = "unexpected message type", - "serf: {}", - msg.ty().as_str() - ); - Self::send_key_response(q, &mut response).await; - return; - } - }, - Err(e) => { - tracing::error!(err=%e, "serf: failed to decode key request"); - Self::send_key_response(q, &mut response).await; - return; - } - }; - - if !q.ctx.this.encryption_enabled() { - tracing::error!( - err = "encryption is not enabled", - "serf: fail to handle install key" - ); - response.message = SmolStr::new("encryption is not enabled"); - Self::send_key_response(q, &mut response).await; - return; - } - - tracing::info!("serf: received install-key query"); - let kr = q.ctx.this.inner.memberlist.keyring(); - match kr { - Some(kr) => { - kr.insert(req.key.unwrap()); - if q.ctx.this.inner.opts.keyring_file.is_some() { - if let Err(e) = q.ctx.this.write_keyring_file().await { - tracing::error!(err=%e, "serf: failed to write keyring file"); - response.message = SmolStr::new(e.to_string()); - Self::send_key_response(q, &mut response).await; - return; - } - } - - response.result = true; - Self::send_key_response(q, &mut response).await; - } - None => { - tracing::error!( - err = "encryption enabled but keyring is empty", - "serf: fail to handle install key" - ); - response.message = SmolStr::new("encryption enabled but keyring is empty"); - Self::send_key_response(q, &mut response).await; - } - } - } - - #[cfg(feature = "encryption")] - async fn handle_use_key(ev: impl AsRef> + Send) { - let q = ev.as_ref(); - let mut response = KeyResponseMessage::default(); - - let req = match crate::types::decode_message::(&q.payload) { - Ok(msg) => match msg { - MessageRef::KeyRequest(req) => req, - msg => { - tracing::error!(err = "unexpected message type", "serf: {}", msg.ty()); - Self::send_key_response(q, &mut response).await; - return; - } - }, - Err(e) => { - tracing::error!(err=%e, "serf: failed to decode key request"); - Self::send_key_response(q, &mut response).await; - return; - } - }; - - if !q.ctx.this.encryption_enabled() { - tracing::error!( - err = "encryption is not enabled", - "serf: fail to handle use key" - ); - response.message = SmolStr::new("encryption is not enabled"); - Self::send_key_response(q, &mut response).await; - return; - } - - tracing::info!("serf: received use-key query"); - let kr = q.ctx.this.inner.memberlist.keyring(); - match kr { - Some(kr) => { - if let Err(e) = kr.use_key(&req.key.unwrap()) { - tracing::error!(err=%e, "serf: failed to change primary key"); - response.message = SmolStr::new(e.to_string()); - Self::send_key_response(q, &mut response).await; - return; - } - - if q.ctx.this.inner.opts.keyring_file.is_some() { - if let Err(e) = q.ctx.this.write_keyring_file().await { - tracing::error!(err=%e, "serf: failed to write keyring file"); - response.message = SmolStr::new(e.to_string()); - Self::send_key_response(q, &mut response).await; - return; - } - } - - response.result = true; - Self::send_key_response(q, &mut response).await; - } - None => { - tracing::error!( - err = "encryption enabled but keyring is empty", - "serf: fail to handle use key" - ); - response.message = SmolStr::new("encryption enabled but keyring is empty"); - Self::send_key_response(q, &mut response).await; - } - } - } - - #[cfg(feature = "encryption")] - async fn handle_remove_key(ev: impl AsRef> + Send) { - let q = ev.as_ref(); - let mut response = KeyResponseMessage::default(); - - let req = match crate::types::decode_message::(&q.payload) { - Ok(msg) => match msg { - MessageRef::KeyRequest(req) => req, - msg => { - tracing::error!(err = "unexpected message type", "serf: {}", msg.ty()); - Self::send_key_response(q, &mut response).await; - return; - } - }, - Err(e) => { - tracing::error!(target="serf", err=%e, "failed to decode key request"); - Self::send_key_response(q, &mut response).await; - return; - } - }; - - if !q.ctx.this.encryption_enabled() { - tracing::error!( - err = "encryption is not enabled", - "serf: fail to handle remove key" - ); - response.message = SmolStr::new("encryption is not enabled"); - Self::send_key_response(q, &mut response).await; - return; - } - - tracing::info!("serf: received remove-key query"); - let kr = q.ctx.this.inner.memberlist.keyring(); - match kr { - Some(kr) => { - if let Err(e) = kr.remove(&req.key.unwrap()) { - tracing::error!(err=%e, "serf: failed to remove key"); - response.message = SmolStr::new(e.to_string()); - Self::send_key_response(q, &mut response).await; - return; - } - - if q.ctx.this.inner.opts.keyring_file.is_some() { - if let Err(e) = q.ctx.this.write_keyring_file().await { - tracing::error!(err=%e, "serf: failed to write keyring file"); - response.message = SmolStr::new(e.to_string()); - Self::send_key_response(q, &mut response).await; - return; - } - } - - response.result = true; - Self::send_key_response(q, &mut response).await; - } - None => { - tracing::error!( - err = "encryption enabled but keyring is empty", - "serf: fail to handle remove key" - ); - response.message = SmolStr::new("encryption enabled but keyring is empty"); - Self::send_key_response(q, &mut response).await; - } - } - } - - /// Invoked when a query is received to return a list of all - /// installed keys the Serf instance knows of. - #[cfg(feature = "encryption")] - async fn handle_list_keys(ev: impl AsRef> + Send) { - let q = ev.as_ref(); - let mut response = KeyResponseMessage::default(); - if !q.ctx.this.encryption_enabled() { - tracing::error!( - err = "encryption is not enabled", - "serf: fail to handle list keys" - ); - response.message = SmolStr::new("encryption is not enabled"); - Self::send_key_response(q, &mut response).await; - return; - } - - tracing::info!("serf: received list-keys query"); - let kr = q.ctx.this.inner.memberlist.keyring(); - match kr { - Some(kr) => { - for k in kr.keys() { - response.keys.push(k); - } - - let primary_key = kr.primary_key(); - response.primary_key = Some(primary_key); - response.result = true; - Self::send_key_response(q, &mut response).await; - } - None => { - tracing::error!( - err = "encryption enabled but keyring is empty", - "serf: fail to handle list keys" - ); - response.message = SmolStr::new("encryption enabled but keyring is empty"); - Self::send_key_response(q, &mut response).await; - } - } - } - - #[cfg(feature = "encryption")] - pub(crate) fn key_list_response_with_correct_size( - q: &QueryEvent, - resp: &mut KeyResponseMessage, - ) -> Result< - ( - Bytes, - crate::types::QueryResponseMessage, - ), - Error, - > { - let actual = resp.keys.len(); - - // if the provided list of keys is smaller then the max allowed, just iterate over it - // to avoid an out of bound access when truncating - let max_list_keys = - (q.ctx.this.inner.opts.query_response_size_limit / MIN_ENCODED_KEY_LENGTH).min(actual); - - for i in (0..=max_list_keys).rev() { - let kraw = crate::types::encode_message_to_bytes(&*resp)?; - - // create response - let qresp = q.create_response(kraw.clone()); - - let encoded_len = crate::types::encoded_message_len(&qresp); - // Check the size limit - if q.check_response_size(encoded_len).is_err() { - resp.keys.drain(i..); - resp.message = SmolStr::new(format!( - "truncated key list response, showing first {} of {} keys", - i, actual - )); - continue; - } - - // encode response - let qraw = crate::types::encode_message_to_bytes(&qresp)?; - - if actual > i { - tracing::warn!("serf: {}", resp.message); - } - return Ok((qraw, qresp)); - } - Err(Error::fail_truncate_response()) - } - - #[cfg(feature = "encryption")] - async fn send_key_response(q: &QueryEvent, resp: &mut KeyResponseMessage) { - match q.name.as_str() { - crate::event::INTERNAL_LIST_KEYS => { - let (raw, qresp) = match Self::key_list_response_with_correct_size(q, resp) { - Ok((raw, qresp)) => (raw, qresp), - Err(e) => { - tracing::error!(target="serf", err=%e); - return; - } - }; - - if let Err(e) = q.respond_with_message_and_response(raw, qresp).await { - tracing::error!(target="serf", err=%e, "failed to respond to key query"); - } - } - _ => match crate::types::encode_message_to_bytes(&*resp) { - Ok(raw) => { - if let Err(e) = q.respond(raw).await { - tracing::error!(target="serf", err=%e, "failed to respond to key query"); - } - } - Err(e) => { - tracing::error!(target="serf", err=%e, "failed to encode key response"); - } - }, - } - } -} diff --git a/legacy/serf-core/src/serf/query.rs b/legacy/serf-core/src/serf/query.rs deleted file mode 100644 index a50ba02b..00000000 --- a/legacy/serf-core/src/serf/query.rs +++ /dev/null @@ -1,602 +0,0 @@ -use std::{ - collections::HashSet, - sync::Arc, - time::{Duration, Instant}, -}; - -use crate::types::FilterRef; -use async_channel::{Receiver, Sender}; -use async_lock::RwLock; -use either::Either; -use futures::{FutureExt, StreamExt, stream::FuturesUnordered}; -use memberlist_core::{ - CheapClone, - bytes::Bytes, - proto::{Data, RepeatedDecoder, SmallVec, TinyVec}, - tracing, - transport::{Node, Transport}, -}; - -use crate::{ - delegate::Delegate, - error::Error, - types::{Filter, LamportTime, Member, MemberStatus, QueryMessage, QueryResponseMessage}, -}; - -use super::Serf; - -/// Provided to [`Serf::query`] to configure the parameters of the -/// query. If not provided, sane defaults will be used. -#[viewit::viewit( - vis_all = "pub(crate)", - getters(vis_all = "pub", style = "ref"), - setters(vis_all = "pub", prefix = "with") -)] -#[derive(Debug, Clone)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub struct QueryParam { - /// The filters to apply to the query. - #[viewit( - getter(const, attrs(doc = "Returns the filters of the query")), - setter(attrs(doc = "Sets the filters of the query")) - )] - filters: TinyVec>, - - /// If true, we are requesting an delivery acknowledgement from - /// every node that meets the filter requirement. This means nodes - /// the receive the message but do not pass the filters, will not - /// send an ack. - #[viewit( - getter( - const, - style = "move", - attrs( - doc = "Returns if we are requesting an delivery acknowledgement from every node that meets the filter requirement. This means nodes the receive the message but do not pass the filters, will not send an ack." - ) - ), - setter(attrs( - doc = "Sets if we are requesting an delivery acknowledgement from every node that meets the filter requirement. This means nodes the receive the message but do not pass the filters, will not send an ack." - )) - )] - request_ack: bool, - - /// Controls the number of duplicate responses to relay - /// back to the sender through other nodes for redundancy. - #[viewit( - getter( - const, - style = "move", - attrs( - doc = "Returns the number of duplicate responses to relay back to the sender through other nodes for redundancy." - ) - ), - setter(attrs( - doc = "Sets the number of duplicate responses to relay back to the sender through other nodes for redundancy." - )) - )] - relay_factor: u8, - - /// The timeout limits how long the query is left open. If not provided, - /// then a default timeout is used based on the configuration of Serf - #[viewit( - getter( - const, - style = "move", - attrs( - doc = "Returns timeout limits how long the query is left open. If not provided, then a default timeout is used based on the configuration of [`Serf`]" - ) - ), - setter(attrs(doc = "Sets timeout limits how long the query is left open.")) - )] - #[cfg_attr(feature = "serde", serde(with = "humantime_serde"))] - timeout: Duration, -} - -struct QueryResponseChannel { - /// Used to send the name of a node for which we've received an ack - ack_ch: Option<(Sender>, Receiver>)>, - /// Used to send a response from a node - resp_ch: (Sender>, Receiver>), -} - -pub(crate) struct QueryResponseCore { - closed: bool, - acks: HashSet>, - responses: HashSet>, -} - -pub(crate) struct QueryResponseInner { - core: RwLock>, - channel: QueryResponseChannel, -} - -/// Returned for each new Query. It is used to collect -/// Ack's as well as responses and to provide those back to a client. -#[viewit::viewit(vis_all = "pub(crate)")] -#[derive(Clone)] -pub struct QueryResponse { - /// The duration of the query - #[viewit( - getter( - style = "move", - const, - attrs(doc = "Returns the ending deadline of the query") - ), - setter(skip) - )] - deadline: Instant, - - /// The query id - #[viewit( - getter(style = "move", const, attrs(doc = "Returns the id of the query")), - setter(skip) - )] - id: u32, - - /// Stores the LTime of the query - #[viewit( - getter( - style = "move", - const, - attrs(doc = "Returns the Lamport Time of the query") - ), - setter(skip) - )] - ltime: LamportTime, - - #[viewit(getter(vis = "pub(crate)", const, style = "ref"), setter(skip))] - inner: Arc>, -} - -impl QueryResponse { - pub(crate) fn from_query(q: &QueryMessage, num_nodes: usize) -> Self { - QueryResponse::new( - q.id(), - q.ltime(), - num_nodes, - Instant::now() + q.timeout(), - q.ack(), - ) - } -} - -impl QueryResponse { - #[inline] - pub(crate) fn new( - id: u32, - ltime: LamportTime, - num_nodes: usize, - deadline: Instant, - ack: bool, - ) -> Self { - let (ack_ch, acks) = if ack { - ( - Some(async_channel::bounded(num_nodes)), - HashSet::with_capacity(num_nodes), - ) - } else { - (None, HashSet::new()) - }; - - Self { - deadline, - id, - ltime, - inner: Arc::new(QueryResponseInner { - core: RwLock::new(QueryResponseCore { - closed: false, - acks, - responses: HashSet::with_capacity(num_nodes), - }), - channel: QueryResponseChannel { - ack_ch, - resp_ch: async_channel::bounded(num_nodes), - }, - }), - } - } - - /// Returns a receiver that can be used to listen for acks. - /// Channel will be closed when the query is finished. This is `None`, - /// if the query did not specify `request_ack`. - #[inline] - pub fn ack_rx(&self) -> Option>> { - self.inner.channel.ack_ch.as_ref().map(|(_, r)| r.clone()) - } - - /// Returns a receiver that can be used to listen for responses. - /// Channel will be closed when the query is finished. - #[inline] - pub fn response_rx(&self) -> async_channel::Receiver> { - self.inner.channel.resp_ch.1.clone() - } - - /// Returns if the query is finished running - #[inline] - pub async fn finished(&self) -> bool { - let c = self.inner.core.read().await; - c.closed || (Instant::now() > self.deadline) - } - - /// Used to close the query, which will close the underlying - /// channels and prevent further deliveries - #[inline] - pub async fn close(&self) { - let mut c = self.inner.core.write().await; - if c.closed { - return; - } - - c.closed = true; - - if let Some((tx, _)) = &self.inner.channel.ack_ch { - tx.close(); - } - - self.inner.channel.resp_ch.0.close(); - } - - #[inline] - pub(crate) async fn handle_query_response( - &self, - resp: QueryResponseMessage, - _local: &T::Id, - #[cfg(feature = "metrics")] metrics_labels: &memberlist_core::proto::MetricLabels, - ) where - I: Eq + std::hash::Hash + CheapClone + core::fmt::Debug, - A: Eq + std::hash::Hash + CheapClone + core::fmt::Debug, - D: Delegate, - T: Transport, - { - // Check if the query is closed - let c = self.inner.core.read().await; - if c.closed || (Instant::now() > self.deadline) { - return; - } - - // Process each type of response - if resp.ack() { - // Exit early if this is a duplicate ack - if c.acks.contains(&resp.from) { - #[cfg(feature = "metrics")] - { - metrics::counter!("serf.query.duplicate_acks", metrics_labels.iter()).increment(1); - } - return; - } - - #[cfg(feature = "metrics")] - { - metrics::counter!("serf.query.acks", metrics_labels.iter()).increment(1); - } - - drop(c); - if let Err(e) = self.send_ack::(&resp).await { - tracing::warn!("serf: {}", e); - } - } else { - // Exit early if this is a duplicate response - if c.responses.contains(&resp.from) { - #[cfg(feature = "metrics")] - { - metrics::counter!("serf.query.duplicate_responses", metrics_labels.iter()).increment(1); - } - return; - } - - #[cfg(feature = "metrics")] - { - metrics::counter!("serf.query.responses", metrics_labels.iter()).increment(1); - } - drop(c); - - if let Err(e) = self - .send_response::(NodeResponse { - from: resp.from, - payload: resp.payload, - }) - .await - { - tracing::warn!("serf: {}", e); - } - } - } - - /// Sends a response on the response channel ensuring the channel is not closed. - #[inline] - pub(crate) async fn send_response(&self, nr: NodeResponse) -> Result<(), Error> - where - I: Eq + std::hash::Hash + CheapClone + core::fmt::Debug, - A: Eq + std::hash::Hash + CheapClone + core::fmt::Debug, - D: Delegate, - T: Transport, - { - let mut c = self.inner.core.write().await; - // Exit early if this is a duplicate ack - if c.responses.contains(&nr.from) { - return Ok(()); - } - - if c.closed { - Ok(()) - } else { - let id = nr.from.cheap_clone(); - futures::select! { - _ = self.inner.channel.resp_ch.0.send(nr).fuse() => { - c.responses.insert(id); - Ok(()) - }, - default => { - Err(Error::query_response_delivery_failed()) - } - } - } - } - - /// Sends a response on the ack channel ensuring the channel is not closed. - #[inline] - pub(crate) async fn send_ack( - &self, - nr: &QueryResponseMessage, - ) -> Result<(), Error> - where - I: Eq + std::hash::Hash + CheapClone, - A: Eq + std::hash::Hash + CheapClone, - D: Delegate, - T: Transport, - { - let mut c = self.inner.core.write().await; - // Exit early if this is a duplicate ack - if c.acks.contains(&nr.from) { - return Ok(()); - } - - if c.closed { - Ok(()) - } else if let Some((tx, _)) = &self.inner.channel.ack_ch { - futures::select! { - _ = tx.send(nr.from.cheap_clone()).fuse() => { - c.acks.insert(nr.from.clone()); - Ok(()) - }, - default => { - Err(Error::query_response_delivery_failed()) - } - } - } else { - Ok(()) - } - } -} - -/// Used to represent a single response from a node -#[viewit::viewit( - vis_all = "pub(crate)", - setters(skip), - getters(vis_all = "pub", style = "ref") -)] -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub struct NodeResponse { - #[viewit(getter(attrs(doc = "Returns the node that sent the response")))] - from: Node, - #[viewit(getter(attrs(doc = "Returns the payload of the response")))] - payload: Bytes, -} - -#[inline] -fn random_members(k: usize, mut members: SmallVec>) -> SmallVec> { - let n = members.len(); - if n == 0 { - return SmallVec::new(); - } - - // The modified Fisher-Yates algorithm, but up to 3*n times to ensure exhaustive search for small n. - let rounds = 3 * n; - let mut i = 0; - - while i < rounds && i < n { - let j = (rand::random::() as usize) % (n - i) + i; - members.swap(i, j); - i += 1; - if i >= k && i >= rounds { - break; - } - } - - members.truncate(k); - members -} - -impl Serf -where - D: Delegate, - T: Transport, -{ - /// Returns the default timeout value for a query - /// Computed as - /// ```text - /// gossip_interval * query_timeout_mult * log(N+1) - /// ``` - pub async fn default_query_timeout(&self) -> Duration { - let n = self.inner.memberlist.num_online_members().await; - let mut timeout = self.inner.opts.memberlist_options.gossip_interval(); - timeout *= self.inner.opts.query_timeout_mult as u32; - timeout *= ((n + 1) as f64).log10().ceil() as u32; // Using ceil approximation - timeout - } - - /// Used to return the default query parameters - pub async fn default_query_param(&self) -> QueryParam { - QueryParam { - filters: TinyVec::new(), - request_ack: false, - relay_factor: 0, - timeout: self.default_query_timeout().await, - } - } - - pub(crate) fn should_process_query( - &self, - filters: Either, &[Filter]>, - ) -> Result { - match filters { - Either::Left(filters) => { - for filter in filters.iter::>() { - let filter = filter?; - match filter { - FilterRef::Id(ids) => { - // Check if we are being targeted - let mut found = false; - for id in ids.iter::() { - let id = id?; - if ::from_ref(id)?.eq(self.inner.memberlist.local_id()) { - found = true; - break; - } - } - if !found { - return Ok(false); - } - } - FilterRef::Tag(tag) => { - // Check if we match this regex - let tags = self.inner.opts.tags.load(); - if !tags.is_empty() { - if let Some(expr) = tags.get(tag.tag()) { - if let Some(re) = tag.expr() { - if !regex::Regex::new(re) - .map_err(|_| memberlist_core::proto::DecodeError::custom("invalid regex"))? - .is_match(expr) - { - return Ok(false); - } - } - } else { - return Ok(false); - } - } else { - return Ok(false); - } - } - } - } - - Ok(true) - } - Either::Right(filters) => { - for filter in filters.iter() { - match &filter { - Filter::Id(nodes) => { - // Check if we are being targeted - let found = nodes - .iter() - .any(|n: &T::Id| n.eq(self.inner.memberlist.local_id())); - if !found { - return Ok(false); - } - } - Filter::Tag(tag) => { - // Check if we match this regex - let tags = self.inner.opts.tags.load(); - if !tags.is_empty() { - if let Some(expr) = tags.get(tag.tag()) { - if let Some(re) = tag.expr() { - if !re.is_match(expr) { - return Ok(false); - } - } - } else { - return Ok(false); - } - } else { - return Ok(false); - } - } - } - } - Ok(true) - } - } - } - - pub(crate) async fn relay_response( - &self, - relay_factor: u8, - node: Node, - resp: QueryResponseMessage, - ) -> Result<(), Error> { - if relay_factor == 0 { - return Ok(()); - } - - // Needs to be worth it; we need to have at least relayFactor *other* - // nodes. If you have a tiny cluster then the relayFactor shouldn't - // be needed. - let members = { - let members = self.inner.members.read().await; - if members.states.len() < relay_factor as usize + 1 { - return Ok(()); - } - members - .states - .iter() - .filter_map(|(id, m)| { - if m.member.status == MemberStatus::Alive && id != self.inner.memberlist.local_id() { - Some(m.member.clone()) - } else { - None - } - }) - .collect::>() - }; - - if members.is_empty() { - return Ok(()); - } - - // Prep the relay message, which is a wrapped version of the original. - let encoded_len = crate::types::encoded_relay_message_len(&resp, &node); - if encoded_len > self.inner.opts.query_response_size_limit { - return Err(Error::relayed_response_too_large( - self.inner.opts.query_response_size_limit, - )); - } - - let raw = crate::types::encode_relay_message_to_bytes(&resp, &node)?; - - // Relay to a random set of peers. - let relay_members = random_members(relay_factor as usize, members); - - let futs: FuturesUnordered<_> = relay_members - .into_iter() - .map(|m| { - let raw = raw.clone(); - async move { - self - .inner - .memberlist - .send(m.node.address(), raw) - .await - .map_err(|e| (m, e)) - } - }) - .collect(); - - let mut errs = TinyVec::new(); - let stream = StreamExt::filter_map(futs, |res| async move { - if let Err((m, e)) = res { - Some((m, e)) - } else { - None - } - }); - futures::pin_mut!(stream); - - while let Some(err) = stream.next().await { - errs.push(err); - } - - Ok(()) - } -} diff --git a/legacy/serf-core/src/snapshot.rs b/legacy/serf-core/src/snapshot.rs deleted file mode 100644 index 7edb9a81..00000000 --- a/legacy/serf-core/src/snapshot.rs +++ /dev/null @@ -1,885 +0,0 @@ -use std::{ - borrow::Cow, - collections::HashSet, - fs::{File, OpenOptions}, - io::{BufReader, BufWriter, Read, Seek, Write}, - mem, - path::PathBuf, - time::Duration, -}; - -#[cfg(unix)] -use std::os::unix::prelude::OpenOptionsExt; - -use crate::types::UserEventMessage; -use async_channel::{Receiver, Sender}; -use byteorder::{LittleEndian, ReadBytesExt}; -use futures::FutureExt; -use memberlist_core::{ - CheapClone, - agnostic_lite::{AsyncSpawner, RuntimeLite}, - bytes::{BufMut, BytesMut}, - proto::{Data, MaybeResolvedAddress, TinyVec}, - tracing, - transport::{Id, Node, Transport}, -}; -use rand::seq::SliceRandom; - -use crate::{ - delegate::Delegate, - event::{CrateEvent, MemberEvent, MemberEventType}, - invalid_data_io_error, - types::{Epoch, LamportClock, LamportTime}, -}; - -/// How often we force a flush of the snapshot file -const FLUSH_INTERVAL: Duration = Duration::from_millis(500); - -/// How often we fetch the current lamport time of the cluster and write to the snapshot file -const CLOCK_UPDATE_INTERVAL: Duration = Duration::from_millis(500); - -/// The extention we use for the temporary file during compaction -const TMP_EXT: &str = "compact"; - -/// How often we attempt to recover from -/// errors writing to the snapshot file. -const SNAPSHOT_ERROR_RECOVERY_INTERVAL: Duration = Duration::from_secs(30); - -/// The size of the event buffers between Serf and the -/// consuming application. If this is exhausted we will block Serf and Memberlist. -const EVENT_CH_SIZE: usize = 2048; - -/// The time limit to write pending events to the snapshot during a shutdown -const SHUTDOWN_FLUSH_TIMEOUT: Duration = Duration::from_millis(250); - -/// An estimated bytes per node to snapshot -const SNAPSHOT_BYTES_PER_NODE: usize = 128; - -/// The threshold we apply to -/// the snapshot size estimate (nodes * bytes per node) before compacting. -const SNAPSHOT_COMPACTION_THRESHOLD: usize = 2; - -/// Errors that can occur while interacting with snapshots -#[derive(Debug, thiserror::Error)] -pub enum SnapshotError { - /// Returned when opening a snapshot fails - #[error("failed to open snapshot: {0}")] - Open(std::io::Error), - /// Returned when opening a new snapshot fails - #[error("failed to open new snapshot: {0}")] - OpenNew(std::io::Error), - /// Returned when flush new snapshot fails - #[error("failed to flush new snapshot: {0}")] - FlushNew(std::io::Error), - /// Returned when flush snapshot fails - #[error("failed to flush snapshot: {0}")] - Flush(std::io::Error), - /// Returned when fsync snapshot fails - #[error("failed to fsync snapshot: {0}")] - Sync(std::io::Error), - /// Returned when stat snapshot fails - #[error("failed to stat snapshot: {0}")] - Stat(std::io::Error), - /// Returned when remove old snapshot fails - #[error("failed to remove old snapshot: {0}")] - Remove(std::io::Error), - /// Returned when installing a new snapshot fails - #[error("failed to install new snapshot: {0}")] - Install(std::io::Error), - /// Returned when writing to a new snapshot fails - #[error("failed to write to new snapshot: {0}")] - WriteNew(std::io::Error), - /// Returned when writing to a snapshot fails - #[error("failed to write to snapshot: {0}")] - Write(std::io::Error), - /// Returned when seek to start of a snapshot fails - #[error("failed to seek to beginning of snapshot: {0}")] - SeekStart(std::io::Error), - /// Returned when seek to end of a snapshot fails - #[error("failed to seek to end of snapshot: {0}")] - SeekEnd(std::io::Error), - /// Returned when replaying a snapshot fails - #[error("failed to replay snapshot: {0}")] - Replay(std::io::Error), - /// Returned when fail to decode snapshot record type. - #[error(transparent)] - UnknownRecordType(#[from] UnknownRecordType), -} - -/// UnknownRecordType is used to indicate that we encountered an unknown -/// record type while reading a snapshot file. -#[derive(Debug, thiserror::Error)] -#[error("unrecognized snapshot record type: {0}")] -pub struct UnknownRecordType(u8); - -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -#[repr(u8)] -enum SnapshotRecordType { - Alive = 0, - NotAlive = 1, - Clock = 2, - EventClock = 3, - QueryClock = 4, - Coordinate = 5, - Leave = 6, - Comment = 7, -} - -impl TryFrom for SnapshotRecordType { - type Error = UnknownRecordType; - - fn try_from(value: u8) -> Result { - match value { - 0 => Ok(Self::Alive), - 1 => Ok(Self::NotAlive), - 2 => Ok(Self::Clock), - 3 => Ok(Self::EventClock), - 4 => Ok(Self::QueryClock), - 5 => Ok(Self::Coordinate), - 6 => Ok(Self::Leave), - 7 => Ok(Self::Comment), - v => Err(UnknownRecordType(v)), - } - } -} - -#[allow(dead_code)] -enum SnapshotRecord<'a, I: Clone, A: Clone> { - Alive(Cow<'a, Node>), - NotAlive(Cow<'a, Node>), - Clock(LamportTime), - EventClock(LamportTime), - QueryClock(LamportTime), - Coordinate, - Leave, - Comment, -} - -const MAX_INLINED_BYTES: usize = 64; - -macro_rules! encode { - ($w:ident.$node: ident::$status: ident) => {{ - let node = $node.as_ref(); - let encoded_node_len = node.encoded_len(); - let encoded_len = 4 + 1 + encoded_node_len; - if encoded_len <= MAX_INLINED_BYTES { - let mut buf = [0u8; MAX_INLINED_BYTES]; - buf[0] = Self::$status; - buf[1..5].copy_from_slice(&(encoded_node_len as u32).to_le_bytes()); - node.encode(&mut buf[5..]).map_err(invalid_data_io_error)?; - $w.write_all(&buf[..encoded_len]).map(|_| encoded_len) - } else { - let mut buf = BytesMut::with_capacity(encoded_len); - buf.put_u8(Self::$status); - buf.put_u32_le(encoded_node_len as u32); - node.encode(&mut buf).map_err(invalid_data_io_error)?; - $w.write_all(&buf).map(|_| encoded_len) - } - }}; - ($w:ident.$t: ident($status: ident)) => {{ - const N: usize = mem::size_of::() + mem::size_of::(); - let mut data = [0u8; N]; - data[0] = Self::$status; - data[1..N].copy_from_slice(&$t.to_le_bytes()); - $w.write_all(&data).map(|_| N) - }}; - ($w:ident.$ident: ident) => {{ $w.write_all(&[Self::$ident]).map(|_| 1) }}; -} - -impl SnapshotRecord<'_, I, A> -where - I: Id + Data, - A: CheapClone + Data + Send + Sync + 'static, -{ - const ALIVE: u8 = 0; - const NOT_ALIVE: u8 = 1; - const CLOCK: u8 = 2; - const EVENT_CLOCK: u8 = 3; - const QUERY_CLOCK: u8 = 4; - const COORDINATE: u8 = 5; - const LEAVE: u8 = 6; - const COMMENT: u8 = 7; - - fn encode(&self, w: &mut W) -> std::io::Result { - match self { - Self::Alive(id) => encode!(w.id::ALIVE), - Self::NotAlive(id) => encode!(w.id::NOT_ALIVE), - Self::Clock(t) => encode!(w.t(CLOCK)), - Self::EventClock(t) => encode!(w.t(EVENT_CLOCK)), - Self::QueryClock(t) => encode!(w.t(QUERY_CLOCK)), - Self::Coordinate => encode!(w.COORDINATE), - Self::Leave => encode!(w.LEAVE), - Self::Comment => encode!(w.COMMENT), - } - } -} - -#[viewit::viewit] -pub(crate) struct ReplayResult { - alive_nodes: HashSet>, - last_clock: LamportTime, - last_event_clock: LamportTime, - last_query_clock: LamportTime, - offset: u64, - fh: File, - path: PathBuf, -} - -pub(crate) fn open_and_replay_snapshot< - I: Id + Data, - A: CheapClone + Data + core::hash::Hash + Eq + Send + Sync + 'static, - P: AsRef, ->( - p: &P, - rejoin_after_leave: bool, -) -> Result, SnapshotError> { - // Try to open the file - #[cfg(unix)] - let fh = OpenOptions::new() - .create(true) - .append(true) - .read(true) - .mode(0o644) - .open(p) - .map_err(SnapshotError::Open)?; - #[cfg(not(unix))] - let fh = OpenOptions::new() - .create(true) - .append(true) - .read(true) - .write(true) - .open(p) - .map_err(SnapshotError::Open)?; - - // Determine the offset - let offset = fh.metadata().map_err(SnapshotError::Stat)?.len(); - - // Read each line - let mut reader = BufReader::new(fh); - let mut buf = Vec::new(); - let mut alive_nodes = HashSet::new(); - let mut last_clock = LamportTime::ZERO; - let mut last_event_clock = LamportTime::ZERO; - let mut last_query_clock = LamportTime::ZERO; - - loop { - let kind = match reader.read_u8() { - Ok(b) => SnapshotRecordType::try_from(b)?, - Err(e) => { - if e.kind() == std::io::ErrorKind::UnexpectedEof { - break; - } - return Err(SnapshotError::Replay(e)); - } - }; - - match kind { - SnapshotRecordType::Alive => { - let len = reader - .read_u32::() - .map_err(SnapshotError::Replay)? as usize; - buf.resize(len, 0); - reader.read_exact(&mut buf).map_err(SnapshotError::Replay)?; - - let (_, node) = as Data>::decode(&buf) - .map_err(|e| SnapshotError::Replay(invalid_data_io_error(e)))?; - alive_nodes.insert(node); - } - SnapshotRecordType::NotAlive => { - let len = reader - .read_u32::() - .map_err(SnapshotError::Replay)? as usize; - buf.resize(len, 0); - reader.read_exact(&mut buf).map_err(SnapshotError::Replay)?; - - let (_, node) = as Data>::decode(&buf) - .map_err(|e| SnapshotError::Replay(invalid_data_io_error(e)))?; - alive_nodes.remove(&node); - } - SnapshotRecordType::Clock => { - let t = reader - .read_u64::() - .map_err(SnapshotError::Replay)?; - last_clock = LamportTime::new(t); - } - SnapshotRecordType::EventClock => { - let t = reader - .read_u64::() - .map_err(SnapshotError::Replay)?; - last_event_clock = LamportTime::new(t); - } - SnapshotRecordType::QueryClock => { - let t = reader - .read_u64::() - .map_err(SnapshotError::Replay)?; - last_query_clock = LamportTime::new(t); - } - SnapshotRecordType::Coordinate => continue, - SnapshotRecordType::Leave => { - // Ignore a leave if we plan on re-joining - if rejoin_after_leave { - tracing::info!("serf: ignoring previous leave in snapshot"); - continue; - } - alive_nodes.clear(); - last_clock = LamportTime::ZERO; - last_event_clock = LamportTime::ZERO; - last_query_clock = LamportTime::ZERO; - } - SnapshotRecordType::Comment => continue, - } - } - - // Seek to the end - let mut f = reader.into_inner(); - - f.seek(std::io::SeekFrom::End(0)) - .map(|_| ReplayResult { - alive_nodes, - last_clock, - last_event_clock, - last_query_clock, - offset, - fh: f, - path: p.as_ref().to_path_buf(), - }) - .map_err(SnapshotError::SeekEnd) -} - -pub(crate) struct SnapshotHandle { - wait_rx: Receiver<()>, - shutdown_rx: Receiver<()>, - leave_tx: Sender<()>, -} - -impl SnapshotHandle { - /// Used to wait until the snapshotter finishes shut down - pub(crate) async fn wait(&self) { - let _ = self.wait_rx.recv().await; - } - - /// Used to remove known nodes to prevent a restart from - /// causing a join. Otherwise nodes will re-join after leaving! - pub(crate) async fn leave(&self) { - futures::select! { - _ = self.leave_tx.send(()).fuse() => {}, - _ = self.shutdown_rx.recv().fuse() => {}, - } - } -} - -/// Responsible for ingesting events and persisting -/// them to disk, and providing a recovery mechanism at start time. -pub(crate) struct Snapshot -where - D: Delegate, - T: Transport, -{ - alive_nodes: HashSet>, - clock: LamportClock, - fh: Option>, - last_flush: Epoch, - last_clock: LamportTime, - last_event_clock: LamportTime, - last_query_clock: LamportTime, - leave_rx: Receiver<()>, - leaving: bool, - min_compact_size: u64, - path: PathBuf, - offset: u64, - rejoin_after_leave: bool, - stream_rx: Receiver>, - shutdown_rx: Receiver<()>, - wait_tx: Sender<()>, - last_attempted_compaction: Epoch, - #[cfg(feature = "metrics")] - metric_labels: std::sync::Arc, -} - -// flushEvent is used to handle writing out an event -macro_rules! stream_flush_event { - ($this:ident <- $event:ident) => {{ - // Stop recording events after a leave is issued - if $this.leaving { - break; - } - - match &$event { - CrateEvent::Member(e) => $this.process_member_event(e), - CrateEvent::User(e) => $this.process_user_event(e), - CrateEvent::Query(e) => $this.process_query_event(e.ltime), - CrateEvent::InternalQuery { query, .. } => $this.process_query_event(query.ltime), - } - }}; -} - -macro_rules! tee_stream_flush_event { - ($stream_tx:ident <- $event:ident -> $out_tx:ident) => {{ - // Forward to the internal stream, do not block - futures::select! { - _ = $stream_tx.send($event.clone()).fuse() => {} - default => {} - } - - // Forward the event immediately, do not block - futures::select! { - _ = $out_tx.send($event).fuse() => {} - default => {} - } - }}; -} - -impl Snapshot -where - D: Delegate, - T: Transport, -{ - #[allow(clippy::type_complexity)] - pub(crate) fn from_replay_result( - replay_result: ReplayResult, - min_compact_size: u64, - rejoin_after_leave: bool, - clock: LamportClock, - out_tx: Sender>, - shutdown_rx: Receiver<()>, - #[cfg(feature = "metrics")] metric_labels: std::sync::Arc, - ) -> Result< - ( - Sender>, - TinyVec>>, - SnapshotHandle, - ), - SnapshotError, - > { - let (in_tx, in_rx) = async_channel::bounded(EVENT_CH_SIZE); - let (stream_tx, stream_rx) = async_channel::bounded(EVENT_CH_SIZE); - let (leave_tx, leave_rx) = async_channel::bounded(1); - let (wait_tx, wait_rx) = async_channel::bounded(1); - - let ReplayResult { - alive_nodes, - last_clock, - last_event_clock, - last_query_clock, - offset, - fh, - path, - } = replay_result; - - // Create the snapshotter - let this = Self { - alive_nodes, - clock, - fh: Some(BufWriter::new(fh)), - last_flush: Epoch::now(), - last_clock, - last_event_clock, - last_query_clock, - leave_rx, - leaving: false, - min_compact_size, - path, - offset, - rejoin_after_leave, - stream_rx, - shutdown_rx: shutdown_rx.clone(), - wait_tx, - last_attempted_compaction: Epoch::now(), - #[cfg(feature = "metrics")] - metric_labels, - }; - - let mut alive_nodes = this - .alive_nodes - .iter() - .map(|n| { - let id = n.id().cheap_clone(); - let addr = n.address().cheap_clone(); - Node::new(id, MaybeResolvedAddress::resolved(addr)) - }) - .collect::>(); - alive_nodes.shuffle(&mut rand::rng()); - - // Start handling new commands - let handle = ::spawn(Self::tee_stream( - in_rx, - stream_tx, - out_tx, - shutdown_rx.clone(), - )); - ::spawn_detach(this.stream(handle)); - - Ok(( - in_tx, - alive_nodes, - SnapshotHandle { - wait_rx, - shutdown_rx, - leave_tx, - }, - )) - } - - /// A long running routine that is used to copy events - /// to the output channel and the internal event handler. - async fn tee_stream( - in_rx: Receiver>, - stream_tx: Sender>, - out_tx: Sender>, - shutdown_rx: Receiver<()>, - ) { - loop { - futures::select! { - ev = in_rx.recv().fuse() => { - if let Ok(ev) = ev { - tee_stream_flush_event!(stream_tx <- ev -> out_tx) - } else { - break; - } - } - _ = shutdown_rx.recv().fuse() => { - break; - } - } - } - - // Drain any remaining events before exiting - loop { - futures::select! { - ev = in_rx.recv().fuse() => { - if let Ok(ev) = ev { - tee_stream_flush_event!(stream_tx <- ev -> out_tx) - } else { - break; - } - } - default => break, - } - } - tracing::debug!("serf: snapshotter tee stream exits"); - } - - fn handle_leave(&mut self) { - self.leaving = true; - - // If we plan to re-join, keep our state - if !self.rejoin_after_leave { - self.alive_nodes.clear(); - } - self.try_append(SnapshotRecord::Leave); - if let Some(fh) = self.fh.as_mut() { - if let Err(e) = fh.flush() { - tracing::error!(target="serf", err=%SnapshotError::Flush(e), "failed to flush leave to snapshot"); - } - - if let Err(e) = fh.get_mut().sync_all() { - tracing::error!(target="serf", err=%SnapshotError::Sync(e), "failed to sync leave to snapshot"); - } - } - } - - /// Long running routine that is used to handle events - async fn stream( - mut self, - tee_handle: <::Spawner as AsyncSpawner>::JoinHandle<()>, - ) { - let mut clock_ticker = ::interval(CLOCK_UPDATE_INTERVAL); - - loop { - futures::select! { - signal = self.leave_rx.recv().fuse() => { - if signal.is_ok() { - self.handle_leave(); - } - } - ev = self.stream_rx.recv().fuse() => { - if let Ok(ev) = ev { - stream_flush_event!(self <- ev) - } else { - break; - } - } - _ = futures::StreamExt::next(&mut clock_ticker).fuse() => { - self.update_clock(); - } - _ = self.shutdown_rx.recv().fuse() => { - break; - } - } - } - - if self.leave_rx.try_recv().is_ok() { - self.handle_leave(); - } - - // Setup a timeout - let flush_timeout = ::sleep(SHUTDOWN_FLUSH_TIMEOUT); - futures::pin_mut!(flush_timeout); - - // snapshot the clock - self.update_clock(); - - // Clear out the buffers - loop { - futures::select! { - ev = self.stream_rx.recv().fuse() => { - if let Ok(ev) = ev { - stream_flush_event!(self <- ev) - } else { - break; - } - } - _ = (&mut flush_timeout).fuse() => { - break; - } - default => { - break; - } - } - } - - if let Some(fh) = self.fh.as_mut() { - if let Err(e) = fh.flush() { - tracing::error!(target="serf", err=%SnapshotError::Flush(e), "failed to flush leave to snapshot"); - } - - if let Err(e) = fh.get_mut().sync_all() { - tracing::error!(target="serf", err=%SnapshotError::Sync(e), "failed to sync leave to snapshot"); - } - } - - self.wait_tx.close(); - if let Err(e) = tee_handle.await { - tracing::error!(target="serf", err=%e, "failed to wait for tee stream to exit"); - } - tracing::debug!("serf: snapshotter stream exits"); - } - - /// Used to handle a single user event - fn process_user_event(&mut self, e: &UserEventMessage) { - // Ignore old clocks - let ltime = e.ltime(); - if ltime <= self.last_event_clock { - return; - } - - self.last_event_clock = ltime; - self.try_append(SnapshotRecord::EventClock(ltime)); - } - - /// Used to handle a single query event - fn process_query_event(&mut self, ltime: LamportTime) { - // Ignore old clocks - if ltime <= self.last_query_clock { - return; - } - - self.last_query_clock = ltime; - self.try_append(SnapshotRecord::QueryClock(ltime)); - } - - /// Used to handle a single member event - fn process_member_event(&mut self, e: &MemberEvent) { - match e.ty { - MemberEventType::Join => { - for m in e.members() { - let node = m.node(); - self.alive_nodes.insert(node.cheap_clone()); - self.try_append(SnapshotRecord::Alive(Cow::Borrowed(node))) - } - } - MemberEventType::Leave | MemberEventType::Failed => { - for m in e.members() { - let node = m.node(); - self.alive_nodes.remove(node); - self.try_append(SnapshotRecord::NotAlive(Cow::Borrowed(node))); - } - } - _ => {} - } - self.update_clock(); - } - - /// Called periodically to check if we should udpate our - /// clock value. This is done after member events but should also be done - /// periodically due to race conditions with join and leave intents - fn update_clock(&mut self) { - let t: u64 = self.clock.time().into(); - let last_seen = LamportTime::from(t.saturating_sub(1)); - if last_seen > self.last_clock { - self.last_clock = last_seen; - self.try_append(SnapshotRecord::Clock(self.last_clock)); - } - } - - fn try_append(&mut self, l: SnapshotRecord<'_, T::Id, T::ResolvedAddress>) { - if let Err(e) = self.append_line(l) { - tracing::error!(err = %e, "serf: failed to update snapshot"); - if self.last_attempted_compaction.elapsed() > SNAPSHOT_ERROR_RECOVERY_INTERVAL { - self.last_attempted_compaction = Epoch::now(); - tracing::info!("serf: attempting compaction to recover from error..."); - if let Err(e) = self.compact() { - tracing::error!(err = %e, "serf: compaction failed, will reattempt after {}s", SNAPSHOT_ERROR_RECOVERY_INTERVAL.as_secs()); - } else { - tracing::info!("serf: finished compaction, successfully recovered from error state"); - } - } - } - } - - fn append_line( - &mut self, - l: SnapshotRecord<'_, T::Id, T::ResolvedAddress>, - ) -> Result<(), SnapshotError> { - #[cfg(feature = "metrics")] - let start = crate::types::Epoch::now(); - - #[cfg(feature = "metrics")] - let metric_labels = self.metric_labels.clone(); - #[cfg(feature = "metrics")] - scopeguard::defer!( - metrics::histogram!("serf.snapshot.append_line", metric_labels.iter()) - .record(start.elapsed().as_millis() as f64) - ); - - let f = self.fh.as_mut().unwrap(); - let n = l.encode(f).map_err(SnapshotError::Write)?; - - // check if we should flush - if self.last_flush.elapsed() > FLUSH_INTERVAL { - self.last_flush = Epoch::now(); - self - .fh - .as_mut() - .unwrap() - .flush() - .map_err(SnapshotError::Flush)?; - } - - // Check if a compaction is necessary - self.offset += n as u64; - if self.offset > self.snapshot_max_size() { - self.compact()?; - } - Ok(()) - } - - /// Computes the maximum size and is used to force periodic compaction. - fn snapshot_max_size(&self) -> u64 { - let nodes = self.alive_nodes.len() as u64; - let est_size = nodes * SNAPSHOT_BYTES_PER_NODE as u64; - let threshold = est_size * SNAPSHOT_COMPACTION_THRESHOLD as u64; - threshold.max(self.min_compact_size) - } - - /// Used to compact the snapshot once it is too large - fn compact(&mut self) -> Result<(), SnapshotError> { - #[cfg(feature = "metrics")] - let start = crate::types::Epoch::now(); - - #[cfg(feature = "metrics")] - let metric_labels = self.metric_labels.clone(); - #[cfg(feature = "metrics")] - scopeguard::defer!( - metrics::histogram!("serf.snapshot.compact", metric_labels.iter()) - .record(start.elapsed().as_millis() as f64) - ); - - // Try to open the file to new file - let new_path = self.path.with_extension(TMP_EXT); - #[cfg(unix)] - let fh = OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .mode(0o755) - .open(&new_path) - .map_err(SnapshotError::OpenNew)?; - - #[cfg(not(unix))] - let fh = OpenOptions::new() - .create(true) - .write(true) - .truncate(true) - .open(&new_path) - .map_err(SnapshotError::OpenNew)?; - - // Create a buffered writer - let mut buf = BufWriter::new(fh); - - // Write out the live nodes - let mut offset = 0u64; - for node in self.alive_nodes.iter() { - offset += SnapshotRecord::Alive(Cow::Borrowed(node)) - .encode(&mut buf) - .map_err(SnapshotError::WriteNew)? as u64; - } - - // Write out the clocks - offset += SnapshotRecord::::Clock(self.last_clock) - .encode(&mut buf) - .map_err(SnapshotError::WriteNew)? as u64; - - offset += SnapshotRecord::::EventClock(self.last_event_clock) - .encode(&mut buf) - .map_err(SnapshotError::WriteNew)? as u64; - - offset += SnapshotRecord::::QueryClock(self.last_query_clock) - .encode(&mut buf) - .map_err(SnapshotError::WriteNew)? as u64; - - // Flush the new snapshot - buf.flush().map_err(SnapshotError::Flush)?; - - // Sync the new snapshot - buf.get_ref().sync_all().map_err(SnapshotError::Sync)?; - drop(buf); - - // We now need to swap the old snapshot file with the new snapshot. - // Turns out, Windows won't let us rename the files if we have - // open handles to them or if the destination already exists. This - // means we are forced to close the existing handles, delete the - // old file, move the new one in place, and then re-open the file - // handles. - - // Flush the existing snapshot, ignoring errors since we will - // delete it momentarily. - let mut old = self.fh.take().unwrap(); - let _ = old.flush(); - drop(old); - - // Delete the old file - if let Err(e) = std::fs::remove_file(&self.path) { - if !matches!(e.kind(), std::io::ErrorKind::NotFound) { - return Err(SnapshotError::Remove(e)); - } - } - - // Move the new file into place - std::fs::rename(&new_path, &self.path).map_err(SnapshotError::Install)?; - - // Open the new snapshot - #[cfg(unix)] - let fh = OpenOptions::new() - .create(true) - .append(true) - .read(true) - .mode(0o755) - .open(&self.path) - .map_err(SnapshotError::Open)?; - - #[cfg(not(unix))] - let fh = OpenOptions::new() - .create(true) - .append(true) - .read(true) - .write(true) - .open(&self.path) - .map_err(SnapshotError::Open)?; - - self.fh = Some(BufWriter::new(fh)); - self.offset = offset; - self.last_flush = Epoch::now(); - Ok(()) - } -} diff --git a/legacy/serf-core/src/types.rs b/legacy/serf-core/src/types.rs deleted file mode 100644 index 8ad419b2..00000000 --- a/legacy/serf-core/src/types.rs +++ /dev/null @@ -1,250 +0,0 @@ -use std::time::Duration; - -pub use memberlist_core::proto::{ - Data, DataRef, DelegateVersion as MemberlistDelegateVersion, Domain, HostAddr, - MaybeResolvedAddress, Node, NodeId, ParseDomainError, ParseHostAddrError, ParseNodeIdError, - ProtocolVersion as MemberlistProtocolVersion, bytes, utils, -}; -pub use smol_str::*; - -#[cfg(feature = "encryption")] -#[cfg_attr(docsrs, doc(cfg(feature = "encryption")))] -pub use memberlist_core::proto::encryption::*; - -#[cfg(any( - feature = "crc32", - feature = "xxhash64", - feature = "xxhash32", - feature = "xxhash3", - feature = "murmur3", -))] -#[cfg_attr( - docsrs, - doc(cfg(any( - feature = "crc32", - feature = "xxhash64", - feature = "xxhash32", - feature = "xxhash3", - feature = "murmur3" - ))) -)] -pub use memberlist_core::proto::checksum::*; - -#[cfg(any( - feature = "zstd", - feature = "lz4", - feature = "snappy", - feature = "brotli", -))] -#[cfg_attr( - docsrs, - doc(cfg(any( - feature = "zstd", - feature = "lz4", - feature = "snappy", - feature = "brotli" - ))) -)] -pub use memberlist_core::proto::compression::*; - -#[cfg(feature = "arbitrary")] -mod arbitrary_impl; - -#[cfg(feature = "quickcheck")] -mod quickcheck_impl; - -#[cfg(test)] -#[cfg(feature = "quickcheck")] -mod tests; - -mod clock; -pub use clock::*; - -/// Vivialdi coordinate implementation -pub mod coordinate; - -mod conflict; -pub(crate) use conflict::*; - -mod filter; -pub(crate) use filter::*; - -mod leave; -pub(crate) use leave::*; - -mod member; -pub use member::*; - -mod message; -pub(crate) use message::*; - -mod join; -pub(crate) use join::*; - -mod tags; -pub use tags::*; - -mod push_pull; -pub(crate) use push_pull::*; - -mod user_event; -pub(crate) use user_event::*; - -mod query; -pub(crate) use query::*; - -mod version; -pub use version::*; - -#[cfg(feature = "encryption")] -mod key; -#[cfg(feature = "encryption")] -#[cfg_attr(docsrs, doc(cfg(feature = "encryption")))] -pub use key::*; - -#[cfg(any(feature = "arbitrary", feature = "quickcheck"))] -#[doc(hidden)] -pub mod fuzzy; - -#[cfg(debug_assertions)] -#[inline] -fn debug_assert_write_eq(actual: usize, expected: usize) { - debug_assert_eq!( - actual, - expected, - "{}: expect writting {expected} bytes, but actual write {actual} bytes", - core::any::type_name::(), - ); -} - -#[cfg(windows)] -pub(crate) type Epoch = system_epoch::SystemTimeEpoch; - -#[cfg(not(windows))] -pub(crate) type Epoch = instant_epoch::InstantEpoch; - -#[cfg(windows)] -mod system_epoch { - use super::*; - use std::time::SystemTime; - - type SystemTimeEpochInner = SystemTime; - - #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] - pub(crate) struct SystemTimeEpoch(SystemTimeEpochInner); - - impl core::fmt::Debug for SystemTimeEpoch { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - self.0.fmt(f) - } - } - - impl core::ops::Sub for SystemTimeEpoch { - type Output = Duration; - - fn sub(self, rhs: Self) -> Duration { - self.0.duration_since(rhs.0).unwrap() - } - } - - impl core::ops::Sub for SystemTimeEpoch { - type Output = Self; - - fn sub(self, rhs: Duration) -> Self { - Self(self.0 - rhs) - } - } - - impl core::ops::SubAssign for SystemTimeEpoch { - fn sub_assign(&mut self, rhs: Duration) { - self.0 -= rhs; - } - } - - impl core::ops::Add for SystemTimeEpoch { - type Output = Self; - - fn add(self, rhs: Duration) -> Self { - SystemTimeEpoch(self.0 + rhs) - } - } - - impl core::ops::AddAssign for SystemTimeEpoch { - fn add_assign(&mut self, rhs: Duration) { - self.0 += rhs; - } - } - - impl SystemTimeEpoch { - pub(crate) fn now() -> Self { - Self(SystemTimeEpochInner::now()) - } - - pub(crate) fn elapsed(&self) -> Duration { - self.0.elapsed().unwrap() - } - } -} - -#[cfg(not(windows))] -mod instant_epoch { - use super::*; - use std::time::Instant; - - type InstantEpochInner = Instant; - - #[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] - pub(crate) struct InstantEpoch(InstantEpochInner); - - impl core::fmt::Debug for InstantEpoch { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - self.0.fmt(f) - } - } - - impl core::ops::Sub for InstantEpoch { - type Output = Duration; - - fn sub(self, rhs: Self) -> Duration { - self.0 - rhs.0 - } - } - - impl core::ops::Sub for InstantEpoch { - type Output = Self; - - fn sub(self, rhs: Duration) -> Self { - Self(self.0 - rhs) - } - } - - impl core::ops::SubAssign for InstantEpoch { - fn sub_assign(&mut self, rhs: Duration) { - self.0 -= rhs; - } - } - - impl core::ops::Add for InstantEpoch { - type Output = Self; - - fn add(self, rhs: Duration) -> Self { - InstantEpoch(self.0 + rhs) - } - } - - impl core::ops::AddAssign for InstantEpoch { - fn add_assign(&mut self, rhs: Duration) { - self.0 += rhs; - } - } - - impl InstantEpoch { - pub(crate) fn now() -> Self { - Self(InstantEpochInner::now()) - } - - pub(crate) fn elapsed(&self) -> Duration { - self.0.elapsed() - } - } -} diff --git a/legacy/serf-core/src/types/arbitrary_impl.rs b/legacy/serf-core/src/types/arbitrary_impl.rs deleted file mode 100644 index bc33614b..00000000 --- a/legacy/serf-core/src/types/arbitrary_impl.rs +++ /dev/null @@ -1,190 +0,0 @@ -use std::{ - collections::{HashMap, HashSet}, - hash::Hash, -}; - -use super::{Filter, MessageType, TagFilter}; -use arbitrary::{Arbitrary, Unstructured}; -use indexmap::{IndexMap, IndexSet}; -use memberlist_core::proto::TinyVec; - -pub(super) fn into<'a, F, T>(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result -where - F: arbitrary::Arbitrary<'a>, - T: From, -{ - u.arbitrary::().map(Into::into) -} - -pub(super) fn arbitrary_indexmap<'a, K, V>( - u: &mut arbitrary::Unstructured<'a>, -) -> arbitrary::Result> -where - K: Arbitrary<'a> + Hash + Eq, - V: Arbitrary<'a>, -{ - let map = u.arbitrary::>()?; - Ok(IndexMap::from_iter(map)) -} - -pub(super) fn arbitrary_indexset<'a, K>( - u: &mut arbitrary::Unstructured<'a>, -) -> arbitrary::Result> -where - K: Arbitrary<'a> + Hash + Eq, -{ - let map = u.arbitrary::>()?; - Ok(IndexSet::from_iter(map)) -} - -impl<'a, I> Arbitrary<'a> for Filter -where - I: Arbitrary<'a>, -{ - fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result { - let kind = u.arbitrary::()?; - Ok(if kind { - Filter::Id(into::, TinyVec<_>>(u)?) - } else { - Filter::Tag( - TagFilter::new() - .with_tag(u.arbitrary()?) - .maybe_expr(if u.arbitrary()? { - let complexity = u.int_in_range(1..=5)?; - let mut patterns = Vec::new(); - - // Basic character classes and quantifiers - let character_classes = vec![ - r"\d", - r"\w", - r"\s", - r"[a-z]", - r"[A-Z]", - r"[0-9]", - r"[a-zA-Z]", - r"[a-zA-Z0-9]", - r".", - ]; - - let quantifiers = vec!["", "*", "+", "?", "{1,3}", "{2,5}"]; - - // Add more complex patterns for higher complexity - let mut extended_classes = character_classes.clone(); - if complexity > 1 { - extended_classes.extend(vec![r"[^a-z]", r"[^0-9]", r"\D", r"\W", r"\S"]); - } - - if complexity > 2 { - // Add a group with random content - let char_class = u.choose(&extended_classes)?; - let quantifier = u.choose(&quantifiers)?; - patterns.push(format!("({}{})", char_class, quantifier)); - } - - // Generate random pattern parts - for _ in 0..complexity { - let char_class = u.choose(&extended_classes)?; - let quantifier = u.choose(&quantifiers)?; - patterns.push(format!("{}{}", char_class, quantifier)); - } - - // Maybe add anchors for higher complexity - if complexity > 2 && u.ratio(7, 10)? { - if u.arbitrary()? { - patterns.insert(0, "^".to_string()); - } - if u.arbitrary()? { - patterns.push("$".to_string()); - } - } - - // Add alternation for even higher complexity - if complexity > 3 && u.ratio(6, 10)? { - let char_class = u.choose(&extended_classes)?; - let quantifier = u.choose(&quantifiers)?; - patterns.push(format!("|{}{}", char_class, quantifier)); - } - - Some(patterns.join("").try_into().unwrap()) - } else { - None - }), - ) - }) - } -} - -impl<'a> Arbitrary<'a> for super::QueryFlag { - fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result { - Ok(if u.arbitrary()? { - Self::NO_BROADCAST - } else { - Self::ACK - }) - } -} - -impl<'a> Arbitrary<'a> for super::ProtocolVersion { - fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result { - u.arbitrary::().map(Into::into) - } -} - -impl<'a> Arbitrary<'a> for super::DelegateVersion { - fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result { - u.arbitrary::().map(Into::into) - } -} - -impl<'a> Arbitrary<'a> for super::MemberStatus { - fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result { - u.arbitrary::().map(Into::into) - } -} - -#[cfg(feature = "encryption")] -impl<'a, I> Arbitrary<'a> for super::KeyResponse -where - I: Arbitrary<'a> + Eq + std::hash::Hash, -{ - fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result { - Ok(Self { - messages: arbitrary_indexmap(u)?, - num_nodes: u.arbitrary()?, - num_resp: u.arbitrary()?, - num_err: u.arbitrary()?, - keys: arbitrary_indexmap(u)?, - primary_keys: arbitrary_indexmap(u)?, - }) - } -} - -impl<'a> Arbitrary<'a> for MessageType { - fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result { - u.arbitrary::() - .map(|val| Self::from(val % Self::ALL.len() as u8)) - } -} - -impl<'a> Arbitrary<'a> for super::coordinate::Coordinate { - fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result { - Ok(Self { - portion: Vec::::arbitrary(u)? - .into_iter() - .map(|f| if f.is_nan() { 0.0 } else { f }) - .collect(), - error: rand_f64_not_nan(u)?, - adjustment: rand_f64_not_nan(u)?, - height: rand_f64_not_nan(u)?, - }) - } -} - -fn rand_f64_not_nan(u: &mut Unstructured<'_>) -> arbitrary::Result { - loop { - let f = f64::arbitrary(u)?; - if !f.is_nan() { - return Ok(f); - } - } -} diff --git a/legacy/serf-core/src/types/clock.rs b/legacy/serf-core/src/types/clock.rs deleted file mode 100644 index 4233ecd5..00000000 --- a/legacy/serf-core/src/types/clock.rs +++ /dev/null @@ -1,191 +0,0 @@ -use std::sync::{ - Arc, - atomic::{AtomicU64, Ordering}, -}; - -use memberlist_core::proto::{Data, DataRef, DecodeError, EncodeError}; - -/// A lamport time is a simple u64 that represents a point in time. -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "serde", serde(transparent))] -#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[repr(transparent)] -pub struct LamportTime(pub(crate) u64); - -impl core::fmt::Display for LamportTime { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - write!(f, "{}", self.0) - } -} - -impl From for LamportTime { - fn from(time: u64) -> Self { - Self(time) - } -} - -impl From for u64 { - fn from(time: LamportTime) -> Self { - time.0 - } -} - -impl LamportTime { - /// Zero lamport time - pub const ZERO: Self = Self(0); - - /// Creates a new lamport time from the given u64 - #[inline] - pub const fn new(time: u64) -> Self { - Self(time) - } - - /// Returns the lamport time as a big endian byte array - #[inline] - pub const fn to_be_bytes(self) -> [u8; 8] { - self.0.to_be_bytes() - } - - /// Returns the lamport time as a little endian byte array - #[inline] - pub const fn to_le_bytes(self) -> [u8; 8] { - self.0.to_le_bytes() - } - - /// Creates a new lamport time from a big endian byte array - #[inline] - pub const fn from_be_bytes(bytes: [u8; 8]) -> Self { - Self(u64::from_be_bytes(bytes)) - } - - /// Creates a new lamport time from a little endian byte array - #[inline] - pub const fn from_le_bytes(bytes: [u8; 8]) -> Self { - Self(u64::from_le_bytes(bytes)) - } -} - -impl core::ops::Add for LamportTime { - type Output = Self; - - #[inline] - fn add(self, rhs: Self) -> Self::Output { - Self(self.0 + rhs.0) - } -} - -impl core::ops::Sub for LamportTime { - type Output = Self; - - #[inline] - fn sub(self, rhs: Self) -> Self::Output { - Self(self.0 - rhs.0) - } -} - -impl core::ops::Rem for LamportTime { - type Output = Self; - - #[inline] - fn rem(self, rhs: Self) -> Self::Output { - Self(self.0 % rhs.0) - } -} - -impl Data for LamportTime { - type Ref<'a> = Self; - - fn from_ref(val: Self::Ref<'_>) -> Result - where - Self: Sized, - { - Ok(val) - } - - fn encoded_len(&self) -> usize { - ::encoded_len(&self.0) - } - - fn encode(&self, buf: &mut [u8]) -> Result { - ::encode(&self.0, buf) - } -} - -impl<'a> DataRef<'a, LamportTime> for LamportTime { - fn decode(src: &'a [u8]) -> Result<(usize, LamportTime), DecodeError> { - >::decode(src).map(|(n, v)| (n, v.into())) - } -} - -/// A thread safe implementation of a lamport clock. It -/// uses efficient atomic operations for all of its functions, falling back -/// to a heavy lock only if there are enough CAS failures. -#[derive(Debug, Clone)] -pub struct LamportClock(Arc); - -impl Default for LamportClock { - fn default() -> Self { - Self::new() - } -} - -impl LamportClock { - /// Creates a new lamport clock with the given initial value - #[inline] - pub fn new() -> Self { - Self(Arc::new(AtomicU64::new(0))) - } - - /// Return the current value of the lamport clock - #[inline] - pub fn time(&self) -> LamportTime { - LamportTime(self.0.load(Ordering::SeqCst)) - } - - /// Increment and return the value of the lamport clock - #[inline] - pub fn increment(&self) -> LamportTime { - LamportTime(self.0.fetch_add(1, Ordering::SeqCst) + 1) - } - - /// Witness is called to update our local clock if necessary after - /// witnessing a clock value received from another process - #[inline] - pub fn witness(&self, time: LamportTime) { - loop { - // If the other value is old, we do not need to do anything - let current = self.0.load(Ordering::SeqCst); - if time.0 < current { - return; - } - - // Ensure that our local clock is at least one ahead. - match self - .0 - .compare_exchange_weak(current, time.0 + 1, Ordering::SeqCst, Ordering::SeqCst) - { - Ok(_) => return, - Err(_) => continue, - } - } - } -} - -#[test] -fn test_lamport_clock() { - let l = LamportClock::new(); - - assert_eq!(l.time(), 0.into()); - assert_eq!(l.increment(), 1.into()); - assert_eq!(l.time(), 1.into()); - - l.witness(41.into()); - assert_eq!(l.time(), 42.into()); - - l.witness(41.into()); - assert_eq!(l.time(), 42.into()); - - l.witness(30.into()); - assert_eq!(l.time(), 42.into()); -} diff --git a/legacy/serf-core/src/types/conflict.rs b/legacy/serf-core/src/types/conflict.rs deleted file mode 100644 index 1482c939..00000000 --- a/legacy/serf-core/src/types/conflict.rs +++ /dev/null @@ -1,161 +0,0 @@ -use memberlist_core::proto::{ - Data, DataRef, DecodeError, EncodeError, WireType, - utils::{merge, skip}, -}; - -use super::*; - -/// A conflict message -#[viewit::viewit(setters(prefix = "with"))] -#[derive(Debug, PartialEq, Clone)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -pub struct ConflictResponseMessage { - /// The member - #[viewit( - getter(const, style = "ref", attrs(doc = "Returns the member")), - setter(attrs(doc = "Sets the member (Builder pattern)")) - )] - member: Member, -} - -/// The borrow type of conflict message -#[viewit::viewit(setters(prefix = "with"))] -#[derive(Debug, PartialEq)] -pub struct ConflictResponseMessageBorrow<'a, I, A> { - /// The member - #[viewit( - getter(const, style = "ref", attrs(doc = "Returns the member")), - setter(attrs(doc = "Sets the member (Builder pattern)")) - )] - member: &'a Member, -} - -impl<'a, I, A> ConflictResponseMessageBorrow<'a, I, A> { - /// Create a new conflict response message - pub fn new(member: &'a Member) -> Self { - Self { member } - } -} - -impl<'a, I, A> From<&'a ConflictResponseMessage> for ConflictResponseMessageBorrow<'a, I, A> { - fn from(val: &'a ConflictResponseMessage) -> Self { - Self::new(&val.member) - } -} - -impl<'a, I, A> From<&'a Member> for ConflictResponseMessageBorrow<'a, I, A> { - fn from(val: &'a Member) -> Self { - Self::new(val) - } -} - -impl ConflictResponseMessageBorrow<'_, I, A> -where - I: Data, - A: Data, -{ - pub(super) fn encoded_len_in(&self) -> usize { - 1 + self.member.encoded_len_with_length_delimited() - } - - pub(super) fn encode_in(&self, buf: &mut [u8]) -> Result { - let mut offset = 0; - - if offset >= buf.len() { - return Err(EncodeError::insufficient_buffer( - self.encoded_len_in(), - buf.len(), - )); - } - - buf[offset] = MEMBER_BYTE; - offset += 1; - offset += self - .member - .encode_length_delimited(&mut buf[offset..]) - .map_err(|e| e.update(self.encoded_len_in(), buf.len()))?; - - #[cfg(debug_assertions)] - super::debug_assert_write_eq::(offset, self.encoded_len_in()); - - Ok(offset) - } -} - -const MEMBER_TAG: u8 = 1; -const MEMBER_BYTE: u8 = merge(WireType::LengthDelimited, MEMBER_TAG); - -/// The reference to a [`ConflictResponseMessage`]. -#[viewit::viewit(getters(style = "ref", vis_all = "pub"), setters(skip), vis_all = "")] -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct ConflictResponseMessageRef<'a, I, A> { - #[viewit(getter(const, style = "ref", attrs(doc = "Returns the member")))] - member: MemberRef<'a, I, A>, -} - -impl<'a, I, A> DataRef<'a, ConflictResponseMessage> - for ConflictResponseMessageRef<'a, I::Ref<'a>, A::Ref<'a>> -where - I: Data, - A: Data, -{ - fn decode(buf: &'a [u8]) -> Result<(usize, Self), DecodeError> - where - Self: Sized, - { - let mut offset = 0; - let mut member = None; - - while offset < buf.len() { - match buf[offset] { - MEMBER_BYTE => { - if member.is_some() { - return Err(DecodeError::duplicate_field( - "ConflictResponseMessage", - "member", - MEMBER_TAG, - )); - } - offset += 1; - - let (len, val) = , A::Ref<'_>> as DataRef<'_, Member>>::decode_length_delimited(&buf[offset..])?; - offset += len; - member = Some(val); - } - _ => offset += skip("ConflictResponseMessage", &buf[offset..])?, - } - } - - let member = member.ok_or(DecodeError::missing_field( - "ConflictResponseMessage", - "member", - ))?; - Ok((offset, Self { member })) - } -} - -impl Data for ConflictResponseMessage -where - I: Data, - A: Data, -{ - type Ref<'a> = ConflictResponseMessageRef<'a, I::Ref<'a>, A::Ref<'a>>; - - fn from_ref(val: Self::Ref<'_>) -> Result - where - Self: Sized, - { - Ok(Self { - member: Member::from_ref(val.member)?, - }) - } - - fn encoded_len(&self) -> usize { - ConflictResponseMessageBorrow::from(self).encoded_len() - } - - fn encode(&self, buf: &mut [u8]) -> Result { - ConflictResponseMessageBorrow::from(self).encode_in(buf) - } -} diff --git a/legacy/serf-core/src/types/coordinate.rs b/legacy/serf-core/src/types/coordinate.rs deleted file mode 100644 index 0915128e..00000000 --- a/legacy/serf-core/src/types/coordinate.rs +++ /dev/null @@ -1,1282 +0,0 @@ -use core::time::Duration; - -use memberlist_core::proto::{ - Data, DataRef, DecodeError, EncodeError, RepeatedDecoder, WireType, - utils::{merge, skip}, -}; -use rand::RngExt; -use smallvec::SmallVec; - -use std::{ - collections::HashMap, - sync::atomic::{AtomicUsize, Ordering}, -}; - -use memberlist_core::CheapClone; -use parking_lot::RwLock; - -/// Used to convert float seconds to nanoseconds. -const SECONDS_TO_NANOSECONDS: f64 = 1.0e9; -/// Used to decide if two coordinates are on top of each -/// other. -const ZERO_THRESHOLD: f64 = 1.0e-6; - -/// The default dimensionality of the coordinate system. -const DEFAULT_DIMENSIONALITY: usize = 8; - -/// The default adjustment window size. -const DEFAULT_ADJUSTMENT_WINDOW_SIZE: usize = 20; - -const DEFAULT_LATENCY_FILTER_SAMPLES_SIZE: usize = 8; - -/// CoordinateOptions is used to set the parameters of the Vivaldi-based coordinate mapping -/// algorithm. -/// -/// The following references are called out at various points in the documentation -/// here: -/// -/// [1] Dabek, Frank, et al. "Vivaldi: A decentralized network coordinate system." -/// ACM SIGCOMM Computer Communication Review. Vol. 34. No. 4. ACM, 2004. -/// [2] Ledlie, Jonathan, Paul Gardner, and Margo I. Seltzer. "Network Coordinates -/// in the Wild." NSDI. Vol. 7. 2007. -/// [3] Lee, Sanghwan, et al. "On suitability of Euclidean embedding for -/// host-based network coordinate systems." Networking, IEEE/ACM Transactions -/// on 18.1 (2010): 27-40. -#[viewit::viewit( - vis_all = "pub(crate)", - getters(style = "ref", vis_all = "pub"), - setters(prefix = "with", vis_all = "pub") -)] -#[derive(Debug, Clone, PartialEq)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub struct CoordinateOptions { - /// The dimensionality of the coordinate system. As discussed in [2], more - /// dimensions improves the accuracy of the estimates up to a point. Per [2] - /// we chose 8 dimensions plus a non-Euclidean height. - #[viewit( - getter( - const, - style = "move", - attrs(doc = "Returns the dimensionality of the coordinate system.") - ), - setter(attrs(doc = "Sets the dimensionality of the coordinate system.")) - )] - dimensionality: usize, - - /// The default error value when a node hasn't yet made - /// any observations. It also serves as an upper limit on the error value in - /// case observations cause the error value to increase without bound. - #[viewit( - getter( - const, - style = "move", - attrs(doc = "Returns the default error value when a node hasn't yet made any observations.") - ), - setter(attrs( - doc = "Sets the default error value when a node hasn't yet made any observations." - )) - )] - vivaldi_error_max: f64, - - /// A tuning factor that controls the maximum impact an - /// observation can have on a node's confidence. See [1] for more details. - #[viewit( - getter( - const, - style = "move", - attrs(doc = "Returns the maximum impact an observation can have on a node's confidence.") - ), - setter(attrs( - doc = "Sets the maximum impact an observation can have on a node's confidence." - )) - )] - vivaldi_ce: f64, - - /// A tuning factor that controls the maximum impact an - /// observation can have on a node's coordinate. See [1] for more details. - #[viewit( - getter( - const, - style = "move", - attrs(doc = "Returns the maximum impact an observation can have on a node's coordinate.") - ), - setter(attrs( - doc = "Sets the maximum impact an observation can have on a node's coordinate." - )) - )] - vivaldi_cc: f64, - - /// A tuning factor that determines how many samples - /// we retain to calculate the adjustment factor as discussed in [3]. Setting - /// this to zero disables this feature. - #[viewit( - getter( - const, - style = "move", - attrs(doc = "Returns how many samples we retain to calculate the adjustment factor.") - ), - setter(attrs(doc = "Sets how many samples we retain to calculate the adjustment factor.")) - )] - adjustment_window_size: usize, - - /// The minimum value of the height parameter. Since this - /// always must be positive, it will introduce a small amount error, so - /// the chosen value should be relatively small compared to "normal" - /// coordinates. - #[viewit( - getter( - const, - style = "move", - attrs(doc = "Returns the minimum value of the height parameter.") - ), - setter(attrs(doc = "Sets the minimum value of the height parameter.")) - )] - height_min: f64, - - /// The maximum number of samples that are retained - /// per node, in order to compute a median. The intent is to ride out blips - /// but still keep the delay low, since our time to probe any given node is - /// pretty infrequent. See [2] for more details. - #[viewit( - getter( - const, - style = "move", - attrs(doc = "Returns the maximum number of samples that are retained per node.") - ), - setter(attrs(doc = "Sets the maximum number of samples that are retained per node.")) - )] - latency_filter_size: usize, - - /// A tuning factor that sets how much gravity has an effect - /// to try to re-center coordinates. See [2] for more details. - #[viewit( - getter( - const, - style = "move", - attrs(doc = "Returns how much gravity has an effect to try to re-center coordinates.") - ), - setter(attrs(doc = "Sets how much gravity has an effect to try to re-center coordinates.")) - )] - gravity_rho: f64, - - #[cfg(feature = "metrics")] - #[viewit( - getter( - const, - style = "ref", - attrs( - cfg(feature = "metrics"), - doc = "The metric labels used to identify the metrics for this coordinate client." - ) - ), - setter(attrs( - cfg(feature = "metrics"), - doc = "Sets the metric labels used to identify the metrics for this coordinate client." - )) - )] - metric_labels: std::sync::Arc, -} - -impl Default for CoordinateOptions { - fn default() -> Self { - Self::new() - } -} - -impl CoordinateOptions { - /// Returns a `CoordinateOptions` that has some default values suitable for - /// basic testing of the algorithm, but not tuned to any particular type of cluster. - #[inline] - pub fn new() -> Self { - Self { - dimensionality: DEFAULT_DIMENSIONALITY, - vivaldi_error_max: 1.5, - vivaldi_ce: 0.25, - vivaldi_cc: 0.25, - adjustment_window_size: 20, - height_min: 10.0e-6, - latency_filter_size: 3, - gravity_rho: 150.0, - #[cfg(feature = "metrics")] - metric_labels: std::sync::Arc::new(memberlist_core::proto::MetricLabels::default()), - } - } -} - -/// A specialized structure for holding network coordinates for the -/// Vivaldi-based coordinate mapping algorithm. All of the fields should be public -/// to enable this to be serialized. All values in here are in units of seconds. -#[viewit::viewit(getters(style = "move"), setters(prefix = "with"))] -#[derive(Debug, Clone, PartialEq)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub struct Coordinate { - /// The Euclidean portion of the coordinate. This is used along - /// with the other fields to provide an overall distance estimate. The - /// units here are seconds. - #[viewit( - getter( - const, - style = "ref", - attrs(doc = "Returns the Euclidean portion of the coordinate.") - ), - setter(attrs(doc = "Sets the Euclidean portion of the coordinate.")) - )] - portion: SmallVec<[f64; DEFAULT_DIMENSIONALITY]>, - /// Reflects the confidence in the given coordinate and is updated - /// dynamically by the Vivaldi Client. This is dimensionless. - #[viewit( - getter(const, attrs(doc = "Returns the confidence in the given coordinate.")), - setter(attrs(doc = "Sets the confidence in the given coordinate.")) - )] - error: f64, - /// A distance offset computed based on a calculation over - /// observations from all other nodes over a fixed window and is updated - /// dynamically by the Vivaldi Client. The units here are seconds. - #[viewit( - getter(const, attrs(doc = "Returns the distance offset.")), - setter(attrs(doc = "Sets the distance offset.")) - )] - adjustment: f64, - /// A distance offset that accounts for non-Euclidean effects - /// which model the access links from nodes to the core Internet. The access - /// links are usually set by bandwidth and congestion, and the core links - /// usually follow distance based on geography. - #[viewit( - getter( - const, - attrs(doc = "Returns the distance offset that accounts for non-Euclidean effects.") - ), - setter(attrs(doc = "Sets the distance offset that accounts for non-Euclidean effects.")) - )] - height: f64, -} - -impl Default for Coordinate { - #[inline] - fn default() -> Self { - Self::new() - } -} - -impl Coordinate { - /// Creates a new coordinate at the origin, using the default options - /// to supply key initial values. - #[inline] - pub fn new() -> Self { - Self::with_options(CoordinateOptions::new()) - } - - /// Creates a new coordinate at the origin, using the given options - /// to supply key initial values. - #[inline] - pub fn with_options(opts: CoordinateOptions) -> Self { - let mut vec = SmallVec::with_capacity(opts.dimensionality); - vec.resize(opts.dimensionality, 0.0); - Self { - portion: vec, - error: opts.vivaldi_error_max, - adjustment: 0.0, - height: opts.height_min, - } - } - - /// Returns true if the coordinate is valid. - #[inline] - pub fn is_valid(&self) -> bool { - self.portion.iter().all(|&f| f.is_finite()) - && self.error.is_finite() - && self.adjustment.is_finite() - && self.height.is_finite() - } - - /// Returns true if the dimensions of the coordinates are compatible. - #[inline] - pub fn is_compatible_with(&self, other: &Self) -> bool { - self.portion.len() == other.portion.len() - } - - /// Returns the result of applying the force from the direction of the - /// other coordinate. - pub fn apply_force(&self, height_min: f64, force: f64, other: &Self) -> Self { - assert!( - self.is_compatible_with(other), - "coordinate dimensionality does not match" - ); - - let mut ret = self.clone(); - let (mut unit, mag) = unit_vector_at(&self.portion, &other.portion); - add_in_place(&mut ret.portion, mul_in_place(&mut unit, force)); - if mag > ZERO_THRESHOLD { - ret.height = (ret.height + other.height) * force / mag + ret.height; - ret.height = ret.height.max(height_min); - } - ret - } - - /// Apply the result of applying the force from the direction of the - /// other coordinate to self. - pub fn apply_force_in_place(&mut self, height_min: f64, force: f64, other: &Self) { - assert!( - self.is_compatible_with(other), - "coordinate dimensionality does not match" - ); - let (mut unit, mag) = unit_vector_at(&self.portion, &other.portion); - add_in_place(&mut self.portion, mul_in_place(&mut unit, force)); - - if mag > ZERO_THRESHOLD { - self.height = (self.height + other.height) * force / mag + self.height; - self.height = self.height.max(height_min); - } - } - - /// Returns the distance between this coordinate and the other - /// coordinate, including adjustments. - pub fn distance_to(&self, other: &Self) -> Duration { - assert!( - self.is_compatible_with(other), - "coordinate dimensionality does not match" - ); - - let dist = self.raw_distance_to(other); - let adjusted_dist = dist + self.adjustment + other.adjustment; - let dist = if adjusted_dist > 0.0 { - adjusted_dist - } else { - dist - }; - Duration::from_nanos((dist * SECONDS_TO_NANOSECONDS) as u64) - } - - #[inline] - pub(crate) fn raw_distance_to(&self, other: &Self) -> f64 { - magnitude_in_place(diff_in_place(&self.portion, &other.portion)) + self.height + other.height - } -} - -#[inline] -fn add_in_place(vec1: &mut [f64], vec2: &[f64]) { - for (x, y) in vec1.iter_mut().zip(vec2.iter()) { - *x += y; - } -} - -/// Returns difference between the vec1 and vec2. This assumes the -/// dimensions have already been checked to be compatible. -#[inline] -fn diff(vec1: &[f64], vec2: &[f64]) -> SmallVec<[f64; DEFAULT_DIMENSIONALITY]> { - vec1.iter().zip(vec2).map(|(x, y)| x - y).collect() -} - -/// computes difference between the vec1 and vec2 in place. This assumes the -/// dimensions have already been checked to be compatible. -#[inline] -fn diff_in_place<'a>(vec1: &'a [f64], vec2: &'a [f64]) -> impl Iterator + 'a { - vec1.iter().zip(vec2).map(|(x, y)| x - y) -} - -/// multiplied by a scalar factor in place. -#[inline] -fn mul_in_place(vec: &mut [f64], factor: f64) -> &mut [f64] { - for x in vec.iter_mut() { - *x *= factor; - } - vec -} - -/// Computes the magnitude of the vec. -#[inline] -fn magnitude_in_place(vec: impl Iterator) -> f64 { - vec.fold(0.0, |acc, x| acc + x * x).sqrt() -} - -/// Returns a unit vector pointing at vec1 from vec2. If the two -/// positions are the same then a random unit vector is returned. We also return -/// the distance between the points for use in the later height calculation. -fn unit_vector_at(vec1: &[f64], vec2: &[f64]) -> (SmallVec<[f64; DEFAULT_DIMENSIONALITY]>, f64) { - let mut ret = diff(vec1, vec2); - - let mag = magnitude_in_place(ret.iter().copied()); - if mag > ZERO_THRESHOLD { - mul_in_place(&mut ret, mag.recip()); - return (ret, mag); - } - - for x in ret.iter_mut() { - *x = rand_f64() - 0.5; - } - - let mag = magnitude_in_place(ret.iter().copied()); - if mag > ZERO_THRESHOLD { - mul_in_place(&mut ret, mag.recip()); - return (ret, 0.0); - } - - // And finally just give up and make a unit vector along the first - // dimension. This should be exceedingly rare. - ret.fill(0.0); - ret[0] = 1.0; - (ret, 0.0) -} - -fn rand_f64() -> f64 { - let mut rng = rand::rng(); - loop { - let f = (rng.random_range::(0..(1u64 << 63u64)) as f64) / ((1u64 << 63u64) as f64); - if f == 1.0 { - continue; - } - return f; - } -} - -const PORTION_TAG: u8 = 1; -const ERROR_TAG: u8 = 2; -const ADJUSTMENT_TAG: u8 = 3; -const HEIGHT_TAG: u8 = 4; -const PORTION_BYTE: u8 = merge(WireType::Fixed64, PORTION_TAG); -const ERROR_BYTE: u8 = merge(WireType::Fixed64, ERROR_TAG); -const ADJUSTMENT_BYTE: u8 = merge(WireType::Fixed64, ADJUSTMENT_TAG); -const HEIGHT_BYTE: u8 = merge(WireType::Fixed64, HEIGHT_TAG); - -/// The reference type to [`Coordinate`]. -#[derive(Copy, Clone, Debug, PartialEq)] -pub struct CoordinateRef<'a> { - portion: RepeatedDecoder<'a>, - error: f64, - adjustment: f64, - height: f64, -} - -impl<'a> DataRef<'a, Coordinate> for CoordinateRef<'a> { - fn decode(buf: &'a [u8]) -> Result<(usize, Self), DecodeError> - where - Self: Sized, - { - let mut offset = 0; - let buf_len = buf.len(); - - let mut portion_offsets = None; - let mut num_portions = 0; - let mut error = None; - let mut adjustment = None; - let mut height = None; - - while offset < buf_len { - match buf[offset] { - PORTION_BYTE => { - let readed = skip("Coordinate", &buf[offset..])?; - if let Some((ref mut fnso, ref mut lnso)) = portion_offsets { - if *fnso > offset { - *fnso = offset; - } - - if *lnso < offset + readed { - *lnso = offset + readed; - } - } else { - portion_offsets = Some((offset, offset + readed)); - } - num_portions += 1; - offset += readed; - } - ERROR_BYTE => { - if error.is_some() { - return Err(DecodeError::duplicate_field( - "Coordinate", - "error", - ERROR_TAG, - )); - } - offset += 1; - - let (len, val) = ::decode(&buf[offset..])?; - offset += len; - error = Some(val); - } - ADJUSTMENT_BYTE => { - if adjustment.is_some() { - return Err(DecodeError::duplicate_field( - "Coordinate", - "adjustment", - ADJUSTMENT_TAG, - )); - } - offset += 1; - - let (len, val) = ::decode(&buf[offset..])?; - offset += len; - adjustment = Some(val); - } - HEIGHT_BYTE => { - if height.is_some() { - return Err(DecodeError::duplicate_field( - "Coordinate", - "height", - HEIGHT_TAG, - )); - } - offset += 1; - - let (len, val) = ::decode(&buf[offset..])?; - offset += len; - height = Some(val); - } - _ => offset += skip("Coordinate", &buf[offset..])?, - } - } - - Ok(( - offset, - Self { - portion: if let Some((start, end)) = portion_offsets { - RepeatedDecoder::new(PORTION_TAG, WireType::Fixed64, buf) - .with_nums(num_portions) - .with_offsets(start, end) - } else { - RepeatedDecoder::new(PORTION_TAG, WireType::Fixed64, buf) - }, - error: error.ok_or_else(|| DecodeError::missing_field("Coordinate", "error"))?, - adjustment: adjustment - .ok_or_else(|| DecodeError::missing_field("Coordinate", "adjustment"))?, - height: height.ok_or_else(|| DecodeError::missing_field("Coordinate", "height"))?, - }, - )) - } -} - -impl Data for Coordinate { - type Ref<'a> = CoordinateRef<'a>; - - fn from_ref(val: Self::Ref<'_>) -> Result - where - Self: Sized, - { - val - .portion - .iter::() - .collect::, _>>() - .map(|portion| Self { - portion, - error: val.error, - adjustment: val.adjustment, - height: val.height, - }) - } - - fn encoded_len(&self) -> usize { - self - .portion - .iter() - .fold(0, |acc, x| acc + 1 + x.encoded_len_with_length_delimited()) - + 1 - + self.error.encoded_len() - + 1 - + self.adjustment.encoded_len() - + 1 - + self.height.encoded_len() - } - - fn encode(&self, buf: &mut [u8]) -> Result { - macro_rules! bail { - ($this:ident($offset:expr, $len:ident)) => { - if $offset >= $len { - return Err(EncodeError::insufficient_buffer(self.encoded_len(), $len)); - } - }; - } - - let mut offset = 0; - let buf_len = buf.len(); - for x in self.portion.iter() { - bail!(self(offset, buf_len)); - buf[offset] = PORTION_BYTE; - offset += 1; - offset += x - .encode(&mut buf[offset..]) - .map_err(|e| e.update(self.encoded_len(), buf_len))?; - } - - bail!(self(offset, buf_len)); - buf[offset] = ERROR_BYTE; - offset += 1; - offset += self.error.encode(&mut buf[offset..])?; - - bail!(self(offset, buf_len)); - buf[offset] = ADJUSTMENT_BYTE; - offset += 1; - offset += self.adjustment.encode(&mut buf[offset..])?; - - bail!(self(offset, buf_len)); - buf[offset] = HEIGHT_BYTE; - offset += 1; - offset += self.height.encode(&mut buf[offset..])?; - - Ok(offset) - } -} - -/// Error type for the [`Coordinate`]. -#[derive(Debug, thiserror::Error, PartialEq, Eq)] -pub enum CoordinateError { - /// Returned when the dimensions of the coordinates are not compatible. - #[error("dimensions aren't compatible")] - DimensionalityMismatch, - /// Returned when the coordinate is invalid. - #[error("invalid coordinate")] - InvalidCoordinate, - /// Returned when the round trip time is not in a valid range. - #[error("round trip time not in valid range, duration {0:?} is not a value less than 10s")] - InvalidRTT(Duration), -} - -/// Used to record events that occur when updating coordinates. -#[viewit::viewit(setters(prefix = "with"))] -#[derive(Debug, Copy, Clone, Eq, PartialEq)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub struct CoordinateClientStats { - /// Incremented any time we reset our local coordinate because - /// our calculations have resulted in an invalid state. - #[viewit( - getter( - const, - style = "move", - attrs( - doc = "Returns the number of times we reset our local coordinate because our calculations have resulted in an invalid state." - ) - ), - setter(attrs( - doc = "Sets the number of times we reset our local coordinate because our calculations have resulted in an invalid state." - )) - )] - resets: usize, -} - -impl Default for CoordinateClientStats { - #[inline] - fn default() -> Self { - Self::new() - } -} - -impl CoordinateClientStats { - #[inline] - const fn new() -> Self { - Self { resets: 0 } - } -} - -struct CoordinateClientInner { - /// The current estimate of the client's network coordinate. - coord: Coordinate, - - /// Origin is a coordinate sitting at the origin. - origin: Coordinate, - - /// Contains the tuning parameters that govern the performance of - /// the algorithm. - opts: CoordinateOptions, - - /// The current index into the adjustmentSamples slice. - adjustment_index: usize, - - /// Used to store samples for the adjustment calculation. - adjustment_samples: SmallVec<[f64; DEFAULT_ADJUSTMENT_WINDOW_SIZE]>, - - /// Used to store the last several RTT samples, - /// keyed by node name. We will use the config's LatencyFilterSamples - /// value to determine how many samples we keep, per node. - latency_filter_samples: HashMap>, -} - -impl CoordinateClientInner -where - I: CheapClone + Eq + core::hash::Hash, -{ - /// Applies a small amount of gravity to pull coordinates towards - /// the center of the coordinate system to combat drift. This assumes that the - /// mutex is locked already. - #[inline] - fn update_gravity(&mut self) { - let dist = self.origin.distance_to(&self.coord).as_secs(); - let force = -f64::powf((dist as f64) / self.opts.gravity_rho, 2.0); - self - .coord - .apply_force_in_place(self.opts.height_min, force, &self.origin); - } - - #[inline] - fn latency_filter(&mut self, node: &I, rtt_seconds: f64) -> f64 { - let samples = self - .latency_filter_samples - .entry(node.cheap_clone()) - .or_insert_with(|| SmallVec::with_capacity(self.opts.latency_filter_size)); - - // Add the new sample and trim the list, if needed. - samples.push(rtt_seconds); - if samples.len() > self.opts.latency_filter_size { - samples.remove(0); - } - // Sort a copy of the samples and return the median. - let mut tmp = SmallVec::<[f64; DEFAULT_LATENCY_FILTER_SAMPLES_SIZE]>::from_slice(samples); - tmp.sort_unstable_by(|a, b| a.partial_cmp(b).unwrap_or(std::cmp::Ordering::Equal)); - tmp[tmp.len() / 2] - } - - /// Updates the Vivaldi portion of the client's coordinate. This - /// assumes that the mutex has been locked already. - fn update_vivaldi(&mut self, other: &Coordinate, mut rtt_seconds: f64) { - const ZERO_THRESHOLD: f64 = 1.0e-6; - - let dist = self.coord.distance_to(other).as_secs_f64(); - rtt_seconds = rtt_seconds.max(ZERO_THRESHOLD); - - let wrongness = ((dist - rtt_seconds) / rtt_seconds).abs(); - - let total_error = (self.coord.error + other.error).max(ZERO_THRESHOLD); - - let weight = self.coord.error / total_error; - self.coord.error = ((self.opts.vivaldi_ce * weight * wrongness) - + (self.coord.error * (1.0 - self.opts.vivaldi_ce * weight))) - .min(self.opts.vivaldi_error_max); - - let force = self.opts.vivaldi_cc * weight * (rtt_seconds - dist); - self - .coord - .apply_force_in_place(self.opts.height_min, force, other); - } - - /// Updates the adjustment portion of the client's coordinate, if - /// the feature is enabled. This assumes that the mutex has been locked already. - fn update_adjustment(&mut self, other: &Coordinate, rtt_seconds: f64) { - if self.opts.adjustment_window_size == 0 { - return; - } - // Note that the existing adjustment factors don't figure in to this - // calculation so we use the raw distance here. - let dist = self.coord.raw_distance_to(other); - self.adjustment_samples[self.adjustment_index] = rtt_seconds - dist; - self.adjustment_index = (self.adjustment_index + 1) % self.opts.adjustment_window_size; - - self.coord.adjustment = - self.adjustment_samples.iter().sum::() / (2.0 * self.opts.adjustment_window_size as f64); - } -} - -/// Manages the estimated network coordinate for a given node, and adjusts -/// it as the node observes round trip times and estimated coordinates from other -/// nodes. The core algorithm is based on Vivaldi, see the documentation for Config -/// for more details. -/// -/// `CoordinateClient` is thread-safe. -// TODO: are there any better ways to avoid using a RwLock? -pub struct CoordinateClient { - inner: RwLock>, - /// Used to record events that occur when updating coordinates. - stats: AtomicUsize, -} - -impl Default for CoordinateClient { - #[inline] - fn default() -> Self { - Self::new() - } -} - -impl CoordinateClient { - /// Creates a new client. - #[inline] - pub fn new() -> Self { - Self { - inner: RwLock::new(CoordinateClientInner { - coord: Coordinate::new(), - origin: Coordinate::new(), - opts: CoordinateOptions::new(), - adjustment_index: 0, - adjustment_samples: SmallVec::from_slice(&[0.0; DEFAULT_ADJUSTMENT_WINDOW_SIZE]), - latency_filter_samples: HashMap::new(), - }), - stats: AtomicUsize::new(0), - } - } - - /// Creates a new client with given options. - #[inline] - pub fn with_options(opts: CoordinateOptions) -> Self { - let mut samples = SmallVec::with_capacity(opts.adjustment_window_size); - samples.resize(opts.adjustment_window_size, 0.0); - Self { - inner: RwLock::new(CoordinateClientInner { - coord: Coordinate::with_options(opts.clone()), - origin: Coordinate::with_options(opts.clone()), - opts, - adjustment_index: 0, - adjustment_samples: samples, - latency_filter_samples: HashMap::new(), - }), - stats: AtomicUsize::new(0), - } - } - - /// Returns a copy of the coordinate for this client. - #[inline] - pub fn get_coordinate(&self) -> Coordinate { - self.inner.read().coord.clone() - } - - /// Forces the client's coordinate to a known state. - #[inline] - pub fn set_coordinate(&self, coord: Coordinate) -> Result<(), CoordinateError> { - let mut l = self.inner.write(); - Self::check_coordinate(&l.coord, &coord).map(|_| l.coord = coord) - } - - /// Returns a copy of stats for the client. - #[inline] - pub fn stats(&self) -> CoordinateClientStats { - CoordinateClientStats { - resets: self.stats.load(Ordering::Relaxed), - } - } - - /// Returns the estimated RTT from the client's coordinate to other, the - /// coordinate for another node. - #[inline] - pub fn distance_to(&self, coord: &Coordinate) -> Duration { - self.inner.read().coord.distance_to(coord) - } - - /// Returns an error if the coordinate isn't compatible with - /// this client, or if the coordinate itself isn't valid. This assumes the mutex - /// has been locked already. - #[inline] - fn check_coordinate(this: &Coordinate, coord: &Coordinate) -> Result<(), CoordinateError> { - if !this.is_compatible_with(coord) { - return Err(CoordinateError::DimensionalityMismatch); - } - - if !coord.is_valid() { - return Err(CoordinateError::InvalidCoordinate); - } - - Ok(()) - } -} - -impl CoordinateClient -where - I: CheapClone + Eq + core::hash::Hash, -{ - /// Removes any client state for the given node. - #[inline] - pub fn forget_node(&self, node: &I) { - self.inner.write().latency_filter_samples.remove(node); - } - - /// Update takes other, a coordinate for another node, and rtt, a round trip - /// time observation for a ping to that node, and updates the estimated position of - /// the client's coordinate. Returns the updated coordinate. - pub fn update( - &self, - node: &I, - other: &Coordinate, - rtt: Duration, - ) -> Result { - let mut l = self.inner.write(); - Self::check_coordinate(&l.coord, other)?; - - // The code down below can handle zero RTTs, which we have seen in - // https://github.com/hashicorp/consul/issues/3789, presumably in - // environments with coarse-grained monotonic clocks (we are still - // trying to pin this down). In any event, this is ok from a code PoV - // so we don't need to alert operators with spammy messages. We did - // add a counter so this is still observable, though. - const MAX_RTT: Duration = Duration::from_secs(10); - - if rtt > MAX_RTT { - return Err(CoordinateError::InvalidRTT(rtt)); - } - - #[cfg(feature = "metrics")] - if rtt.is_zero() { - metrics::counter!("serf.coordinate.zero-rtt", l.opts.metric_labels.iter()).increment(1); - } - - let rtt_seconds = l.latency_filter(node, rtt.as_secs_f64()); - l.update_vivaldi(other, rtt_seconds); - l.update_adjustment(other, rtt_seconds); - l.update_gravity(); - - if !l.coord.is_valid() { - self.stats.fetch_add(1, Ordering::Acquire); - l.coord = Coordinate::with_options(l.opts.clone()); - } - - Ok(l.coord.clone()) - } -} - -#[cfg(test)] -mod tests { - use smol_str::SmolStr; - - use super::*; - - fn verify_equal_floats(f1: f64, f2: f64) { - if (f1 - f2).abs() > ZERO_THRESHOLD { - panic!("Equal assertion fail, {:9.6} != {:9.6}", f1, f2); - } - } - - fn verify_equal_vectors(vec1: &[f64], vec2: &[f64]) { - if vec1.len() != vec2.len() { - panic!("Vector length mismatch, {} != {}", vec1.len(), vec2.len()); - } - - for (v1, v2) in vec1.iter().zip(vec2.iter()) { - verify_equal_floats(*v1, *v2); - } - } - - #[test] - fn test_client_update() { - let cfg = CoordinateOptions::default().with_dimensionality(3); - - let client = CoordinateClient::with_options(cfg.clone()); - - let c = client.get_coordinate(); - verify_equal_vectors(&c.portion, [0.0, 0.0, 0.0].as_slice()); - - // Place a node right above the client and observe an RTT longer than the - // client expects, given its distance. - let mut other = Coordinate::with_options(cfg.clone()); - other.portion[2] = 0.001; - - let rtt = Duration::from_nanos((2.0 * other.portion[2] * 1.0e9) as u64); - let mut c = client.update(&SmolStr::from("other"), &other, rtt).unwrap(); - - // The client should have scooted down to get away from it. - assert!(c.portion[2] < 0.0); - - // Set the coordinate to a known state. - c.portion[2] = 99.0; - client.set_coordinate(c.clone()).unwrap(); - let c = client.get_coordinate(); - verify_equal_floats(c.portion[2], 99.0); - } - - #[test] - fn test_client_invalid_in_ping_values() { - let cfg = CoordinateOptions::default().with_dimensionality(3); - - let client = CoordinateClient::with_options(cfg.clone()); - - // Place another node - let mut other = Coordinate::with_options(cfg); - other.portion[2] = 0.001; - let dist = client.distance_to(&other); - - // Update with a series of invalid ping periods, should return an error and estimated rtt remains unchanged - let pings = [9223372036854775807f64, -35f64, 11f64]; - for p in pings { - client - .update( - &SmolStr::from("node"), - &other, - Duration::from_nanos((p as i64).wrapping_mul(SECONDS_TO_NANOSECONDS as i64) as u64), - ) - .unwrap_err(); - - let dist_new = client.distance_to(&other); - assert_eq!(dist_new, dist); - } - } - - #[test] - fn test_client_distance_to() { - let cfg = CoordinateOptions::default() - .with_dimensionality(3) - .with_height_min(0f64); - - let client = CoordinateClient::::with_options(cfg.clone()); - - // Fiddle a raw coordinate to put it a specific number of seconds away. - let mut other = Coordinate::with_options(cfg); - other.portion[2] = 12.345; - let expected = Duration::from_nanos((other.portion[2] * SECONDS_TO_NANOSECONDS) as u64); - let dist = client.distance_to(&other); - assert_eq!(dist, expected); - } - - #[test] - fn test_client_latency_filter() { - let cfg = CoordinateOptions::default().with_latency_filter_size(3); - - let client = CoordinateClient::with_options(cfg); - - // Make sure we get the median, and that things age properly. - verify_equal_floats( - client - .inner - .write() - .latency_filter(&SmolStr::from("alice"), 0.201), - 0.201, - ); - verify_equal_floats( - client - .inner - .write() - .latency_filter(&SmolStr::from("alice"), 0.200), - 0.201, - ); - verify_equal_floats( - client - .inner - .write() - .latency_filter(&SmolStr::from("alice"), 0.207), - 0.201, - ); - - // This glitch will get median-ed out and never seen by Vivaldi. - verify_equal_floats( - client - .inner - .write() - .latency_filter(&SmolStr::from("alice"), 1.9), - 0.207, - ); - verify_equal_floats( - client - .inner - .write() - .latency_filter(&SmolStr::from("alice"), 0.203), - 0.207, - ); - verify_equal_floats( - client - .inner - .write() - .latency_filter(&SmolStr::from("alice"), 0.199), - 0.203, - ); - verify_equal_floats( - client - .inner - .write() - .latency_filter(&SmolStr::from("alice"), 0.211), - 0.203, - ); - - // Make sure different nodes are not coupled. - verify_equal_floats( - client - .inner - .write() - .latency_filter(&SmolStr::from("bob"), 0.310), - 0.310, - ); - - // Make sure we don't leak coordinates for nodes that leave. - client.forget_node(&SmolStr::from("alice")); - verify_equal_floats( - client - .inner - .write() - .latency_filter(&SmolStr::from("alice"), 0.888), - 0.888, - ); - } - - #[test] - fn test_client_nan_defense() { - let cfg = CoordinateOptions::default().with_dimensionality(3); - - let client = CoordinateClient::with_options(cfg.clone()); - - // Block a bad coordinate from coming in. - let mut other = Coordinate::with_options(cfg.clone()); - other.portion[0] = f64::NAN; - assert!(!other.is_valid()); - - let rtt = Duration::from_millis(250); - let c = client - .update(&SmolStr::from("node"), &other, rtt) - .unwrap_err(); - assert_eq!(c, CoordinateError::InvalidCoordinate); - let c = client.get_coordinate(); - assert!(c.is_valid()); - - // Block setting an incompatible coordinate directly. - other.portion.resize(other.portion.len() * 2, 0.0); - let e = client.set_coordinate(other).unwrap_err(); - assert_eq!(e, CoordinateError::DimensionalityMismatch); - let c = client.get_coordinate(); - assert!(c.is_valid()); - - // Poison the internal state and make sure we reset on an update. - client.inner.write().coord.portion[0] = f64::NAN; - let other = Coordinate::with_options(cfg); - let c = client.update(&SmolStr::from("node"), &other, rtt).unwrap(); - assert!(c.is_valid()); - assert_eq!(client.stats().resets, 1); - } - - #[test] - fn test_coordinate_new() { - let opts = CoordinateOptions::default(); - let c = Coordinate::with_options(opts.clone()); - assert_eq!(opts.dimensionality, c.portion.len()); - } - - #[test] - fn test_coordinate_is_valid() { - let c = Coordinate::new(); - let mut fields = vec![]; - for i in 0..c.portion.len() { - fields.push(c.portion[i]); - } - fields.push(c.error); - fields.push(c.adjustment); - fields.push(c.height); - - for field in fields.iter_mut() { - assert!(c.is_valid()); - *field = f64::NAN; - } - } - - #[test] - fn test_coordinate_is_compatible_with() { - let cfg = CoordinateOptions::default().with_dimensionality(3); - - let c1 = Coordinate::with_options(cfg.clone()); - let c2 = Coordinate::with_options(cfg.clone()); - let cfg = cfg.with_dimensionality(2); - let alien = Coordinate::with_options(cfg); - - assert!(c1.is_compatible_with(&c2)); - assert!(!c1.is_compatible_with(&alien)); - assert!(c2.is_compatible_with(&c1)); - assert!(!c2.is_compatible_with(&alien)); - } - - #[test] - #[should_panic(expected = "coordinate dimensionality does not match")] - fn test_coordinate_apply_force() { - let cfg = CoordinateOptions::default() - .with_dimensionality(3) - .with_height_min(0f64); - - let origin = Coordinate::with_options(cfg.clone()); - - // This proves that we normalize, get the direction right, and apply the - // force multiplier correctly. - let mut above = Coordinate::with_options(cfg.clone()); - above.portion[0] = 0.0; - above.portion[1] = 0.0; - above.portion[2] = 2.9; - let c = origin.apply_force(cfg.height_min, 5.3, &above); - - verify_equal_vectors(&c.portion, [0.0, 0.0, -5.3].as_slice()); - - // Scoot a point not starting at the origin to make sure there's nothing - // special there. - let mut right = Coordinate::with_options(cfg.clone()); - right.portion[0] = 3.4; - right.portion[1] = 0.0; - right.portion[2] = -5.3; - let c = c.apply_force(cfg.height_min, 2.0, &right); - verify_equal_vectors(&c.portion, [-2.0, 0.0, -5.3].as_slice()); - - // If the points are right on top of each other, then we should end up - // in a random direction, one unit away. This makes sure the unit vector - // build up doesn't divide by zero. - let c = origin.apply_force(cfg.height_min, 1.0, &origin); - verify_equal_floats(origin.distance_to(&c).as_secs_f64(), 1.0); - - // Enable a minimum height and make sure that gets factored in properly. - let cfg = cfg.with_height_min(10.0e-6); - let origin = Coordinate::with_options(cfg.clone()); - let c = origin.apply_force(cfg.height_min, 5.3, &above); - verify_equal_vectors(&c.portion, [0.0, 0.0, -5.3].as_slice()); - verify_equal_floats(c.height, cfg.height_min + 5.3 * cfg.height_min / 2.9); - - // Make sure the height minimum is enforced. - let c = origin.apply_force(cfg.height_min, -5.3, &above); - verify_equal_vectors(&c.portion, [0.0, 0.0, 5.3].as_slice()); - verify_equal_floats(c.height, cfg.height_min); - - // Shenanigans should get called if the dimensions don't match. - let mut bad = c.clone(); - bad.portion = SmallVec::from_slice(&vec![0.0; bad.portion.len() + 1]); - c.apply_force(cfg.height_min, 1.0, &bad); - } - - #[test] - fn test_coordinate_add() { - let mut vec1 = [1.0, -3.0, 3.0]; - let vec2 = [-4.0, 5.0, 6.0]; - add_in_place(&mut vec1, &vec2); - verify_equal_vectors(&vec1, [-3.0, 2.0, 9.0].as_slice()); - - let zero = [0.0; 3]; - let mut vec1 = [1.0, -3.0, 3.0]; - add_in_place(&mut vec1, &zero); - verify_equal_vectors(&[1.0, -3.0, 3.0], vec1.as_slice()); - } - - #[test] - fn test_coordinate_diff() { - let vec1 = [1.0, -3.0, 3.0]; - let vec2 = [-4.0, 5.0, 6.0]; - verify_equal_vectors(diff(&vec1, &vec2).as_slice(), [5.0, -8.0, -3.0].as_slice()); - - let zero = [0.0; 3]; - verify_equal_vectors(diff(&vec1, &zero).as_slice(), vec1.as_slice()); - } - - #[test] - fn test_coordinate_diff_in_place() { - let vec1 = [1.0, -3.0, 3.0]; - let vec2 = [-4.0, 5.0, 6.0]; - verify_equal_vectors( - &diff_in_place(&vec1, &vec2).collect::>(), - [5.0, -8.0, -3.0].as_slice(), - ); - - let zero = [0.0; 3]; - verify_equal_vectors( - &diff_in_place(&vec1, &zero).collect::>(), - vec1.as_slice(), - ); - } - - #[test] - fn test_coordinate_magnitude() { - let zero = [0.0; 3]; - verify_equal_floats(magnitude_in_place(zero.into_iter()), 0.0); - - let vec = [1.0, -2.0, 3.0]; - verify_equal_floats(magnitude_in_place(vec.into_iter()), 3.7416573867739413); - } - - #[test] - fn test_coordinate_unit_vector_at() { - let vec1 = [1.0, 2.0, 3.0]; - let vec2 = [0.5, 0.6, 0.7]; - let (u, mag) = unit_vector_at(&vec1, &vec2); - verify_equal_vectors( - &u, - [0.18257418583505536, 0.511207720338155, 0.8398412548412546].as_slice(), - ); - verify_equal_floats(magnitude_in_place(u.iter().copied()), 1.0); - let vec1 = [1.0, 2.0, 3.0]; - verify_equal_floats(mag, magnitude_in_place(diff(&vec1, &vec2).into_iter())); - - // If we give positions that are equal we should get a random unit vector - // returned to us, rather than a divide by zero. - let vec1 = [1.0, 2.0, 3.0]; - let (u, mag) = unit_vector_at(&vec1, &vec1); - verify_equal_floats(mag, 0.0); - verify_equal_floats(magnitude_in_place(u.iter().copied()), 1.0); - } -} diff --git a/legacy/serf-core/src/types/filter.rs b/legacy/serf-core/src/types/filter.rs deleted file mode 100644 index bf258d25..00000000 --- a/legacy/serf-core/src/types/filter.rs +++ /dev/null @@ -1,272 +0,0 @@ -use memberlist_core::proto::{ - Data, DataRef, DecodeError, EncodeError, RepeatedDecoder, TinyVec, WireType, - utils::{merge, skip}, -}; - -pub use tag_filter::*; -mod tag_filter; - -/// The type of filter -#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash, derive_more::IsVariant, derive_more::Display)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[non_exhaustive] -pub enum FilterType { - /// Filter by node ids - #[display("id")] - Id, - /// Filter by tag - #[display("tag")] - Tag, - /// Unknown filter type - #[display("unknown({_0})")] - Unknown(u8), -} - -impl FilterType { - /// Get the string representation of the filter type - #[inline] - pub fn as_str(&self) -> std::borrow::Cow<'static, str> { - std::borrow::Cow::Borrowed(match self { - Self::Id => "id", - Self::Tag => "tag", - Self::Unknown(val) => return std::borrow::Cow::Owned(format!("unknown({})", val)), - }) - } -} - -impl From for FilterType { - fn from(value: u8) -> Self { - match value { - 0 => Self::Id, - 1 => Self::Tag, - val => Self::Unknown(val), - } - } -} - -impl From for u8 { - fn from(val: FilterType) -> Self { - match val { - FilterType::Id => 0, - FilterType::Tag => 1, - FilterType::Unknown(val) => val, - } - } -} - -/// Used with a queryFilter to specify the type of -/// filter we are sending -#[derive( - Debug, - Clone, - PartialEq, - Eq, - derive_more::IsVariant, - derive_more::From, - derive_more::Unwrap, - derive_more::TryUnwrap, -)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[unwrap(ref, ref_mut)] -#[try_unwrap(ref, ref_mut)] -#[non_exhaustive] -pub enum Filter { - /// Filter by node ids - Id(TinyVec), - /// Filter by tag - Tag(TagFilter), -} - -impl Filter { - /// Returns the type of filter - #[inline] - pub const fn ty(&self) -> FilterType { - match self { - Self::Id(_) => FilterType::Id, - Self::Tag { .. } => FilterType::Tag, - } - } -} - -const FILTER_ID_TAG: u8 = 1; -const FILTER_TAG_TAG: u8 = 2; - -/// The reference type to [`Filter`] -#[derive(Clone, Copy, Debug)] -pub enum FilterRef<'a> { - /// Filter by node ids - Id(RepeatedDecoder<'a>), - /// Filter by tag - Tag(TagFilterRef<'a>), -} - -impl<'a, I> DataRef<'a, Filter> for FilterRef<'a> -where - I: Data, -{ - fn decode(buf: &'a [u8]) -> Result<(usize, Self), DecodeError> - where - Self: Sized, - { - let buf_len = buf.len(); - let mut offset = 0; - let mut ids_offsets = None; - let mut num_ids = 0; - let mut f = None; - - while offset < buf_len { - match buf[offset] { - val if val == Filter::::id_byte() => { - let readed = skip("Filter", &buf[offset..])?; - if let Some((ref mut fnso, ref mut lnso)) = ids_offsets { - if *fnso > offset { - *fnso = offset; - } - - if *lnso < offset + readed { - *lnso = offset + readed; - } - } else { - ids_offsets = Some((offset, offset + readed)); - } - num_ids += 1; - offset += readed; - } - val if val == Filter::::tag_byte() => { - if let Some(Self::Tag(_)) = f { - return Err(DecodeError::duplicate_field( - "Filter", - "tag", - FILTER_TAG_TAG, - )); - } - - if ids_offsets.is_some() { - return Err(DecodeError::duplicate_field("Filter", "id", FILTER_ID_TAG)); - } - - offset += 1; - let (read, tag) = - >::decode_length_delimited(&buf[offset..])?; - offset += read; - f = Some(FilterRef::Tag(tag)); - } - _ => offset += skip("Filter", &buf[offset..])?, - } - } - - Ok(( - offset, - if let Some(tag) = f { - tag - } else if let Some((start, end)) = ids_offsets { - Self::Id( - RepeatedDecoder::new(FILTER_ID_TAG, I::WIRE_TYPE, buf) - .with_nums(num_ids) - .with_offsets(start, end), - ) - } else { - Self::Id(RepeatedDecoder::new(FILTER_ID_TAG, I::WIRE_TYPE, buf).with_nums(0)) - }, - )) - } -} - -impl Filter -where - I: Data, -{ - const fn id_byte() -> u8 { - merge(I::WIRE_TYPE, FILTER_ID_TAG) - } - - const fn tag_byte() -> u8 { - merge(WireType::LengthDelimited, FILTER_TAG_TAG) - } -} - -impl Data for Filter -where - I: Data, -{ - type Ref<'a> = FilterRef<'a>; - - fn from_ref(val: Self::Ref<'_>) -> Result - where - Self: Sized, - { - match val { - FilterRef::Id(decoder) => decoder - .iter::() - .map(|res| res.and_then(I::from_ref)) - .collect::>() - .map(Self::Id), - FilterRef::Tag(tag) => TagFilter::from_ref(tag).map(Self::Tag), - } - } - - fn encoded_len(&self) -> usize { - match self { - Filter::Id(ids) => ids - .iter() - .map(|id| 1 + id.encoded_len_with_length_delimited()) - .sum::(), - Filter::Tag(tag) => 1 + tag.encoded_len_with_length_delimited(), - } - } - - fn encode(&self, buf: &mut [u8]) -> Result { - let buf_len = buf.len(); - if buf_len < 1 { - return Err(EncodeError::insufficient_buffer( - self.encoded_len(), - buf_len, - )); - } - - let mut offset = 0; - - match self { - Filter::Id(ids) => { - ids - .iter() - .try_fold(&mut offset, |offset, id| { - if *offset >= buf_len { - return Err(EncodeError::insufficient_buffer( - self.encoded_len(), - buf_len, - )); - } - - buf[*offset] = Self::id_byte(); - *offset += 1; - - *offset += id.encode_length_delimited(&mut buf[*offset..])?; - - Ok(offset) - }) - .map_err(|e: EncodeError| e.update(self.encoded_len(), buf_len))?; - - Ok(offset) - } - Filter::Tag(tag) => { - buf[offset] = Self::tag_byte(); - offset += 1; - - if offset > buf_len { - return Err(EncodeError::insufficient_buffer( - self.encoded_len(), - buf_len, - )); - } - - offset += tag - .encode_length_delimited(&mut buf[offset..]) - .map_err(|e| e.update(self.encoded_len(), buf_len))?; - - Ok(offset) - } - } - } -} diff --git a/legacy/serf-core/src/types/filter/tag_filter.rs b/legacy/serf-core/src/types/filter/tag_filter.rs deleted file mode 100644 index 7d69dcdb..00000000 --- a/legacy/serf-core/src/types/filter/tag_filter.rs +++ /dev/null @@ -1,228 +0,0 @@ -use memberlist_core::proto::{ - Data, DataRef, DecodeError, EncodeError, WireType, - utils::{merge, skip}, -}; -use regex::Regex; -use smol_str::SmolStr; - -const TAG_TAG: u8 = 1; -const EXPR_TAG: u8 = 2; -const TAG_BYTE: u8 = merge(WireType::LengthDelimited, TAG_TAG); -const EXPR_BYTE: u8 = merge(WireType::LengthDelimited, EXPR_TAG); - -#[viewit::viewit(vis_all = "", getters(vis_all = "pub"), setters(skip))] -/// The reference type of the [`TagFilter`] type -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct TagFilterRef<'a> { - #[viewit(getter(const, attrs(doc = "Returns the tag")))] - tag: &'a str, - #[viewit(getter(const, attrs(doc = "Returns the expression")))] - expr: Option<&'a str>, -} - -impl<'a> DataRef<'a, TagFilter> for TagFilterRef<'a> { - fn decode(src: &'a [u8]) -> Result<(usize, Self), DecodeError> - where - Self: Sized, - { - let mut offset = 0; - let buf_len = src.len(); - let mut tag = None; - let mut expr = None; - - while offset < buf_len { - match src[offset] { - TAG_BYTE => { - if tag.is_some() { - return Err(DecodeError::duplicate_field("TagFilter", "tag", TAG_TAG)); - } - offset += 1; - - let (read, value) = - <&str as DataRef<'_, SmolStr>>::decode_length_delimited(&src[offset..])?; - offset += read; - tag = Some(value); - } - EXPR_BYTE => { - if expr.is_some() { - return Err(DecodeError::duplicate_field("TagFilter", "expr", EXPR_TAG)); - } - offset += 1; - - let (read, value) = - <&str as DataRef<'_, SmolStr>>::decode_length_delimited(&src[offset..])?; - offset += read; - expr = Some(value); - } - _ => offset += skip("TagFilter", &src[offset..])?, - } - } - - Ok(( - offset, - Self { - tag: tag.unwrap_or(""), - expr: expr.and_then(|expr| if expr.is_empty() { None } else { Some(expr) }), - }, - )) - } -} - -/// The tag filter -#[viewit::viewit( - vis_all = "", - getters(vis_all = "pub", style = "ref"), - setters(vis_all = "pub", prefix = "with") -)] -#[derive(Debug, Clone)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub struct TagFilter { - #[viewit( - getter(const, attrs(doc = "Returns the tag")), - setter(attrs(doc = "Sets the tag (Builder pattern)")) - )] - tag: SmolStr, - #[cfg_attr(feature = "serde", serde(with = "serde_regex"))] - #[viewit( - getter( - const, - attrs(doc = "Returns the expression"), - result(converter(fn = "Option::as_ref"), type = "Option<&Regex>"), - ), - setter( - rename = "maybe_expr", - attrs(doc = "Sets the expression (Builder pattern)") - ) - )] - expr: Option, -} - -impl Default for TagFilter { - fn default() -> Self { - Self::new() - } -} - -impl TagFilter { - /// Creates a new tag filter - #[inline] - pub const fn new() -> Self { - Self { - tag: SmolStr::new_inline(""), - expr: None, - } - } - - /// Set the expression for the tag filter - #[inline] - pub fn with_expr(mut self, expr: Regex) -> Self { - self.expr = Some(expr); - self - } -} - -impl PartialEq for TagFilter { - fn eq(&self, other: &Self) -> bool { - self.tag == other.tag - && self.expr.as_ref().map(|re| re.as_str()) == other.expr.as_ref().map(|re| re.as_str()) - } -} - -impl Eq for TagFilter {} - -impl Data for TagFilter { - type Ref<'a> = TagFilterRef<'a>; - - fn from_ref(val: Self::Ref<'_>) -> Result - where - Self: Sized, - { - Ok(Self { - tag: SmolStr::from(val.tag), - expr: val - .expr - .map(|expr| Regex::new(expr).map_err(|e| DecodeError::custom(e.to_string()))) - .transpose()?, - }) - } - - fn encoded_len(&self) -> usize { - 1 + self.tag.encoded_len_with_length_delimited() - + match self.expr.as_ref() { - Some(re) => { - let re = re.as_str(); - let len = re.len(); - 1 + (len as u32).encoded_len() + len - } - None => 0, - } - } - - fn encode(&self, buf: &mut [u8]) -> Result { - let buf_len = buf.len(); - let mut offset = 0; - - if buf_len <= offset { - return Err(EncodeError::insufficient_buffer( - self.encoded_len(), - buf_len, - )); - } - - buf[offset] = TAG_BYTE; - offset += 1; - offset += self - .tag - .encode_length_delimited(&mut buf[offset..]) - .map_err(|e| e.update(self.encoded_len(), buf_len))?; - - if let Some(re) = self.expr.as_ref() { - let re = re.as_str(); - let len = re.len(); - if buf_len < offset + 1 + (len as u32).encoded_len() + len { - return Err(EncodeError::insufficient_buffer( - self.encoded_len(), - buf_len, - )); - } - - buf[offset] = EXPR_BYTE; - offset += 1; - offset += (len as u32).encode(&mut buf[offset..])?; - buf[offset..offset + len].copy_from_slice(re.as_bytes()); - offset += len; - } - - #[cfg(debug_assertions)] - super::super::debug_assert_write_eq::(offset, self.encoded_len()); - - Ok(offset) - } -} - -#[cfg(feature = "serde")] -mod serde_regex { - use regex::Regex; - use serde::{de, ser}; - - pub fn serialize(value: &Option, serializer: S) -> Result - where - S: ser::Serializer, - { - match value { - Some(re) => serializer.serialize_str(re.as_str()), - None => serializer.serialize_none(), - } - } - - pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error> - where - D: de::Deserializer<'de>, - { - let s = as de::Deserialize<'_>>::deserialize(deserializer)?; - match s { - Some(s) => s.try_into().map(Some).map_err(de::Error::custom), - None => Ok(None), - } - } -} diff --git a/legacy/serf-core/src/types/fuzzy.rs b/legacy/serf-core/src/types/fuzzy.rs deleted file mode 100644 index d1399b16..00000000 --- a/legacy/serf-core/src/types/fuzzy.rs +++ /dev/null @@ -1,200 +0,0 @@ -use super::*; - -use core::hash::Hash; - -use crate::types::bytes::Bytes; -use memberlist_core::proto::Data; - -/// Message for fuzzy testing -#[derive(Clone, Debug)] -pub enum Message { - /// Leave message - Leave(LeaveMessage), - /// Join message - Join(JoinMessage), - /// PushPull message - PushPull(PushPullMessage), - /// UserEvent message - UserEvent(UserEventMessage), - /// Query message - Query(QueryMessage), - /// QueryResponse message - QueryResponse(QueryResponseMessage), - /// ConflictResponse message - ConflictResponse(ConflictResponseMessage), - #[cfg(feature = "encryption")] - /// KeyRequest message - KeyRequest(KeyRequestMessage), - #[cfg(feature = "encryption")] - /// KeyResponse message - KeyResponse(KeyResponseMessage), -} - -#[cfg(feature = "arbitrary")] -const _: () = { - use arbitrary::{Arbitrary, Unstructured}; - - impl<'a, I, A> Arbitrary<'a> for Message - where - I: Arbitrary<'a> + Hash + Eq, - A: Arbitrary<'a>, - { - fn arbitrary(g: &mut Unstructured<'a>) -> arbitrary::Result { - let variant = MessageType::arbitrary(g)?; - - Ok(match variant { - MessageType::ConflictResponse => Message::ConflictResponse(Arbitrary::arbitrary(g)?), - MessageType::Join => Message::Join(Arbitrary::arbitrary(g)?), - MessageType::Leave => Message::Leave(Arbitrary::arbitrary(g)?), - MessageType::PushPull => Message::PushPull(Arbitrary::arbitrary(g)?), - MessageType::Query => Message::Query(Arbitrary::arbitrary(g)?), - MessageType::QueryResponse => Message::QueryResponse(Arbitrary::arbitrary(g)?), - MessageType::UserEvent => Message::UserEvent(Arbitrary::arbitrary(g)?), - #[cfg(feature = "encryption")] - MessageType::KeyRequest => Message::KeyRequest(Arbitrary::arbitrary(g)?), - #[cfg(feature = "encryption")] - MessageType::KeyResponse => Message::KeyResponse(Arbitrary::arbitrary(g)?), - _ => Message::Query(QueryMessage::arbitrary(g)?), - }) - } - } -}; - -#[cfg(feature = "quickcheck")] -const _: () = { - use quickcheck::{Arbitrary, Gen}; - - impl Arbitrary for Message - where - I: Arbitrary + Hash + Eq, - A: Arbitrary, - { - fn arbitrary(g: &mut Gen) -> Self { - loop { - let variant = MessageType::arbitrary(g); - - return match variant { - MessageType::ConflictResponse => Message::ConflictResponse(Arbitrary::arbitrary(g)), - MessageType::Join => Message::Join(Arbitrary::arbitrary(g)), - MessageType::Leave => Message::Leave(Arbitrary::arbitrary(g)), - MessageType::PushPull => Message::PushPull(Arbitrary::arbitrary(g)), - MessageType::Query => Message::Query(Arbitrary::arbitrary(g)), - MessageType::QueryResponse => Message::QueryResponse(Arbitrary::arbitrary(g)), - MessageType::UserEvent => Message::UserEvent(Arbitrary::arbitrary(g)), - #[cfg(feature = "encryption")] - MessageType::KeyRequest => Message::KeyRequest(Arbitrary::arbitrary(g)), - #[cfg(feature = "encryption")] - MessageType::KeyResponse => Message::KeyResponse(Arbitrary::arbitrary(g)), - _ => continue, - }; - } - } - } -}; - -fn encode(data: &T) -> Bytes { - encode_message_to_bytes(data).unwrap() -} - -fn encode_relay(data: &T, node: &Node) -> Bytes -where - I: Data, - A: Data, - T: Encodable, -{ - encode_relay_message_to_bytes(data, node).unwrap() -} - -pub fn encodable_round_trip(msg: Message, node: Option>) -> bool -where - I: Data + Eq + Hash, - A: Data + PartialEq, -{ - macro_rules! encode_variant { - (< $($g:ty), +$(,)? > $variant:ident ($ty:ty) <- $input:ident) => {{ - let data = encode(&$input); - assert_eq!(data.len(), encoded_message_len(&$input), "message: length mismatch"); - let decoded = super::decode_message :: < $($g),* > (&data).unwrap(); - let MessageRef::$variant(decoded) = decoded else { return false }; - - let owned = <$ty as Data>::from_ref(decoded).unwrap(); - assert_eq!($input, owned, "message: decoded mismatch"); - true - }}; - (@relay< $($g:ty), +$(,)? > $variant:ident ($ty:ty) <- ($input:ident, $node:ident)) => {{ - let data = encode_relay(&$input, &$node); - assert_eq!(data.len(), encoded_relay_message_len(&$input, &$node), "relay message: length mismatch"); - let decoded = super::decode_message :: < $($g),* > (&data).unwrap(); - let MessageRef::Relay(RelayMessageRef { node, payload, .. }) = decoded else { return false }; - assert_eq!( as Data>::from_ref(node).unwrap(), $node, "relay message: node mismatch"); - - let decoded = super::decode_message :: < $($g),* > (&payload).unwrap(); - let MessageRef::$variant(decoded) = decoded else { return false }; - - let owned = <$ty as Data>::from_ref(decoded).unwrap(); - assert_eq!($input, owned, "relay message: decoded mismatch"); - true - }}; - } - - match node { - Some(node) => match msg { - Message::Leave(leave_message) => { - encode_variant!(@relay Leave(LeaveMessage) <- (leave_message, node)) - } - Message::Join(join_message) => { - encode_variant!(@relay Join(JoinMessage) <- (join_message, node)) - } - Message::PushPull(push_pull_message) => { - encode_variant!(@relay PushPull(PushPullMessage) <- (push_pull_message, node)) - } - Message::UserEvent(user_event_message) => { - encode_variant!(@relay UserEvent(UserEventMessage) <- (user_event_message, node)) - } - Message::Query(query_message) => { - encode_variant!(@relay Query(QueryMessage) <- (query_message, node)) - } - Message::QueryResponse(query_response_message) => { - encode_variant!(@relay QueryResponse(QueryResponseMessage) <- (query_response_message, node)) - } - Message::ConflictResponse(conflict_response_message) => { - encode_variant!(@relay ConflictResponse(ConflictResponseMessage) <- (conflict_response_message, node)) - } - #[cfg(feature = "encryption")] - Message::KeyRequest(key_request_message) => { - encode_variant!(@relay KeyRequest(KeyRequestMessage) <- (key_request_message, node)) - } - #[cfg(feature = "encryption")] - Message::KeyResponse(key_response_message) => { - encode_variant!(@relay KeyResponse(KeyResponseMessage) <- (key_response_message, node)) - } - }, - None => match msg { - Message::Leave(msg) => encode_variant!( Leave(LeaveMessage) <- msg), - Message::Join(join_message) => encode_variant!( Join(JoinMessage) <- join_message), - Message::PushPull(push_pull_message) => { - encode_variant!( PushPull(PushPullMessage) <- push_pull_message) - } - Message::UserEvent(user_event_message) => { - encode_variant!( UserEvent(UserEventMessage) <- user_event_message) - } - Message::Query(query_message) => { - encode_variant!( Query(QueryMessage) <- query_message) - } - Message::QueryResponse(query_response_message) => { - encode_variant!( QueryResponse(QueryResponseMessage) <- query_response_message) - } - Message::ConflictResponse(conflict_response_message) => { - encode_variant!( ConflictResponse(ConflictResponseMessage) <- conflict_response_message) - } - #[cfg(feature = "encryption")] - Message::KeyRequest(key_request_message) => { - encode_variant!( KeyRequest(KeyRequestMessage) <- key_request_message) - } - #[cfg(feature = "encryption")] - Message::KeyResponse(key_response_message) => { - encode_variant!( KeyResponse(KeyResponseMessage) <- key_response_message) - } - }, - } -} diff --git a/legacy/serf-core/src/types/join.rs b/legacy/serf-core/src/types/join.rs deleted file mode 100644 index f71e0552..00000000 --- a/legacy/serf-core/src/types/join.rs +++ /dev/null @@ -1,159 +0,0 @@ -use memberlist_core::proto::{ - Data, DataRef, DecodeError, EncodeError, WireType, - utils::{merge, skip}, -}; - -use super::LamportTime; - -const LTIME_TAG: u8 = 1; -const LTIME_BYTE: u8 = merge(WireType::Varint, LTIME_TAG); -const ID_TAG: u8 = 2; - -/// The message broadcasted after we join to -/// associated the node with a lamport clock -#[viewit::viewit(setters(prefix = "with"))] -#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -pub struct JoinMessage { - /// The lamport time - #[viewit( - getter( - const, - style = "move", - attrs(doc = "Returns the lamport time for this message") - ), - setter( - const, - attrs(doc = "Sets the lamport time for this message (Builder pattern)") - ) - )] - ltime: LamportTime, - /// The id of the node - #[viewit( - getter(const, style = "ref", attrs(doc = "Returns the node")), - setter(attrs(doc = "Sets the node (Builder pattern)")) - )] - id: I, -} - -impl JoinMessage { - /// Create a new join message - pub fn new(ltime: LamportTime, id: I) -> Self { - Self { ltime, id } - } - - const fn id_byte() -> u8 - where - I: Data, - { - merge(I::WIRE_TYPE, ID_TAG) - } -} - -impl<'a, I> DataRef<'a, JoinMessage> for JoinMessage> -where - I: Data, -{ - fn decode(buf: &'a [u8]) -> Result<(usize, Self), DecodeError> - where - Self: Sized, - { - let mut offset = 0; - let mut ltime = None; - let mut id = None; - - while offset < buf.len() { - match buf[offset] { - LTIME_BYTE => { - if ltime.is_some() { - return Err(DecodeError::duplicate_field( - "JoinMessage", - "ltime", - LTIME_TAG, - )); - } - offset += 1; - - let (read, value) = >::decode(&buf[offset..])?; - offset += read; - ltime = Some(value); - } - b if b == JoinMessage::::id_byte() => { - if id.is_some() { - return Err(DecodeError::duplicate_field("JoinMessage", "id", ID_TAG)); - } - offset += 1; - - let (read, value) = - as DataRef<'_, I>>::decode_length_delimited(&buf[offset..])?; - offset += read; - id = Some(value); - } - _ => offset += skip("JoinMessage", &buf[offset..])?, - } - } - - Ok(( - offset, - Self { - ltime: ltime.ok_or_else(|| DecodeError::missing_field("JoinMessage", "ltime"))?, - id: id.ok_or_else(|| DecodeError::missing_field("JoinMessage", "id"))?, - }, - )) - } -} - -impl Data for JoinMessage -where - I: Data, -{ - type Ref<'a> = JoinMessage>; - - fn from_ref(val: Self::Ref<'_>) -> Result - where - Self: Sized, - { - I::from_ref(val.id).map(|id| Self { - ltime: val.ltime, - id, - }) - } - - fn encoded_len(&self) -> usize { - 1 + self.ltime.encoded_len() + 1 + self.id.encoded_len_with_length_delimited() - } - - fn encode(&self, buf: &mut [u8]) -> Result { - let buf_len = buf.len(); - let mut offset = 0; - - if buf_len < 1 { - return Err(EncodeError::insufficient_buffer( - self.encoded_len(), - buf_len, - )); - } - - buf[offset] = LTIME_BYTE; - offset += 1; - offset += self.ltime.encode(&mut buf[offset..])?; - - if buf_len <= offset { - return Err(EncodeError::insufficient_buffer( - self.encoded_len(), - buf_len, - )); - } - - buf[offset] = Self::id_byte(); - offset += 1; - - offset += self.id.encode_length_delimited(&mut buf[offset..])?; - - #[cfg(debug_assertions)] - super::debug_assert_write_eq::(offset, self.encoded_len()); - - Ok(offset) - } -} diff --git a/legacy/serf-core/src/types/key.rs b/legacy/serf-core/src/types/key.rs deleted file mode 100644 index 040d70f6..00000000 --- a/legacy/serf-core/src/types/key.rs +++ /dev/null @@ -1,452 +0,0 @@ -use indexmap::IndexMap; -use memberlist_core::proto::{ - Data, DataRef, DecodeError, EncodeError, RepeatedDecoder, SecretKey, SecretKeys, WireType, - utils::{merge, skip}, -}; -use smol_str::SmolStr; - -/// KeyRequest is used to contain input parameters which get broadcasted to all -/// nodes as part of a key query operation. -#[viewit::viewit(setters(prefix = "with"))] -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -#[repr(transparent)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "serde", serde(transparent))] -#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -pub struct KeyRequestMessage { - /// The secret key - #[viewit( - getter(const, attrs(doc = "Returns the secret key")), - setter(const, attrs(doc = "Sets the secret key (Builder pattern)")) - )] - key: Option, -} - -const KEY_REQ_KEY_TAG: u8 = 1; -const KEY_REQ_KEY_BYTE: u8 = merge(WireType::LengthDelimited, KEY_REQ_KEY_TAG); - -impl DataRef<'_, Self> for KeyRequestMessage { - fn decode(buf: &'_ [u8]) -> Result<(usize, Self), DecodeError> - where - Self: Sized, - { - let mut offset = 0; - let buf_len = buf.len(); - - let mut key = None; - - while offset < buf_len { - match buf[offset] { - KEY_REQ_KEY_BYTE => { - offset += 1; - - let (bytes_read, val) = ::decode_length_delimited(&buf[offset..])?; - offset += bytes_read; - key = Some(val); - } - _ => offset += skip("KeyRequestMessage", &buf[offset..])?, - } - } - - Ok((offset, Self { key })) - } -} - -impl Data for KeyRequestMessage { - type Ref<'a> = Self; - - fn from_ref(val: Self::Ref<'_>) -> Result - where - Self: Sized, - { - Ok(val) - } - - fn encoded_len(&self) -> usize { - let mut len = 0; - if let Some(key) = &self.key { - len += 1 + key.encoded_len_with_length_delimited(); - } - len - } - - fn encode(&self, buf: &mut [u8]) -> Result { - let buf_len = buf.len(); - let mut offset = 0; - - if let Some(key) = &self.key { - if buf_len < offset + 1 { - return Err(EncodeError::insufficient_buffer( - self.encoded_len(), - buf_len, - )); - } - buf[offset] = KEY_REQ_KEY_BYTE; - offset += 1; - - let bytes_written = key.encode_length_delimited(&mut buf[offset..])?; - offset += bytes_written; - } - - Ok(offset) - } -} - -/// Key response message -#[viewit::viewit(setters(prefix = "with"))] -#[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] -#[cfg(feature = "encryption")] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -pub struct KeyResponseMessage { - /// Indicates true/false if there were errors or not - #[viewit( - getter(const, attrs(doc = "Returns true/false if there were errors or not")), - setter( - const, - attrs(doc = "Sets true/false if there were errors or not (Builder pattern)") - ) - )] - result: bool, - /// Contains error messages or other information - #[viewit( - getter( - const, - style = "ref", - attrs(doc = "Returns the error messages or other information") - ), - setter(attrs(doc = "Sets the error messages or other information (Builder pattern)")) - )] - message: SmolStr, - /// Used in listing queries to relay a list of installed keys - #[viewit( - getter(const, style = "ref", attrs(doc = "Returns a list of installed keys")), - setter(attrs(doc = "Sets the the list of installed keys (Builder pattern)")) - )] - keys: SecretKeys, - /// Used in listing queries to relay the primary key - #[viewit( - getter(const, attrs(doc = "Returns the primary key")), - setter(attrs(doc = "Sets the primary key (Builder pattern)")) - )] - primary_key: Option, -} - -impl KeyResponseMessage { - /// Adds a key to the list of keys - #[inline] - pub fn add_key(&mut self, key: SecretKey) -> &mut Self { - self.keys.push(key); - self - } -} - -const KEY_RESPONSE_RESULT_TAG: u8 = 1; -const KEY_RESPONSE_RESULT_BYTE: u8 = merge(WireType::Byte, KEY_RESPONSE_RESULT_TAG); -const KEY_RESPONSE_MESSAGE_TAG: u8 = 2; -const KEY_RESPONSE_MESSAGE_BYTE: u8 = merge(WireType::LengthDelimited, KEY_RESPONSE_MESSAGE_TAG); -const KEY_RESPONSE_KEYS_TAG: u8 = 3; -const KEY_RESPONSE_KEYS_BYTE: u8 = merge(WireType::LengthDelimited, KEY_RESPONSE_KEYS_TAG); -const KEY_RESPONSE_PRIMARY_KEY_TAG: u8 = 4; -const KEY_RESPONSE_PRIMARY_KEY_BYTE: u8 = - merge(WireType::LengthDelimited, KEY_RESPONSE_PRIMARY_KEY_TAG); - -/// The reference type for [`KeyResponseMessage`]. -#[viewit::viewit(getters(style = "ref", vis_all = "pub"), setters(skip), vis_all = "")] -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub struct KeyResponseMessageRef<'a> { - #[viewit(getter( - const, - style = "move", - attrs(doc = "Returns true/false if there were errors or not") - ))] - result: bool, - #[viewit(getter( - const, - style = "move", - attrs(doc = "Returns the error messages or other information") - ))] - message: &'a str, - #[viewit(getter(const, attrs(doc = "Returns a list of installed keys")))] - keys: RepeatedDecoder<'a>, - #[viewit(getter( - const, - attrs(doc = "Returns the primary key"), - result(converter(fn = "Option::as_ref"), type = "Option<&SecretKey>"), - ))] - primary_key: Option, -} - -impl<'a> DataRef<'a, KeyResponseMessage> for KeyResponseMessageRef<'a> { - fn decode(buf: &'a [u8]) -> Result<(usize, Self), DecodeError> - where - Self: Sized, - { - let mut offset = 0; - let buf_len = buf.len(); - - let mut result = None; - let mut message = None; - let mut keys_offsets = None; - let mut num_keys = 0; - let mut primary_key = None; - - while offset < buf_len { - match buf[offset] { - KEY_RESPONSE_RESULT_BYTE => { - if result.is_some() { - return Err(DecodeError::duplicate_field( - "KeyResponseMessage", - "result", - KEY_RESPONSE_RESULT_TAG, - )); - } - offset += 1; - if offset >= buf_len { - return Err(DecodeError::buffer_underflow()); - } - result = Some(buf[offset] != 0); - offset += 1; - } - KEY_RESPONSE_MESSAGE_BYTE => { - if message.is_some() { - return Err(DecodeError::duplicate_field( - "KeyResponseMessage", - "message", - KEY_RESPONSE_MESSAGE_TAG, - )); - } - - offset += 1; - let (bytes_read, val) = - <&str as DataRef<'_, SmolStr>>::decode_length_delimited(&buf[offset..])?; - offset += bytes_read; - message = Some(val); - } - KEY_RESPONSE_KEYS_BYTE => { - let readed = skip("KeyResponseMessage", &buf[offset..])?; - if let Some((ref mut fnso, ref mut lnso)) = keys_offsets { - if *fnso > offset { - *fnso = offset; - } - - if *lnso < offset + readed { - *lnso = offset + readed; - } - } else { - keys_offsets = Some((offset, offset + readed)); - } - num_keys += 1; - offset += readed; - } - KEY_RESPONSE_PRIMARY_KEY_BYTE => { - if primary_key.is_some() { - return Err(DecodeError::duplicate_field( - "KeyResponseMessage", - "primary_key", - KEY_RESPONSE_PRIMARY_KEY_TAG, - )); - } - - offset += 1; - let (bytes_read, val) = ::decode_length_delimited(&buf[offset..])?; - offset += bytes_read; - primary_key = Some(val); - } - _ => offset += skip("KeyResponseMessage", &buf[offset..])?, - } - } - - Ok(( - offset, - Self { - result: result.unwrap_or_default(), - message: message.unwrap_or_default(), - keys: if let Some((start, end)) = keys_offsets { - RepeatedDecoder::new(KEY_RESPONSE_KEYS_TAG, WireType::LengthDelimited, buf) - .with_nums(num_keys) - .with_offsets(start, end) - } else { - RepeatedDecoder::new(KEY_RESPONSE_KEYS_TAG, WireType::LengthDelimited, buf) - }, - primary_key, - }, - )) - } -} - -impl Data for KeyResponseMessage { - type Ref<'a> = KeyResponseMessageRef<'a>; - - fn from_ref(val: Self::Ref<'_>) -> Result - where - Self: Sized, - { - val - .keys - .iter::() - .map(|res| res.and_then(Data::from_ref)) - .collect::>() - .map(|keys| Self { - result: val.result, - message: SmolStr::new(val.message), - keys, - primary_key: val.primary_key, - }) - } - - fn encoded_len(&self) -> usize { - let mut len = 0; - if self.result { - len += 1 + 1; - } - - if !self.message.is_empty() { - len += 1 + self.message.encoded_len_with_length_delimited(); - } - - len += self - .keys - .iter() - .map(|key| 1 + key.encoded_len_with_length_delimited()) - .sum::(); - - if let Some(key) = &self.primary_key { - len += 1 + key.encoded_len_with_length_delimited(); - } - - len - } - - fn encode(&self, buf: &mut [u8]) -> Result { - macro_rules! bail { - ($this:ident($offset:expr, $len:ident)) => { - if $offset >= $len { - return Err(EncodeError::insufficient_buffer(self.encoded_len(), $len)); - } - }; - } - - let buf_len = buf.len(); - let mut offset = 0; - - if self.result { - bail!(self(offset, buf_len)); - buf[offset] = KEY_RESPONSE_RESULT_BYTE; - offset += 1; - bail!(self(offset, buf_len)); - buf[offset] = 1; - offset += 1; - } - - if !self.message.is_empty() { - bail!(self(offset, buf_len)); - buf[offset] = KEY_RESPONSE_MESSAGE_BYTE; - offset += 1; - offset += self.message.encode_length_delimited(&mut buf[offset..])?; - } - - for key in self.keys.iter() { - bail!(self(offset, buf_len)); - buf[offset] = KEY_RESPONSE_KEYS_BYTE; - offset += 1; - offset += key.encode_length_delimited(&mut buf[offset..])?; - } - - if let Some(key) = &self.primary_key { - bail!(self(offset, buf_len)); - buf[offset] = KEY_RESPONSE_PRIMARY_KEY_BYTE; - offset += 1; - offset += key.encode_length_delimited(&mut buf[offset..])?; - } - - #[cfg(debug_assertions)] - super::debug_assert_write_eq::(offset, self.encoded_len()); - - Ok(offset) - } -} - -/// KeyResponse is used to relay a query for a list of all keys in use. -#[viewit::viewit(setters(prefix = "with"))] -#[derive(Default)] -pub struct KeyResponse { - /// Map of node id to response message - #[viewit( - getter( - const, - style = "ref", - attrs(doc = "Returns the map of node id to response message") - ), - setter(attrs(doc = "Sets the map of node id to response message (Builder pattern)")) - )] - messages: IndexMap, - /// Total nodes memberlist knows of - #[viewit( - getter(const, attrs(doc = "Returns the total nodes memberlist knows of")), - setter( - const, - attrs(doc = "Sets total nodes memberlist knows of (Builder pattern)") - ) - )] - num_nodes: usize, - /// Total responses received - #[viewit( - getter(const, attrs(doc = "Returns the total responses received")), - setter( - const, - attrs(doc = "Sets the total responses received (Builder pattern)") - ) - )] - num_resp: usize, - /// Total errors from request - #[viewit( - getter(const, attrs(doc = "Returns the total errors from request")), - setter( - const, - attrs(doc = "Sets the total errors from request (Builder pattern)") - ) - )] - num_err: usize, - - /// A mapping of the value of the key bytes to the - /// number of nodes that have the key installed. - #[viewit( - getter( - const, - style = "ref", - attrs( - doc = "Returns a mapping of the value of the key bytes to the number of nodes that have the key installed." - ) - ), - setter(attrs( - doc = "Sets a mapping of the value of the key bytes to the number of nodes that have the key installed (Builder pattern)" - )) - )] - keys: IndexMap, - - /// A mapping of the value of the primary - /// key bytes to the number of nodes that have the key installed. - #[viewit( - getter( - const, - style = "ref", - attrs( - doc = "Returns a mapping of the value of the primary key bytes to the number of nodes that have the key installed." - ) - ), - setter(attrs( - doc = "Sets a mapping of the value of the primary key bytes to the number of nodes that have the key installed. (Builder pattern)" - )) - )] - primary_keys: IndexMap, -} - -/// KeyRequestOptions is used to contain optional parameters for a keyring operation -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -pub struct KeyRequestOptions { - /// The number of duplicate query responses to send by relaying through - /// other nodes, for redundancy - pub relay_factor: u8, -} diff --git a/legacy/serf-core/src/types/leave.rs b/legacy/serf-core/src/types/leave.rs deleted file mode 100644 index ff721a41..00000000 --- a/legacy/serf-core/src/types/leave.rs +++ /dev/null @@ -1,196 +0,0 @@ -use memberlist_core::proto::{ - Data, DataRef, DecodeError, EncodeError, WireType, - utils::{merge, skip}, -}; - -use super::LamportTime; - -const LTIME_TAG: u8 = 1; -const PRUNE_TAG: u8 = 2; -const ID_TAG: u8 = 3; - -const LTIME_BYTE: u8 = merge(WireType::Varint, LTIME_TAG); -const PRUNE_BYTE: u8 = merge(WireType::Byte, PRUNE_TAG); - -/// The message broadcasted to signal the intentional to -/// leave. -#[viewit::viewit(setters(prefix = "with"))] -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -pub struct LeaveMessage { - /// The lamport time - #[viewit( - getter(const, attrs(doc = "Returns the lamport time for this message")), - setter( - const, - attrs(doc = "Sets the lamport time for this message (Builder pattern)") - ) - )] - ltime: LamportTime, - /// The id of the node - #[viewit( - getter(const, style = "ref", attrs(doc = "Returns the node")), - setter(attrs(doc = "Sets the node (Builder pattern)")) - )] - id: I, - - /// If prune or not - #[viewit( - getter(const, style = "ref", attrs(doc = "Returns if prune or not")), - setter(attrs(doc = "Sets prune or not (Builder pattern)")) - )] - prune: bool, -} - -impl LeaveMessage { - const fn id_byte() -> u8 - where - I: Data, - { - merge(I::WIRE_TYPE, ID_TAG) - } -} - -impl<'a, I> DataRef<'a, LeaveMessage> for LeaveMessage> -where - I: Data, -{ - fn decode(buf: &'a [u8]) -> Result<(usize, Self), DecodeError> - where - Self: Sized, - { - let mut offset = 0; - let buf_len = buf.len(); - - let mut ltime = None; - let mut id = None; - let mut prune = None; - - while offset < buf_len { - match buf[offset] { - LTIME_BYTE => { - if ltime.is_some() { - return Err(DecodeError::duplicate_field( - "LeaveMessage", - "ltime", - LTIME_TAG, - )); - } - offset += 1; - - let (read, value) = >::decode(&buf[offset..])?; - offset += read; - ltime = Some(value); - } - PRUNE_BYTE => { - if prune.is_some() { - return Err(DecodeError::duplicate_field( - "LeaveMessage", - "prune", - PRUNE_TAG, - )); - } - offset += 1; - - let (read, value) = >::decode(&buf[offset..])?; - offset += read; - prune = Some(value); - } - val if val == LeaveMessage::::id_byte() => { - offset += 1; - let (read, id_ref) = I::Ref::decode_length_delimited(&buf[offset..])?; - offset += read; - id = Some(id_ref); - } - _ => offset += skip("LeaveMessage", &buf[offset..])?, - } - } - - Ok(( - offset, - Self { - ltime: ltime.ok_or_else(|| DecodeError::missing_field("LeaveMessage", "ltime"))?, - id: id.ok_or_else(|| DecodeError::missing_field("LeaveMessage", "id"))?, - prune: prune.unwrap_or_default(), - }, - )) - } -} - -impl Data for LeaveMessage -where - I: Data, -{ - type Ref<'a> = LeaveMessage>; - - fn from_ref(val: Self::Ref<'_>) -> Result - where - Self: Sized, - { - I::from_ref(val.id).map(|id| Self { - ltime: val.ltime, - id, - prune: val.prune, - }) - } - - fn encoded_len(&self) -> usize { - 1 + self.ltime.encoded_len() - + if self.prune { 1 + 1 } else { 0 } - + 1 - + self.id.encoded_len_with_length_delimited() - } - - fn encode(&self, buf: &mut [u8]) -> Result { - let mut offset = 0; - let buf_len = buf.len(); - - if offset >= buf_len { - return Err(EncodeError::insufficient_buffer( - self.encoded_len(), - buf_len, - )); - } - - buf[offset] = LTIME_BYTE; - offset += 1; - offset += self - .ltime - .encode(&mut buf[offset..]) - .map_err(|e| e.update(self.encoded_len(), buf_len))?; - - if self.prune { - if offset >= buf_len { - return Err(EncodeError::insufficient_buffer( - self.encoded_len(), - buf_len, - )); - } - - buf[offset] = PRUNE_BYTE; - offset += 1; - offset += ::encode(&true, &mut buf[offset..]) - .map_err(|e| e.update(self.encoded_len(), buf_len))?; - } - - if offset >= buf_len { - return Err(EncodeError::insufficient_buffer( - self.encoded_len(), - buf_len, - )); - } - - buf[offset] = Self::id_byte(); - offset += 1; - offset += self - .id - .encode_length_delimited(&mut buf[offset..]) - .map_err(|e| e.update(self.encoded_len(), buf_len))?; - - #[cfg(debug_assertions)] - super::debug_assert_write_eq::(offset, self.encoded_len()); - - Ok(offset) - } -} diff --git a/legacy/serf-core/src/types/member.rs b/legacy/serf-core/src/types/member.rs deleted file mode 100644 index 44ac15fd..00000000 --- a/legacy/serf-core/src/types/member.rs +++ /dev/null @@ -1,497 +0,0 @@ -use std::sync::Arc; - -use memberlist_core::proto::{ - CheapClone, Data, DataRef, DecodeError, EncodeError, OneOrMore, WireType, - utils::{merge, skip}, -}; - -use super::{ - DelegateVersion, Epoch, LamportTime, MemberlistDelegateVersion, MemberlistProtocolVersion, - MessageType, Node, ProtocolVersion, Tags, TagsRef, -}; - -use std::collections::HashMap; - -/// Used to track members that are no longer active due to -/// leaving, failing, partitioning, etc. It tracks the member along with -/// when that member was marked as leaving. -#[viewit::viewit] -#[derive(Clone, Debug)] -pub(crate) struct MemberState { - member: Member, - /// lamport clock time of last received message - status_time: LamportTime, - /// wall clock time of leave - leave_time: Option, -} - -/// Used to buffer intents for out-of-order deliveries. -#[derive(Debug)] -pub(crate) struct NodeIntent { - pub(crate) ty: MessageType, - pub(crate) wall_time: Epoch, - pub(crate) ltime: LamportTime, -} - -pub(crate) struct Members { - pub(crate) states: HashMap>, - pub(crate) recent_intents: HashMap, - pub(crate) left_members: OneOrMore>, - pub(crate) failed_members: OneOrMore>, -} - -impl Default for Members { - fn default() -> Self { - Self { - states: Default::default(), - recent_intents: Default::default(), - left_members: Default::default(), - failed_members: Default::default(), - } - } -} - -const MEMBER_STATUS_NONE: u8 = 0; -const MEMBER_STATUS_ALIVE: u8 = 1; -const MEMBER_STATUS_LEAVING: u8 = 2; -const MEMBER_STATUS_LEFT: u8 = 3; -const MEMBER_STATUS_FAILED: u8 = 4; - -/// The member status. -#[derive( - Debug, Default, Copy, Clone, Eq, PartialEq, Hash, derive_more::IsVariant, derive_more::Display, -)] -#[repr(u8)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[non_exhaustive] -pub enum MemberStatus { - /// None status - #[display("none")] - #[default] - None, - /// Alive status - #[display("alive")] - Alive, - /// Leaving status - #[display("leaving")] - Leaving, - /// Left status - #[display("left")] - Left, - /// Failed status - #[display("failed")] - Failed, - /// Unknown state (used for forwards and backwards compatibility) - #[display("unknown({_0})")] - Unknown(u8), -} - -impl From for MemberStatus { - fn from(value: u8) -> Self { - match value { - MEMBER_STATUS_NONE => Self::None, - MEMBER_STATUS_ALIVE => Self::Alive, - MEMBER_STATUS_LEAVING => Self::Leaving, - MEMBER_STATUS_LEFT => Self::Left, - MEMBER_STATUS_FAILED => Self::Failed, - val => Self::Unknown(val), - } - } -} - -impl From for u8 { - fn from(val: MemberStatus) -> Self { - match val { - MemberStatus::None => MEMBER_STATUS_NONE, - MemberStatus::Alive => MEMBER_STATUS_ALIVE, - MemberStatus::Leaving => MEMBER_STATUS_LEAVING, - MemberStatus::Left => MEMBER_STATUS_LEFT, - MemberStatus::Failed => MEMBER_STATUS_FAILED, - MemberStatus::Unknown(val) => val, - } - } -} - -impl MemberStatus { - /// Get the string representation of the member status - #[inline] - pub fn as_str(&self) -> std::borrow::Cow<'static, str> { - std::borrow::Cow::Borrowed(match self { - Self::None => "none", - Self::Alive => "alive", - Self::Leaving => "leaving", - Self::Left => "left", - Self::Failed => "failed", - Self::Unknown(val) => return format!("unknown({})", val).into(), - }) - } -} - -/// A single member of the Serf cluster. -#[viewit::viewit(setters(prefix = "with"))] -#[derive(Debug, PartialEq)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -pub struct Member { - /// The node - #[viewit( - getter(const, style = "ref", attrs(doc = "Returns the node")), - setter(attrs(doc = "Sets the node (Builder pattern)")) - )] - node: Node, - /// The tags - #[viewit( - getter(const, style = "ref", attrs(doc = "Returns the tags")), - setter(attrs(doc = "Sets the tags (Builder pattern)")) - )] - tags: Arc, - /// The status - #[viewit( - getter(const, style = "ref", attrs(doc = "Returns the status")), - setter(attrs(doc = "Sets the status (Builder pattern)")) - )] - status: MemberStatus, - /// The memberlist protocol version - #[viewit( - getter(const, attrs(doc = "Returns the memberlist protocol version")), - setter( - const, - attrs(doc = "Sets the memberlist protocol version (Builder pattern)") - ) - )] - memberlist_protocol_version: MemberlistProtocolVersion, - /// The memberlist delegate version - #[viewit( - getter(const, attrs(doc = "Returns the memberlist delegate version")), - setter( - const, - attrs(doc = "Sets the memberlist delegate version (Builder pattern)") - ) - )] - memberlist_delegate_version: MemberlistDelegateVersion, - - /// The serf protocol version - #[viewit( - getter(const, attrs(doc = "Returns the serf protocol version")), - setter(const, attrs(doc = "Sets the serf protocol version (Builder pattern)")) - )] - protocol_version: ProtocolVersion, - /// The serf delegate version - #[viewit( - getter(const, attrs(doc = "Returns the serf delegate version")), - setter(const, attrs(doc = "Sets the serf delegate version (Builder pattern)")) - )] - delegate_version: DelegateVersion, -} - -impl Member { - /// Create a new member with the given node, tags, and status. - /// Other fields are set to their default values. - #[inline] - pub fn new(node: Node, tags: Tags, status: MemberStatus) -> Self { - Self { - node, - tags: Arc::new(tags), - status, - memberlist_protocol_version: MemberlistProtocolVersion::V1, - memberlist_delegate_version: MemberlistDelegateVersion::V1, - protocol_version: ProtocolVersion::V1, - delegate_version: DelegateVersion::V1, - } - } -} - -impl Clone for Member { - fn clone(&self) -> Self { - Self { - node: self.node.clone(), - tags: self.tags.clone(), - status: self.status, - memberlist_protocol_version: self.memberlist_protocol_version, - memberlist_delegate_version: self.memberlist_delegate_version, - protocol_version: self.protocol_version, - delegate_version: self.delegate_version, - } - } -} - -impl CheapClone for Member { - fn cheap_clone(&self) -> Self { - Self { - node: self.node.cheap_clone(), - tags: self.tags.cheap_clone(), - status: self.status, - memberlist_protocol_version: self.memberlist_protocol_version, - memberlist_delegate_version: self.memberlist_delegate_version, - protocol_version: self.protocol_version, - delegate_version: self.delegate_version, - } - } -} - -const NODE_TAG: u8 = 1; -const TAGS_TAG: u8 = 2; -const STATUS_TAG: u8 = 3; -const MEMBERLIST_PROTOCOL_VERSION_TAG: u8 = 4; -const MEMBERLIST_DELEGATE_VERSION_TAG: u8 = 5; -const PROTOCOL_VERSION_TAG: u8 = 6; -const DELEGATE_VERSION_TAG: u8 = 7; - -const NODE_BYTE: u8 = merge(WireType::LengthDelimited, NODE_TAG); -const TAGS_BYTE: u8 = merge(WireType::LengthDelimited, TAGS_TAG); -const STATUS_BYTE: u8 = merge(WireType::Byte, STATUS_TAG); -const MEMBERLIST_PROTOCOL_VERSION_BYTE: u8 = merge(WireType::Byte, MEMBERLIST_PROTOCOL_VERSION_TAG); -const MEMBERLIST_DELEGATE_VERSION_BYTE: u8 = merge(WireType::Byte, MEMBERLIST_DELEGATE_VERSION_TAG); -const PROTOCOL_VERSION_BYTE: u8 = merge(WireType::Byte, PROTOCOL_VERSION_TAG); -const DELEGATE_VERSION_BYTE: u8 = merge(WireType::Byte, DELEGATE_VERSION_TAG); - -/// A reference type to [`Member`] -#[viewit::viewit(vis_all = "", getters(vis_all = "pub"), setters(skip))] -#[derive(Debug, Clone, Copy, PartialEq)] -pub struct MemberRef<'a, I, A> { - /// The node - #[viewit(getter(const, style = "ref", attrs(doc = "Returns the node")))] - node: Node, - /// The tags - #[viewit(getter(const, style = "ref", attrs(doc = "Returns the tags")))] - tags: TagsRef<'a>, - /// The status - #[viewit(getter(const, style = "ref", attrs(doc = "Returns the status")))] - status: MemberStatus, - /// The memberlist protocol version - #[viewit(getter(const, attrs(doc = "Returns the memberlist protocol version")))] - memberlist_protocol_version: MemberlistProtocolVersion, - /// The memberlist delegate version - #[viewit(getter(const, attrs(doc = "Returns the memberlist delegate version")))] - memberlist_delegate_version: MemberlistDelegateVersion, - /// The serf protocol version - #[viewit(getter(const, attrs(doc = "Returns the serf protocol version")))] - protocol_version: ProtocolVersion, - /// The serf delegate version - #[viewit(getter(const, attrs(doc = "Returns the serf delegate version")))] - delegate_version: DelegateVersion, -} - -impl<'a, I, A> DataRef<'a, Member> for MemberRef<'a, I::Ref<'a>, A::Ref<'a>> -where - I: Data, - A: Data, -{ - fn decode(buf: &'a [u8]) -> Result<(usize, Self), DecodeError> - where - Self: Sized, - { - let mut offset = 0; - let buf_len = buf.len(); - - let mut node = None; - let mut tags = None; - let mut status = None; - let mut memberlist_protocol_version = None; - let mut memberlist_delegate_version = None; - let mut protocol_version = None; - let mut delegate_version = None; - - while offset < buf_len { - match buf[offset] { - NODE_BYTE => { - if node.is_some() { - return Err(DecodeError::duplicate_field("Member", "node", NODE_TAG)); - } - offset += 1; - let (size, val) = - , A::Ref<'_>> as DataRef<'_, Node>>::decode_length_delimited( - &buf[offset..], - )?; - node = Some(val); - offset += size; - } - TAGS_BYTE => { - if tags.is_some() { - return Err(DecodeError::duplicate_field("Member", "tags", TAGS_TAG)); - } - offset += 1; - let (size, val) = - as DataRef<'_, Tags>>::decode_length_delimited(&buf[offset..])?; - tags = Some(val); - offset += size; - } - STATUS_BYTE => { - if status.is_some() { - return Err(DecodeError::duplicate_field("Member", "status", STATUS_TAG)); - } - offset += 1; - status = Some(buf[offset].into()); - offset += 1; - } - MEMBERLIST_PROTOCOL_VERSION_BYTE => { - if memberlist_protocol_version.is_some() { - return Err(DecodeError::duplicate_field( - "Member", - "memberlist_protocol_version", - MEMBERLIST_PROTOCOL_VERSION_TAG, - )); - } - offset += 1; - memberlist_protocol_version = Some(buf[offset].into()); - offset += 1; - } - MEMBERLIST_DELEGATE_VERSION_BYTE => { - if memberlist_delegate_version.is_some() { - return Err(DecodeError::duplicate_field( - "Member", - "memberlist_delegate_version", - MEMBERLIST_DELEGATE_VERSION_TAG, - )); - } - offset += 1; - memberlist_delegate_version = Some(buf[offset].into()); - offset += 1; - } - PROTOCOL_VERSION_BYTE => { - if protocol_version.is_some() { - return Err(DecodeError::duplicate_field( - "Member", - "protocol_version", - PROTOCOL_VERSION_TAG, - )); - } - offset += 1; - protocol_version = Some(buf[offset].into()); - offset += 1; - } - DELEGATE_VERSION_BYTE => { - if delegate_version.is_some() { - return Err(DecodeError::duplicate_field( - "Member", - "delegate_version", - DELEGATE_VERSION_TAG, - )); - } - offset += 1; - delegate_version = Some(buf[offset].into()); - offset += 1; - } - _ => offset += skip("Member", &buf[offset..])?, - } - } - - Ok(( - offset, - Self { - node: node.ok_or_else(|| DecodeError::missing_field("Member", "node"))?, - tags: tags.ok_or_else(|| DecodeError::missing_field("Member", "tags"))?, - status: status.ok_or_else(|| DecodeError::missing_field("Member", "status"))?, - memberlist_protocol_version: memberlist_protocol_version - .ok_or_else(|| DecodeError::missing_field("Member", "memberlist_protocol_version"))?, - memberlist_delegate_version: memberlist_delegate_version - .ok_or_else(|| DecodeError::missing_field("Member", "memberlist_delegate_version"))?, - protocol_version: protocol_version - .ok_or_else(|| DecodeError::missing_field("Member", "protocol_version"))?, - delegate_version: delegate_version - .ok_or_else(|| DecodeError::missing_field("Member", "delegate_version"))?, - }, - )) - } -} - -impl Data for Member -where - I: Data, - A: Data, -{ - type Ref<'a> = MemberRef<'a, I::Ref<'a>, A::Ref<'a>>; - - fn from_ref(val: Self::Ref<'_>) -> Result - where - Self: Sized, - { - Ok(Self { - node: Node::from_ref(val.node)?, - tags: Tags::from_ref(val.tags)?.into(), - status: val.status, - memberlist_protocol_version: val.memberlist_protocol_version, - memberlist_delegate_version: val.memberlist_delegate_version, - protocol_version: val.protocol_version, - delegate_version: val.delegate_version, - }) - } - - fn encoded_len(&self) -> usize { - let mut len = 0; - len += 1 + self.node.encoded_len_with_length_delimited(); - len += 1 + self.tags.encoded_len_with_length_delimited(); - len += 1 + 1; // status - len += 1 + 1; // memberlist_protocol_version - len += 1 + 1; // memberlist_delegate_version - len += 1 + 1; // protocol_version - len += 1 + 1; // delegate_version - len - } - - fn encode(&self, buf: &mut [u8]) -> Result { - macro_rules! bail { - ($this:ident($offset:expr, $len:ident)) => { - if $offset >= $len { - return Err(EncodeError::insufficient_buffer(self.encoded_len(), $len)); - } - }; - } - - let buf_len = buf.len(); - let mut offset = 0; - bail!(self(offset, buf_len)); - - buf[offset] = NODE_BYTE; - offset += 1; - offset += self.node.encode_length_delimited(&mut buf[offset..])?; - - bail!(self(offset, buf_len)); - buf[offset] = TAGS_BYTE; - offset += 1; - offset += self.tags.encode_length_delimited(&mut buf[offset..])?; - - bail!(self(offset, buf_len)); - buf[offset] = STATUS_BYTE; - offset += 1; - - bail!(self(offset, buf_len)); - buf[offset] = self.status.into(); - offset += 1; - - bail!(self(offset, buf_len)); - buf[offset] = MEMBERLIST_PROTOCOL_VERSION_BYTE; - offset += 1; - - bail!(self(offset, buf_len)); - buf[offset] = self.memberlist_protocol_version.into(); - offset += 1; - - bail!(self(offset, buf_len)); - buf[offset] = MEMBERLIST_DELEGATE_VERSION_BYTE; - offset += 1; - - bail!(self(offset, buf_len)); - buf[offset] = self.memberlist_delegate_version.into(); - offset += 1; - - bail!(self(offset, buf_len)); - buf[offset] = PROTOCOL_VERSION_BYTE; - offset += 1; - - bail!(self(offset, buf_len)); - buf[offset] = self.protocol_version.into(); - offset += 1; - - bail!(self(offset, buf_len)); - buf[offset] = DELEGATE_VERSION_BYTE; - offset += 1; - buf[offset] = self.delegate_version.into(); - offset += 1; - - #[cfg(debug_assertions)] - super::debug_assert_write_eq::(offset, self.encoded_len()); - - Ok(offset) - } -} diff --git a/legacy/serf-core/src/types/message.rs b/legacy/serf-core/src/types/message.rs deleted file mode 100644 index 1d5813f7..00000000 --- a/legacy/serf-core/src/types/message.rs +++ /dev/null @@ -1,757 +0,0 @@ -use memberlist_core::proto::{ - Data, DataRef, DecodeError, EncodeError, Node, WireType, - bytes::Bytes, - utils::{merge, skip}, -}; - -use super::{ - ConflictResponseMessage, ConflictResponseMessageBorrow, ConflictResponseMessageRef, JoinMessage, - LeaveMessage, PushPullMessage, PushPullMessageBorrow, PushPullMessageRef, QueryMessage, - QueryMessageRef, QueryResponseMessage, QueryResponseMessageRef, UserEventMessage, - UserEventMessageRef, -}; - -#[cfg(feature = "encryption")] -use super::{KeyRequestMessage, KeyResponseMessage, KeyResponseMessageRef}; - -const LEAVE_MESSAGE_TAG: u8 = 1; -const JOIN_MESSAGE_TAG: u8 = 2; -const PUSH_PULL_MESSAGE_TAG: u8 = 3; -const USER_EVENT_MESSAGE_TAG: u8 = 4; -const QUERY_MESSAGE_TAG: u8 = 5; -const QUERY_RESPONSE_MESSAGE_TAG: u8 = 6; -const CONFLICT_RESPONSE_MESSAGE_TAG: u8 = 7; -const RELAY_MESSAGE_TAG: u8 = 8; -#[cfg(feature = "encryption")] -const KEY_REQUEST_MESSAGE_TAG: u8 = 9; -#[cfg(feature = "encryption")] -const KEY_RESPONSE_MESSAGE_TAG: u8 = 10; - -const LEAVE_MESSAGE_BYTE: u8 = merge(WireType::LengthDelimited, LEAVE_MESSAGE_TAG); -const JOIN_MESSAGE_BYTE: u8 = merge(WireType::LengthDelimited, JOIN_MESSAGE_TAG); -const PUSH_PULL_MESSAGE_BYTE: u8 = merge(WireType::LengthDelimited, PUSH_PULL_MESSAGE_TAG); -const USER_EVENT_MESSAGE_BYTE: u8 = merge(WireType::LengthDelimited, USER_EVENT_MESSAGE_TAG); -const QUERY_MESSAGE_BYTE: u8 = merge(WireType::LengthDelimited, QUERY_MESSAGE_TAG); -const QUERY_RESPONSE_MESSAGE_BYTE: u8 = - merge(WireType::LengthDelimited, QUERY_RESPONSE_MESSAGE_TAG); -const CONFLICT_RESPONSE_MESSAGE_BYTE: u8 = - merge(WireType::LengthDelimited, CONFLICT_RESPONSE_MESSAGE_TAG); -const RELAY_MESSAGE_BYTE: u8 = merge(WireType::LengthDelimited, RELAY_MESSAGE_TAG); -#[cfg(feature = "encryption")] -const KEY_REQUEST_MESSAGE_BYTE: u8 = merge(WireType::LengthDelimited, KEY_REQUEST_MESSAGE_TAG); -#[cfg(feature = "encryption")] -const KEY_RESPONSE_MESSAGE_BYTE: u8 = merge(WireType::LengthDelimited, KEY_RESPONSE_MESSAGE_TAG); - -/// The types of gossip messages Serf will send along -/// memberlist. -#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash, derive_more::Display, derive_more::IsVariant)] -#[repr(u8)] -#[non_exhaustive] -pub enum MessageType { - /// Leave message - #[display("leave")] - Leave, - /// Join message - #[display("join")] - Join, - /// PushPull message - #[display("push_pull")] - PushPull, - /// UserEvent message - #[display("user_event")] - UserEvent, - /// Query message - #[display("query")] - Query, - /// QueryResponse message - #[display("query_response")] - QueryResponse, - /// ConflictResponse message - #[display("conflict_response")] - ConflictResponse, - /// Relay message - #[display("relay")] - Relay, - /// KeyRequest message - #[cfg(feature = "encryption")] - #[display("key_request")] - KeyRequest, - /// KeyResponse message - #[cfg(feature = "encryption")] - #[display("key_response")] - KeyResponse, - /// Unknown message type, used for forwards and backwards compatibility - #[display("unknown({_0})")] - Unknown(u8), -} - -impl MessageType { - /// All message types in order - pub const ALL: &[Self] = &[ - Self::Leave, - Self::Join, - Self::PushPull, - Self::UserEvent, - Self::Query, - Self::QueryResponse, - Self::ConflictResponse, - Self::Relay, - #[cfg(feature = "encryption")] - Self::KeyRequest, - #[cfg(feature = "encryption")] - Self::KeyResponse, - ]; - - /// Get the string representation of the message type - #[inline] - pub fn as_str(&self) -> std::borrow::Cow<'static, str> { - std::borrow::Cow::Borrowed(match self { - Self::Leave => "leave", - Self::Join => "join", - Self::PushPull => "push_pull", - Self::UserEvent => "user_event", - Self::Query => "query", - Self::QueryResponse => "query_response", - Self::ConflictResponse => "conflict_response", - Self::Relay => "relay", - #[cfg(feature = "encryption")] - Self::KeyRequest => "key_request", - #[cfg(feature = "encryption")] - Self::KeyResponse => "key_response", - Self::Unknown(val) => return std::borrow::Cow::Owned(format!("unknown({val})")), - }) - } -} - -impl From for MessageType { - fn from(value: u8) -> Self { - match value { - LEAVE_MESSAGE_TAG => Self::Leave, - JOIN_MESSAGE_TAG => Self::Join, - PUSH_PULL_MESSAGE_TAG => Self::PushPull, - USER_EVENT_MESSAGE_TAG => Self::UserEvent, - QUERY_MESSAGE_TAG => Self::Query, - QUERY_RESPONSE_MESSAGE_TAG => Self::QueryResponse, - CONFLICT_RESPONSE_MESSAGE_TAG => Self::ConflictResponse, - RELAY_MESSAGE_TAG => Self::Relay, - #[cfg(feature = "encryption")] - KEY_REQUEST_MESSAGE_TAG => Self::KeyRequest, - #[cfg(feature = "encryption")] - KEY_RESPONSE_MESSAGE_TAG => Self::KeyResponse, - val => Self::Unknown(val), - } - } -} - -impl From for u8 { - fn from(val: MessageType) -> Self { - match val { - MessageType::Leave => LEAVE_MESSAGE_TAG, - MessageType::Join => JOIN_MESSAGE_TAG, - MessageType::PushPull => PUSH_PULL_MESSAGE_TAG, - MessageType::UserEvent => USER_EVENT_MESSAGE_TAG, - MessageType::Query => QUERY_MESSAGE_TAG, - MessageType::QueryResponse => QUERY_RESPONSE_MESSAGE_TAG, - MessageType::ConflictResponse => CONFLICT_RESPONSE_MESSAGE_TAG, - MessageType::Relay => RELAY_MESSAGE_TAG, - #[cfg(feature = "encryption")] - MessageType::KeyRequest => KEY_REQUEST_MESSAGE_TAG, - #[cfg(feature = "encryption")] - MessageType::KeyResponse => KEY_RESPONSE_MESSAGE_TAG, - MessageType::Unknown(val) => val, - } - } -} - -macro_rules! bail { - ($this:ident($offset:expr, $len:ident)) => { - if $offset >= $len { - return Err(EncodeError::insufficient_buffer( - encoded_message_len($this), - $len, - )); - } - }; - (@relay $this:ident($offset:expr, $len:ident, $node:ident)) => { - if $offset >= $len { - return Err(EncodeError::insufficient_buffer( - encoded_relay_message_len($this, $node), - $len, - )); - } - }; -} - -const RELAY_NODE_TAG: u8 = 1; -const RELAY_MSG_TAG: u8 = 2; - -const RELAY_NODE_BYTE: u8 = merge(WireType::LengthDelimited, RELAY_NODE_TAG); -const RELAY_MSG_BYTE: u8 = merge(WireType::LengthDelimited, RELAY_MSG_TAG); - -/// A trait for encoding messages. -pub trait Encodable { - const ID: u8; - - /// Encodes the message into a buffer. - fn encode(&self, buf: &mut [u8]) -> Result; - - /// Returns the encoded length of the message. - fn encoded_len(&self) -> usize; -} - -impl Encodable for &T { - const ID: u8 = T::ID; - - fn encode(&self, buf: &mut [u8]) -> Result { - (*self).encode(buf) - } - - fn encoded_len(&self) -> usize { - (*self).encoded_len() - } -} - -macro_rules! impl_encodable { - ( - $( - $(#[$attr:meta])* - $type:ident $(<$($generic:ident), +$(,)?>)? = $id:expr, - )* - ) => { - $( - $(#[$attr])* - impl $(<$($generic), +>)? Encodable for $type $(<$($generic), +>)? - $( - where - $($generic: Data,)+ - )? - { - const ID: u8 = $id; - - fn encode(&self, buf: &mut [u8]) -> Result { - Data::encode(self, buf) - } - - fn encoded_len(&self) -> usize { - Data::encoded_len(self) - } - } - )* - }; -} - -impl_encodable!( - LeaveMessage = LEAVE_MESSAGE_BYTE, - JoinMessage = JOIN_MESSAGE_BYTE, - UserEventMessage = USER_EVENT_MESSAGE_BYTE, - QueryMessage = QUERY_MESSAGE_BYTE, - QueryResponseMessage = QUERY_RESPONSE_MESSAGE_BYTE, - ConflictResponseMessage = CONFLICT_RESPONSE_MESSAGE_BYTE, - #[cfg(feature = "encryption")] - KeyRequestMessage = KEY_REQUEST_MESSAGE_BYTE, - #[cfg(feature = "encryption")] - KeyResponseMessage = KEY_RESPONSE_MESSAGE_BYTE, -); - -impl super::Encodable for ConflictResponseMessageBorrow<'_, I, A> -where - I: Data, - A: Data, -{ - const ID: u8 = CONFLICT_RESPONSE_MESSAGE_BYTE; - - fn encode(&self, buf: &mut [u8]) -> Result { - self.encode_in(buf) - } - - fn encoded_len(&self) -> usize { - self.encoded_len_in() - } -} - -impl super::Encodable for PushPullMessage -where - I: Data + Eq + core::hash::Hash, -{ - const ID: u8 = PUSH_PULL_MESSAGE_BYTE; - - fn encode(&self, buf: &mut [u8]) -> Result { - Data::encode(self, buf) - } - - fn encoded_len(&self) -> usize { - Data::encoded_len(self) - } -} - -impl super::Encodable for PushPullMessageBorrow<'_, I> -where - I: Data, -{ - const ID: u8 = PUSH_PULL_MESSAGE_BYTE; - - fn encode(&self, buf: &mut [u8]) -> Result { - self.encode_in(buf) - } - - fn encoded_len(&self) -> usize { - self.encoded_len_in() - } -} - -/// A reference type to a relay message. -#[viewit::viewit(vis_all = "pub(crate)", getters(vis_all = "pub"), setters(skip))] -#[derive(Debug, Clone, Copy)] -pub struct RelayMessageRef<'a, I, A> { - /// The node - #[viewit(getter(style = "ref", attrs(doc = "Get the node to relay to")))] - node: Node, - /// The offset of the payload to the original buffer - #[viewit(getter( - style = "move", - attrs(doc = "Get the offset of the payload to the original buffer") - ))] - payload_offset: usize, - /// The relay message payload - #[viewit(getter(style = "move", attrs(doc = "Get the relay message payload")))] - payload: &'a [u8], -} - -/// A reference to a message. -#[derive(Debug, derive_more::IsVariant, derive_more::Unwrap, derive_more::TryUnwrap)] -#[unwrap(ref)] -#[try_unwrap(ref)] -#[non_exhaustive] -pub enum MessageRef<'a, I, A> { - /// Leave message - Leave(LeaveMessage), - /// Join message - Join(JoinMessage), - /// PushPull message - PushPull(PushPullMessageRef<'a, I>), - /// UserEvent message - UserEvent(UserEventMessageRef<'a>), - /// Query message - Query(QueryMessageRef<'a, I, A>), - /// QueryResponse message - QueryResponse(QueryResponseMessageRef<'a, I, A>), - /// ConflictResponse message - ConflictResponse(ConflictResponseMessageRef<'a, I, A>), - /// Relay message - Relay(RelayMessageRef<'a, I, A>), - #[cfg(feature = "encryption")] - /// KeyRequest message - KeyRequest(KeyRequestMessage), - #[cfg(feature = "encryption")] - /// KeyResponse message - KeyResponse(KeyResponseMessageRef<'a>), -} - -impl MessageRef<'_, I, A> { - /// Returns the message type. - #[inline] - pub fn ty(&self) -> MessageType { - match self { - Self::Leave(_) => MessageType::Leave, - Self::Join(_) => MessageType::Join, - Self::PushPull(_) => MessageType::PushPull, - Self::UserEvent(_) => MessageType::UserEvent, - Self::Query(_) => MessageType::Query, - Self::QueryResponse(_) => MessageType::QueryResponse, - Self::ConflictResponse(_) => MessageType::ConflictResponse, - Self::Relay { .. } => MessageType::Relay, - #[cfg(feature = "encryption")] - Self::KeyRequest(_) => MessageType::KeyRequest, - #[cfg(feature = "encryption")] - Self::KeyResponse(_) => MessageType::KeyResponse, - } - } -} - -/// Encode a message into a Bytes. -pub fn encode_message_to_bytes(msg: &T) -> Result -where - T: Encodable, -{ - let len = encoded_message_len(msg); - let mut buf = vec![0; len]; - encode_message(msg, &mut buf).map(|_| Bytes::from(buf)) -} - -/// Encode a relay message into a Bytes. -pub fn encode_relay_message_to_bytes( - msg: &T, - node: &Node, -) -> Result -where - T: Encodable, - I: Data, - A: Data, -{ - let len = encoded_relay_message_len(msg, node); - let mut buf = vec![0; len]; - encode_relay_message(msg, node, &mut buf).map(|_| Bytes::from(buf)) -} - -/// Encode a message into a buffer. -pub fn encode_message(msg: &T, buf: &mut [u8]) -> Result -where - T: Encodable, -{ - let mut offset = 0; - let buf_len = buf.len(); - bail!(msg(offset, buf_len)); - - buf[offset] = T::ID; - offset += 1; - - let encoded_len = msg.encoded_len(); - if encoded_len > u32::MAX as usize { - return Err(EncodeError::TooLarge); - } - - offset += (encoded_len as u32) - .encode(&mut buf[offset..]) - .map_err(|e| e.update(encoded_message_len(msg), buf_len))?; - - offset += msg - .encode(&mut buf[offset..]) - .map_err(|e| e.update(encoded_message_len(msg), buf_len))?; - - #[cfg(debug_assertions)] - { - struct Message(core::marker::PhantomData); - super::debug_assert_write_eq::>(offset, encoded_message_len(msg)); - } - - Ok(offset) -} - -/// Encode a relay message into a buffer. -pub fn encode_relay_message( - msg: &T, - node: &Node, - buf: &mut [u8], -) -> Result -where - T: Encodable, - I: Data, - A: Data, -{ - let mut offset = 0; - let buf_len = buf.len(); - bail!(@relay msg(offset, buf_len, node)); - - buf[offset] = RELAY_MESSAGE_BYTE; - offset += 1; - - bail!(@relay msg(offset, buf_len, node)); - buf[offset] = RELAY_NODE_BYTE; - offset += 1; - offset += node - .encode_length_delimited(&mut buf[offset..]) - .map_err(|e| e.update(encoded_relay_message_len(msg, node), buf_len))?; - - bail!(@relay msg(offset, buf_len, node)); - buf[offset] = RELAY_MSG_BYTE; - offset += 1; - - bail!(@relay msg(offset, buf_len, node)); - buf[offset] = T::ID; - offset += 1; - - let encoded_len = msg.encoded_len(); - if encoded_len > u32::MAX as usize { - return Err(EncodeError::TooLarge); - } - - offset += (encoded_len as u32) - .encode(&mut buf[offset..]) - .map_err(|e| e.update(encoded_relay_message_len(msg, node), buf_len))?; - offset += msg - .encode(&mut buf[offset..]) - .map_err(|e| e.update(encoded_relay_message_len(msg, node), buf_len))?; - - #[cfg(debug_assertions)] - { - struct Message(core::marker::PhantomData); - super::debug_assert_write_eq::>(offset, encoded_relay_message_len(msg, node)); - } - - Ok(offset) -} - -/// Returns the encoded length of a message. -pub fn encoded_message_len(msg: &T) -> usize -where - T: Encodable, -{ - let encoded_len = msg.encoded_len(); - 1 + (encoded_len as u32).encoded_len() + encoded_len -} - -/// Returns the encoded length of the relay message. -pub fn encoded_relay_message_len(msg: &T, node: &Node) -> usize -where - T: Encodable, - I: Data, - A: Data, -{ - 1 + 1 + node.encoded_len_with_length_delimited() + 1 + { - let encoded_len = msg.encoded_len(); - 1 + (encoded_len as u32).encoded_len() + encoded_len - } -} - -/// Decode a message from a buffer. -pub fn decode_message( - buf: &[u8], -) -> Result, A::Ref<'_>>, DecodeError> -where - I: Data + Eq + core::hash::Hash, - A: Data, -{ - let mut offset = 0; - let buf_len = buf.len(); - let mut msg = None; - - while offset < buf_len { - match buf[offset] { - LEAVE_MESSAGE_BYTE => { - if msg.is_some() { - return Err(DecodeError::duplicate_field( - "Message", - "value", - LEAVE_MESSAGE_TAG, - )); - } - offset += 1; - - let (len, val) = - > as DataRef<'_, LeaveMessage>>::decode_length_delimited( - &buf[offset..], - )?; - offset += len; - msg = Some(MessageRef::Leave(val)); - } - JOIN_MESSAGE_BYTE => { - if msg.is_some() { - return Err(DecodeError::duplicate_field( - "Message", - "value", - JOIN_MESSAGE_TAG, - )); - } - - offset += 1; - let (len, val) = - > as DataRef<'_, JoinMessage>>::decode_length_delimited( - &buf[offset..], - )?; - offset += len; - msg = Some(MessageRef::Join(val)); - } - PUSH_PULL_MESSAGE_BYTE => { - if msg.is_some() { - return Err(DecodeError::duplicate_field( - "Message", - "value", - PUSH_PULL_MESSAGE_TAG, - )); - } - - offset += 1; - let (len, val) = > as DataRef<'_, PushPullMessage>>::decode_length_delimited(&buf[offset..])?; - offset += len; - msg = Some(MessageRef::PushPull(val)); - } - USER_EVENT_MESSAGE_BYTE => { - if msg.is_some() { - return Err(DecodeError::duplicate_field( - "Message", - "value", - USER_EVENT_MESSAGE_TAG, - )); - } - - offset += 1; - let (len, val) = - as DataRef<'_, UserEventMessage>>::decode_length_delimited( - &buf[offset..], - )?; - offset += len; - msg = Some(MessageRef::UserEvent(val)); - } - QUERY_MESSAGE_BYTE => { - if msg.is_some() { - return Err(DecodeError::duplicate_field( - "Message", - "value", - QUERY_MESSAGE_TAG, - )); - } - offset += 1; - let (len, val) = , A::Ref<'_>> as DataRef< - '_, - QueryMessage, - >>::decode_length_delimited(&buf[offset..])?; - offset += len; - msg = Some(MessageRef::Query(val)); - } - QUERY_RESPONSE_MESSAGE_BYTE => { - if msg.is_some() { - return Err(DecodeError::duplicate_field( - "Message", - "value", - QUERY_RESPONSE_MESSAGE_TAG, - )); - } - offset += 1; - let (len, val) = , A::Ref<'_>> as DataRef< - '_, - QueryResponseMessage, - >>::decode_length_delimited(&buf[offset..])?; - offset += len; - msg = Some(MessageRef::QueryResponse(val)); - } - CONFLICT_RESPONSE_MESSAGE_BYTE => { - if msg.is_some() { - return Err(DecodeError::duplicate_field( - "Message", - "value", - CONFLICT_RESPONSE_MESSAGE_TAG, - )); - } - offset += 1; - let (len, val) = , A::Ref<'_>> as DataRef< - '_, - ConflictResponseMessage, - >>::decode_length_delimited(&buf[offset..])?; - offset += len; - msg = Some(MessageRef::ConflictResponse(val)); - } - RELAY_MESSAGE_BYTE => { - if msg.is_some() { - return Err(DecodeError::duplicate_field( - "Message", - "value", - RELAY_MESSAGE_TAG, - )); - } - offset += 1; - let (readed, (node, payload)) = decode_relay::(&buf[offset..])?; - offset += readed; - msg = Some(MessageRef::Relay(RelayMessageRef { - node, - payload, - payload_offset: offset - payload.len(), - })); - } - #[cfg(feature = "encryption")] - KEY_REQUEST_MESSAGE_BYTE => { - if msg.is_some() { - return Err(DecodeError::duplicate_field( - "Message", - "value", - KEY_REQUEST_MESSAGE_TAG, - )); - } - - offset += 1; - let (len, val) = - >::decode_length_delimited( - &buf[offset..], - )?; - offset += len; - msg = Some(MessageRef::KeyRequest(val)); - } - #[cfg(feature = "encryption")] - KEY_RESPONSE_MESSAGE_BYTE => { - if msg.is_some() { - return Err(DecodeError::duplicate_field( - "Message", - "value", - KEY_RESPONSE_MESSAGE_TAG, - )); - } - - offset += 1; - let (len, val) = - as DataRef<'_, KeyResponseMessage>>::decode_length_delimited( - &buf[offset..], - )?; - offset += len; - msg = Some(MessageRef::KeyResponse(val)); - } - _ => offset += skip("Message", &buf[offset..])?, - } - } - - let msg = msg.ok_or(DecodeError::missing_field("Message", "value"))?; - Ok(msg) -} - -fn decode_relay( - buf: &[u8], -) -> Result<(usize, (Node, A::Ref<'_>>, &[u8])), DecodeError> -where - I: Data, - A: Data, -{ - let mut offset = 0; - let buf_len = buf.len(); - - let mut node = None; - let mut msg = None; - - while offset < buf_len { - match buf[offset] { - RELAY_NODE_BYTE => { - if node.is_some() { - return Err(DecodeError::duplicate_field( - "RelayMessage", - "node", - RELAY_NODE_TAG, - )); - } - offset += 1; - - let (len, val) = - , A::Ref<'_>> as DataRef<'_, Node>>::decode_length_delimited( - &buf[offset..], - )?; - offset += len; - node = Some(val); - } - RELAY_MSG_BYTE => { - if msg.is_some() { - return Err(DecodeError::duplicate_field( - "RelayMessage", - "msg", - RELAY_MSG_TAG, - )); - } - offset += 1; - - // Skip length-delimited field by reading the length and skipping the payload - if buf[offset..].len() < 2 { - return Err(DecodeError::buffer_underflow()); - } - - let start_offset = offset; - let _ = buf[offset]; - offset += 1; - - let (read, length) = ::decode(&buf[offset..])?; - offset += read; - offset += length as usize; - msg = Some(&buf[start_offset..offset]); - } - _ => offset += skip("RelayMessage", &buf[offset..])?, - } - } - - let node = node.ok_or(DecodeError::missing_field("RelayMessage", "node"))?; - - Ok((offset, (node, msg.unwrap_or_default()))) -} diff --git a/legacy/serf-core/src/types/push_pull.rs b/legacy/serf-core/src/types/push_pull.rs deleted file mode 100644 index ef649dcb..00000000 --- a/legacy/serf-core/src/types/push_pull.rs +++ /dev/null @@ -1,587 +0,0 @@ -use indexmap::{IndexMap, IndexSet}; -use memberlist_core::proto::{ - Data, DataRef, DecodeError, EncodeError, RepeatedDecoder, TinyVec, TupleEncoder, WireType, - utils::{merge, skip}, -}; - -use super::{LamportTime, UserEvents}; - -/// Used when doing a state exchange. This -/// is a relatively large message, but is sent infrequently -#[viewit::viewit(setters(prefix = "with"))] -#[derive(Debug, Clone)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr( - feature = "serde", - serde(bound( - serialize = "I: core::cmp::Eq + core::hash::Hash + serde::Serialize", - deserialize = "I: core::cmp::Eq + core::hash::Hash + serde::Deserialize<'de>" - )) -)] -#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -#[cfg_attr( - feature = "arbitrary", - arbitrary(bound = "I: arbitrary::Arbitrary<'arbitrary> + core::cmp::Eq + core::hash::Hash") -)] -pub struct PushPullMessage { - /// Current node lamport time - #[viewit( - getter(const, style = "move", attrs(doc = "Returns the lamport time")), - setter(const, attrs(doc = "Sets the lamport time (Builder pattern)")) - )] - ltime: LamportTime, - /// Maps the node to its status time - #[viewit( - getter( - const, - style = "ref", - attrs(doc = "Returns the maps the node to its status time") - ), - setter(attrs(doc = "Sets the maps the node to its status time (Builder pattern)")) - )] - #[cfg_attr(feature = "arbitrary", arbitrary(with = crate::types::arbitrary_impl::arbitrary_indexmap))] - status_ltimes: IndexMap, - /// List of left nodes - #[viewit( - getter(const, style = "ref", attrs(doc = "Returns the list of left nodes")), - setter(attrs(doc = "Sets the list of left nodes (Builder pattern)")) - )] - #[cfg_attr(feature = "arbitrary", arbitrary(with = crate::types::arbitrary_impl::arbitrary_indexset))] - left_members: IndexSet, - /// Lamport time for event clock - #[viewit( - getter( - const, - style = "move", - attrs(doc = "Returns the lamport time for event clock") - ), - setter( - const, - attrs(doc = "Sets the lamport time for event clock (Builder pattern)") - ) - )] - event_ltime: LamportTime, - /// Recent events - #[viewit( - getter(const, style = "ref", attrs(doc = "Returns the recent events")), - setter(attrs(doc = "Sets the recent events (Builder pattern)")) - )] - #[cfg_attr(feature = "arbitrary", arbitrary(with = crate::types::arbitrary_impl::into::, TinyVec>))] - events: TinyVec, - /// Lamport time for query clock - #[viewit( - getter( - const, - style = "move", - attrs(doc = "Returns the lamport time for query clock") - ), - setter( - const, - attrs(doc = "Sets the lamport time for query clock (Builder pattern)") - ) - )] - query_ltime: LamportTime, -} - -impl PartialEq for PushPullMessage -where - I: core::hash::Hash + Eq, -{ - fn eq(&self, other: &Self) -> bool { - self.ltime == other.ltime - && self.status_ltimes == other.status_ltimes - && self.left_members == other.left_members - && self.event_ltime == other.event_ltime - && self.events == other.events - && self.query_ltime == other.query_ltime - } -} - -const LTIME_TAG: u8 = 1; -const STATUS_LTIMES_TAG: u8 = 2; -const LEFT_MEMBERS_TAG: u8 = 3; -const EVENT_LTIME_TAG: u8 = 4; -const EVENTS_TAG: u8 = 5; -const QUERY_LTIME_TAG: u8 = 6; - -const LTIME_BYTE: u8 = merge(WireType::Varint, LTIME_TAG); -const STATUS_LTIMES_BYTE: u8 = merge(WireType::LengthDelimited, STATUS_LTIMES_TAG); -const EVENT_LTIME_BYTE: u8 = merge(WireType::Varint, EVENT_LTIME_TAG); -const EVENTS_BYTE: u8 = merge(WireType::LengthDelimited, EVENTS_TAG); -const QUERY_LTIME_BYTE: u8 = merge(WireType::Varint, QUERY_LTIME_TAG); - -#[inline] -const fn left_members_byte() -> u8 { - merge(I::WIRE_TYPE, LEFT_MEMBERS_TAG) -} - -/// Used when doing a state exchange. This -/// is a relatively large message, but is sent infrequently -#[viewit::viewit(vis_all = "", getters(vis_all = "pub"), setters(skip))] -#[derive(Debug)] -pub struct PushPullMessageRef<'a, I> { - /// Current node lamport time - #[viewit(getter(const, style = "move", attrs(doc = "Returns the lamport time")))] - ltime: LamportTime, - /// Maps the node to its status time - #[viewit(getter( - const, - style = "ref", - attrs(doc = "Returns the maps the node to its status time") - ))] - status_ltimes: RepeatedDecoder<'a>, - /// List of left nodes - #[viewit(getter(const, style = "ref", attrs(doc = "Returns the list of left nodes")))] - left_members: RepeatedDecoder<'a>, - /// Lamport time for event clock - #[viewit(getter( - const, - style = "move", - attrs(doc = "Returns the lamport time for event clock") - ))] - event_ltime: LamportTime, - /// Recent events - #[viewit(getter(const, style = "ref", attrs(doc = "Returns the recent events")))] - events: RepeatedDecoder<'a>, - /// Lamport time for query clock - #[viewit(getter( - const, - style = "move", - attrs(doc = "Returns the lamport time for query clock") - ))] - query_ltime: LamportTime, - #[viewit(getter(skip))] - _m: core::marker::PhantomData, -} - -impl Clone for PushPullMessageRef<'_, I> { - fn clone(&self) -> Self { - *self - } -} - -impl Copy for PushPullMessageRef<'_, I> {} - -impl<'a, I> DataRef<'a, PushPullMessage> for PushPullMessageRef<'a, I::Ref<'a>> -where - I: Data + Eq + core::hash::Hash, -{ - fn decode(buf: &'a [u8]) -> Result<(usize, Self), DecodeError> - where - Self: Sized, - { - let mut offset = 0; - let buf_len = buf.len(); - - let mut ltime = None; - let mut status_ltimes_offsets = None; - let mut num_status_ltimes = 0; - let mut left_members_offsets = None; - let mut num_left_members = 0; - let mut event_ltime = None; - let mut events_offsets = None; - let mut num_events = 0; - let mut query_ltime = None; - - let left_members_byte = left_members_byte::(); - - while offset < buf_len { - match buf[offset] { - LTIME_BYTE => { - if ltime.is_some() { - return Err(DecodeError::duplicate_field( - "PushPullMessage", - "ltime", - LTIME_TAG, - )); - } - - offset += 1; - let (o, v) = >::decode(&buf[offset..])?; - offset += o; - ltime = Some(v); - } - STATUS_LTIMES_BYTE => { - let readed = skip("PushPull", &buf[offset..])?; - if let Some((ref mut fnso, ref mut lnso)) = status_ltimes_offsets { - if *fnso > offset { - *fnso = offset; - } - - if *lnso < offset + readed { - *lnso = offset + readed; - } - } else { - status_ltimes_offsets = Some((offset, offset + readed)); - } - num_status_ltimes += 1; - offset += readed; - } - b if b == left_members_byte => { - let readed = skip("PushPull", &buf[offset..])?; - if let Some((ref mut fnso, ref mut lnso)) = left_members_offsets { - if *fnso > offset { - *fnso = offset; - } - - if *lnso < offset + readed { - *lnso = offset + readed; - } - } else { - left_members_offsets = Some((offset, offset + readed)); - } - num_left_members += 1; - offset += readed; - } - EVENT_LTIME_BYTE => { - if event_ltime.is_some() { - return Err(DecodeError::duplicate_field( - "PushPullMessage", - "event_ltime", - EVENT_LTIME_TAG, - )); - } - - offset += 1; - let (o, v) = >::decode(&buf[offset..])?; - offset += o; - event_ltime = Some(v); - } - EVENTS_BYTE => { - let readed = skip("PushPull", &buf[offset..])?; - if let Some((ref mut fnso, ref mut lnso)) = events_offsets { - if *fnso > offset { - *fnso = offset; - } - - if *lnso < offset + readed { - *lnso = offset + readed; - } - } else { - events_offsets = Some((offset, offset + readed)); - } - num_events += 1; - offset += readed; - } - QUERY_LTIME_BYTE => { - if query_ltime.is_some() { - return Err(DecodeError::duplicate_field( - "PushPullMessage", - "query_ltime", - QUERY_LTIME_TAG, - )); - } - - offset += 1; - let (o, v) = >::decode(&buf[offset..])?; - offset += o; - query_ltime = Some(v); - } - _ => offset += skip("PushPull", &buf[offset..])?, - } - } - - Ok(( - offset, - Self { - ltime: ltime.ok_or_else(|| DecodeError::missing_field("PushPullMessage", "ltime"))?, - status_ltimes: if let Some((start, end)) = status_ltimes_offsets { - RepeatedDecoder::new(STATUS_LTIMES_TAG, WireType::LengthDelimited, buf) - .with_nums(num_status_ltimes) - .with_offsets(start, end) - } else { - RepeatedDecoder::new(STATUS_LTIMES_TAG, WireType::LengthDelimited, buf) - }, - left_members: if let Some((start, end)) = left_members_offsets { - RepeatedDecoder::new(LEFT_MEMBERS_TAG, I::WIRE_TYPE, buf) - .with_nums(num_left_members) - .with_offsets(start, end) - } else { - RepeatedDecoder::new(LEFT_MEMBERS_TAG, I::WIRE_TYPE, buf) - }, - event_ltime: event_ltime - .ok_or_else(|| DecodeError::missing_field("PushPullMessage", "event_ltime"))?, - events: if let Some((start, end)) = events_offsets { - RepeatedDecoder::new(EVENTS_TAG, WireType::LengthDelimited, buf) - .with_nums(num_events) - .with_offsets(start, end) - } else { - RepeatedDecoder::new(EVENTS_TAG, WireType::LengthDelimited, buf) - }, - query_ltime: query_ltime - .ok_or_else(|| DecodeError::missing_field("PushPullMessage", "query_ltime"))?, - _m: core::marker::PhantomData, - }, - )) - } -} - -impl Data for PushPullMessage -where - I: Data + Eq + core::hash::Hash, -{ - type Ref<'a> = PushPullMessageRef<'a, I::Ref<'a>>; - - fn from_ref(val: Self::Ref<'_>) -> Result - where - Self: Sized, - { - Ok(Self { - ltime: val.ltime, - status_ltimes: val - .status_ltimes - .iter::<(I, LamportTime)>() - .map(|res| res.and_then(Data::from_ref)) - .collect::, DecodeError>>()?, - left_members: val - .left_members - .iter::() - .map(|res| res.and_then(Data::from_ref)) - .collect::, DecodeError>>()?, - event_ltime: val.event_ltime, - events: val - .events - .iter::() - .map(|res| res.and_then(Data::from_ref)) - .collect::, DecodeError>>()?, - query_ltime: val.query_ltime, - }) - } - - fn encoded_len(&self) -> usize { - let mut len = 0usize; - - len += 1 + self.ltime.encoded_len(); - - len += self - .status_ltimes - .iter() - .map(|(k, v)| 1 + TupleEncoder::new(k, v).encoded_len_with_length_delimited()) - .sum::(); - - len += self - .left_members - .iter() - .map(|id| 1 + id.encoded_len_with_length_delimited()) - .sum::(); - - len += 1 + self.event_ltime.encoded_len(); - - len += self - .events - .iter() - .map(|e| 1 + e.encoded_len_with_length_delimited()) - .sum::(); - - len += 1 + self.query_ltime.encoded_len(); - - len - } - - fn encode(&self, buf: &mut [u8]) -> Result { - macro_rules! bail { - ($this:ident($offset:expr, $len:ident)) => { - if $offset >= $len { - return Err(EncodeError::insufficient_buffer($this.encoded_len(), $len)); - } - }; - } - - let mut offset = 0; - let buf_len = buf.len(); - - bail!(self(offset, buf_len)); - buf[offset] = LTIME_BYTE; - offset += 1; - offset += self.ltime.encode(&mut buf[offset..])?; - - self - .status_ltimes - .iter() - .try_fold(&mut offset, |off, (k, v)| { - bail!(self(*off, buf_len)); - buf[*off] = STATUS_LTIMES_BYTE; - *off += 1; - *off += TupleEncoder::new(k, v).encode_with_length_delimited(&mut buf[*off..])?; - Ok(off) - }) - .map_err(|e: EncodeError| e.update(self.encoded_len(), buf_len))?; - - let left_members_byte = left_members_byte::(); - self - .left_members - .iter() - .try_fold(&mut offset, |off, id| { - bail!(self(*off, buf_len)); - buf[*off] = left_members_byte; - *off += 1; - *off += id.encode_length_delimited(&mut buf[*off..])?; - Ok(off) - }) - .map_err(|e: EncodeError| e.update(self.encoded_len(), buf_len))?; - - bail!(self(offset, buf_len)); - buf[offset] = EVENT_LTIME_BYTE; - offset += 1; - offset += self.event_ltime.encode(&mut buf[offset..])?; - - self - .events - .iter() - .try_fold(&mut offset, |off, e| { - bail!(self(*off, buf_len)); - buf[*off] = EVENTS_BYTE; - *off += 1; - *off += e.encode_length_delimited(&mut buf[*off..])?; - Ok(off) - }) - .map_err(|e: EncodeError| e.update(self.encoded_len(), buf_len))?; - - bail!(self(offset, buf_len)); - buf[offset] = QUERY_LTIME_BYTE; - offset += 1; - offset += self.query_ltime.encode(&mut buf[offset..])?; - - #[cfg(debug_assertions)] - super::debug_assert_write_eq::(offset, self.encoded_len()); - - Ok(offset) - } -} - -/// Used when doing a state exchange. This -/// is a relatively large message, but is sent infrequently -#[viewit::viewit(getters(skip), setters(skip))] -#[derive(Debug)] -#[cfg_attr(feature = "serde", derive(serde::Serialize))] -pub struct PushPullMessageBorrow<'a, I> { - /// Current node lamport time - ltime: LamportTime, - /// Maps the node to its status time - status_ltimes: &'a IndexMap, - /// List of left nodes - left_members: &'a IndexSet, - /// Lamport time for event clock - event_ltime: LamportTime, - /// Recent events - events: &'a [Option], - /// Lamport time for query clock - query_ltime: LamportTime, -} - -impl Clone for PushPullMessageBorrow<'_, I> { - fn clone(&self) -> Self { - *self - } -} - -impl Copy for PushPullMessageBorrow<'_, I> {} - -impl PushPullMessageBorrow<'_, I> -where - I: Data, -{ - pub(super) fn encoded_len_in(&self) -> usize { - let mut len = 0usize; - - len += 1 + self.ltime.encoded_len(); - - len += self - .status_ltimes - .iter() - .map(|(k, v)| 1 + TupleEncoder::new(k, v).encoded_len_with_length_delimited()) - .sum::(); - - len += self - .left_members - .iter() - .map(|id| 1 + id.encoded_len_with_length_delimited()) - .sum::(); - len += 1 + self.event_ltime.encoded_len(); - len += self - .events - .iter() - .filter_map(|e| { - e.as_ref() - .map(|e| 1 + e.encoded_len_with_length_delimited()) - }) - .sum::(); - len += 1 + self.query_ltime.encoded_len(); - - len - } - - pub(super) fn encode_in(&self, buf: &mut [u8]) -> Result { - macro_rules! bail { - ($this:ident($offset:expr, $len:ident)) => { - if $offset >= $len { - return Err(EncodeError::insufficient_buffer( - $this.encoded_len_in(), - $len, - )); - } - }; - } - - let mut offset = 0; - let buf_len = buf.len(); - - bail!(self(offset, buf_len)); - buf[offset] = LTIME_BYTE; - offset += 1; - offset += self.ltime.encode(&mut buf[offset..])?; - - self - .status_ltimes - .iter() - .try_fold(&mut offset, |off, (k, v)| { - bail!(self(*off, buf_len)); - buf[*off] = STATUS_LTIMES_BYTE; - *off += 1; - *off += TupleEncoder::new(k, v).encode_with_length_delimited(&mut buf[*off..])?; - Ok(off) - }) - .map_err(|e: EncodeError| e.update(self.encoded_len_in(), buf_len))?; - - let left_members_byte = left_members_byte::(); - self - .left_members - .iter() - .try_fold(&mut offset, |off, id| { - bail!(self(*off, buf_len)); - buf[*off] = left_members_byte; - *off += 1; - *off += id.encode_length_delimited(&mut buf[*off..])?; - Ok(off) - }) - .map_err(|e: EncodeError| e.update(self.encoded_len_in(), buf_len))?; - - bail!(self(offset, buf_len)); - buf[offset] = EVENT_LTIME_BYTE; - offset += 1; - offset += self.event_ltime.encode(&mut buf[offset..])?; - - self - .events - .iter() - .filter_map(|e| e.as_ref()) - .try_fold(&mut offset, |off, e| { - bail!(self(*off, buf_len)); - buf[*off] = EVENTS_BYTE; - *off += 1; - *off += e.encode_length_delimited(&mut buf[*off..])?; - Ok(off) - }) - .map_err(|e: EncodeError| e.update(self.encoded_len_in(), buf_len))?; - - bail!(self(offset, buf_len)); - buf[offset] = QUERY_LTIME_BYTE; - offset += 1; - offset += self.query_ltime.encode(&mut buf[offset..])?; - - #[cfg(debug_assertions)] - super::debug_assert_write_eq::(offset, self.encoded_len_in()); - - Ok(offset) - } -} diff --git a/legacy/serf-core/src/types/query.rs b/legacy/serf-core/src/types/query.rs deleted file mode 100644 index 3209612b..00000000 --- a/legacy/serf-core/src/types/query.rs +++ /dev/null @@ -1,527 +0,0 @@ -use smol_str::SmolStr; - -use std::time::Duration; - -use memberlist_core::proto::{ - Data, DataRef, DecodeError, EncodeError, Node, RepeatedDecoder, TinyVec, WireType, - bytes::Bytes, - utils::{merge, skip}, -}; - -use super::{Filter, LamportTime}; - -pub use response::*; - -mod response; - -bitflags::bitflags! { - /// Flags for query message - #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] - #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] - #[cfg_attr(feature = "serde", serde(transparent))] - pub struct QueryFlag: u32 { - /// Ack flag is used to force receiver to send an ack back - const ACK = 1 << 0; - /// NoBroadcast is used to prevent re-broadcast of a query. - /// this can be used to selectively send queries to individual members - const NO_BROADCAST = 1 << 1; - } -} - -const LTIME_TAG: u8 = 1; -const ID_TAG: u8 = 2; -const FROM_TAG: u8 = 3; -const FILTERS_TAG: u8 = 4; -const FLAGS_TAG: u8 = 5; -const RELAY_FACTOR_TAG: u8 = 6; -const TIMEOUT_TAG: u8 = 7; -const NAME_TAG: u8 = 8; -const PAYLOAD_TAG: u8 = 9; - -const LTIME_BYTE: u8 = merge(WireType::Varint, LTIME_TAG); -const ID_BYTE: u8 = merge(WireType::Varint, ID_TAG); -const FROM_BYTE: u8 = merge(WireType::LengthDelimited, FROM_TAG); -const FILTERS_BYTE: u8 = merge(WireType::LengthDelimited, FILTERS_TAG); -const FLAGS_BYTE: u8 = merge(WireType::Varint, FLAGS_TAG); -const RELAY_FACTOR_BYTE: u8 = merge(WireType::Varint, RELAY_FACTOR_TAG); -const TIMEOUT_BYTE: u8 = merge(WireType::Varint, TIMEOUT_TAG); -const NAME_BYTE: u8 = merge(WireType::LengthDelimited, NAME_TAG); -const PAYLOAD_BYTE: u8 = merge(WireType::LengthDelimited, PAYLOAD_TAG); - -/// Query message -#[viewit::viewit(getters(style = "ref"), setters(prefix = "with"))] -#[derive(Debug, Clone, Eq, PartialEq)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -pub struct QueryMessage { - /// Event lamport time - #[viewit( - getter(const, style = "move", attrs(doc = "Returns the event lamport time")), - setter(const, attrs(doc = "Sets the event lamport time (Builder pattern)")) - )] - ltime: LamportTime, - /// query id, randomly generated - #[viewit( - getter(const, style = "move", attrs(doc = "Returns the query id")), - setter(attrs(doc = "Sets the query id (Builder pattern)")) - )] - id: u32, - /// source node - #[viewit( - getter(const, attrs(doc = "Returns the from node")), - setter(attrs(doc = "Sets the from node (Builder pattern)")) - )] - from: Node, - /// Potential query filters - #[viewit( - getter(const, attrs(doc = "Returns the potential query filters")), - setter(attrs(doc = "Sets the potential query filters (Builder pattern)")) - )] - #[cfg_attr(feature = "arbitrary", arbitrary(with = crate::types::arbitrary_impl::into::>, TinyVec>>))] - filters: TinyVec>, - /// Used to provide various flags - #[viewit( - getter(const, style = "move", attrs(doc = "Returns the flags")), - setter(attrs(doc = "Sets the flags (Builder pattern)")) - )] - flags: QueryFlag, - /// Used to set the number of duplicate relayed responses - #[viewit( - getter( - const, - style = "move", - attrs(doc = "Returns the number of duplicate relayed responses") - ), - setter(attrs(doc = "Sets the number of duplicate relayed responses (Builder pattern)")) - )] - relay_factor: u8, - /// Maximum time between delivery and response - #[viewit( - getter( - const, - style = "move", - attrs(doc = "Returns the maximum time between delivery and response") - ), - setter(attrs(doc = "Sets the maximum time between delivery and response (Builder pattern)")) - )] - timeout: Duration, - /// Query nqme - #[viewit( - getter(const, style = "ref", attrs(doc = "Returns the name of the query")), - setter(attrs(doc = "Sets the name of the query (Builder pattern)")) - )] - name: SmolStr, - /// Query payload - #[viewit( - getter(const, style = "ref", attrs(doc = "Returns the payload")), - setter(attrs(doc = "Sets the payload (Builder pattern)")) - )] - #[cfg_attr(feature = "arbitrary", arbitrary(with = crate::types::arbitrary_impl::into::, Bytes>))] - payload: Bytes, -} - -impl QueryMessage { - /// Checks if the ack flag is set - #[inline] - pub fn ack(&self) -> bool { - self.flags.contains(QueryFlag::ACK) - } - - /// Checks if the no broadcast flag is set - #[inline] - pub fn no_broadcast(&self) -> bool { - self.flags.contains(QueryFlag::NO_BROADCAST) - } -} - -/// The reference type of [`QueryMessage`] -#[viewit::viewit(vis_all = "", getters(vis_all = "pub", style = "ref"), setters(skip))] -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub struct QueryMessageRef<'a, I, A> { - /// Event lamport time - #[viewit(getter(const, style = "move", attrs(doc = "Returns the event lamport time")))] - ltime: LamportTime, - /// query id, randomly generated - #[viewit(getter(const, style = "move", attrs(doc = "Returns the query id")))] - id: u32, - /// source node - #[viewit(getter(const, attrs(doc = "Returns the from node")))] - from: Node, - /// Potential query filters - #[viewit(getter(const, attrs(doc = "Returns the potential query filters")))] - filters: RepeatedDecoder<'a>, - /// Used to provide various flags - #[viewit(getter(const, style = "move", attrs(doc = "Returns the flags")))] - flags: QueryFlag, - /// Used to set the number of duplicate relayed responses - #[viewit(getter( - const, - style = "move", - attrs(doc = "Returns the number of duplicate relayed responses") - ))] - relay_factor: u8, - /// Maximum time between delivery and response - #[viewit(getter( - const, - style = "move", - attrs(doc = "Returns the maximum time between delivery and response") - ))] - timeout: Duration, - /// Query nqme - #[viewit(getter(const, style = "move", attrs(doc = "Returns the name of the query")))] - name: &'a str, - /// Query payload - #[viewit(getter(const, style = "move", attrs(doc = "Returns the payload")))] - payload: &'a [u8], -} - -impl QueryMessageRef<'_, I, A> { - /// Checks if the ack flag is set - #[inline] - pub fn ack(&self) -> bool { - self.flags.contains(QueryFlag::ACK) - } - - /// Checks if the no broadcast flag is set - #[inline] - pub fn no_broadcast(&self) -> bool { - self.flags.contains(QueryFlag::NO_BROADCAST) - } -} - -impl<'a, I, A> DataRef<'a, QueryMessage> for QueryMessageRef<'a, I::Ref<'a>, A::Ref<'a>> -where - I: Data, - A: Data, -{ - fn decode(buf: &'a [u8]) -> Result<(usize, Self), DecodeError> - where - Self: Sized, - { - let mut offset = 0; - let buf_len = buf.len(); - - let mut ltime = None; - let mut id = None; - let mut from = None; - let mut filters_offsets = None; - let mut num_filters = 0; - let mut flags = None; - let mut relay_factor = None; - let mut timeout = None; - let mut name = None; - let mut payload = None; - - while offset < buf_len { - match buf[offset] { - LTIME_BYTE => { - if ltime.is_some() { - return Err(DecodeError::duplicate_field( - "QueryMessage", - "ltime", - LTIME_TAG, - )); - } - - offset += 1; - let (o, v) = >::decode(&buf[offset..])?; - offset += o; - ltime = Some(v); - } - ID_BYTE => { - if id.is_some() { - return Err(DecodeError::duplicate_field("QueryMessage", "id", ID_TAG)); - } - - offset += 1; - let (o, v) = >::decode(&buf[offset..])?; - offset += o; - id = Some(v); - } - FROM_BYTE => { - if from.is_some() { - return Err(DecodeError::duplicate_field( - "QueryMessage", - "from", - FROM_TAG, - )); - } - - offset += 1; - let (o, v) = - , A::Ref<'_>> as DataRef<'_, Node>>::decode_length_delimited( - &buf[offset..], - )?; - offset += o; - from = Some(v); - } - FILTERS_BYTE => { - let readed = skip("QueryMessage", &buf[offset..])?; - if let Some((ref mut fnso, ref mut lnso)) = filters_offsets { - if *fnso > offset { - *fnso = offset; - } - - if *lnso < offset + readed { - *lnso = offset + readed; - } - } else { - filters_offsets = Some((offset, offset + readed)); - } - num_filters += 1; - offset += readed; - } - FLAGS_BYTE => { - if flags.is_some() { - return Err(DecodeError::duplicate_field( - "QueryMessage", - "flags", - FLAGS_TAG, - )); - } - - offset += 1; - let (o, v) = >::decode(&buf[offset..])?; - offset += o; - flags = Some(QueryFlag::from_bits_truncate(v)); - } - RELAY_FACTOR_BYTE => { - if relay_factor.is_some() { - return Err(DecodeError::duplicate_field( - "QueryMessage", - "relay_factor", - RELAY_FACTOR_TAG, - )); - } - - offset += 1; - let (o, v) = >::decode(&buf[offset..])?; - offset += o; - relay_factor = Some(v); - } - TIMEOUT_BYTE => { - if timeout.is_some() { - return Err(DecodeError::duplicate_field( - "QueryMessage", - "timeout", - TIMEOUT_TAG, - )); - } - - offset += 1; - let (o, v) = >::decode(&buf[offset..])?; - offset += o; - timeout = Some(v); - } - NAME_BYTE => { - if name.is_some() { - return Err(DecodeError::duplicate_field( - "QueryMessage", - "name", - NAME_TAG, - )); - } - - offset += 1; - let (o, v) = <&str as DataRef<'_, SmolStr>>::decode_length_delimited(&buf[offset..])?; - offset += o; - name = Some(v); - } - PAYLOAD_BYTE => { - if payload.is_some() { - return Err(DecodeError::duplicate_field( - "QueryMessage", - "payload", - PAYLOAD_TAG, - )); - } - - offset += 1; - let (o, v) = <&[u8] as DataRef<'_, Bytes>>::decode_length_delimited(&buf[offset..])?; - offset += o; - payload = Some(v); - } - _ => offset += skip("QueryMessage", &buf[offset..])?, - } - } - - let filters = - RepeatedDecoder::new(FILTERS_TAG, WireType::LengthDelimited, buf).with_nums(num_filters); - - Ok(( - offset, - Self { - ltime: ltime.ok_or_else(|| DecodeError::missing_field("QueryMessage", "ltime"))?, - id: id.ok_or_else(|| DecodeError::missing_field("QueryMessage", "id"))?, - from: from.ok_or_else(|| DecodeError::missing_field("QueryMessage", "from"))?, - filters: if let Some((start, end)) = filters_offsets { - filters.with_offsets(start, end) - } else { - filters - }, - flags: flags.ok_or_else(|| DecodeError::missing_field("QueryMessage", "flags"))?, - relay_factor: relay_factor - .ok_or_else(|| DecodeError::missing_field("QueryMessage", "relay_factor"))?, - timeout: timeout.ok_or_else(|| DecodeError::missing_field("QueryMessage", "timeout"))?, - name: name.unwrap_or_default(), - payload: payload.unwrap_or_default(), - }, - )) - } -} - -impl Data for QueryMessage -where - I: Data, - A: Data, -{ - type Ref<'a> = QueryMessageRef<'a, I::Ref<'a>, A::Ref<'a>>; - - fn from_ref(val: Self::Ref<'_>) -> Result - where - Self: Sized, - { - val - .filters - .iter::>() - .map(|res| res.and_then(Data::from_ref)) - .collect::, DecodeError>>() - .and_then(|filters| { - Ok(Self { - ltime: val.ltime, - id: val.id, - from: Node::from_ref(val.from)?, - filters, - flags: val.flags, - relay_factor: val.relay_factor, - timeout: val.timeout, - name: SmolStr::from(val.name), - payload: Bytes::copy_from_slice(val.payload), - }) - }) - } - - fn encoded_len(&self) -> usize { - let mut len = 0; - - len += 1 + self.ltime.encoded_len(); - len += 1 + self.id.encoded_len(); - len += 1 + self.from.encoded_len_with_length_delimited(); - len += self - .filters - .iter() - .map(|f| 1 + f.encoded_len_with_length_delimited()) - .sum::(); - - len += 1 + self.flags.bits().encoded_len(); - len += 1 + self.relay_factor.encoded_len(); - len += 1 + self.timeout.encoded_len(); - - let nlen = self.name.len(); - - if nlen != 0 { - len += 1 + self.name.encoded_len_with_length_delimited(); - } - - let plen = self.payload.len(); - - if plen != 0 { - len += 1 + self.payload.encoded_len_with_length_delimited(); - } - - len - } - - fn encode(&self, buf: &mut [u8]) -> Result { - macro_rules! bail { - ($this:ident($offset:expr, $len:ident)) => { - if $offset >= $len { - return Err(EncodeError::insufficient_buffer(self.encoded_len(), $len)); - } - }; - } - - let mut offset = 0; - let buf_len = buf.len(); - - bail!(self(offset, buf_len)); - buf[offset] = LTIME_BYTE; - offset += 1; - offset += self - .ltime - .encode(&mut buf[offset..]) - .map_err(|e| e.update(self.encoded_len(), buf_len))?; - - bail!(self(offset, buf_len)); - buf[offset] = ID_BYTE; - offset += 1; - offset += self - .id - .encode(&mut buf[offset..]) - .map_err(|e| e.update(self.encoded_len(), buf_len))?; - - bail!(self(offset, buf_len)); - buf[offset] = FROM_BYTE; - offset += 1; - offset += self - .from - .encode_length_delimited(&mut buf[offset..]) - .map_err(|e| e.update(self.encoded_len(), buf_len))?; - - for filter in self.filters.iter() { - bail!(self(offset, buf_len)); - buf[offset] = FILTERS_BYTE; - offset += 1; - - offset += filter - .encode_length_delimited(&mut buf[offset..]) - .map_err(|e| e.update(self.encoded_len(), buf_len))?; - } - - bail!(self(offset, buf_len)); - buf[offset] = FLAGS_BYTE; - offset += 1; - offset += ::encode(&self.flags.bits(), &mut buf[offset..]) - .map_err(|e| e.update(self.encoded_len(), buf_len))?; - - bail!(self(offset, buf_len)); - buf[offset] = RELAY_FACTOR_BYTE; - offset += 1; - bail!(self(offset, buf_len)); - buf[offset] = self.relay_factor; - offset += 1; - - bail!(self(offset, buf_len)); - buf[offset] = TIMEOUT_BYTE; - offset += 1; - offset += self - .timeout - .encode(&mut buf[offset..]) - .map_err(|e| e.update(self.encoded_len(), buf_len))?; - - if !self.name.is_empty() { - bail!(self(offset, buf_len)); - buf[offset] = NAME_BYTE; - offset += 1; - offset += self - .name - .encode_length_delimited(&mut buf[offset..]) - .map_err(|e| e.update(self.encoded_len(), buf_len))?; - } - - if !self.payload.is_empty() { - bail!(self(offset, buf_len)); - buf[offset] = PAYLOAD_BYTE; - offset += 1; - offset += self - .payload - .encode_length_delimited(&mut buf[offset..]) - .map_err(|e| e.update(self.encoded_len(), buf_len))?; - } - - #[cfg(debug_assertions)] - super::debug_assert_write_eq::(offset, self.encoded_len()); - - Ok(offset) - } -} diff --git a/legacy/serf-core/src/types/query/response.rs b/legacy/serf-core/src/types/query/response.rs deleted file mode 100644 index e79364b0..00000000 --- a/legacy/serf-core/src/types/query/response.rs +++ /dev/null @@ -1,303 +0,0 @@ -use memberlist_core::proto::{ - Data, DataRef, DecodeError, EncodeError, Node, WireType, - bytes::Bytes, - utils::{merge, skip}, -}; - -use super::{LamportTime, QueryFlag}; - -const LTIME_TAG: u8 = 1; -const ID_TAG: u8 = 2; -const FROM_TAG: u8 = 3; -const FLAGS_TAG: u8 = 4; -const PAYLOAD_TAG: u8 = 5; - -const LTIME_BYTE: u8 = merge(WireType::Varint, LTIME_TAG); -const ID_BYTE: u8 = merge(WireType::Varint, ID_TAG); -const FROM_BYTE: u8 = merge(WireType::LengthDelimited, FROM_TAG); -const FLAGS_BYTE: u8 = merge(WireType::Varint, FLAGS_TAG); -const PAYLOAD_BYTE: u8 = merge(WireType::LengthDelimited, PAYLOAD_TAG); - -/// Query response message -#[viewit::viewit(getters(style = "ref"), setters(prefix = "with"))] -#[derive(Debug, Clone, Eq, PartialEq)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -pub struct QueryResponseMessage { - /// Event lamport time - #[viewit( - getter(const, attrs(doc = "Returns the lamport time for this message")), - setter( - const, - attrs(doc = "Sets the lamport time for this message (Builder pattern)") - ) - )] - ltime: LamportTime, - /// query id - #[viewit( - getter(const, attrs(doc = "Returns the query id")), - setter(attrs(doc = "Sets the query id (Builder pattern)")) - )] - id: u32, - /// node - #[viewit( - getter(const, attrs(doc = "Returns the from node")), - setter(attrs(doc = "Sets the from node (Builder pattern)")) - )] - from: Node, - /// Used to provide various flags - #[viewit( - getter(const, style = "ref", attrs(doc = "Returns the flags")), - setter(attrs(doc = "Sets the flags (Builder pattern)")) - )] - flags: QueryFlag, - /// Optional response payload - #[viewit( - getter(const, style = "ref", attrs(doc = "Returns the payload")), - setter(attrs(doc = "Sets the payload (Builder pattern)")) - )] - #[cfg_attr(feature = "arbitrary", arbitrary(with = crate::types::arbitrary_impl::into::, Bytes>))] - payload: Bytes, -} - -impl QueryResponseMessage { - /// Checks if the ack flag is set - #[inline] - pub fn ack(&self) -> bool { - self.flags.contains(QueryFlag::ACK) - } -} - -/// The reference type to a query response message -#[viewit::viewit( - vis_all = "pub(crate)", - getters(vis_all = "pub", style = "ref"), - setters(skip) -)] -#[derive(Debug, Copy, Clone, Eq, PartialEq)] -pub struct QueryResponseMessageRef<'a, I, A> { - /// Event lamport time - #[viewit(getter( - const, - style = "move", - attrs(doc = "Returns the lamport time for this message") - ))] - ltime: LamportTime, - /// query id - #[viewit(getter(const, style = "move", attrs(doc = "Returns the query id")))] - id: u32, - /// node - #[viewit(getter(const, attrs(doc = "Returns the from node")))] - from: Node, - /// Used to provide various flags - #[viewit(getter(const, style = "move", attrs(doc = "Returns the flags")))] - flags: QueryFlag, - /// Optional response payload - #[viewit(getter(const, style = "move", attrs(doc = "Returns the payload")))] - payload: &'a [u8], -} - -impl<'a, I, A> DataRef<'a, QueryResponseMessage> - for QueryResponseMessageRef<'a, I::Ref<'a>, A::Ref<'a>> -where - I: Data, - A: Data, -{ - fn decode(buf: &'a [u8]) -> Result<(usize, Self), DecodeError> - where - Self: Sized, - { - let mut offset = 0; - let buf_len = buf.len(); - - let mut ltime = None; - let mut id = None; - let mut from = None; - let mut flags = None; - let mut payload = None; - - while offset < buf_len { - match buf[offset] { - LTIME_BYTE => { - if ltime.is_some() { - return Err(DecodeError::duplicate_field( - "QueryResponseMessage", - "ltime", - LTIME_TAG, - )); - } - - offset += 1; - let (o, v) = >::decode(&buf[offset..])?; - offset += o; - ltime = Some(v); - } - ID_BYTE => { - if id.is_some() { - return Err(DecodeError::duplicate_field( - "QueryResponseMessage", - "id", - ID_TAG, - )); - } - - offset += 1; - let (o, v) = >::decode(&buf[offset..])?; - offset += o; - id = Some(v); - } - FROM_BYTE => { - if from.is_some() { - return Err(DecodeError::duplicate_field( - "QueryResponseMessage", - "from", - FROM_TAG, - )); - } - - offset += 1; - let (o, v) = - , A::Ref<'_>> as DataRef<'_, Node>>::decode_length_delimited( - &buf[offset..], - )?; - offset += o; - from = Some(v); - } - FLAGS_BYTE => { - if flags.is_some() { - return Err(DecodeError::duplicate_field( - "QueryResponseMessage", - "flags", - FLAGS_TAG, - )); - } - - offset += 1; - let (o, v) = >::decode(&buf[offset..])?; - offset += o; - flags = Some(QueryFlag::from_bits_retain(v)); - } - PAYLOAD_BYTE => { - if payload.is_some() { - return Err(DecodeError::duplicate_field( - "QueryResponseMessage", - "payload", - PAYLOAD_TAG, - )); - } - - offset += 1; - let (o, v) = <&[u8] as DataRef<'_, Bytes>>::decode_length_delimited(&buf[offset..])?; - offset += o; - payload = Some(v); - } - _ => offset += skip("QueryResponseMessage", &buf[offset..])?, - } - } - - Ok(( - offset, - Self { - ltime: ltime.ok_or_else(|| DecodeError::missing_field("QueryResponseMessage", "ltime"))?, - id: id.ok_or_else(|| DecodeError::missing_field("QueryResponseMessage", "id"))?, - from: from.ok_or_else(|| DecodeError::missing_field("QueryResponseMessage", "from"))?, - flags: flags.ok_or_else(|| DecodeError::missing_field("QueryResponseMessage", "flags"))?, - payload: payload.unwrap_or_default(), - }, - )) - } -} - -impl Data for QueryResponseMessage -where - I: Data, - A: Data, -{ - type Ref<'a> = QueryResponseMessageRef<'a, I::Ref<'a>, A::Ref<'a>>; - - fn from_ref(val: Self::Ref<'_>) -> Result - where - Self: Sized, - { - Ok(Self { - ltime: val.ltime, - id: val.id, - from: Node::from_ref(val.from)?, - flags: val.flags, - payload: Bytes::copy_from_slice(val.payload), - }) - } - - fn encoded_len(&self) -> usize { - let mut len = 1 + self.ltime.encoded_len(); - len += 1 + self.id.encoded_len(); - len += 1 + self.from.encoded_len_with_length_delimited(); - len += 1 + self.flags.bits().encoded_len(); // flags - let plen = self.payload.len(); - if plen > 0 { - len += 1 + self.payload.encoded_len_with_length_delimited(); - } - - len - } - - fn encode(&self, buf: &mut [u8]) -> Result { - macro_rules! bail { - ($this:ident($offset:expr, $len:ident)) => { - if $offset >= $len { - return Err(EncodeError::insufficient_buffer(self.encoded_len(), $len)); - } - }; - } - - let buf_len = buf.len(); - let mut offset = 0; - - bail!(self(offset, buf_len)); - buf[offset] = LTIME_BYTE; - offset += 1; - offset += self - .ltime - .encode(&mut buf[offset..]) - .map_err(|e| e.update(self.encoded_len(), buf_len))?; - - bail!(self(offset, buf_len)); - buf[offset] = ID_BYTE; - offset += 1; - offset += self - .id - .encode(&mut buf[offset..]) - .map_err(|e| e.update(self.encoded_len(), buf_len))?; - - bail!(self(offset, buf_len)); - buf[offset] = FROM_BYTE; - offset += 1; - - offset += self - .from - .encode_length_delimited(&mut buf[offset..]) - .map_err(|e| e.update(self.encoded_len(), buf_len))?; - - bail!(self(offset, buf_len)); - buf[offset] = FLAGS_BYTE; - offset += 1; - - offset += ::encode(&self.flags.bits(), &mut buf[offset..]) - .map_err(|e| e.update(self.encoded_len(), buf_len))?; - - if !self.payload.is_empty() { - bail!(self(offset, buf_len)); - buf[offset] = PAYLOAD_BYTE; - offset += 1; - - offset += self - .payload - .encode_length_delimited(&mut buf[offset..]) - .map_err(|e| e.update(self.encoded_len(), buf_len))?; - } - - #[cfg(debug_assertions)] - super::super::debug_assert_write_eq::(offset, self.encoded_len()); - Ok(offset) - } -} diff --git a/legacy/serf-core/src/types/quickcheck_impl.rs b/legacy/serf-core/src/types/quickcheck_impl.rs deleted file mode 100644 index 61faba06..00000000 --- a/legacy/serf-core/src/types/quickcheck_impl.rs +++ /dev/null @@ -1,324 +0,0 @@ -use core::hash::Hash; - -use quickcheck::{Arbitrary, Gen}; -use smol_str::SmolStr; - -use super::{ - ConflictResponseMessage, DelegateVersion, Filter, JoinMessage, LamportTime, LeaveMessage, Member, - MemberStatus, MessageType, ProtocolVersion, PushPullMessage, QueryFlag, QueryMessage, - QueryResponseMessage, TagFilter, Tags, UserEvent, UserEventMessage, UserEvents, - coordinate::Coordinate, -}; - -#[cfg(feature = "encryption")] -use super::{KeyRequestMessage, KeyResponseMessage}; - -impl Arbitrary for ProtocolVersion { - fn arbitrary(g: &mut Gen) -> Self { - ProtocolVersion::from(u8::arbitrary(g)) - } -} - -impl Arbitrary for DelegateVersion { - fn arbitrary(g: &mut Gen) -> Self { - DelegateVersion::from(u8::arbitrary(g)) - } -} - -impl Arbitrary for UserEvent { - fn arbitrary(g: &mut Gen) -> Self { - Self { - name: String::arbitrary(g).into(), - payload: Vec::arbitrary(g).into(), - } - } -} - -impl Arbitrary for UserEvents { - fn arbitrary(g: &mut Gen) -> Self { - Self { - ltime: u64::arbitrary(g).into(), - events: Vec::arbitrary(g).into(), - } - } -} - -impl Arbitrary for UserEventMessage { - fn arbitrary(g: &mut Gen) -> Self { - Self { - ltime: u64::arbitrary(g).into(), - name: String::arbitrary(g).into(), - payload: Vec::arbitrary(g).into(), - cc: bool::arbitrary(g), - } - } -} - -impl Arbitrary for LamportTime { - fn arbitrary(g: &mut Gen) -> Self { - LamportTime::from(u64::arbitrary(g)) - } -} - -impl Arbitrary for TagFilter { - fn arbitrary(g: &mut Gen) -> Self { - Self::new() - .with_tag(String::arbitrary(g).into()) - .maybe_expr(if Arbitrary::arbitrary(g) { - let complexity = *g.choose(&[1, 2, 3, 4, 5]).unwrap(); - let mut patterns = Vec::new(); - - // Basic character classes and quantifiers - let character_classes = vec![ - r"\d", - r"\w", - r"\s", - r"[a-z]", - r"[A-Z]", - r"[0-9]", - r"[a-zA-Z]", - r"[a-zA-Z0-9]", - r".", - ]; - - let quantifiers = vec!["", "*", "+", "?", "{1,3}", "{2,5}"]; - - // Add more complex patterns for higher complexity - let mut extended_classes = character_classes.clone(); - if complexity > 1 { - extended_classes.extend(vec![r"[^a-z]", r"[^0-9]", r"\D", r"\W", r"\S"]); - } - - if complexity > 2 { - // Add a group with random content - let char_class = *g.choose(&extended_classes).unwrap(); - let quantifier = *g.choose(&quantifiers).unwrap(); - patterns.push(format!("({}{})", char_class, quantifier)); - } - - // Generate random pattern parts - for _ in 0..complexity { - let char_class = *g.choose(&extended_classes).unwrap(); - let quantifier = *g.choose(&quantifiers).unwrap(); - patterns.push(format!("{}{}", char_class, quantifier)); - } - - // Maybe add anchors for higher complexity - if complexity > 2 && rand::random_ratio(7, 10) { - if Arbitrary::arbitrary(g) { - patterns.insert(0, "^".to_string()); - } - if Arbitrary::arbitrary(g) { - patterns.push("$".to_string()); - } - } - - // Add alternation for even higher complexity - if complexity > 3 && rand::random_ratio(6, 10) { - let char_class = *g.choose(&extended_classes).unwrap(); - let quantifier = *g.choose(&quantifiers).unwrap(); - patterns.push(format!("|{}{}", char_class, quantifier)); - } - - Some(patterns.join("").try_into().unwrap()) - } else { - None - }) - } -} - -impl Arbitrary for Tags { - fn arbitrary(g: &mut Gen) -> Self { - Self::from_iter( - Vec::<(String, String)>::arbitrary(g) - .into_iter() - .map(|(k, v)| (SmolStr::from(k), SmolStr::from(v))), - ) - } -} - -impl Arbitrary for QueryMessage -where - I: Arbitrary, - A: Arbitrary, -{ - fn arbitrary(g: &mut Gen) -> Self { - Self { - ltime: Arbitrary::arbitrary(g), - flags: Arbitrary::arbitrary(g), - id: Arbitrary::arbitrary(g), - from: Arbitrary::arbitrary(g), - filters: Vec::arbitrary(g).into(), - relay_factor: Arbitrary::arbitrary(g), - timeout: Arbitrary::arbitrary(g), - name: String::arbitrary(g).into(), - payload: Vec::arbitrary(g).into(), - } - } -} - -impl Arbitrary for QueryFlag { - fn arbitrary(g: &mut Gen) -> Self { - if bool::arbitrary(g) { - QueryFlag::NO_BROADCAST - } else { - QueryFlag::ACK - } - } -} - -impl Arbitrary for QueryResponseMessage -where - I: Arbitrary, - A: Arbitrary, -{ - fn arbitrary(g: &mut Gen) -> Self { - Self { - ltime: Arbitrary::arbitrary(g), - id: Arbitrary::arbitrary(g), - from: Arbitrary::arbitrary(g), - flags: Arbitrary::arbitrary(g), - payload: Vec::arbitrary(g).into(), - } - } -} - -impl Arbitrary for Filter -where - I: Arbitrary, -{ - fn arbitrary(g: &mut Gen) -> Self { - if bool::arbitrary(g) { - Filter::Id(Vec::::arbitrary(g).into()) - } else { - Filter::Tag(TagFilter::arbitrary(g)) - } - } -} - -impl Arbitrary for PushPullMessage -where - I: Arbitrary + Hash + Eq, -{ - fn arbitrary(g: &mut Gen) -> Self { - Self { - ltime: Arbitrary::arbitrary(g), - status_ltimes: Vec::<(I, LamportTime)>::arbitrary(g).into_iter().collect(), - left_members: Vec::::arbitrary(g).into_iter().collect(), - event_ltime: Arbitrary::arbitrary(g), - events: Vec::::arbitrary(g).into(), - query_ltime: Arbitrary::arbitrary(g), - } - } -} - -impl Arbitrary for MemberStatus { - fn arbitrary(g: &mut Gen) -> Self { - MemberStatus::from(u8::arbitrary(g)) - } -} - -impl Arbitrary for Member -where - I: Arbitrary, - A: Arbitrary, -{ - fn arbitrary(g: &mut Gen) -> Self { - Self { - node: Arbitrary::arbitrary(g), - tags: Tags::arbitrary(g).into(), - status: Arbitrary::arbitrary(g), - memberlist_protocol_version: Arbitrary::arbitrary(g), - memberlist_delegate_version: Arbitrary::arbitrary(g), - protocol_version: Arbitrary::arbitrary(g), - delegate_version: Arbitrary::arbitrary(g), - } - } -} - -impl Arbitrary for LeaveMessage -where - I: Arbitrary, -{ - fn arbitrary(g: &mut Gen) -> Self { - Self { - ltime: Arbitrary::arbitrary(g), - prune: bool::arbitrary(g), - id: Arbitrary::arbitrary(g), - } - } -} - -#[cfg(feature = "encryption")] -impl Arbitrary for KeyRequestMessage { - fn arbitrary(g: &mut Gen) -> Self { - Self { - key: Arbitrary::arbitrary(g), - } - } -} - -#[cfg(feature = "encryption")] -impl Arbitrary for KeyResponseMessage { - fn arbitrary(g: &mut Gen) -> Self { - Self { - result: Arbitrary::arbitrary(g), - message: String::arbitrary(g).into(), - keys: Vec::arbitrary(g).into(), - primary_key: Arbitrary::arbitrary(g), - } - } -} - -impl Arbitrary for JoinMessage -where - I: Arbitrary, -{ - fn arbitrary(g: &mut Gen) -> Self { - Self { - ltime: Arbitrary::arbitrary(g), - id: Arbitrary::arbitrary(g), - } - } -} - -impl Arbitrary for ConflictResponseMessage -where - I: Arbitrary, - A: Arbitrary, -{ - fn arbitrary(g: &mut Gen) -> Self { - Self { - member: Arbitrary::arbitrary(g), - } - } -} - -impl Arbitrary for MessageType { - fn arbitrary(g: &mut Gen) -> Self { - MessageType::from(u8::arbitrary(g) % Self::ALL.len() as u8) - } -} - -impl Arbitrary for Coordinate { - fn arbitrary(g: &mut Gen) -> Self { - Self { - portion: Vec::::arbitrary(g) - .into_iter() - .map(|f| if f.is_nan() { 0.0 } else { f }) - .collect(), - error: rand_f64_not_nan(g), - adjustment: rand_f64_not_nan(g), - height: rand_f64_not_nan(g), - } - } -} - -fn rand_f64_not_nan(u: &mut Gen) -> f64 { - loop { - let f = f64::arbitrary(u); - if !f.is_nan() { - return f; - } - } -} diff --git a/legacy/serf-core/src/types/tags.rs b/legacy/serf-core/src/types/tags.rs deleted file mode 100644 index 31f5c673..00000000 --- a/legacy/serf-core/src/types/tags.rs +++ /dev/null @@ -1,156 +0,0 @@ -use indexmap::IndexMap; -use memberlist_core::proto::{ - Data, DataRef, DecodeError, EncodeError, RepeatedDecoder, TupleEncoder, WireType, - utils::{merge, skip}, -}; -use smol_str::SmolStr; - -const TAGS_TAG: u8 = 1; -const TAGS_BYTE: u8 = merge(WireType::LengthDelimited, TAGS_TAG); - -/// Tags of a node -#[derive( - Debug, - Default, - PartialEq, - Clone, - derive_more::From, - derive_more::Into, - derive_more::Deref, - derive_more::DerefMut, - derive_more::AsRef, - derive_more::AsMut, - derive_more::IntoIterator, -)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "serde", serde(transparent))] -#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -pub struct Tags( - #[cfg_attr(feature = "arbitrary", arbitrary(with = crate::types::arbitrary_impl::arbitrary_indexmap))] - IndexMap, -); - -impl FromIterator<(K, V)> for Tags -where - K: Into, - V: Into, -{ - fn from_iter>(iter: T) -> Self { - Self( - iter - .into_iter() - .map(|(k, v)| (k.into(), v.into())) - .collect(), - ) - } -} - -impl Tags { - /// Create a new Tags - #[inline] - pub fn new() -> Self { - Self(IndexMap::new()) - } - - /// Create a new Tags with a capacity - pub fn with_capacity(cap: usize) -> Self { - Self(IndexMap::with_capacity(cap)) - } -} - -/// The reference type to [`Tags`], which is an iterator and yields a reference to the key and value -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct TagsRef<'a> { - src: RepeatedDecoder<'a>, -} - -impl<'a> DataRef<'a, Tags> for TagsRef<'a> { - fn decode(src: &'a [u8]) -> Result<(usize, Self), DecodeError> - where - Self: Sized, - { - let mut offset = 0; - let buf_len = src.len(); - - let mut tags_offsets = None; - let mut num_tags = 0; - - while offset < buf_len { - match src[offset] { - TAGS_BYTE => { - let readed = skip("Tags", &src[offset..])?; - if let Some((ref mut fnso, ref mut lnso)) = tags_offsets { - if *fnso > offset { - *fnso = offset; - } - - if *lnso < offset + readed { - *lnso = offset + readed; - } - } else { - tags_offsets = Some((offset, offset + readed)); - } - num_tags += 1; - offset += readed; - } - _ => offset += skip("Tags", &src[offset..])?, - } - } - - let decoder = - RepeatedDecoder::new(TAGS_TAG, WireType::LengthDelimited, src).with_nums(num_tags); - - Ok(( - offset, - Self { - src: if let Some((fnso, lnso)) = tags_offsets { - decoder.with_offsets(fnso, lnso) - } else { - decoder - }, - }, - )) - } -} - -impl Data for Tags { - type Ref<'a> = TagsRef<'a>; - - fn from_ref(val: Self::Ref<'_>) -> Result - where - Self: Sized, - { - val - .src - .iter::<(SmolStr, SmolStr)>() - .map(|res| res.and_then(Data::from_ref)) - .collect::, DecodeError>>() - .map(Self) - } - - fn encoded_len(&self) -> usize { - self - .0 - .iter() - .map(|(k, v)| 1 + TupleEncoder::new(k, v).encoded_len_with_length_delimited()) - .sum::() - } - - fn encode(&self, buf: &mut [u8]) -> Result { - let buf_len = buf.len(); - self - .0 - .iter() - .try_fold(0, |mut offset, (k, v)| { - if offset >= buf_len { - return Err(EncodeError::insufficient_buffer(1, 0)); - } - - buf[offset] = TAGS_BYTE; - offset += 1; - offset += TupleEncoder::new(k, v).encode_with_length_delimited(&mut buf[offset..])?; - Ok(offset) - }) - .map_err(|e: EncodeError| e.update(self.encoded_len(), buf.len())) - } -} diff --git a/legacy/serf-core/src/types/tests.rs b/legacy/serf-core/src/types/tests.rs deleted file mode 100644 index f630a9d1..00000000 --- a/legacy/serf-core/src/types/tests.rs +++ /dev/null @@ -1,310 +0,0 @@ -use memberlist_core::proto::{Data, DataRef}; - -use super::{ - coordinate::Coordinate, - fuzzy::{Message, encodable_round_trip}, - *, -}; - -fn data_round_trip(data: &T) { - let mut buf = vec![0; data.encoded_len() + 2]; - let len = data.encode(&mut buf).unwrap(); - let buf = &buf[..len]; - let (readed, decoded) = DataRef::decode(&buf[..len]).unwrap(); - let decoded = T::from_ref(decoded).unwrap(); - assert_eq!(len, readed); - assert_eq!(data, &decoded); - - let mut buf = vec![0; data.encoded_len_with_length_delimited() + 2]; - let len = data.encode_length_delimited(&mut buf).unwrap(); - let buf = &buf[..len]; - let (readed, decoded) = DataRef::decode_length_delimited(&buf[..len]).unwrap(); - let decoded = T::from_ref(decoded).unwrap(); - assert_eq!(len, readed); - assert_eq!(data, &decoded); -} - -macro_rules! data_round_trip { - ($( - $(#[$attr:meta])* - $ty:ty - ),+$(,)?) => { - $( - paste::paste! { - $(#[$attr])* - #[quickcheck_macros::quickcheck] - fn [< data_round_trip_ $ty:snake >](data: $ty) { - data_round_trip(&data); - } - } - )* - }; -} - -type StringFilter = Filter; -type U64Filter = Filter; - -type QueryMessageStringString = QueryMessage; -type QueryMessageU64String = QueryMessage; -type QueryMessageStringU64 = QueryMessage; -type QueryMessageU64U64 = QueryMessage; - -type PushPullMessageString = PushPullMessage; -type PushPullMessageU64 = PushPullMessage; - -type MemberStringString = Member; -type MemberU64String = Member; -type MemberStringU64 = Member; -type MemberU64U64 = Member; - -type LeaveMessageString = LeaveMessage; -type LeaveMessageU64 = LeaveMessage; - -type JoinMessageString = JoinMessage; -type JoinMessageU64 = JoinMessage; - -type ConflictResponseMessageStringString = ConflictResponseMessage; -type ConflictResponseMessageU64U64 = ConflictResponseMessage; -type ConflictResponseMessageStringU64 = ConflictResponseMessage; - -type QueryResponseMessageStringString = QueryResponseMessage; -type QueryResponseMessageU64U64 = QueryResponseMessage; -type QueryResponseMessageStringU64 = QueryResponseMessage; - -data_round_trip! { - Coordinate, -} - -data_round_trip! { - // Coordinate, - ConflictResponseMessageStringString, - ConflictResponseMessageU64U64, - ConflictResponseMessageStringU64, - JoinMessageString, - JoinMessageU64, - LeaveMessageString, - LeaveMessageU64, - MemberStringString, - MemberU64String, - MemberStringU64, - MemberU64U64, - Tags, - TagFilter, - StringFilter, - U64Filter, - UserEvent, - UserEvents, - UserEventMessage, - PushPullMessageU64, - PushPullMessageString, - QueryMessageStringString, - QueryMessageU64String, - QueryMessageStringU64, - QueryMessageU64U64, - QueryResponseMessageStringString, - QueryResponseMessageU64U64, - QueryResponseMessageStringU64, - #[cfg(feature = "encryption")] - KeyRequestMessage, - #[cfg(feature = "encryption")] - KeyResponseMessage, -} - -macro_rules! encodable_round_trip { - (@message $(<$a:ty, $b:ty>),+$(,)?) => { - $( - paste::paste! { - #[quickcheck_macros::quickcheck] - fn [< message _encodable_round_trip_ $a:snake _ $b:snake >](msg: Message<$a, $b>) -> bool { - encodable_round_trip(msg, None) - } - } - )* - }; - (@relay_message $(<$a:ty, $b:ty>),+$(,)?) => { - $( - paste::paste! { - #[quickcheck_macros::quickcheck] - fn [< relay_message _encodable_round_trip_ $a:snake _ $b:snake >](msg: Message<$a, $b>, node: Node<$a, $b>) -> bool { - encodable_round_trip(msg, Some(node)) - } - } - )* - }; - (@query_message $(<$a:ty, $b:ty>),+$(,)?) => { - $( - paste::paste! { - #[quickcheck_macros::quickcheck] - fn [< query_message _encodable_round_trip_ $a:snake _ $b:snake >](msg: QueryMessage<$a, $b>, node: Option>) -> bool { - encodable_round_trip(Message::Query(msg), node) - } - } - )* - }; - (@query_response_message $(<$a:ty, $b:ty>),+$(,)?) => { - $( - paste::paste! { - #[quickcheck_macros::quickcheck] - fn [< query_response_message _encodable_round_trip_ $a:snake _ $b:snake >](msg: QueryResponseMessage<$a, $b>, node: Option>) -> bool { - encodable_round_trip(Message::QueryResponse(msg), node) - } - } - )* - }; - (@join_message $(<$a:ty, $b:ty>),+$(,)?) => { - $( - paste::paste! { - #[quickcheck_macros::quickcheck] - fn [< join_message _encodable_round_trip_ $a:snake _ $b:snake >](msg: JoinMessage<$a>, node: Option>) -> bool { - encodable_round_trip(Message::Join(msg), node) - } - } - )* - }; - (@push_pull_message $(<$a:ty>),+$(,)?) => { - $( - paste::paste! { - #[quickcheck_macros::quickcheck] - fn [< push_pull_message _encodable_round_trip_ $a:snake >](msg: PushPullMessage<$a>, node: Option>) -> bool { - encodable_round_trip(Message::PushPull(msg), node) - } - } - )* - }; - (@leave_message $(<$a:ty>),+$(,)?) => { - $( - paste::paste! { - #[quickcheck_macros::quickcheck] - fn [< leave_message _encodable_round_trip_ $a:snake >](msg: LeaveMessage<$a>, node: Option>) -> bool { - encodable_round_trip(Message::Leave(msg), node) - } - } - )* - }; - (@conflict_response_message $(<$a:ty, $b:ty>),+$(,)?) => { - $( - paste::paste! { - #[quickcheck_macros::quickcheck] - fn [< conflict_response_message _encodable_round_trip_ $a:snake _ $b:snake >](msg: ConflictResponseMessage<$a, $b>, node: Option>) -> bool { - encodable_round_trip(Message::ConflictResponse(msg), node) - } - } - )* - }; - (@user_event_message $(<$a:ty, $b:ty>),+$(,)?) => { - $( - paste::paste! { - #[quickcheck_macros::quickcheck] - fn [< user_event_message _encodable_round_trip_ $a:snake _ $b:snake >](msg: UserEventMessage, node: Option>) -> bool { - encodable_round_trip(Message::UserEvent(msg), node) - } - } - )* - }; - (@key_request_message $(<$a:ty, $b:ty>),+$(,)?) => { - $( - paste::paste! { - #[quickcheck_macros::quickcheck] - fn [< key_request_message _encodable_round_trip_ $a:snake _ $b:snake >](msg: KeyRequestMessage, node: Option>) -> bool { - encodable_round_trip(Message::KeyRequest(msg), node) - } - } - )* - }; - (@key_response_message $(<$a:ty, $b:ty>),+$(,)?) => { - $( - paste::paste! { - #[quickcheck_macros::quickcheck] - fn [< key_response_message _encodable_round_trip_ $a:snake _ $b:snake >](msg: KeyResponseMessage, node: Option>) -> bool { - encodable_round_trip(Message::KeyResponse(msg), node) - } - } - )* - }; -} - -encodable_round_trip!( - @message - , - , - , - , -); - -encodable_round_trip!( - @relay_message - , - , - , - , -); - -encodable_round_trip!( - @query_message - , - , - , - , -); - -encodable_round_trip!( - @query_response_message - , - , - , - , -); - -encodable_round_trip!( - @conflict_response_message - , - , - , - , -); - -encodable_round_trip!( - @join_message - , - , - , - , -); - -encodable_round_trip!( - @push_pull_message - , - , -); - -encodable_round_trip!( - @leave_message - , - , -); - -encodable_round_trip!( - @user_event_message - , - , - , - , -); - -#[cfg(feature = "encryption")] -encodable_round_trip!( - @key_request_message - , - , - , - , -); - -#[cfg(feature = "encryption")] -encodable_round_trip!( - @key_response_message - , - , - , - , -); diff --git a/legacy/serf-core/src/types/user_event.rs b/legacy/serf-core/src/types/user_event.rs deleted file mode 100644 index 6a38e027..00000000 --- a/legacy/serf-core/src/types/user_event.rs +++ /dev/null @@ -1,155 +0,0 @@ -use memberlist_core::proto::{ - Data, DataRef, DecodeError, EncodeError, WireType, - bytes::Bytes, - utils::{merge, skip}, -}; -use smol_str::SmolStr; - -pub use message::*; -pub use user_events::*; - -mod message; -mod user_events; - -const NAME_TAG: u8 = 1; -const PAYLOAD_TAG: u8 = 2; - -const NAME_BYTE: u8 = merge(WireType::LengthDelimited, NAME_TAG); -const PAYLOAD_BYTE: u8 = merge(WireType::LengthDelimited, PAYLOAD_TAG); - -/// Stores all the user events at a specific time -#[viewit::viewit(getters(style = "ref"), setters(prefix = "with"))] -#[derive(Debug, Clone, Eq, PartialEq)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -pub struct UserEvent { - /// The name of the event - #[viewit( - getter(const, attrs(doc = "Returns the name of the event")), - setter(attrs(doc = "Sets the name of the event (Builder pattern)")) - )] - name: SmolStr, - /// The payload of the event - #[viewit( - getter(const, attrs(doc = "Returns the payload of the event")), - setter(attrs(doc = "Sets the payload of the event (Builder pattern)")) - )] - #[cfg_attr(feature = "arbitrary", arbitrary(with = crate::types::arbitrary_impl::into::, Bytes>))] - payload: Bytes, -} - -/// The reference to a [`UserEvent`]. -#[viewit::viewit(getters(style = "ref", vis_all = "pub"), setters(skip), vis_all = "")] -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub struct UserEventRef<'a> { - #[viewit(getter(const, attrs(doc = "Returns the name of the event")))] - name: &'a str, - #[viewit(getter(const, attrs(doc = "Returns the payload of the event")))] - payload: &'a [u8], -} - -impl<'a> DataRef<'a, UserEvent> for UserEventRef<'a> { - fn decode(buf: &'a [u8]) -> Result<(usize, Self), DecodeError> - where - Self: Sized, - { - let mut offset = 0; - let buf_len = buf.len(); - - let mut name = None; - let mut payload = None; - - while offset < buf_len { - match buf[offset] { - NAME_BYTE => { - if name.is_some() { - return Err(DecodeError::duplicate_field("UserEvent", "name", NAME_TAG)); - } - offset += 1; - let (size, val) = - <&str as DataRef<'_, SmolStr>>::decode_length_delimited(&buf[offset..])?; - name = Some(val); - offset += size; - } - PAYLOAD_BYTE => { - if payload.is_some() { - return Err(DecodeError::duplicate_field( - "UserEvent", - "payload", - PAYLOAD_TAG, - )); - } - offset += 1; - let (size, val) = <&[u8] as DataRef<'_, Bytes>>::decode_length_delimited(&buf[offset..])?; - payload = Some(val); - offset += size; - } - _ => offset += skip("UserEvent", &buf[offset..])?, - } - } - - Ok(( - offset, - Self { - name: name.unwrap_or_default(), - payload: payload.unwrap_or_default(), - }, - )) - } -} - -impl Data for UserEvent { - type Ref<'a> = UserEventRef<'a>; - - fn from_ref(val: Self::Ref<'_>) -> Result - where - Self: Sized, - { - Ok(Self { - name: SmolStr::from(val.name), - payload: Bytes::copy_from_slice(val.payload), - }) - } - - fn encoded_len(&self) -> usize { - let mut len = 0; - if !self.name.is_empty() { - len += 1 + self.name.encoded_len_with_length_delimited(); - } - - if !self.payload.is_empty() { - len += 1 + self.payload.encoded_len_with_length_delimited(); - } - len - } - - fn encode(&self, buf: &mut [u8]) -> Result { - let mut offset = 0; - let buf_len = buf.len(); - if !self.name.is_empty() { - if offset >= buf_len { - return Err(EncodeError::insufficient_buffer( - self.encoded_len(), - buf_len, - )); - } - buf[offset] = NAME_BYTE; - offset += 1; - offset += self.name.encode_length_delimited(&mut buf[offset..])?; - } - - if !self.payload.is_empty() { - if offset >= buf_len { - return Err(EncodeError::insufficient_buffer( - self.encoded_len(), - buf_len, - )); - } - buf[offset] = PAYLOAD_BYTE; - offset += 1; - offset += self.payload.encode_length_delimited(&mut buf[offset..])?; - } - - Ok(offset) - } -} diff --git a/legacy/serf-core/src/types/user_event/message.rs b/legacy/serf-core/src/types/user_event/message.rs deleted file mode 100644 index 85b2e95b..00000000 --- a/legacy/serf-core/src/types/user_event/message.rs +++ /dev/null @@ -1,272 +0,0 @@ -use memberlist_core::proto::{ - CheapClone, Data, DataRef, DecodeError, EncodeError, WireType, - bytes::Bytes, - utils::{merge, skip}, -}; -use smol_str::SmolStr; - -use super::super::LamportTime; - -/// Used for user-generated events -#[viewit::viewit(setters(prefix = "with"))] -#[derive(Debug, Default, Clone, Eq, PartialEq)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -pub struct UserEventMessage { - /// The lamport time - #[viewit( - getter( - const, - style = "move", - attrs(doc = "Returns the lamport time for this message") - ), - setter( - const, - attrs(doc = "Sets the lamport time for this message (Builder pattern)") - ) - )] - ltime: LamportTime, - /// The name of the event - #[viewit( - getter(const, attrs(doc = "Returns the name of the event")), - setter(attrs(doc = "Sets the name of the event (Builder pattern)")) - )] - name: SmolStr, - /// The payload of the event - #[viewit( - getter(const, attrs(doc = "Returns the payload of the event")), - setter(attrs(doc = "Sets the payload of the event (Builder pattern)")) - )] - #[cfg_attr(feature = "arbitrary", arbitrary(with = crate::types::arbitrary_impl::into::, Bytes>))] - payload: Bytes, - /// "Can Coalesce". - #[viewit( - getter( - const, - style = "move", - attrs(doc = "Returns if this message can be coalesced") - ), - setter( - const, - attrs(doc = "Sets if this message can be coalesced (Builder pattern)") - ) - )] - cc: bool, -} - -impl CheapClone for UserEventMessage { - fn cheap_clone(&self) -> Self { - Self { - ltime: self.ltime, - name: self.name.cheap_clone(), - payload: self.payload.clone(), - cc: self.cc, - } - } -} - -const LTIME_TAG: u8 = 1; -const CC_TAG: u8 = 2; -const NAME_TAG: u8 = 3; -const PAYLOAD_TAG: u8 = 4; - -const LTIME_BYTE: u8 = merge(WireType::Varint, LTIME_TAG); -const CC_BYTE: u8 = merge(WireType::Byte, CC_TAG); -const NAME_BYTE: u8 = merge(WireType::LengthDelimited, NAME_TAG); -const PAYLOAD_BYTE: u8 = merge(WireType::LengthDelimited, PAYLOAD_TAG); - -/// The reference type of [`UserEventMessage`] -#[viewit::viewit(vis_all = "", getters(vis_all = "pub", style = "ref"), setters(skip))] -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -pub struct UserEventMessageRef<'a> { - /// The lamport time - #[viewit(getter( - const, - style = "move", - attrs(doc = "Returns the lamport time for this message") - ))] - ltime: LamportTime, - /// The name of the event - #[viewit(getter(const, style = "move", attrs(doc = "Returns the name of the event")))] - name: &'a str, - /// The payload of the event - #[viewit(getter(const, style = "move", attrs(doc = "Returns the payload of the event")))] - payload: &'a [u8], - /// "Can Coalesce". - #[viewit(getter( - const, - style = "move", - attrs(doc = "Returns if this message can be coalesced") - ))] - cc: bool, -} - -impl<'a> DataRef<'a, UserEventMessage> for UserEventMessageRef<'a> { - fn decode(buf: &'a [u8]) -> Result<(usize, Self), DecodeError> - where - Self: Sized, - { - let mut offset = 0; - let buf_len = buf.len(); - - let mut ltime = None; - let mut name = None; - let mut payload = None; - let mut cc = None; - - while offset < buf_len { - match buf[offset] { - LTIME_BYTE => { - if ltime.is_some() { - return Err(DecodeError::duplicate_field( - "UserEventMessage", - "ltime", - LTIME_TAG, - )); - } - - offset += 1; - let (o, v) = >::decode(&buf[offset..])?; - offset += o; - ltime = Some(v); - } - CC_BYTE => { - if cc.is_some() { - return Err(DecodeError::duplicate_field( - "UserEventMessage", - "cc", - CC_TAG, - )); - } - - offset += 1; - let (o, v) = >::decode(&buf[offset..])?; - offset += o; - cc = Some(v); - } - NAME_BYTE => { - if name.is_some() { - return Err(DecodeError::duplicate_field( - "UserEventMessage", - "name", - NAME_TAG, - )); - } - offset += 1; - let (o, v) = <&str as DataRef<'_, SmolStr>>::decode_length_delimited(&buf[offset..])?; - offset += o; - name = Some(v); - } - PAYLOAD_BYTE => { - if payload.is_some() { - return Err(DecodeError::duplicate_field( - "UserEventMessage", - "payload", - PAYLOAD_TAG, - )); - } - - offset += 1; - let (o, v) = <&[u8] as DataRef<'_, Bytes>>::decode_length_delimited(&buf[offset..])?; - offset += o; - payload = Some(v); - } - _ => offset += skip("UserEventMessage", &buf[offset..])?, - } - } - - Ok(( - offset, - Self { - ltime: ltime.ok_or_else(|| DecodeError::missing_field("UserEventMessage", "ltime"))?, - name: name.unwrap_or_default(), - payload: payload.unwrap_or_default(), - cc: cc.unwrap_or_default(), - }, - )) - } -} - -impl Data for UserEventMessage { - type Ref<'a> = UserEventMessageRef<'a>; - - fn from_ref(val: Self::Ref<'_>) -> Result - where - Self: Sized, - { - Ok(Self { - ltime: val.ltime, - name: SmolStr::from(val.name), - payload: Bytes::copy_from_slice(val.payload), - cc: val.cc, - }) - } - - fn encoded_len(&self) -> usize { - let mut len = 1 + self.ltime.encoded_len(); - let nlen = self.name.len(); - if nlen > 0 { - len += 1 + self.name.encoded_len_with_length_delimited(); - } - - let plen = self.payload.len(); - if plen > 0 { - len += 1 + self.payload.encoded_len_with_length_delimited(); - } - - if self.cc { - len += 1 + 1; - } - - len - } - - fn encode(&self, buf: &mut [u8]) -> Result { - macro_rules! bail { - ($this:ident($offset:expr, $len:ident)) => { - if $offset >= $len { - return Err(EncodeError::insufficient_buffer(self.encoded_len(), $len)); - } - }; - } - - let buf_len = buf.len(); - let mut offset = 0; - - bail!(self(offset, buf_len)); - buf[offset] = LTIME_BYTE; - offset += 1; - offset += self - .ltime - .encode(&mut buf[offset..]) - .map_err(|e| e.update(self.encoded_len(), buf_len))?; - - if self.cc { - bail!(self(offset, buf_len)); - buf[offset] = CC_BYTE; - offset += 1; - bail!(self(offset, buf_len)); - buf[offset] = 1; - offset += 1; - } - - if !self.name.is_empty() { - bail!(self(offset, buf_len)); - buf[offset] = NAME_BYTE; - offset += 1; - offset += self.name.encode_length_delimited(&mut buf[offset..])?; - } - - if !self.payload.is_empty() { - bail!(self(offset, buf_len)); - buf[offset] = PAYLOAD_BYTE; - offset += 1; - offset += self.payload.encode_length_delimited(&mut buf[offset..])?; - } - - #[cfg(debug_assertions)] - super::super::debug_assert_write_eq::(offset, self.encoded_len()); - - Ok(offset) - } -} diff --git a/legacy/serf-core/src/types/user_event/user_events.rs b/legacy/serf-core/src/types/user_event/user_events.rs deleted file mode 100644 index c1cd05c7..00000000 --- a/legacy/serf-core/src/types/user_event/user_events.rs +++ /dev/null @@ -1,181 +0,0 @@ -use memberlist_core::proto::{ - Data, DataRef, DecodeError, EncodeError, OneOrMore, RepeatedDecoder, WireType, - utils::{merge, skip}, -}; - -use super::{super::LamportTime, UserEvent}; - -const LTIME_TAG: u8 = 1; -const EVENTS_TAG: u8 = 2; - -const LTIME_BYTE: u8 = merge(WireType::Varint, LTIME_TAG); -const EVENTS_BYTE: u8 = merge(WireType::LengthDelimited, EVENTS_TAG); - -/// Used to buffer events to prevent re-delivery -#[viewit::viewit(setters(prefix = "with"))] -#[derive(Debug, Clone, Eq, PartialEq)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -#[cfg_attr(feature = "arbitrary", derive(arbitrary::Arbitrary))] -pub struct UserEvents { - /// The lamport time - #[viewit( - getter(const, attrs(doc = "Returns the lamport time for this message")), - setter( - const, - attrs(doc = "Sets the lamport time for this message (Builder pattern)") - ) - )] - ltime: LamportTime, - - /// The user events - #[viewit( - getter(const, style = "ref", attrs(doc = "Returns the user events")), - setter(attrs(doc = "Sets the user events (Builder pattern)")) - )] - #[cfg_attr(feature = "arbitrary", arbitrary(with = crate::types::arbitrary_impl::into::, OneOrMore>))] - events: OneOrMore, -} - -/// The reference type for [`UserEvents`] -#[viewit::viewit(vis_all = "", getters(vis_all = "pub", style = "ref"), setters(skip))] -#[derive(Debug, Clone, Copy)] -pub struct UserEventsRef<'a> { - /// The lamport time - #[viewit(getter(const, attrs(doc = "Returns the lamport time for this message")))] - ltime: LamportTime, - - /// The user events - #[viewit(getter(const, style = "ref", attrs(doc = "Returns the bu user events")))] - events: RepeatedDecoder<'a>, -} - -impl<'a> DataRef<'a, UserEvents> for UserEventsRef<'a> { - fn decode(buf: &'a [u8]) -> Result<(usize, Self), DecodeError> - where - Self: Sized, - { - let mut offset = 0; - let buf_len = buf.len(); - - let mut ltime = None; - let mut events_offsets = None; - let mut num_events = 0; - - while offset < buf_len { - match buf[offset] { - LTIME_BYTE => { - if ltime.is_some() { - return Err(DecodeError::duplicate_field( - "UserEvents", - "ltime", - LTIME_TAG, - )); - } - offset += 1; - let (size, val) = >::decode(&buf[offset..])?; - ltime = Some(val); - offset += size; - } - EVENTS_BYTE => { - let readed = super::skip("UserEvents", &buf[offset..])?; - if let Some((ref mut fnso, ref mut lnso)) = events_offsets { - if *fnso > offset { - *fnso = offset; - } - - if *lnso < offset + readed { - *lnso = offset + readed; - } - } else { - events_offsets = Some((offset, offset + readed)); - } - num_events += 1; - offset += readed; - } - _ => offset += skip("UserEvents", &buf[offset..])?, - } - } - - Ok(( - offset, - Self { - ltime: ltime.ok_or_else(|| DecodeError::missing_field("UserEvents", "ltime"))?, - events: if let Some((start, end)) = events_offsets { - RepeatedDecoder::new(EVENTS_TAG, WireType::LengthDelimited, buf) - .with_nums(num_events) - .with_offsets(start, end) - } else { - RepeatedDecoder::new(EVENTS_TAG, WireType::LengthDelimited, buf) - }, - }, - )) - } -} - -impl Data for UserEvents { - type Ref<'a> = UserEventsRef<'a>; - - fn from_ref(val: Self::Ref<'_>) -> Result - where - Self: Sized, - { - val - .events - .iter::() - .map(|ev| ev.and_then(UserEvent::from_ref)) - .collect::, DecodeError>>() - .map(|events| Self { - ltime: val.ltime, - events, - }) - } - - fn encoded_len(&self) -> usize { - let mut len = 0; - len += 1 + self.ltime.encoded_len(); - len += self - .events - .iter() - .map(|e| 1 + e.encoded_len_with_length_delimited()) - .sum::(); - len - } - - fn encode(&self, buf: &mut [u8]) -> Result { - let buf_len = buf.len(); - let mut offset = 0; - - if offset >= buf_len { - return Err(EncodeError::insufficient_buffer( - self.encoded_len(), - buf_len, - )); - } - - buf[offset] = LTIME_BYTE; - offset += 1; - - offset += self - .ltime - .encode(&mut buf[offset..]) - .map_err(|e| e.update(self.encoded_len(), buf_len))?; - - self - .events - .iter() - .try_fold(&mut offset, |offset, ev| { - if *offset >= buf_len { - return Err(EncodeError::insufficient_buffer( - self.encoded_len(), - buf_len, - )); - } - buf[*offset] = EVENTS_BYTE; - *offset += 1; - *offset += ev.encode_length_delimited(&mut buf[*offset..])?; - Ok(offset) - }) - .map(|offset| *offset) - .map_err(|e: EncodeError| e.update(self.encoded_len(), buf_len)) - } -} diff --git a/legacy/serf-core/src/types/version.rs b/legacy/serf-core/src/types/version.rs deleted file mode 100644 index b603f7d6..00000000 --- a/legacy/serf-core/src/types/version.rs +++ /dev/null @@ -1,151 +0,0 @@ -use memberlist_core::proto::{Data, DataRef, DecodeError, EncodeError, WireType}; - -/// Delegate version -#[derive( - Debug, Default, Copy, Clone, PartialEq, Eq, Hash, derive_more::IsVariant, derive_more::Display, -)] -#[cfg_attr(feature = "serde", derive(::serde::Serialize, ::serde::Deserialize))] -#[non_exhaustive] -pub enum DelegateVersion { - /// Version 1 - #[default] - #[display("v1")] - V1, - /// Unknown version (used for forwards and backwards compatibility) - #[display("unknown({_0})")] - Unknown(u8), -} - -impl From for DelegateVersion { - fn from(v: u8) -> Self { - match v { - 1 => Self::V1, - val => Self::Unknown(val), - } - } -} - -impl From for u8 { - fn from(v: DelegateVersion) -> Self { - match v { - DelegateVersion::V1 => 1, - DelegateVersion::Unknown(val) => val, - } - } -} - -/// Protocol version -#[derive( - Debug, Default, Copy, Clone, PartialEq, Eq, Hash, derive_more::IsVariant, derive_more::Display, -)] -#[cfg_attr(feature = "serde", derive(::serde::Serialize, ::serde::Deserialize))] -#[non_exhaustive] -pub enum ProtocolVersion { - /// Version 1 - #[default] - #[display("v1")] - V1, - /// Unknown version (used for forwards and backwards compatibility) - #[display("unknown({_0})")] - Unknown(u8), -} - -impl From for ProtocolVersion { - fn from(v: u8) -> Self { - match v { - 1 => Self::V1, - val => Self::Unknown(val), - } - } -} - -impl From for u8 { - fn from(v: ProtocolVersion) -> Self { - match v { - ProtocolVersion::V1 => 1, - ProtocolVersion::Unknown(val) => val, - } - } -} - -macro_rules! impl_data { - ($($ty:ty),+$(,)?) => { - $( - impl<'a> DataRef<'a, Self> for $ty { - fn decode(src: &'a [u8]) -> Result<(usize, Self), DecodeError> { - if src.is_empty() { - return Err(DecodeError::buffer_underflow()); - } - - Ok((1, Self::from(src[0]))) - } - } - - impl Data for $ty { - const WIRE_TYPE: WireType = WireType::Byte; - - type Ref<'a> = Self; - - fn from_ref(val: Self::Ref<'_>) -> Result { - Ok(val) - } - - #[inline] - fn encoded_len(&self) -> usize { - 1 - } - - #[inline] - fn encode(&self, buf: &mut [u8]) -> Result { - if buf.is_empty() { - return Err(EncodeError::insufficient_buffer(1, 0)); - } - - buf[0] = u8::from(*self); - Ok(1) - } - } - )* - }; -} - -impl_data!(DelegateVersion, ProtocolVersion); - -#[cfg(test)] -mod tests { - use super::*; - - #[cfg(feature = "arbitrary")] - use arbitrary::{Arbitrary, Unstructured}; - - #[test] - #[cfg(feature = "arbitrary")] - fn test_delegate_version() { - let mut buf = [0; 64]; - rand::fill(&mut buf[..]); - - let mut data = Unstructured::new(&buf); - let _ = DelegateVersion::arbitrary(&mut data).unwrap(); - - assert_eq!(u8::from(DelegateVersion::V1), 1u8); - assert_eq!(DelegateVersion::V1.to_string(), "v1"); - assert_eq!(DelegateVersion::Unknown(2).to_string(), "unknown(2)"); - assert_eq!(DelegateVersion::from(1), DelegateVersion::V1); - assert_eq!(DelegateVersion::from(2), DelegateVersion::Unknown(2)); - } - - #[test] - #[cfg(feature = "arbitrary")] - fn test_protocol_version() { - let mut buf = [0; 64]; - rand::fill(&mut buf[..]); - - let mut data = Unstructured::new(&buf); - let _ = ProtocolVersion::arbitrary(&mut data).unwrap(); - assert_eq!(u8::from(ProtocolVersion::V1), 1); - assert_eq!(ProtocolVersion::V1.to_string(), "v1"); - assert_eq!(ProtocolVersion::Unknown(2).to_string(), "unknown(2)"); - assert_eq!(ProtocolVersion::from(1), ProtocolVersion::V1); - assert_eq!(ProtocolVersion::from(2), ProtocolVersion::Unknown(2)); - } -} diff --git a/legacy/serf/Cargo.toml b/legacy/serf/Cargo.toml deleted file mode 100644 index 7ff8f94f..00000000 --- a/legacy/serf/Cargo.toml +++ /dev/null @@ -1,74 +0,0 @@ -[package] -name = "serf" -version.workspace = true -rust-version.workspace = true -edition.workspace = true -license.workspace = true -description.workspace = true -repository.workspace = true -readme.workspace = true -homepage.workspace = true -keywords.workspace = true -categories.workspace = true - -[package.metadata.docs.rs] -all-features = true -rustdoc-args = ["--cfg", "docsrs"] - -[features] -default = ["tokio", "compression", "encryption", "dns", "tcp", "quic"] - -tokio = ["memberlist/tokio"] -smol = ["memberlist/smol"] - -serde = [ - "memberlist/serde", - "serf-core/serde", -] - -metrics = [ - "memberlist/metrics", - "serf-core/metrics", -] - -compression = [] - -encryption = ["memberlist/encryption", "serf-core/encryption"] - -quic = ["memberlist/quic"] -quinn = ["memberlist/quinn", "quic"] - -net = ["memberlist/net"] -tcp = ["memberlist/tcp", "net"] -tls = ["memberlist/tls", "net"] - -# enable DNS node address resolver -dns = ["memberlist/dns"] -quic-ring = ["memberlist/quic-ring"] -quic-aws-lc-rs = ["memberlist/quic-aws-lc-rs"] -h3-ring = ["memberlist/h3-ring"] -h3-aws-lc-rs = ["memberlist/h3-aws-lc-rs"] -https-ring = ["memberlist/https-ring"] -https-aws-lc-rs = ["memberlist/https-aws-lc-rs"] -tls-ring = ["memberlist/tls-ring"] -tls-aws-lc-rs = ["memberlist/tls-aws-lc-rs"] -webpki-roots = ["memberlist/webpki-roots"] -rustls-platform-verifier = ["memberlist/rustls-platform-verifier"] -dnssec-ring = ["memberlist/dnssec-ring"] -dnssec-aws-lc-rs = ["memberlist/dnssec-aws-lc-rs"] - -test = ["memberlist/test", "serf-core/test"] - -[dependencies] -memberlist.workspace = true -serf-core = { workspace = true, default-features = false } - -[dev-dependencies] -tokio = { version = "1", features = ["full"] } -smol_str = "0.3" -paste = "1" - -[[test]] -name = "main" -path = "test/main.rs" -required-features = ["test"] diff --git a/legacy/serf/README.md b/legacy/serf/README.md deleted file mode 120000 index 32d46ee8..00000000 --- a/legacy/serf/README.md +++ /dev/null @@ -1 +0,0 @@ -../README.md \ No newline at end of file diff --git a/legacy/serf/src/lib.rs b/legacy/serf/src/lib.rs deleted file mode 100644 index d3b17242..00000000 --- a/legacy/serf/src/lib.rs +++ /dev/null @@ -1,27 +0,0 @@ -#![doc = include_str!("../README.md")] -#![doc(html_logo_url = "https://raw.githubusercontent.com/al8n/serf/main/art/logo_72x72.png")] -#![forbid(unsafe_code)] -#![deny(warnings, missing_docs)] -#![allow(clippy::type_complexity)] -#![cfg_attr(docsrs, feature(doc_cfg))] -#![cfg_attr(docsrs, allow(unused_attributes))] - -pub use serf_core::*; - -pub use memberlist::{agnostic, transport}; - -#[cfg(feature = "net")] -pub use memberlist::net; - -#[cfg(feature = "quic")] -pub use memberlist::quic; - -/// [`Serf`] for `tokio` runtime. -#[cfg(feature = "tokio")] -#[cfg_attr(docsrs, doc(cfg(feature = "tokio")))] -pub mod tokio; - -/// [`Serf`] for `smol` runtime. -#[cfg(feature = "smol")] -#[cfg_attr(docsrs, doc(cfg(feature = "smol")))] -pub mod smol; diff --git a/legacy/serf/src/smol.rs b/legacy/serf/src/smol.rs deleted file mode 100644 index 97c1fc04..00000000 --- a/legacy/serf/src/smol.rs +++ /dev/null @@ -1,19 +0,0 @@ -pub use memberlist::smol::*; - -/// [`Serf`](super::Serf) type alias for using [`NetTransport`](memberlist::net::NetTransport) and [`Tcp`](memberlist::net::stream_layer::tcp::Tcp) stream layer with `smol` runtime. -#[cfg(all(any(feature = "tcp", feature = "tls",), not(target_family = "wasm")))] -#[cfg_attr( - docsrs, - doc(cfg(all(any(feature = "tcp", feature = "tls",), not(target_family = "wasm")))) -)] -pub type SmolTcpSerf = serf_core::Serf, D>; - -/// [`Serf`](super::Serf) type alias for using [`NetTransport`](memberlist::net::NetTransport) and [`Tls`](memberlist::net::stream_layer::tls::Tls) stream layer with `smol` runtime. -#[cfg(all(feature = "tls", not(target_family = "wasm")))] -#[cfg_attr(docsrs, doc(cfg(all(feature = "tls", not(target_family = "wasm")))))] -pub type SmolTlsSerf = serf_core::Serf, D>; - -/// [`Serf`](super::Serf) type alias for using [`QuicTransport`](memberlist::quic::QuicTransport) and [`Quinn`](memberlist::quic::stream_layer::quinn::Quinn) stream layer with `smol` runtime. -#[cfg(all(feature = "quinn", not(target_family = "wasm")))] -#[cfg_attr(docsrs, doc(cfg(all(feature = "quinn", not(target_family = "wasm")))))] -pub type SmolQuicSerf = serf_core::Serf, D>; diff --git a/legacy/serf/src/tokio.rs b/legacy/serf/src/tokio.rs deleted file mode 100644 index c43f0e23..00000000 --- a/legacy/serf/src/tokio.rs +++ /dev/null @@ -1,19 +0,0 @@ -pub use memberlist::tokio::*; - -/// [`Serf`](super::Serf) type alias for using [`NetTransport`](memberlist::net::NetTransport) and [`Tcp`](memberlist::net::stream_layer::tcp::Tcp) stream layer with `tokio` runtime. -#[cfg(all(any(feature = "tcp", feature = "tls",), not(target_family = "wasm")))] -#[cfg_attr( - docsrs, - doc(cfg(all(any(feature = "tcp", feature = "tls",), not(target_family = "wasm")))) -)] -pub type TokioTcpSerf = serf_core::Serf, D>; - -/// [`Serf`](super::Serf) type alias for using [`NetTransport`](memberlist::net::NetTransport) and [`Tls`](memberlist::net::stream_layer::tls::Tls) stream layer with `tokio` runtime. -#[cfg(all(feature = "tls", not(target_family = "wasm")))] -#[cfg_attr(docsrs, doc(cfg(all(feature = "tls", not(target_family = "wasm")))))] -pub type TokioTlsSerf = serf_core::Serf, D>; - -/// [`Serf`](super::Serf) type alias for using [`QuicTransport`](memberlist::quic::QuicTransport) and [`Quinn`](memberlist::quic::stream_layer::quinn::Quinn) stream layer with `tokio` runtime. -#[cfg(all(feature = "quinn", not(target_family = "wasm")))] -#[cfg_attr(docsrs, doc(cfg(all(feature = "quinn", not(target_family = "wasm")))))] -pub type TokioQuicSerf = serf_core::Serf, D>; diff --git a/legacy/serf/test/main.rs b/legacy/serf/test/main.rs deleted file mode 100644 index 6655e378..00000000 --- a/legacy/serf/test/main.rs +++ /dev/null @@ -1,22 +0,0 @@ -use core::future::Future; -use serf_core::tests::run as run_unit_test; - -#[cfg(feature = "net")] -#[path = "./main/net.rs"] -mod net; - -#[cfg(feature = "tokio")] -fn tokio_run(fut: impl Future) { - let runtime = ::tokio::runtime::Builder::new_multi_thread() - .worker_threads(32) - .enable_all() - .build() - .unwrap(); - run_unit_test(|fut| runtime.block_on(fut), fut) -} - -#[cfg(feature = "smol")] -fn smol_run(fut: impl Future) { - use serf::agnostic::{RuntimeLite, smol::SmolRuntime}; - run_unit_test(SmolRuntime::block_on, fut); -} diff --git a/legacy/serf/test/main/net.rs b/legacy/serf/test/main/net.rs deleted file mode 100644 index 7e8941c8..00000000 --- a/legacy/serf/test/main/net.rs +++ /dev/null @@ -1,57 +0,0 @@ -#[path = "./net/delegate.rs"] -mod delegate; - -#[path = "./net/event.rs"] -mod event; - -#[path = "./net/leave.rs"] -mod leave; - -#[path = "./net/reap.rs"] -mod reap; - -#[path = "./net/reconnect.rs"] -mod reconnect; - -#[path = "./net/remove.rs"] -mod remove; - -#[path = "./net/snapshot.rs"] -mod snapshot; - -#[path = "./net/update.rs"] -mod update; - -#[path = "./net/role.rs"] -mod role; - -#[path = "./net/set_tags.rs"] -mod set_tags; - -#[path = "./net/get_queue_max.rs"] -mod get_queue_max; - -#[path = "./net/local_member.rs"] -mod local_member; - -#[path = "./net/num_nodes.rs"] -mod num_nodes; - -#[path = "./net/state.rs"] -mod state; - -#[path = "./net/stats.rs"] -mod stats; - -#[path = "./net/coordinates.rs"] -mod coordinates; - -#[path = "./net/name_resolution.rs"] -mod name_resolution; - -#[path = "./net/join.rs"] -mod join; - -#[cfg(feature = "encryption")] -#[path = "./net/write_keyring_file.rs"] -mod write_keyring_file; diff --git a/legacy/serf/test/main/net/coordinates.rs b/legacy/serf/test/main/net/coordinates.rs deleted file mode 100644 index 2746b604..00000000 --- a/legacy/serf/test/main/net/coordinates.rs +++ /dev/null @@ -1,75 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{serf_coordinates, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_coordinates_v4() { - let name = "serf_coordinates1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_coordinates2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_coordinates3_v4"; - let mut opts3 = NetTransportOptions::new(SmolStr::new(name)); - opts3.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_coordinates::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2, opts3)); - } - - #[test] - fn test_serf_coordinates_v6() { - let name = "serf_coordinates1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_coordinates2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - let name = "serf_coordinates3_v6"; - let mut opts3 = NetTransportOptions::new(SmolStr::new(name)); - opts3.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_coordinates::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2, opts3)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/delegate.rs b/legacy/serf/test/main/net/delegate.rs deleted file mode 100644 index db882bd8..00000000 --- a/legacy/serf/test/main/net/delegate.rs +++ /dev/null @@ -1,11 +0,0 @@ -#[path = "./delegate/nodemeta.rs"] -mod nodemeta; - -#[path = "./delegate/local_state.rs"] -mod local_state; - -#[path = "./delegate/remote_state.rs"] -mod remote_state; - -#[path = "./delegate/ping_delegate.rs"] -mod ping_delegate; diff --git a/legacy/serf/test/main/net/delegate/local_state.rs b/legacy/serf/test/main/net/delegate/local_state.rs deleted file mode 100644 index e6d75f6d..00000000 --- a/legacy/serf/test/main/net/delegate/local_state.rs +++ /dev/null @@ -1,66 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{delegate::delegate_local_state, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_delegate_local_state_v4() { - let name = "delegate_nodemeta_local_state1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "delegate_nodemeta_local_state2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](delegate_local_state::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - - #[test] - fn test_delegate_local_state_v6() { - let name = "delegate_nodemeta_local_state1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "delegate_nodemeta_local_state2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](delegate_local_state::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/delegate/nodemeta.rs b/legacy/serf/test/main/net/delegate/nodemeta.rs deleted file mode 100644 index 9f222569..00000000 --- a/legacy/serf/test/main/net/delegate/nodemeta.rs +++ /dev/null @@ -1,91 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{delegate::{delegate_nodemeta, delegate_nodemeta_panic}, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_delegate_nodemeta_v4() { - let name = "delegate_nodemeta_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - [< $rt:snake _run >](delegate_nodemeta::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - - #[test] - fn test_delegate_nodemeta_v6() { - let name = "delegate_nodemeta_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - [< $rt:snake _run >](delegate_nodemeta::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - - #[test] - #[should_panic] - fn test_delegate_nodemeta_panic_v4() { - let name = "delegate_nodemeta_panic_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - [< $rt:snake _run >](delegate_nodemeta_panic::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - - #[test] - #[should_panic] - fn test_delegate_nodemeta_panic_v6() { - let name = "delegate_nodemeta_panic_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - [< $rt:snake _run >](delegate_nodemeta_panic::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/delegate/ping_delegate.rs b/legacy/serf/test/main/net/delegate/ping_delegate.rs deleted file mode 100644 index 35c9cd8f..00000000 --- a/legacy/serf/test/main/net/delegate/ping_delegate.rs +++ /dev/null @@ -1,109 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{delegate::{serf_ping_delegate_versioning, serf_ping_delegate_rogue_coordinate}, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_ping_delegate_versioning_v4() { - let name = "serf_ping_delegate_versioning1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_ping_delegate_versioning2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_ping_delegate_versioning::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - - #[test] - fn test_delegate_serf_ping_delegate_versioning_v6() { - let name = "serf_ping_delegate_versioning1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_ping_delegate_versioning2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_ping_delegate_versioning::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - - #[test] - fn test_serf_ping_delegate_rogue_coordinate_v4() { - let name = "serf_ping_delegate_rogue_coordinate1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_ping_delegate_rogue_coordinate2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_ping_delegate_rogue_coordinate::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - - #[test] - fn test_serf_ping_delegate_rogue_coordinate_v6() { - let name = "serf_ping_delegate_rogue_coordinate1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_ping_delegate_rogue_coordinate2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_ping_delegate_rogue_coordinate::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/delegate/remote_state.rs b/legacy/serf/test/main/net/delegate/remote_state.rs deleted file mode 100644 index 8f9960ae..00000000 --- a/legacy/serf/test/main/net/delegate/remote_state.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{delegate::delegate_merge_remote_state, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_delegate_merge_remote_state_v4() { - let name = "delegate_nodemeta_merge_remote_state_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](delegate_merge_remote_state::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - - #[test] - fn test_delegate_merge_remote_state_v6() { - let name = "delegate_nodemeta_merge_remote_state_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](delegate_merge_remote_state::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/event.rs b/legacy/serf/test/main/net/event.rs deleted file mode 100644 index a832f841..00000000 --- a/legacy/serf/test/main/net/event.rs +++ /dev/null @@ -1,53 +0,0 @@ -#[path = "./event/default_query.rs"] -mod default_query; - -#[path = "./event/event_user_size_limit.rs"] -mod event_user_size_limit; - -#[path = "./event/event_user.rs"] -mod event_user; - -#[path = "./event/events_failed.rs"] -mod events_failed; - -#[path = "./event/events_join.rs"] -mod events_join; - -#[path = "./event/events_leave_avoid_infinite_rebroadcast.rs"] -mod events_leave_avoid_infinite_rebroadcast; - -#[path = "./event/events_leave.rs"] -mod events_leave; - -#[path = "./event/query_deduplicate.rs"] -mod query_deduplicate; - -#[path = "./event/query_filter.rs"] -mod query_filter; - -#[path = "./event/query_old_message.rs"] -mod query_old_message; - -#[path = "./event/query.rs"] -mod query; - -#[path = "./event/query_same_clock.rs"] -mod query_same_clock; - -#[path = "./event/query_size_limit.rs"] -mod query_size_limit; - -#[path = "./event/query_size_limit_increased.rs"] -mod query_size_limit_increased; - -#[path = "./event/remove_failed_events_leave.rs"] -mod remove_failed_events_leave; - -#[path = "./event/should_process.rs"] -mod should_process; - -#[path = "./event/user_event_old_message.rs"] -mod user_event_old_message; - -#[path = "./event/user_event_same_clock.rs"] -mod user_event_same_clock; diff --git a/legacy/serf/test/main/net/event/default_query.rs b/legacy/serf/test/main/net/event/default_query.rs deleted file mode 100644 index 0a5dfdea..00000000 --- a/legacy/serf/test/main/net/event/default_query.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{event::default_query, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_default_query_v4() { - let name = "default_query_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](default_query::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - - #[test] - fn test_default_query_v6() { - let name = "default_query_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](default_query::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/event/event_user.rs b/legacy/serf/test/main/net/event/event_user.rs deleted file mode 100644 index 88f095da..00000000 --- a/legacy/serf/test/main/net/event/event_user.rs +++ /dev/null @@ -1,67 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{event::serf_event_user, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_event_user_v4() { - let name = "serf_event_user1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_event_user2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_event_user::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - - #[test] - fn test_serf_event_user_v6() { - let name = "serf_event_user1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_event_user2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_event_user::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/event/event_user_size_limit.rs b/legacy/serf/test/main/net/event/event_user_size_limit.rs deleted file mode 100644 index 2728fe27..00000000 --- a/legacy/serf/test/main/net/event/event_user_size_limit.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{event::serf_event_user_size_limit, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_event_user_size_limit_v4() { - let name = "serf_event_user_size_limit_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_event_user_size_limit::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - - #[test] - fn test_serf_event_user_size_limit_v6() { - let name = "serf_event_user_size_limit_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_event_user_size_limit::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/event/events_failed.rs b/legacy/serf/test/main/net/event/events_failed.rs deleted file mode 100644 index cc047fa4..00000000 --- a/legacy/serf/test/main/net/event/events_failed.rs +++ /dev/null @@ -1,67 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{event::serf_events_failed, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_events_failed_v4() { - let name = "serf_events_failed1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_events_failed2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_events_failed::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - - #[test] - fn test_serf_events_failed_v6() { - let name = "serf_events_failed1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_events_failed2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_events_failed::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/event/events_join.rs b/legacy/serf/test/main/net/event/events_join.rs deleted file mode 100644 index 95cc3fc2..00000000 --- a/legacy/serf/test/main/net/event/events_join.rs +++ /dev/null @@ -1,67 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{event::serf_events_join, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_events_join_v4() { - let name = "serf_events_join1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_events_join2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_events_join::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - - #[test] - fn test_serf_events_join_v6() { - let name = "serf_events_join1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_events_join2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_events_join::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/event/events_leave.rs b/legacy/serf/test/main/net/event/events_leave.rs deleted file mode 100644 index a5701100..00000000 --- a/legacy/serf/test/main/net/event/events_leave.rs +++ /dev/null @@ -1,67 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{event::serf_events_leave, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_events_leave_v4() { - let name = "serf_events_leave1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_events_leave2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_events_leave::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - - #[test] - fn test_serf_events_leave_v6() { - let name = "serf_events_leave1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_events_leave2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_events_leave::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/event/events_leave_avoid_infinite_rebroadcast.rs b/legacy/serf/test/main/net/event/events_leave_avoid_infinite_rebroadcast.rs deleted file mode 100644 index 8cae3cfd..00000000 --- a/legacy/serf/test/main/net/event/events_leave_avoid_infinite_rebroadcast.rs +++ /dev/null @@ -1,93 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{event::serf_events_leave_avoid_infinite_rebroadcast, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_events_leave_avoid_infinite_rebroadcast_v4() { - let name = "serf_events_leave_avoid_infinite_rebroadcast1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_events_leave_avoid_infinite_rebroadcast2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_events_leave_avoid_infinite_rebroadcast3_v4"; - let mut opts3 = NetTransportOptions::new(SmolStr::new(name)); - opts3.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_events_leave_avoid_infinite_rebroadcast4_v4"; - let mut opts4 = NetTransportOptions::new(SmolStr::new(name)); - opts4.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_events_leave_avoid_infinite_rebroadcast::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - _, - >(opts, opts2, opts3, opts4, |id, addr| async move { - let mut opts2 = NetTransportOptions::new(id); - opts2.add_bind_address(addr); - opts2 - })); - } - - #[test] - fn test_serf_events_leave_avoid_infinite_rebroadcast_v6() { - let name = "serf_events_leave_avoid_infinite_rebroadcast1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_events_leave_avoid_infinite_rebroadcast2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - let name = "serf_events_leave_avoid_infinite_rebroadcast3_v6"; - let mut opts3 = NetTransportOptions::new(SmolStr::new(name)); - opts3.add_bind_address(next_socket_addr_v6()); - - let name = "serf_events_leave_avoid_infinite_rebroadcast4_v6"; - let mut opts4 = NetTransportOptions::new(SmolStr::new(name)); - opts4.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_events_leave_avoid_infinite_rebroadcast::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - _, - >(opts, opts2, opts3, opts4, |id, addr| async move { - let mut opts2 = NetTransportOptions::new(id); - opts2.add_bind_address(addr); - opts2 - })); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/event/query.rs b/legacy/serf/test/main/net/event/query.rs deleted file mode 100644 index 5cbef87b..00000000 --- a/legacy/serf/test/main/net/event/query.rs +++ /dev/null @@ -1,67 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{event::serf_query, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_query_v4() { - let name = "serf_query1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_query2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_query::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - - #[test] - fn test_serf_query_v6() { - let name = "serf_query1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_query2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_query::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/event/query_deduplicate.rs b/legacy/serf/test/main/net/event/query_deduplicate.rs deleted file mode 100644 index d92a2a5f..00000000 --- a/legacy/serf/test/main/net/event/query_deduplicate.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{event::serf_query_deduplicate, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_query_deduplicate_v4() { - let name = "serf_query_deduplicate_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_query_deduplicate::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - - #[test] - fn test_serf_query_deduplicate_v6() { - let name = "serf_query_deduplicate_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_query_deduplicate::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/event/query_filter.rs b/legacy/serf/test/main/net/event/query_filter.rs deleted file mode 100644 index f35ccd45..00000000 --- a/legacy/serf/test/main/net/event/query_filter.rs +++ /dev/null @@ -1,75 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{event::serf_query_filter, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_query_filter_v4() { - let name = "serf_query_filter1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_query_filter2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_query_filter3_v4"; - let mut opts3 = NetTransportOptions::new(SmolStr::new(name)); - opts3.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_query_filter::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2, opts3)); - } - - #[test] - fn test_serf_query_filter_v6() { - let name = "serf_query_filter1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_query_filter2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - let name = "serf_query_filter3_v6"; - let mut opts3 = NetTransportOptions::new(SmolStr::new(name)); - opts3.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_query_filter::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2, opts3)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/event/query_old_message.rs b/legacy/serf/test/main/net/event/query_old_message.rs deleted file mode 100644 index a741bb2e..00000000 --- a/legacy/serf/test/main/net/event/query_old_message.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - transport::Node, - }; - use serf_core::tests::{event::query_old_message, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_query_old_message_v4() { - let name = "query_old_message_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](query_old_message::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, Node::new("fake1".into(), next_socket_addr_v4(0)))); - } - - #[test] - fn test_query_old_message_v6() { - let name = "query_old_message_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](query_old_message::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, Node::new("fake1".into(), next_socket_addr_v4(0)))); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/event/query_same_clock.rs b/legacy/serf/test/main/net/event/query_same_clock.rs deleted file mode 100644 index 16acef46..00000000 --- a/legacy/serf/test/main/net/event/query_same_clock.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - transport::Node, - }; - use serf_core::tests::{event::query_same_clock, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_query_same_clock_v4() { - let name = "query_same_clock_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](query_same_clock::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, Node::new("fake1".into(), next_socket_addr_v4(0)))); - } - - #[test] - fn test_query_same_clock_v6() { - let name = "query_same_clock_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](query_same_clock::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, Node::new("fake1".into(), next_socket_addr_v4(0)))); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/event/query_size_limit.rs b/legacy/serf/test/main/net/event/query_size_limit.rs deleted file mode 100644 index fae5ac3f..00000000 --- a/legacy/serf/test/main/net/event/query_size_limit.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{event::serf_query_size_limit, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_query_size_limit_v4() { - let name = "serf_query_size_limit_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_query_size_limit::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - - #[test] - fn test_serf_query_size_limit_v6() { - let name = "serf_query_size_limit_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_query_size_limit::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/event/query_size_limit_increased.rs b/legacy/serf/test/main/net/event/query_size_limit_increased.rs deleted file mode 100644 index f6f38a6c..00000000 --- a/legacy/serf/test/main/net/event/query_size_limit_increased.rs +++ /dev/null @@ -1,60 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{event::serf_query_size_limit_increased, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_query_size_limit_increased_v4() { - let name = "serf_query_size_limit_increased_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_query_size_limit_increased::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - - #[test] - fn test_serf_query_size_limit_increased_v6() { - let name = "serf_query_size_limit_increased_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_query_size_limit_increased::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/event/remove_failed_events_leave.rs b/legacy/serf/test/main/net/event/remove_failed_events_leave.rs deleted file mode 100644 index 95c42f13..00000000 --- a/legacy/serf/test/main/net/event/remove_failed_events_leave.rs +++ /dev/null @@ -1,67 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{event::serf_remove_failed_events_leave, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_remove_failed_events_leave_v4() { - let name = "serf_remove_failed_events_leave1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_remove_failed_events_leave2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_remove_failed_events_leave::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - - #[test] - fn test_serf_remove_failed_events_leave_v6() { - let name = "serf_remove_failed_events_leave1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_remove_failed_events_leave2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_remove_failed_events_leave::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/event/should_process.rs b/legacy/serf/test/main/net/event/should_process.rs deleted file mode 100644 index 6f530b43..00000000 --- a/legacy/serf/test/main/net/event/should_process.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{event::should_process, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_should_process_v4() { - let name = "should_process_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](should_process::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - - #[test] - fn test_should_process_v6() { - let name = "should_process_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](should_process::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/event/user_event_old_message.rs b/legacy/serf/test/main/net/event/user_event_old_message.rs deleted file mode 100644 index afbcff3e..00000000 --- a/legacy/serf/test/main/net/event/user_event_old_message.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{event::user_event_old_message, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_user_event_old_message_v4() { - let name = "user_event_old_message_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](user_event_old_message::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - - #[test] - fn test_user_event_old_message_v6() { - let name = "user_event_old_message_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](user_event_old_message::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/event/user_event_same_clock.rs b/legacy/serf/test/main/net/event/user_event_same_clock.rs deleted file mode 100644 index 78a52bcb..00000000 --- a/legacy/serf/test/main/net/event/user_event_same_clock.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{event::user_event_same_clock, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_user_event_same_clock_v4() { - let name = "user_event_same_clock_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](user_event_same_clock::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - - #[test] - fn test_user_event_same_clock_v6() { - let name = "user_event_same_clock_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](user_event_same_clock::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/get_queue_max.rs b/legacy/serf/test/main/net/get_queue_max.rs deleted file mode 100644 index 8966f9af..00000000 --- a/legacy/serf/test/main/net/get_queue_max.rs +++ /dev/null @@ -1,63 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{serf_get_queue_max, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_get_queue_max_v4() { - let name = "serf_get_queue_max_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_get_queue_max::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, |idx| { - format!("127.0.0.1:{idx}").parse().unwrap() - })); - } - - #[test] - fn test_serf_get_queue_max_v6() { - let name = "serf_get_queue_max_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_get_queue_max::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, |idx| { - format!("127.0.0.1:{idx}").parse().unwrap() - })); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/join.rs b/legacy/serf/test/main/net/join.rs deleted file mode 100644 index 71f84d8d..00000000 --- a/legacy/serf/test/main/net/join.rs +++ /dev/null @@ -1,32 +0,0 @@ -#[path = "./join/intent_buffer_early.rs"] -mod intent_buffer_early; - -#[path = "./join/intent_newer.rs"] -mod intent_newer; - -#[path = "./join/intent_old_message.rs"] -mod intent_old_message; - -#[path = "./join/intent_reset_leaving.rs"] -mod intent_reset_leaving; - -#[path = "./join/join_cancel.rs"] -mod join_cancel; - -#[path = "./join/join_ignore_old.rs"] -mod join_ignore_old; - -#[path = "./join/join_leave_join.rs"] -mod join_leave_join; - -#[path = "./join/join_leave.rs"] -mod join_leave; - -#[path = "./join/leave_ltime.rs"] -mod leave_ltime; - -#[path = "./join/pending_intent.rs"] -mod pending_intent; - -#[path = "./join/pending_intents.rs"] -mod pending_intents; diff --git a/legacy/serf/test/main/net/join/intent_buffer_early.rs b/legacy/serf/test/main/net/join/intent_buffer_early.rs deleted file mode 100644 index 790f44ff..00000000 --- a/legacy/serf/test/main/net/join/intent_buffer_early.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{join::join_intent_buffer_early, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_join_intent_buffer_early_v4() { - let name = "join_intent_buffer_early_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](join_intent_buffer_early::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - - #[test] - fn test_join_intent_buffer_early_v6() { - let name = "join_intent_buffer_early_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](join_intent_buffer_early::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/join/intent_newer.rs b/legacy/serf/test/main/net/join/intent_newer.rs deleted file mode 100644 index 67bea317..00000000 --- a/legacy/serf/test/main/net/join/intent_newer.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{join::join_intent_newer, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_join_intent_newer_v4() { - let name = "join_intent_newer_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](join_intent_newer::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, next_socket_addr_v4(0))); - } - - #[test] - fn test_join_intent_newer_v6() { - let name = "join_intent_newer_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](join_intent_newer::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, next_socket_addr_v6())); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/join/intent_old_message.rs b/legacy/serf/test/main/net/join/intent_old_message.rs deleted file mode 100644 index 436f3373..00000000 --- a/legacy/serf/test/main/net/join/intent_old_message.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{join::join_intent_old_message, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_join_intent_old_message_v4() { - let name = "join_intent_old_message_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](join_intent_old_message::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, next_socket_addr_v4(0))); - } - - #[test] - fn test_join_intent_old_message_v6() { - let name = "join_intent_old_message_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](join_intent_old_message::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, next_socket_addr_v6())); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/join/intent_reset_leaving.rs b/legacy/serf/test/main/net/join/intent_reset_leaving.rs deleted file mode 100644 index 30b32b66..00000000 --- a/legacy/serf/test/main/net/join/intent_reset_leaving.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{join::join_intent_reset_leaving, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_join_intent_reset_leaving_v4() { - let name = "join_intent_reset_leaving_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](join_intent_reset_leaving::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, next_socket_addr_v4(0))); - } - - #[test] - fn test_join_intent_reset_leaving_v6() { - let name = "join_intent_reset_leaving_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](join_intent_reset_leaving::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, next_socket_addr_v6())); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/join/join_cancel.rs b/legacy/serf/test/main/net/join/join_cancel.rs deleted file mode 100644 index 705b875d..00000000 --- a/legacy/serf/test/main/net/join/join_cancel.rs +++ /dev/null @@ -1,67 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{join::serf_join_cancel, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_join_cancel_v4() { - let name = "serf_join_cancel1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_join_cancel2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_join_cancel::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - - #[test] - fn test_serf_join_cancel_v6() { - let name = "serf_join_cancel1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_join_cancel2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_join_cancel::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/join/join_ignore_old.rs b/legacy/serf/test/main/net/join/join_ignore_old.rs deleted file mode 100644 index bd69643d..00000000 --- a/legacy/serf/test/main/net/join/join_ignore_old.rs +++ /dev/null @@ -1,67 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{join::serf_join_ignore_old, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_join_ignore_old_v4() { - let name = "serf_join_ignore_old1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_join_ignore_old2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_join_ignore_old::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - - #[test] - fn test_serf_join_ignore_old_v6() { - let name = "serf_join_ignore_old1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_join_ignore_old2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_join_ignore_old::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/join/join_leave.rs b/legacy/serf/test/main/net/join/join_leave.rs deleted file mode 100644 index ae9f2ee8..00000000 --- a/legacy/serf/test/main/net/join/join_leave.rs +++ /dev/null @@ -1,67 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{join::serf_join_leave, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_join_leave_v4() { - let name = "serf_join_leave1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_join_leave2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_join_leave::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - - #[test] - fn test_serf_join_leave_v6() { - let name = "serf_join_leave1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_join_leave2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_join_leave::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/join/join_leave_join.rs b/legacy/serf/test/main/net/join/join_leave_join.rs deleted file mode 100644 index 7dd61046..00000000 --- a/legacy/serf/test/main/net/join/join_leave_join.rs +++ /dev/null @@ -1,67 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{join::serf_join_leave_join, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_join_leave_join_v4() { - let name = "serf_join_leave_join1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_join_leave_join2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_join_leave_join::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - - #[test] - fn test_serf_join_leave_join_v6() { - let name = "serf_join_leave_join1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_join_leave_join2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_join_leave_join::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/join/leave_ltime.rs b/legacy/serf/test/main/net/join/leave_ltime.rs deleted file mode 100644 index 95c223b5..00000000 --- a/legacy/serf/test/main/net/join/leave_ltime.rs +++ /dev/null @@ -1,67 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{join::join_leave_ltime, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_join_leave_ltime_v4() { - let name = "join_leave_ltime1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "join_leave_ltime2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](join_leave_ltime::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - - #[test] - fn test_join_leave_ltime_v6() { - let name = "join_leave_ltime1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "join_leave_ltime2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](join_leave_ltime::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/join/pending_intent.rs b/legacy/serf/test/main/net/join/pending_intent.rs deleted file mode 100644 index 57633fbc..00000000 --- a/legacy/serf/test/main/net/join/pending_intent.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{join::join_pending_intent, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_join_pending_intent_v4() { - let name = "join_pending_intent_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](join_pending_intent::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, next_socket_addr_v4(0))); - } - - #[test] - fn test_join_pending_intent_v6() { - let name = "join_pending_intent_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](join_pending_intent::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, next_socket_addr_v6())); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/join/pending_intents.rs b/legacy/serf/test/main/net/join/pending_intents.rs deleted file mode 100644 index d3409f7f..00000000 --- a/legacy/serf/test/main/net/join/pending_intents.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{join::join_pending_intents, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_join_pending_intents_v4() { - let name = "join_pending_intents_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](join_pending_intents::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, next_socket_addr_v4(0))); - } - - #[test] - fn test_join_pending_intents_v6() { - let name = "join_pending_intents_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](join_pending_intents::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, next_socket_addr_v6())); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/leave.rs b/legacy/serf/test/main/net/leave.rs deleted file mode 100644 index a674696c..00000000 --- a/legacy/serf/test/main/net/leave.rs +++ /dev/null @@ -1,23 +0,0 @@ -#[path = "./leave/force_leave_failed.rs"] -mod force_leave_failed; - -#[path = "./leave/force_leave_leaving.rs"] -mod force_leave_leaving; - -#[path = "./leave/force_leave_left.rs"] -mod force_leave_left; - -#[path = "./leave/intent_buffer_early.rs"] -mod intent_buffer_early; - -#[path = "./leave/intent_newer.rs"] -mod intent_newer; - -#[path = "./leave/intent_old_message.rs"] -mod intent_old_message; - -#[path = "./leave/rejoin_different_role.rs"] -mod rejoin_different_role; - -#[path = "./leave/snapshot_recovery.rs"] -mod snapshot_recovery; diff --git a/legacy/serf/test/main/net/leave/force_leave_failed.rs b/legacy/serf/test/main/net/leave/force_leave_failed.rs deleted file mode 100644 index 4d6f8a81..00000000 --- a/legacy/serf/test/main/net/leave/force_leave_failed.rs +++ /dev/null @@ -1,75 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{leave::serf_force_leave_failed, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_force_leave_failed_v4() { - let name = "serf_force_leave_failed1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_force_leave_failed2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_force_leave_failed3_v4"; - let mut opts3 = NetTransportOptions::new(SmolStr::new(name)); - opts3.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_force_leave_failed::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2, opts3)); - } - - #[test] - fn test_serf_force_leave_failed_v6() { - let name = "serf_force_leave_failed1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_force_leave_failed2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - let name = "serf_force_leave_failed3_v6"; - let mut opts3 = NetTransportOptions::new(SmolStr::new(name)); - opts3.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_force_leave_failed::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2, opts3)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/leave/force_leave_leaving.rs b/legacy/serf/test/main/net/leave/force_leave_leaving.rs deleted file mode 100644 index ddfba17f..00000000 --- a/legacy/serf/test/main/net/leave/force_leave_leaving.rs +++ /dev/null @@ -1,75 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{leave::serf_force_leave_leaving, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_force_leave_leaving_v4() { - let name = "serf_force_leave_leaving1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_force_leave_leaving2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_force_leave_leaving3_v4"; - let mut opts3 = NetTransportOptions::new(SmolStr::new(name)); - opts3.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_force_leave_leaving::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2, opts3)); - } - - #[test] - fn test_serf_force_leave_leaving_v6() { - let name = "serf_force_leave_leaving1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_force_leave_leaving2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - let name = "serf_force_leave_leaving3_v6"; - let mut opts3 = NetTransportOptions::new(SmolStr::new(name)); - opts3.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_force_leave_leaving::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2, opts3)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/leave/force_leave_left.rs b/legacy/serf/test/main/net/leave/force_leave_left.rs deleted file mode 100644 index 621f3f42..00000000 --- a/legacy/serf/test/main/net/leave/force_leave_left.rs +++ /dev/null @@ -1,75 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{leave::serf_force_leave_left, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_force_leave_left_v4() { - let name = "serf_force_leave_left1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_force_leave_left2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_force_leave_left3_v4"; - let mut opts3 = NetTransportOptions::new(SmolStr::new(name)); - opts3.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_force_leave_left::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2, opts3)); - } - - #[test] - fn test_serf_force_leave_left_v6() { - let name = "serf_force_leave_left1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_force_leave_left2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - let name = "serf_force_leave_left3_v6"; - let mut opts3 = NetTransportOptions::new(SmolStr::new(name)); - opts3.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_force_leave_left::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2, opts3)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/leave/intent_buffer_early.rs b/legacy/serf/test/main/net/leave/intent_buffer_early.rs deleted file mode 100644 index d95f66c1..00000000 --- a/legacy/serf/test/main/net/leave/intent_buffer_early.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{leave::leave_intent_buffer_early, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_leave_intent_buffer_early_v4() { - let name = "leave_intent_buffer_early_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](leave_intent_buffer_early::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - - #[test] - fn test_leave_intent_buffer_early_v6() { - let name = "leave_intent_buffer_early_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](leave_intent_buffer_early::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/leave/intent_newer.rs b/legacy/serf/test/main/net/leave/intent_newer.rs deleted file mode 100644 index 909d5b01..00000000 --- a/legacy/serf/test/main/net/leave/intent_newer.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{leave::leave_intent_newer, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_leave_intent_newer_v4() { - let name = "leave_intent_newer_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](leave_intent_newer::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, next_socket_addr_v4(0))); - } - - #[test] - fn test_leave_intent_newer_v6() { - let name = "leave_intent_newer_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](leave_intent_newer::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, next_socket_addr_v6())); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/leave/intent_old_message.rs b/legacy/serf/test/main/net/leave/intent_old_message.rs deleted file mode 100644 index 193dba51..00000000 --- a/legacy/serf/test/main/net/leave/intent_old_message.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{leave::leave_intent_old_message, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_leave_intent_old_message_v4() { - let name = "leave_intent_old_message_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](leave_intent_old_message::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, next_socket_addr_v4(0))); - } - - #[test] - fn test_leave_intent_old_message_v6() { - let name = "leave_intent_old_message_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](leave_intent_old_message::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, next_socket_addr_v6())); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/leave/rejoin_different_role.rs b/legacy/serf/test/main/net/leave/rejoin_different_role.rs deleted file mode 100644 index e5d58859..00000000 --- a/legacy/serf/test/main/net/leave/rejoin_different_role.rs +++ /dev/null @@ -1,67 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{leave::serf_leave_rejoin_different_role, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_leave_rejoin_different_role_v4() { - let name = "serf_leave_rejoin_different_role1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_leave_rejoin_different_role2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_leave_rejoin_different_role::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - - #[test] - fn test_serf_leave_rejoin_different_role_v6() { - let name = "serf_leave_rejoin_different_role1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_leave_rejoin_different_role2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_leave_rejoin_different_role::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/leave/snapshot_recovery.rs b/legacy/serf/test/main/net/leave/snapshot_recovery.rs deleted file mode 100644 index b7f0220a..00000000 --- a/legacy/serf/test/main/net/leave/snapshot_recovery.rs +++ /dev/null @@ -1,77 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{leave::serf_leave_snapshot_recovery, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_leave_snapshot_recovery_v4() { - let name = "serf_leave_snapshot_recovery1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_leave_snapshot_recovery2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_leave_snapshot_recovery::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - _ - >(opts, opts2, |id, addr| async move { - let mut opts2 = NetTransportOptions::new(id); - opts2.add_bind_address(addr); - opts2 - })); - } - - #[test] - fn test_serf_leave_snapshot_recovery_v6() { - let name = "serf_leave_snapshot_recovery1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_leave_snapshot_recovery2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_leave_snapshot_recovery::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - _ - >(opts, opts2, |id, addr| async move { - let mut opts2 = NetTransportOptions::new(id); - opts2.add_bind_address(addr); - opts2 - })); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/local_member.rs b/legacy/serf/test/main/net/local_member.rs deleted file mode 100644 index 02e5e0af..00000000 --- a/legacy/serf/test/main/net/local_member.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{serf_local_member, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_local_member_v4() { - let name = "serf_local_member_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_local_member::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - - #[test] - fn test_serf_local_member_v6() { - let name = "serf_local_member_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_local_member::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/name_resolution.rs b/legacy/serf/test/main/net/name_resolution.rs deleted file mode 100644 index 0ae76b83..00000000 --- a/legacy/serf/test/main/net/name_resolution.rs +++ /dev/null @@ -1,75 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{serf_name_resolution, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_name_resolution_v4() { - let name = "serf_name_resolution1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_name_resolution2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_name_resolution3_v4"; - let mut opts3 = NetTransportOptions::new(SmolStr::new(name)); - opts3.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_name_resolution::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2, opts3, |opts, id| opts.with_id(id))); - } - - #[test] - fn test_serf_name_resolution_v6() { - let name = "serf_name_resolution1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_name_resolution2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - let name = "serf_name_resolution3_v6"; - let mut opts3 = NetTransportOptions::new(SmolStr::new(name)); - opts3.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_name_resolution::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2, opts3, |opts, id| opts.with_id(id))); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/num_nodes.rs b/legacy/serf/test/main/net/num_nodes.rs deleted file mode 100644 index c8ebc71e..00000000 --- a/legacy/serf/test/main/net/num_nodes.rs +++ /dev/null @@ -1,67 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{serf_num_nodes, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_num_nodes_v4() { - let name = "serf_num_nodes1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_num_nodes2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_num_nodes::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - - #[test] - fn test_serf_num_nodes_v6() { - let name = "serf_num_nodes1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_num_nodes2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_num_nodes::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/reap.rs b/legacy/serf/test/main/net/reap.rs deleted file mode 100644 index a428f1a0..00000000 --- a/legacy/serf/test/main/net/reap.rs +++ /dev/null @@ -1,9 +0,0 @@ -#[allow(clippy::module_inception)] -#[path = "./reap/reap.rs"] -mod reap; - -#[path = "./reap/handler.rs"] -mod handler; - -#[path = "./reap/handler_shutdown.rs"] -mod handler_shutdown; diff --git a/legacy/serf/test/main/net/reap/handler.rs b/legacy/serf/test/main/net/reap/handler.rs deleted file mode 100644 index e96c9886..00000000 --- a/legacy/serf/test/main/net/reap/handler.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{reap::serf_reap_handler, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_reap_handler_v4() { - let name = "serf_reap_handler_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_reap_handler::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, next_socket_addr_v4(0))); - } - - #[test] - fn test_serf_reap_handler_v6() { - let name = "serf_reap_handler_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_reap_handler::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, next_socket_addr_v6())); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/reap/handler_shutdown.rs b/legacy/serf/test/main/net/reap/handler_shutdown.rs deleted file mode 100644 index 98903510..00000000 --- a/legacy/serf/test/main/net/reap/handler_shutdown.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{reap::serf_reap_handler_shutdown, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_reap_handler_shutdown_v4() { - let name = "serf_reap_handler_shutdown_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_reap_handler_shutdown::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - - #[test] - fn test_serf_reap_handler_shutdown_v6() { - let name = "serf_reap_handler_shutdown_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_reap_handler_shutdown::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/reap/reap.rs b/legacy/serf/test/main/net/reap/reap.rs deleted file mode 100644 index 4d1ebc79..00000000 --- a/legacy/serf/test/main/net/reap/reap.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{reap::serf_reap, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_reap_v4() { - let name = "serf_reap_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_reap::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, next_socket_addr_v4(0))); - } - - #[test] - fn test_serf_reap_v6() { - let name = "serf_reap_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_reap::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, next_socket_addr_v6())); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/reconnect.rs b/legacy/serf/test/main/net/reconnect.rs deleted file mode 100644 index 89dc4117..00000000 --- a/legacy/serf/test/main/net/reconnect.rs +++ /dev/null @@ -1,9 +0,0 @@ -#[allow(clippy::module_inception)] -#[path = "./reconnect/reconnect.rs"] -mod reconnect; - -#[path = "./reconnect/same_ip.rs"] -mod same_ip; - -#[path = "./reconnect/timeout.rs"] -mod timeout; diff --git a/legacy/serf/test/main/net/reconnect/reconnect.rs b/legacy/serf/test/main/net/reconnect/reconnect.rs deleted file mode 100644 index 331c7ba0..00000000 --- a/legacy/serf/test/main/net/reconnect/reconnect.rs +++ /dev/null @@ -1,77 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{reconnect::serf_reconnect, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_reconnect_v4() { - let name = "serf_reconnect1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_reconnect2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_reconnect::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - _, - >(opts, opts2, |id, addr| async move { - let mut opts2 = NetTransportOptions::new(id); - opts2.add_bind_address(addr); - opts2 - })); - } - - #[test] - fn test_serf_reconnect_v6() { - let name = "serf_reconnect1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_reconnect2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_reconnect::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - _, - >(opts, opts2, |id, addr| async move { - let mut opts2 = NetTransportOptions::new(id); - opts2.add_bind_address(addr); - opts2 - })); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/reconnect/same_ip.rs b/legacy/serf/test/main/net/reconnect/same_ip.rs deleted file mode 100644 index b7168410..00000000 --- a/legacy/serf/test/main/net/reconnect/same_ip.rs +++ /dev/null @@ -1,75 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{reconnect::serf_reconnect_same_ip, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_reconnect_same_ip_v4() { - let name = "serf_reconnect_same_ip1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_reconnect_same_ip2_v4"; - - [< $rt:snake _run >](serf_reconnect_same_ip::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - _, - _ - >(opts, SmolStr::new(name), |id, addr| async move { - let mut opts2 = NetTransportOptions::new(id); - opts2.add_bind_address(addr); - opts2 - })); - } - - #[test] - fn test_serf_reconnect_same_ip_v6() { - let name = "serf_reconnect_same_ip1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_reconnect_same_ip2_v6"; - - [< $rt:snake _run >](serf_reconnect_same_ip::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - _, - _, - >(opts, SmolStr::new(name), |id, addr| async move { - let mut opts2 = NetTransportOptions::new(id); - opts2.add_bind_address(addr); - opts2 - })); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/reconnect/timeout.rs b/legacy/serf/test/main/net/reconnect/timeout.rs deleted file mode 100644 index c8044f36..00000000 --- a/legacy/serf/test/main/net/reconnect/timeout.rs +++ /dev/null @@ -1,67 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{reconnect::serf_per_node_reconnect_timeout, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_per_node_reconnect_timeout_v4() { - let name = "serf_per_node_reconnect_timeout1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_per_node_reconnect_timeout2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_per_node_reconnect_timeout::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - - #[test] - fn test_serf_per_node_reconnect_timeout_v6() { - let name = "serf_per_node_reconnect_timeout1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_per_node_reconnect_timeout2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_per_node_reconnect_timeout::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/remove.rs b/legacy/serf/test/main/net/remove.rs deleted file mode 100644 index 632b6bed..00000000 --- a/legacy/serf/test/main/net/remove.rs +++ /dev/null @@ -1,8 +0,0 @@ -#[path = "./remove/failed_node.rs"] -mod failed_node; - -#[path = "./remove/failed_node_prune.rs"] -mod failed_node_prune; - -#[path = "./remove/failed_node_ourself.rs"] -mod failed_node_ourself; diff --git a/legacy/serf/test/main/net/remove/failed_node.rs b/legacy/serf/test/main/net/remove/failed_node.rs deleted file mode 100644 index 8d908fd8..00000000 --- a/legacy/serf/test/main/net/remove/failed_node.rs +++ /dev/null @@ -1,75 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{remove::serf_remove_failed_node, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_remove_failed_node_v4() { - let name = "serf_remove_failed_node1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_remove_failed_node2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_remove_failed_node3_v4"; - let mut opts3 = NetTransportOptions::new(SmolStr::new(name)); - opts3.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_remove_failed_node::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2, opts3)); - } - - #[test] - fn test_serf_remove_failed_node_v6() { - let name = "serf_remove_failed_node1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_remove_failed_node2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - let name = "serf_remove_failed_node3_v6"; - let mut opts3 = NetTransportOptions::new(SmolStr::new(name)); - opts3.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_remove_failed_node::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2, opts3)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/remove/failed_node_ourself.rs b/legacy/serf/test/main/net/remove/failed_node_ourself.rs deleted file mode 100644 index 817f07fa..00000000 --- a/legacy/serf/test/main/net/remove/failed_node_ourself.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{remove::serf_remove_failed_node_ourself, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_remove_failed_node_ourself_v4() { - let name = "serf_remove_failed_node_ourself_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_remove_failed_node_ourself::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - - #[test] - fn test_serf_remove_failed_node_ourself_v6() { - let name = "serf_remove_failed_node_ourself_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_remove_failed_node_ourself::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/remove/failed_node_prune.rs b/legacy/serf/test/main/net/remove/failed_node_prune.rs deleted file mode 100644 index afb09be2..00000000 --- a/legacy/serf/test/main/net/remove/failed_node_prune.rs +++ /dev/null @@ -1,75 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{remove::serf_remove_failed_node_prune, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_remove_failed_node_prune_v4() { - let name = "serf_remove_failed_node_prune1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_remove_failed_node_prune2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_remove_failed_node_prune3_v4"; - let mut opts3 = NetTransportOptions::new(SmolStr::new(name)); - opts3.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_remove_failed_node_prune::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2, opts3)); - } - - #[test] - fn test_serf_remove_failed_node_prune_v6() { - let name = "serf_remove_failed_node_prune1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_remove_failed_node_prune2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - let name = "serf_remove_failed_node_prune3_v6"; - let mut opts3 = NetTransportOptions::new(SmolStr::new(name)); - opts3.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_remove_failed_node_prune::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2, opts3)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/role.rs b/legacy/serf/test/main/net/role.rs deleted file mode 100644 index 16c40306..00000000 --- a/legacy/serf/test/main/net/role.rs +++ /dev/null @@ -1,67 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{serf_role, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_role_v4() { - let name = "serf_role1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_role2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_role::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - - #[test] - fn test_serf_role_v6() { - let name = "serf_role1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_role2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_role::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/set_tags.rs b/legacy/serf/test/main/net/set_tags.rs deleted file mode 100644 index fae8a087..00000000 --- a/legacy/serf/test/main/net/set_tags.rs +++ /dev/null @@ -1,67 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{serf_set_tags, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_set_tags_v4() { - let name = "serf_set_tags1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_set_tags2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_set_tags::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - - #[test] - fn test_serf_set_tags_v6() { - let name = "serf_set_tags1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_set_tags2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_set_tags::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, opts2)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/snapshot.rs b/legacy/serf/test/main/net/snapshot.rs deleted file mode 100644 index d681f494..00000000 --- a/legacy/serf/test/main/net/snapshot.rs +++ /dev/null @@ -1,14 +0,0 @@ -#[path = "./snapshot/snapshoter.rs"] -mod snapshoter; - -#[path = "./snapshot/snapshoter_leave.rs"] -mod snapshoter_leave; - -#[path = "./snapshot/snapshoter_leave_rejoin.rs"] -mod snapshoter_leave_rejoin; - -#[path = "./snapshot/snapshoter_recovery.rs"] -mod snapshoter_recovery; - -#[path = "./snapshot/snapshoter_force_compact.rs"] -mod snapshoter_force_compact; diff --git a/legacy/serf/test/main/net/snapshot/snapshoter.rs b/legacy/serf/test/main/net/snapshot/snapshoter.rs deleted file mode 100644 index 31209f9b..00000000 --- a/legacy/serf/test/main/net/snapshot/snapshoter.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{snapshot::snapshoter, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_snapshoter_v4() { - let name = "snapshoter_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](snapshoter::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, next_socket_addr_v4(0))); - } - - #[test] - fn test_snapshoter_v6() { - let name = "snapshoter_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](snapshoter::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, next_socket_addr_v6())); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/snapshot/snapshoter_force_compact.rs b/legacy/serf/test/main/net/snapshot/snapshoter_force_compact.rs deleted file mode 100644 index d3ff112f..00000000 --- a/legacy/serf/test/main/net/snapshot/snapshoter_force_compact.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{snapshot::snapshoter_force_compact, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_snapshoter_force_compact_v4() { - let name = "snapshoter_force_compact_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](snapshoter_force_compact::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, next_socket_addr_v4(0))); - } - - #[test] - fn test_snapshoter_force_compact_v6() { - let name = "snapshoter_force_compact_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](snapshoter_force_compact::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, next_socket_addr_v6())); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/snapshot/snapshoter_leave.rs b/legacy/serf/test/main/net/snapshot/snapshoter_leave.rs deleted file mode 100644 index f1cb93ad..00000000 --- a/legacy/serf/test/main/net/snapshot/snapshoter_leave.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{snapshot::snapshoter_leave, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_snapshoter_leave_v4() { - let name = "snapshoter_leave_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](snapshoter_leave::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, next_socket_addr_v4(0))); - } - - #[test] - fn test_snapshoter_leave_v6() { - let name = "snapshoter_leave_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](snapshoter_leave::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, next_socket_addr_v6())); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/snapshot/snapshoter_leave_rejoin.rs b/legacy/serf/test/main/net/snapshot/snapshoter_leave_rejoin.rs deleted file mode 100644 index 41815fd8..00000000 --- a/legacy/serf/test/main/net/snapshot/snapshoter_leave_rejoin.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{snapshot::snapshoter_leave_rejoin, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_snapshoter_leave_rejoin_v4() { - let name = "snapshoter_leave_rejoin_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](snapshoter_leave_rejoin::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, next_socket_addr_v4(0))); - } - - #[test] - fn test_snapshoter_leave_rejoin_v6() { - let name = "snapshoter_leave_rejoin_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](snapshoter_leave_rejoin::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts, next_socket_addr_v6())); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/snapshot/snapshoter_recovery.rs b/legacy/serf/test/main/net/snapshot/snapshoter_recovery.rs deleted file mode 100644 index cd5bcf64..00000000 --- a/legacy/serf/test/main/net/snapshot/snapshoter_recovery.rs +++ /dev/null @@ -1,77 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{snapshot::serf_snapshot_recovery, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_snapshot_recovery_v4() { - let name = "serf_snapshot_recovery1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_snapshot_recovery2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_snapshot_recovery::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - _ - >(opts, opts2, |id, addr| async move { - let mut opts2 = NetTransportOptions::new(id); - opts2.add_bind_address(addr); - opts2 - })); - } - - #[test] - fn test_serf_snapshot_recovery_v6() { - let name = "serf_snapshot_recovery1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_snapshot_recovery2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_snapshot_recovery::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - _, - >(opts, opts2, |id, addr| async move { - let mut opts2 = NetTransportOptions::new(id); - opts2.add_bind_address(addr); - opts2 - })); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/state.rs b/legacy/serf/test/main/net/state.rs deleted file mode 100644 index 8f0b7667..00000000 --- a/legacy/serf/test/main/net/state.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{serf_state, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_state_v4() { - let name = "serf_state_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_state::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - - #[test] - fn test_serf_state_v6() { - let name = "serf_state_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_state::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/stats.rs b/legacy/serf/test/main/net/stats.rs deleted file mode 100644 index 69b0aa88..00000000 --- a/legacy/serf/test/main/net/stats.rs +++ /dev/null @@ -1,59 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{serf_stats, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_stats_v4() { - let name = "serf_stats_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_stats::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - - #[test] - fn test_serf_stats_v6() { - let name = "serf_stats_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_stats::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(opts)); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/update.rs b/legacy/serf/test/main/net/update.rs deleted file mode 100644 index 043d429d..00000000 --- a/legacy/serf/test/main/net/update.rs +++ /dev/null @@ -1,77 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::tests::{serf_update, next_socket_addr_v4, next_socket_addr_v6}; - use smol_str::SmolStr; - - #[test] - fn test_serf_update_v4() { - let name = "serf_update1_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - let name = "serf_update2_v4"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_update::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - _, - >(opts, opts2, |id, addr| async move { - let mut opts2 = NetTransportOptions::new(id); - opts2.add_bind_address(addr); - opts2 - })); - } - - #[test] - fn test_serf_update_v6() { - let name = "serf_update1_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - let name = "serf_update2_v6"; - let mut opts2 = NetTransportOptions::new(SmolStr::new(name)); - opts2.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_update::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - _, - >(opts, opts2, |id, addr| async move { - let mut opts2 = NetTransportOptions::new(id); - opts2.add_bind_address(addr); - opts2 - })); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); diff --git a/legacy/serf/test/main/net/write_keyring_file.rs b/legacy/serf/test/main/net/write_keyring_file.rs deleted file mode 100644 index 8cd441e2..00000000 --- a/legacy/serf/test/main/net/write_keyring_file.rs +++ /dev/null @@ -1,63 +0,0 @@ -macro_rules! test_mod { - ($rt:ident) => { - paste::paste! { - mod [< $rt:snake >] { - use crate::[< $rt:snake _run >]; - use serf::{ - net::{ - resolver::socket_addr::SocketAddrResolver, stream_layer::tcp::Tcp, NetTransport, - NetTransportOptions, - }, - [< $rt:snake >]::[< $rt:camel Runtime >], - - }; - use serf_core::{tests::{serf_write_keyring_file, next_socket_addr_v4, next_socket_addr_v6}, MemberlistOptions, types::EncryptionAlgorithm}; - use smol_str::SmolStr; - - #[test] - fn test_serf_write_keyring_file_v4() { - let name = "serf_write_keyring_file_v4"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v4(0)); - - [< $rt:snake _run >](serf_write_keyring_file::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(|kr| { - (opts, MemberlistOptions::lan().with_primary_key(kr).with_gossip_verify_outgoing(true).with_encryption_algo(EncryptionAlgorithm::default())) - })); - } - - #[test] - fn test_serf_write_keyring_file_v6() { - let name = "serf_write_keyring_file_v6"; - let mut opts = NetTransportOptions::new(SmolStr::new(name)); - opts.add_bind_address(next_socket_addr_v6()); - - [< $rt:snake _run >](serf_write_keyring_file::< - NetTransport< - SmolStr, - SocketAddrResolver<[< $rt:camel Runtime >]>, - Tcp<[< $rt:camel Runtime >]>, - - [< $rt:camel Runtime >], - >, - >(|kr| { - (opts, MemberlistOptions::lan().with_primary_key(kr).with_gossip_verify_outgoing(true).with_encryption_algo(EncryptionAlgorithm::default())) - })); - } - } - } - }; -} - -#[cfg(feature = "tokio")] -test_mod!(tokio); - -#[cfg(feature = "smol")] -test_mod!(smol); From e1aaceb2ad4c52a4f84d210e4443c67a1e1b2524 Mon Sep 17 00:00:00 2001 From: Al Liu Date: Mon, 13 Jul 2026 15:37:43 +1200 Subject: [PATCH 126/128] test(serf-proto): guard the serf#56 query-ring-wraparound dedup (#94) --- serf-proto/src/endpoint/tests.rs | 40 ++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/serf-proto/src/endpoint/tests.rs b/serf-proto/src/endpoint/tests.rs index c30bb212..dc388a89 100644 --- a/serf-proto/src/endpoint/tests.rs +++ b/serf-proto/src/endpoint/tests.rs @@ -4237,6 +4237,46 @@ fn query_buffer_slot_capped_at_max_query_ids_per_ltime() { ); } +#[test] +fn query_ring_wraparound_reowns_slot_and_still_dedups() { + // The subtle half of serf#56: two ltimes a ring-size apart map to the SAME + // slot. A query at the newer ltime that wraps onto a slot still holding the + // OLDER ltime must start a FRESH record (own the slot at the new ltime), not + // append its id onto the stale entry — appending would leave the slot's ltime + // stale, so a re-gossip of the new query would never dedup and would be + // reprocessed forever. + let mut buf = QueryBuffer::new(4); + + // ltime=1 → idx 1. First sight is fresh. + assert!( + buf.witness_query(2, 1, 100), + "first query at ltime 1 is fresh" + ); + + // ltime=5 → idx 5 % 4 = 1 (same slot, different ltime). Not too old + // (cur_time 6 - ring 4 = 2; ltime 5 is well above), so it must be admitted as + // FRESH and re-own the slot — never mistaken for the stale ltime-1 entry. + assert!( + buf.witness_query(6, 5, 100), + "a query at a wrapped ltime must be fresh, not a dup of the stale slot" + ); + match buf.buffer[1].as_ref() { + Some(q) => assert_eq!( + q.ltime.0, 5, + "the wrapped slot must be re-owned by the new ltime" + ), + None => panic!("expected a slot at idx 1"), + } + + // A re-gossip of (ltime=5, id=100) must now be deduped — which only holds if + // the slot was re-owned by ltime=5 above (the pre-#56 append-onto-stale bug + // would leave the slot at ltime=1 and reprocess this). + assert!( + !buf.witness_query(6, 5, 100), + "a re-gossiped query at the wrapped ltime must be deduped" + ); +} + #[test] fn endpoint_query_buffer_slot_capped_via_adapter() { // Same check via the Endpoint adapter to confirm the cap is enforced in the From ec1e0f3ce994675f584516ddc43c68a23f1e5087 Mon Sep 17 00:00:00 2001 From: Al Liu Date: Mon, 13 Jul 2026 17:03:03 +1200 Subject: [PATCH 127/128] feat(serf): umbrella facade crate, per-crate READMEs, and crate logos (#95) --- .github/workflows/ci-tokio.yml | 9 ++ .github/workflows/coverage.yml | 88 ++++++++++++++ .github/workflows/doc.yml | 5 + .github/workflows/loc.yml | 14 ++- .gitignore | 4 + Cargo.toml | 2 +- README.md | 2 +- serf-compio/README.md | 150 +++++++++++++++++++++++ serf-compio/src/lib.rs | 34 +----- serf-driver/README.md | 108 +++++++++++++++++ serf-driver/src/lib.rs | 11 +- serf-embassy/README.md | 27 ++++- serf-embassy/src/lib.rs | 1 + serf-embedded/README.md | 19 +++ serf-embedded/src/lib.rs | 1 + serf-proto/README.md | 136 +++++++++++++++++++++ serf-proto/src/lib.rs | 41 +------ serf-reactor/README.md | 145 +++++++++++++++++++++++ serf-reactor/src/lib.rs | 45 +------ serf-smoltcp/README.md | 18 +++ serf-smoltcp/src/lib.rs | 1 + serf/Cargo.toml | 121 +++++++++++++++++++ serf/README.md | 209 +++++++++++++++++++++++++++++++++ serf/src/compio.rs | 2 + serf/src/embassy.rs | 2 + serf/src/embedded.rs | 2 + serf/src/lib.rs | 40 +++++++ serf/src/reactor.rs | 2 + serf/src/smol.rs | 17 +++ serf/src/smoltcp.rs | 2 + serf/src/tokio.rs | 17 +++ serf/tests/smoke.rs | 75 ++++++++++++ 32 files changed, 1219 insertions(+), 131 deletions(-) create mode 100644 .github/workflows/coverage.yml create mode 100644 serf-compio/README.md create mode 100644 serf-driver/README.md create mode 100644 serf-proto/README.md create mode 100644 serf-reactor/README.md create mode 100644 serf/Cargo.toml create mode 100644 serf/README.md create mode 100644 serf/src/compio.rs create mode 100644 serf/src/embassy.rs create mode 100644 serf/src/embedded.rs create mode 100644 serf/src/lib.rs create mode 100644 serf/src/reactor.rs create mode 100644 serf/src/smol.rs create mode 100644 serf/src/smoltcp.rs create mode 100644 serf/src/tokio.rs create mode 100644 serf/tests/smoke.rs diff --git a/.github/workflows/ci-tokio.yml b/.github/workflows/ci-tokio.yml index 4530d14c..b0482f9b 100644 --- a/.github/workflows/ci-tokio.yml +++ b/.github/workflows/ci-tokio.yml @@ -80,3 +80,12 @@ jobs: - name: Test run: cargo test -p serf-reactor --features "$FEATURES" + + # The umbrella facade crate otherwise has no CI; exercise its smoke test, + # which drives two loopback nodes through the runtime-pinned `serf::tokio` + # handle. + - name: Clippy umbrella (serf) + run: cargo clippy -p serf --all-targets --features tokio,tcp --no-deps -- -D warnings + + - name: Test umbrella (serf) + run: cargo test -p serf --features tokio,tcp diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100644 index 00000000..39eddcdd --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,88 @@ +name: coverage + +on: + push: + branches: + - main + - refactor/sansio + paths-ignore: + - "README.md" + - "COPYRIGHT" + - "LICENSE*" + - "**.md" + - "**.txt" + - "art" + pull_request: + paths-ignore: + - "README.md" + - "COPYRIGHT" + - "LICENSE*" + - "**.md" + - "**.txt" + - "art" + workflow_dispatch: + +env: + CARGO_TERM_COLOR: always + +jobs: + coverage: + name: coverage + runs-on: ubuntu-latest + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@v7 + + - name: Install Rust + uses: dtolnay/rust-toolchain@master + with: + toolchain: nightly + components: llvm-tools-preview + + - uses: Swatinem/rust-cache@v2 + + # serf-proto's build.rs runs protoc (via buffa-build) for the wire codegen, + # and its memberlist-proto dependency's build.rs does too. + - name: Install protoc + uses: arduino/setup-protoc@v3 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@v2 + with: + tool: cargo-llvm-cov + + - name: Collect coverage + run: | + # Accumulate per-crate runs into a shared profile, then emit one lcov + # report. Each crate runs the same feature combo its own CI workflow + # gates, so the coverage build is a combination already known to compile + # and pass. A per-crate `--all-features` sweep is not usable: the rustls + # backends (ring / aws-lc-rs) are alternatives, not additives. + # + # serf-embedded carries no host test suite of its own — it is exercised + # through the smoltcp and embassy drivers' host suites, which is why it + # has no run of its own here. The `serf` umbrella is pure re-exports with + # no coverable logic; ci-tokio gates its smoke test instead. + cargo llvm-cov --no-report -p serf-proto \ + --features "tcp,tls,quic-rustls-ring,coordinates,aes-gcm,chacha20-poly1305,tag-regex" + cargo llvm-cov --no-report -p serf-driver --all-features + cargo llvm-cov --no-report -p serf-reactor \ + --features "tokio,tcp,tls-rustls-ring,quic-rustls-ring,aes-gcm,chacha20-poly1305,coordinates,tag-regex,tracing,serde,clap,dns,getifs,test" + cargo llvm-cov --no-report -p serf-compio \ + --features "tcp,tls,quic-rustls-ring,aes-gcm,chacha20-poly1305,coordinates,tag-regex,tracing,serde,clap,dns,getifs" + cargo llvm-cov --no-report -p serf-smoltcp -p serf-embassy \ + --features "aes-gcm,cidr" + cargo llvm-cov report --lcov --output-path lcov.info + + - name: Upload to codecov.io + uses: codecov/codecov-action@v7 + with: + files: lcov.info + fail_ci_if_error: false + slug: ${{ github.repository }} + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index 8532cef8..ba0b8231 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -54,3 +54,8 @@ jobs: run: > cargo doc -p serf-compio --no-deps --features tcp,tls,quic-rustls-ring,coordinates,aes-gcm,chacha20-poly1305,tag-regex,tracing,serde,clap,dns,getifs + + # The umbrella's docs.rs metadata is `all-features`, so build exactly that: + # every driver facade and both rustls providers at once. + - name: Doc serf (umbrella) + run: cargo doc -p serf --no-deps --all-features diff --git a/.github/workflows/loc.yml b/.github/workflows/loc.yml index 439c4614..bdd64dab 100644 --- a/.github/workflows/loc.yml +++ b/.github/workflows/loc.yml @@ -43,7 +43,7 @@ jobs: set -euo pipefail tokeit mv tokeit.json loc-serf.json - for c in serf-proto serf-driver serf-compio serf-reactor; do + for c in serf-proto serf-driver serf-compio serf-reactor serf-smoltcp serf-embassy serf-embedded; do ( cd "$c" && tokeit ) mv "$c/tokeit.json" "loc-$c.json" done @@ -55,15 +55,19 @@ jobs: script: | const fs = require('fs'); const gistId = 'd29ceff54c025fe4e8b144a51efb9324'; - // Gist file `serf` is the workspace total (the main README's LoC - // badge reads …/raw/serf); the rest are per-crate, ready for - // per-crate READMEs. The gists API accepts every file in one PATCH, - // so one request avoids the per-crate rate-limit storm. + // Gist file `serf` is the workspace total — both the project README + // and the `serf` umbrella crate's README read …/raw/serf; the rest + // are per-crate and back the per-crate READMEs' LoC badges. The + // gists API accepts every file in one PATCH, so one request avoids + // the per-crate rate-limit storm. const crates = [ 'serf-proto', 'serf-driver', 'serf-compio', 'serf-reactor', + 'serf-smoltcp', + 'serf-embassy', + 'serf-embedded', ]; const files = { serf: { content: fs.readFileSync('loc-serf.json', 'utf8') }, diff --git a/.gitignore b/.gitignore index f5842c58..e3354072 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,7 @@ /Cargo.lock /docs/ /legacy/Cargo.lock + +# cargo-llvm-cov artifacts from a local reproduction of the coverage workflow. +**.profraw +lcov.info diff --git a/Cargo.toml b/Cargo.toml index f289341f..817789ce 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [workspace] members = [ + "serf", "serf-proto", "serf-embedded", "serf-driver", @@ -67,7 +68,6 @@ memberlist-proto = { version = "0.3", default-features = false } memberlist-core = { version = "0.8.5", default-features = false } memberlist = { version = "0.8.5", default-features = false } -serf-core = { path = "serf-core", version = "0.5", default-features = false } serf = { path = "serf", version = "0.5", default-features = false } serf-proto = { path = "serf-proto", default-features = false } serf-driver = { path = "serf-driver", default-features = false } diff --git a/README.md b/README.md index ac294f55..45a27ad6 100644 --- a/README.md +++ b/README.md @@ -181,7 +181,7 @@ Copyright (c) 2025 Al Liu. Copyright (c) 2013 HashiCorp, Inc. [Github-url]: https://github.com/al8n/serf/ -[CI-url]: https://github.com/al8n/serf/actions/workflows/ci.yml +[CI-url]: https://github.com/al8n/serf/actions/workflows/coverage.yml [doc-url]: https://docs.rs/serf [crates-url]: https://crates.io/crates/serf [codecov-url]: https://app.codecov.io/gh/al8n/serf/ diff --git a/serf-compio/README.md b/serf-compio/README.md new file mode 100644 index 00000000..934c8f5e --- /dev/null +++ b/serf-compio/README.md @@ -0,0 +1,150 @@ +
+

serf-compio

+
+
+ +`compio`-native async serf driver — the completion-based (io_uring / IOCP), +thread-per-core, `!Send` counterpart to [`serf-reactor`]. + +[github][Github-url] +LoC +[Build][CI-url] +[codecov][codecov-url] + +[docs.rs][doc-url] +[crates.io][crates-url] +[crates.io][crates-url] +license + +[Discord][discord] + +
+ +## Introduction + +`serf-compio` binds serf's pure Sans-I/O machine ([`serf-proto`]) to the +[`compio`] async runtime — completion-based I/O (io_uring on Linux, IOCP on +Windows). It owns the driver task, the command channel, the +observation-delegate dispatch, and the transport plumbing (TCP, TLS, QUIC) +connecting the stateless machine to real I/O. + +It is the `!Send`, thread-per-core counterpart to the `Send`/`agnostic` +[`serf-reactor`] driver: no `Arc`, no `Mutex`, no cross-thread channel on the +hot path — handles hold `Rc<...>`, and every observation-delegate hook is a +native, unboxed `async fn` fired on the compio thread that owns the +endpoint. Both drivers share [`serf-driver`]'s engines (the observable +snapshot, the append-only snapshotter, the key-management apply logic), so +only the I/O substrate differs. For the `tokio` / `smol` runtimes, use +[`serf-reactor`] instead — a future [`serf`] umbrella facade will wrap the +choice between them. + +## Installation + +```toml +[dependencies] +serf-compio = { version = "0.5", features = ["tcp"] } +serf-proto = "0.5" +``` + +## Example + +Run inside a `compio` runtime (the driver is `!Send`, thread-per-core): + +```rust,ignore +use std::net::SocketAddr; +use memberlist_proto::MaybeResolved; +use serf_proto::options::Options as SerfOptions; +use serf_compio::{ + FirstAddrResolver, RuntimeOptions, Serf, SocketAddrResolver, TcpTransport, + TcpTransportOptions, VoidDelegate, gossip_rng, +}; +use smol_str::SmolStr; + +async fn run() -> Result<(), Box> { + let advertise: SocketAddr = "127.0.0.1:7946".parse()?; + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new("node-a")) + .with_advertise_addr(MaybeResolved::Resolved(advertise)); + + let node = Serf::new::< + TcpTransport, SocketAddrResolver, FirstAddrResolver, _, _, + >( + opts, &SocketAddrResolver, &FirstAddrResolver, + VoidDelegate::::new(), RuntimeOptions::new(), + SerfOptions::new(), gossip_rng()?, None, None, None, // reconnect / merge / snapshot + ) + .await?; + + let seed: SocketAddr = "127.0.0.1:7947".parse()?; + // join returns the seed address reached, or errs once the join deadline + // elapses with no contact. + let reached = node + .join(&SocketAddrResolver, MaybeResolved::Resolved(seed), false) + .await?; + println!("{} members online (joined via {reached})", node.num_members()); + + node.leave().await?; + Ok(()) +} + +#[compio::main] +async fn main() -> Result<(), Box> { + run().await +} +``` + +## Feature flags + +| Feature | Description | +|---------|-------------| +| `tcp` *(default)* | plain-TCP reliable coordinator | +| `tls` | TLS-over-TCP reliable coordinator, via compio's own `rustls` integration | +| `quic` + `quic-rustls-ring` | QUIC reliable coordinator (streams + datagrams) | +| `coordinates` | Vivaldi network coordinate estimation | +| `aes-gcm` / `chacha20-poly1305` | AEAD encryption (gossip + reliable planes) plus serf's key-management surface | +| `tag-regex` *(default)* | regex-backed tag-filter matching | +| `serde` | `Serialize` / `Deserialize` on the `*Options` config types | +| `clap` | `clap::Args` on the `*Options` config types (CLI flags + env vars) | +| `tracing` | emit `tracing` spans around the public driver operations | +| `dns` | DNS address resolution (via `hickory-proto`) | +| `getifs` | auto-detect the advertise address from the host's interfaces | + +## Design + +- `!Send` thread-per-core: no `Arc`, no `Mutex`, no cross-thread channel on + the hot path; handles hold `Rc<…>` and read the published snapshot under + short, non-`.await` borrows. +- One spawned driver task per endpoint owns the compio `TcpListener` + + `UdpSocket`(s) (or the single `UdpSocket` for QUIC) and pumps the + [`serf-proto`] super-machine over completion-based I/O. +- The async observation composite (`MemberDelegate` / `UserEventDelegate` / + `QueryDelegate`) is a native, unboxed `async fn` — no `Send` bound — fired + directly on the compio thread; the synchronous `MergeDelegate` push/pull + veto and, under encryption, the `KeyringDelegate` rotation observer run + inline on the pump, the same contract as [`serf-reactor`]. +- A per-iteration command-channel fairness budget bounds how many queued + commands the pump drains relative to I/O completions each tick, so a + command burst cannot starve socket progress. + +## License + +`serf-compio` is under the terms of the MPL-2.0 license. + +See [LICENSE] for details. + +Copyright (c) 2025 Al Liu. + +Copyright (c) 2013 HashiCorp, Inc. + +[`compio`]: https://crates.io/crates/compio +[`serf`]: https://crates.io/crates/serf +[`serf-proto`]: https://crates.io/crates/serf-proto +[`serf-driver`]: https://crates.io/crates/serf-driver +[`serf-reactor`]: https://crates.io/crates/serf-reactor +[LICENSE]: https://github.com/al8n/serf/blob/main/LICENSE +[Github-url]: https://github.com/al8n/serf/ +[CI-url]: https://github.com/al8n/serf/actions/workflows/ci-compio.yml +[codecov-url]: https://app.codecov.io/gh/al8n/serf/ +[doc-url]: https://docs.rs/serf-compio +[crates-url]: https://crates.io/crates/serf-compio +[discord]: https://discord.gg/4JyVhKFcrt diff --git a/serf-compio/src/lib.rs b/serf-compio/src/lib.rs index 56864dcc..44e75d3f 100644 --- a/serf-compio/src/lib.rs +++ b/serf-compio/src/lib.rs @@ -1,35 +1,5 @@ -//! compio-based async driver for the Sans-I/O serf machine. -//! -//! `serf-compio` binds the pure [`serf-proto`] Sans-I/O serf machine to the -//! [compio](https://github.com/compio-rs/compio) async runtime. It owns the -//! driver task, the command channel, the observation delegate dispatch, and the -//! transport plumbing — everything that connects the stateless machine to real -//! I/O. The machine itself (`serf-proto`) carries no runtime reference and runs -//! identically regardless of which driver crate wraps it. -//! -//! # Feature flags -//! -//! Transport (at least one required): -//! - `tcp` (default) — plain-TCP reliable coordinator. -//! - `tls` — TLS-over-TCP reliable coordinator (implies `tcp`). -//! - `quic` — QUIC coordinator. -//! - `quic-rustls-ring` — QUIC with the rustls+ring TLS backend (implies `quic`). -//! -//! Optional protocol extensions: -//! - `coordinates` — Vivaldi network coordinate estimation. -//! - `aes-gcm` — AES-256-GCM encryption backend. -//! - `chacha20-poly1305` — ChaCha20-Poly1305 encryption backend. -//! - `tag-regex` (default) — regex-backed tag-filter matching. -//! -//! Optional config layering: -//! - `serde` — `Serialize`/`Deserialize` on `*Options` types. -//! - `clap` — `clap::Args` on `*Options` types (CLI flags + env vars). -//! - `tracing` — emit `tracing` spans around public driver operations. -//! -//! Optional resolver support: -//! - `dns` — `DnsResolver` backed by `hickory-proto`. -//! - `getifs` — auto-detect the advertise address from host interfaces. - +#![doc = include_str!("../README.md")] +#![doc(html_logo_url = "https://raw.githubusercontent.com/al8n/serf/main/art/logo_72x72.png")] #![deny(missing_docs)] #![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(docsrs, allow(unused_attributes))] diff --git a/serf-driver/README.md b/serf-driver/README.md new file mode 100644 index 00000000..140ca955 --- /dev/null +++ b/serf-driver/README.md @@ -0,0 +1,108 @@ +
+

serf-driver

+
+
+ +Runtime-agnostic glue shared by the serf async driver crates — the pieces every +runtime needs and none of them should implement twice. + +[github][Github-url] +LoC +[Build][CI-url] +[codecov][codecov-url] + +[docs.rs][doc-url] +[crates.io][crates-url] +[crates.io][crates-url] +license + +[Discord][discord] + +
+ +## Introduction + +A serf driver binds [`serf-proto`]'s Sans-I/O `Endpoint` to a real async runtime: +it owns a run loop, a channel/cell substrate, and delegate dispatch, and drives the +machine's `handle_*` / `poll_*` surface from there. `serf-driver` holds everything +around that binding which has no opinion about which runtime is doing the driving, +so [`serf-reactor`] (tokio / smol) and [`serf-compio`] (compio) share one +implementation instead of maintaining two: the observable `SerfSnapshot`, the common +driver error payloads, the on-disk snapshot engine, the keyring-file persistence +engine, and the observation-channel accounting. + +"Runtime-agnostic" does not mean I/O-free. The snapshot engine appends to its file +synchronously, inline on the driver's own poll call; the keyring-file engine instead +hands each rotation to its own background thread so a write never blocks the pump. +Either way the I/O is plain `std::fs`, not a runtime's async primitives, so the same +code runs unmodified under tokio, smol, or compio. What stays out of this crate is +anything runtime-shaped: the run loop, the channel/cell substrate, and delegate +dispatch live in each driver crate, not here. + +## What lives here + +| Provides | Description | +|----------|--------------| +| `SerfSnapshot` / `SerfStats` | an immutable point-in-time view of the observable membership and the three Lamport clocks, republished by a driver after every change | +| `error` | shared driver error payloads (`GossipMtuTooSmall`, `InvalidOption`, `JoinFailed`) | +| `Snapshotter` | the on-disk snapshot-file engine: appends a durable record per membership change, replays them at construction, compacts once the file crosses a size threshold | +| `KeyringFilePersistence` *(`unix`)* | the file-persistence engine behind each runtime's `FileKeyringDelegate`: one lowercase-hex-encoded key per line (primary first), durable rotation via a temp file + atomic rename | +| `apply_key_request` and friends | pure read-modify-write logic that applies an inbound key-management request to the live wire keyring, shared by every driver's key-management handler | +| `observation_payload_bytes` | the byte-backstop weight of one serf event, for the observation channel's application-flood bound | + +## The snapshot & keyring-file engines + +- **`Snapshotter`** buffers and flushes an append-only record per surfaced + membership change — a change is durable once the pump's poll returns — and + rewrites the file to just the live alive-set and clock floors once it grows past + `DEFAULT_SNAPSHOT_COMPACT_THRESHOLD` bytes. `Snapshotter::open` tolerates a + truncated tail (a crash mid-append) but rejects a malformed record earlier in the + file. +- **`KeyringFilePersistence`** hands every rotation to a dedicated background + thread over a channel, so `keyring_updated` never blocks the driver pump; each + write goes through an exclusively-created, owner-only temp file, an `fsync`, and + an atomic rename, and the pump can gate a key-management response on the + acknowledgement via `KeyringPersistence::Pending`. It is Unix-only — the + durability contract is directory-rename durability, which has no safe portable + API on Windows — so a Windows application supplies its own `KeyringDelegate`. + +## Feature flags + +| Feature | Description | +|---------|-------------| +| `tcp` / `tls` / `quic` / `quic-rustls-ring` | forwarded to `serf-proto`; gate the transport-only surface (`SerfSnapshot`, `observation_payload_bytes`, …) | +| `coordinates` | forwards Vivaldi network-coordinate support to `serf-proto` | +| `aes-gcm` / `chacha20-poly1305` | AEAD backend; enables the keyring apply logic and, on `unix`, `KeyringFilePersistence` | +| `tag-regex` *(default)* | forwards regex-backed tag-filter matching to `serf-proto` | +| `tracing` | the snapshotter and keyring-file engine emit `tracing` warnings on a persistence failure | + +## Installation + +```toml +[dependencies] +serf-driver = "0.5" +``` + +`serf-driver` is a building block for a driver crate, not something most +applications depend on directly — reach for [`serf-reactor`], [`serf-compio`], or +the [`serf`] facade instead. + +## License + +`serf-driver` is under the terms of the MPL-2.0 license. + +See [LICENSE] for details. + +Copyright (c) 2025 Al Liu. + +[`serf-proto`]: https://crates.io/crates/serf-proto +[`serf-reactor`]: https://crates.io/crates/serf-reactor +[`serf-compio`]: https://crates.io/crates/serf-compio +[`serf`]: https://crates.io/crates/serf +[LICENSE]: https://github.com/al8n/serf/blob/main/LICENSE +[Github-url]: https://github.com/al8n/serf/ +[CI-url]: https://github.com/al8n/serf/actions/workflows/ci-core.yml +[codecov-url]: https://app.codecov.io/gh/al8n/serf/ +[doc-url]: https://docs.rs/serf-driver +[crates-url]: https://crates.io/crates/serf-driver +[discord]: https://discord.gg/4JyVhKFcrt diff --git a/serf-driver/src/lib.rs b/serf-driver/src/lib.rs index d28af2bc..65de9e99 100644 --- a/serf-driver/src/lib.rs +++ b/serf-driver/src/lib.rs @@ -1,12 +1,5 @@ -//! Runtime-agnostic glue shared by the serf async driver crates. -//! -//! A driver binds `serf-proto`'s Sans-I/O endpoint to a real async runtime. This crate holds -//! the runtime-independent pieces both the compio and reactor drivers need — the observable -//! [`SerfSnapshot`] (when a transport feature is enabled), the common driver error payloads, -//! and small pure helpers — so they live in one place instead of being copied per runtime. -//! The run loops, the channel and cell substrate, and the delegate dispatch stay in each -//! runtime crate. - +#![doc = include_str!("../README.md")] +#![doc(html_logo_url = "https://raw.githubusercontent.com/al8n/serf/main/art/logo_72x72.png")] #![deny(missing_docs)] #![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(docsrs, allow(unused_attributes))] diff --git a/serf-embassy/README.md b/serf-embassy/README.md index cdeeb603..25baab2e 100644 --- a/serf-embassy/README.md +++ b/serf-embassy/README.md @@ -1,8 +1,25 @@ -# serf-embassy +
+

serf-embassy

+
+
An async `no_std` [serf](https://github.com/al8n/serf) driver over the [embassy-net](https://github.com/embassy-rs/embassy) network stack. +[github][Github-url] +LoC +[Build][CI-url] +[codecov][codecov-url] + +[docs.rs][doc-url] +[crates.io][crates-url] +[crates.io][crates-url] +license + +[Discord][discord] + +
+ `serf-embassy` drives [`serf-embedded`](../serf-embedded)'s transport-agnostic `SerfEngine` on an embassy-net TCP/IP stack. It mirrors [`memberlist-embassy`](https://github.com/al8n/memberlist)'s architecture — @@ -39,3 +56,11 @@ back the handle paired with a `Runner` to spawn as a task. ## License Licensed under the MPL-2.0 license. + +[Github-url]: https://github.com/al8n/serf/ +[CI-url]: https://github.com/al8n/serf/actions/workflows/embedded.yml +[codecov-url]: https://app.codecov.io/gh/al8n/serf/ +[doc-url]: https://docs.rs/serf-embassy +[crates-url]: https://crates.io/crates/serf-embassy +[LICENSE]: https://github.com/al8n/serf/blob/main/LICENSE +[discord]: https://discord.gg/4JyVhKFcrt diff --git a/serf-embassy/src/lib.rs b/serf-embassy/src/lib.rs index ac3a69ba..14d53bab 100644 --- a/serf-embassy/src/lib.rs +++ b/serf-embassy/src/lib.rs @@ -1,4 +1,5 @@ #![doc = include_str!("../README.md")] +#![doc(html_logo_url = "https://raw.githubusercontent.com/al8n/serf/main/art/logo_72x72.png")] #![cfg_attr(not(feature = "std"), no_std)] #![forbid(unsafe_code)] #![deny(missing_docs)] diff --git a/serf-embedded/README.md b/serf-embedded/README.md index 9acd1818..351a8ad9 100644 --- a/serf-embedded/README.md +++ b/serf-embedded/README.md @@ -5,6 +5,18 @@ The transport-agnostic `no_std` driving core shared by serf's embedded drivers. +[github][Github-url] +LoC +[Build][CI-url] +[codecov][codecov-url] + +[docs.rs][doc-url] +[crates.io][crates-url] +[crates.io][crates-url] +license + +[Discord][discord] + `serf-embedded` is the link-layer-independent core the future serf smoltcp / @@ -32,3 +44,10 @@ independent, and the core builds on bare-metal targets such as `thumbv7em-none-e [`ReliablePlane`]: https://docs.rs/memberlist-embedded/latest/memberlist_embedded/reliable/struct.ReliablePlane.html [`SerfEngine`]: https://docs.rs/serf-embedded/latest/serf_embedded/struct.SerfEngine.html [`SerfEngine::pump`]: https://docs.rs/serf-embedded/latest/serf_embedded/struct.SerfEngine.html#method.pump +[Github-url]: https://github.com/al8n/serf/ +[CI-url]: https://github.com/al8n/serf/actions/workflows/embedded.yml +[codecov-url]: https://app.codecov.io/gh/al8n/serf/ +[doc-url]: https://docs.rs/serf-embedded +[crates-url]: https://crates.io/crates/serf-embedded +[LICENSE]: https://github.com/al8n/serf/blob/main/LICENSE +[discord]: https://discord.gg/4JyVhKFcrt diff --git a/serf-embedded/src/lib.rs b/serf-embedded/src/lib.rs index 8926f417..99fccd7f 100644 --- a/serf-embedded/src/lib.rs +++ b/serf-embedded/src/lib.rs @@ -1,4 +1,5 @@ #![doc = include_str!("../README.md")] +#![doc(html_logo_url = "https://raw.githubusercontent.com/al8n/serf/main/art/logo_72x72.png")] #![cfg_attr(not(feature = "std"), no_std)] #![forbid(unsafe_code)] #![deny(missing_docs)] diff --git a/serf-proto/README.md b/serf-proto/README.md new file mode 100644 index 00000000..2a384c9b --- /dev/null +++ b/serf-proto/README.md @@ -0,0 +1,136 @@ +
+

serf-proto

+
+
+ +Sans-I/O state machine and wire codec for the serf cluster-orchestration protocol — +`no_std`-capable, composed over [`memberlist-proto`]'s reliable coordinator. + +[github][Github-url] +LoC +[Build][CI-url] +[codecov][codecov-url] + +[docs.rs][doc-url] +[crates.io][crates-url] +[crates.io][crates-url] +license + +[Discord][discord] + +
+ +## Introduction + +`serf-proto` implements serf's cluster-orchestration layer — join/leave intents, +user events, queries and query responses, push-pull anti-entropy, and (optionally) +Vivaldi network coordinates — as a deterministic state machine that performs **no +I/O of its own**. It depends on [`memberlist-proto`] for the `Data`/`DataRef` codec +primitives and defines serf's own message set on top of them: a [`buffa`]-generated +protobuf wire codec (`[TAG_BYTE][VARINT_LEN][BUFFA_BODY]` framing) plus the typed +bridge that converts between serf's Rust-native message shapes and the generated +codec types. + +Serf logic composes with a memberlist reliable coordinator into one Sans-I/O +super-machine, in the shape [`quinn-proto`] popularized: feed inbound bytes and +timer ticks in through `handle_*` / `handle_timeout`, drain outbound transmits and +events out through `poll_transmit` / `poll_event`. `StreamEndpoint` pairs the +serf-logic core with memberlist's TCP/TLS stream coordinator; `QuicEndpoint` pairs +the same core with memberlist's QUIC coordinator. Both hold the core and the +coordinator as disjoint fields and reach the coordinator only through a narrow +`Reliable` seam, so the serf-logic core itself never names a concrete transport. + +This crate is the core the driver crates are thin layers over. [`serf-driver`] holds +the runtime-independent glue shared by the async drivers; [`serf-reactor`] (tokio / +smol) and [`serf-compio`] (compio) bind it to a real runtime; the bare-metal +`serf-embedded` stack drives it on `serf-smoltcp` / `serf-embassy` with no runtime at +all. Most applications want one of those, or the [`serf`] facade, rather than this +core directly. + +## Wire evolution + +The legacy Go serf carried two negotiation knobs — `protocol_version` and +`delegate_version` — so mixed-version clusters could gate features at runtime. This +stack deliberately carries neither: the wire forms new↔new clusters only, and there +is no per-message version field to dispatch on. + +- **Additive evolution rides proto3 semantics.** Every message body is a proto3 + message; a new optional field decodes as its default on nodes that predate it and + is skipped (not erred) by nodes that do not know it. Never reuse or renumber a + field, change its wire type, or make an optional field required — those are + breaking changes. The framing envelope is additive the same way: an unknown + message tag is dropped with its body length consumed, so a new message type + degrades to a no-op on old nodes rather than a parse failure. +- **Breaking changes are a new cluster generation, fenced by the cluster label.** + The gossip codec stamps every packet and stream with the configured label and + ingress drops anything mismatched, so a layout that cannot be expressed + additively ships as a new deployment under a new label and never meets the old + one on a socket. +- **Delegates are a compile-time surface.** The delegate traits are Rust API + versioned by the crate's semver; there is nothing to negotiate on the wire. + +## Feature tiers + +Unlike a pure wire-codec crate, serf's membership/event/query state is +intrinsically heap-backed (`Vec` / `Box` / `String` / maps), so a build must select +at least one of: + +| Features | Environment | +|----------|-------------| +| `std` *(default)* | `std` hosts (tokio, smol, compio) | +| `alloc` | `no_std` with a global allocator (embassy, smoltcp, …) | + +## Transports & options + +| Feature | Adds | +|---------|------| +| `tcp` | `StreamEndpoint` over memberlist's plain-TCP reliable coordinator (`no_std` + `alloc`) | +| `tls` | rustls record layer on top of `tcp` (implies `tcp`; `std`-only) | +| `quic` + `quic-rustls-ring` | `QuicEndpoint` over memberlist's QUIC (quinn-proto) coordinator (`std`-only) | +| `coordinates` | Vivaldi network-coordinate estimation (`f64` transcendentals; `std`-only) | +| `tag-regex` *(default)* | regex-backed `Filter::Tag` matching; falls back to exact-equality without it | +| `aes-gcm` / `chacha20-poly1305` | key-management messages (`KeyRequest` / `KeyResponse`) for the matching AEAD backend; `encryption` enables both | + +## Installation + +```toml +[dependencies] +serf-proto = "0.5" # std (default) + +# no_std + alloc, with the plain-tcp coordinator: +serf-proto = { version = "0.5", default-features = false, features = ["alloc", "tcp"] } +``` + +## The serf family + +The crates split protocol logic from I/O, mirroring the memberlist layering: + +[`serf`] (facade) · **`serf-proto`** (this crate) · [`serf-driver`] (shared driver +glue) · [`serf-reactor`] (tokio / smol driver) · [`serf-compio`] (compio driver) · +`serf-embedded` (shared `no_std` core) · `serf-smoltcp` (smoltcp driver) · +`serf-embassy` (embassy driver). + +## License + +`serf-proto` is under the terms of the MPL-2.0 license. + +See [LICENSE] for details. + +Copyright (c) 2025 Al Liu. + +Copyright (c) 2013 HashiCorp, Inc. + +[`quinn-proto`]: https://crates.io/crates/quinn-proto +[`memberlist-proto`]: https://crates.io/crates/memberlist-proto +[`buffa`]: https://crates.io/crates/buffa +[`serf`]: https://crates.io/crates/serf +[`serf-driver`]: https://crates.io/crates/serf-driver +[`serf-reactor`]: https://crates.io/crates/serf-reactor +[`serf-compio`]: https://crates.io/crates/serf-compio +[LICENSE]: https://github.com/al8n/serf/blob/main/LICENSE +[Github-url]: https://github.com/al8n/serf/ +[CI-url]: https://github.com/al8n/serf/actions/workflows/ci-core.yml +[codecov-url]: https://app.codecov.io/gh/al8n/serf/ +[doc-url]: https://docs.rs/serf-proto +[crates-url]: https://crates.io/crates/serf-proto +[discord]: https://discord.gg/4JyVhKFcrt diff --git a/serf-proto/src/lib.rs b/serf-proto/src/lib.rs index b89720ff..5bcc24b5 100644 --- a/serf-proto/src/lib.rs +++ b/serf-proto/src/lib.rs @@ -1,42 +1,5 @@ -//! The serf wire codec and Sans-I/O state machine — pure, no-I/O types shared -//! by the serf driver crates. -//! -//! Depends on `memberlist-proto` for the `Data`/`DataRef` codec primitives; defines serf's -//! own message set and framing on top of them. -//! -//! # Wire evolution -//! -//! The legacy serf carried two negotiation knobs — `protocol_version` and -//! `delegate_version` — so mixed-version clusters could gate features at -//! runtime. This stack deliberately carries neither; the wire forms new↔new -//! clusters only, and there is no per-message version field to dispatch on. -//! What replaces them: -//! -//! - **Additive evolution rides proto3 semantics.** Every message body is a -//! proto3 message, and the typed bridge enforces presence only for the -//! fields the protocol requires. A new optional field decodes as its -//! default on nodes that predate it and is skipped (not erred) by nodes -//! that do not know it, so a feature with a sound default posture for old -//! peers ships with no negotiation and no flag day. Never reuse or -//! renumber a field, change a field's wire type, or make an optional field -//! required — those are breaking changes and take the generation path -//! below. The framing envelope is additive the same way: an unknown -//! message tag is dropped with its body length consumed, so a new message -//! type degrades to a no-op on old nodes rather than a parse failure. -//! -//! - **Breaking changes are a new cluster generation, fenced by the cluster -//! label.** The gossip codec stamps every packet and stream with the -//! configured label and ingress drops anything mismatched. A change that -//! cannot be expressed additively ships as a new deployment under a new -//! label, stood up beside the old one and cut over blue/green: nodes of -//! different generations never exchange state, so incompatible layouts -//! never meet on a socket — the label check does the work version -//! negotiation used to approximate, without mixed-version protocol paths -//! to test and maintain. -//! -//! - **Delegates are a compile-time surface.** The delegate traits are Rust -//! API versioned by the crate's semver; there is nothing to negotiate on -//! the wire. +#![doc = include_str!("../README.md")] +#![doc(html_logo_url = "https://raw.githubusercontent.com/al8n/serf/main/art/logo_72x72.png")] #![cfg_attr(not(feature = "std"), no_std)] #![deny(missing_docs)] #![cfg_attr(docsrs, feature(doc_cfg))] diff --git a/serf-reactor/README.md b/serf-reactor/README.md new file mode 100644 index 00000000..7c42d301 --- /dev/null +++ b/serf-reactor/README.md @@ -0,0 +1,145 @@ +
+

serf-reactor

+
+
+ +Runtime-agnostic async serf driver over TCP, TLS, and QUIC — drives the Sans-I/O +[`serf-proto`] machine on `tokio` or `smol` via [`agnostic`]. + +[github][Github-url] +LoC +[Build][CI-url] +[codecov][codecov-url] + +[docs.rs][doc-url] +[crates.io][crates-url] +[crates.io][crates-url] +license + +[Discord][discord] + +
+ +## Introduction + +`serf-reactor` binds serf's pure Sans-I/O machine ([`serf-proto`]) to a +readiness-based ("reactor") async runtime, generic over an [`agnostic`] +`Runtime` so the same driver runs on `tokio` and `smol` with no change to +protocol behavior. It owns the driver task, the command queue, the +observation-delegate dispatch, and the transport plumbing (TCP, TLS, QUIC) +connecting the stateless machine to real I/O. + +It is the `Send`/`Arc` sibling of the `!Send` [`serf-compio`] driver — every +shared value is an `Arc` with atomic / `arc_swap` interior mutability, and +the observation-delegate hooks return `Send` futures — and shares +[`serf-driver`]'s engines (the observable snapshot, the append-only +snapshotter, the key-management apply logic) with it, so only the I/O +substrate differs. `serf-reactor` is the primary async serf driver for most +applications; a future [`serf`] umbrella facade will wrap the runtime choice +the way [`memberlist`] already does for [`memberlist-reactor`]. + +## Installation + +```toml +[dependencies] +serf-reactor = { version = "0.5", features = ["tcp", "tokio"] } +serf-proto = "0.5" +``` + +## Example + +```rust,ignore +use core::net::SocketAddr; +use agnostic::tokio::TokioRuntime; +use serf_proto::options::Options as SerfOptions; +use serf_reactor::{ + FirstAddrResolver, MaybeResolved, RuntimeOptions, Serf, SocketAddrResolver, + TcpTransportOptions, VoidDelegate, +}; +use smol_str::SmolStr; + +#[tokio::main] +async fn main() -> Result<(), Box> { + let advertise: SocketAddr = "127.0.0.1:7946".parse()?; + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new("node-a")) + .with_advertise_addr(MaybeResolved::Resolved(advertise)); + + // The runtime is a type parameter; pick `TokioRuntime` or `SmolRuntime`. + let node = Serf::::tcp( + opts, &SocketAddrResolver, &FirstAddrResolver, + VoidDelegate::::new(), RuntimeOptions::new(), + SerfOptions::new(), None, None, None, // reconnect / merge / snapshot + ) + .await?; + + let seed: SocketAddr = "127.0.0.1:7947".parse()?; + // join returns the seed address reached, or errs once the join deadline + // elapses with no contact. + let reached = node + .join(&SocketAddrResolver, MaybeResolved::Resolved(seed), false) + .await?; + println!("{} members online (joined via {reached})", node.num_members()); + + node.leave().await?; + Ok(()) +} +``` + +## Feature flags + +| Feature | Description | +|---------|-------------| +| `tcp` *(default)* | plain-TCP reliable coordinator | +| `tls` + `tls-rustls-ring` / `-aws-lc-rs` | TLS-over-TCP reliable coordinator via `rustls` | +| `quic` + `quic-rustls-ring` | QUIC reliable coordinator (streams + datagrams) | +| `coordinates` | Vivaldi network coordinate estimation | +| `aes-gcm` / `chacha20-poly1305` | AEAD encryption (gossip + reliable planes) plus serf's key-management surface | +| `tag-regex` *(default)* | regex-backed tag-filter matching | +| `tokio` / `smol` | pull in the concrete [`agnostic`] runtime implementation | +| `serde` | `Serialize` / `Deserialize` on the `*Options` config types | +| `clap` | `clap::Args` on the `*Options` config types (CLI flags + env vars) | +| `tracing` | emit `tracing` spans around the public driver operations | +| `dns` | DNS address resolution (via `hickory-proto`) | +| `getifs` | auto-detect the advertise address from the host's interfaces | + +## Design + +- A quinn-style poll-pump driver: one spawned task per endpoint pumps the + [`serf-proto`] super-machine (`StreamEndpoint` / `QuicEndpoint`) — feeding + inbound packets / timers, draining transmit and event output — over the + runtime's sockets. +- A command queue + one-shot replies bridge the public `Serf` handle to the + driver task; membership reads go through a lock-free published snapshot + and never block the pump. +- The async observation composite (`MemberDelegate` / `UserEventDelegate` / + `QueryDelegate`) returns `Send` futures fired from the driver task; the + synchronous `MergeDelegate` push/pull veto and, under encryption, the + `KeyringDelegate` rotation observer run inline on the pump instead. +- Owns its own `Resolver` / `AdvertiseAddrResolver` boundary, so resolution + stays outside the Sans-I/O core. + +## License + +`serf-reactor` is under the terms of the MPL-2.0 license. + +See [LICENSE] for details. + +Copyright (c) 2025 Al Liu. + +Copyright (c) 2013 HashiCorp, Inc. + +[`agnostic`]: https://github.com/al8n/agnostic +[`memberlist`]: https://crates.io/crates/memberlist +[`memberlist-reactor`]: https://crates.io/crates/memberlist-reactor +[`serf`]: https://crates.io/crates/serf +[`serf-proto`]: https://crates.io/crates/serf-proto +[`serf-driver`]: https://crates.io/crates/serf-driver +[`serf-compio`]: https://crates.io/crates/serf-compio +[LICENSE]: https://github.com/al8n/serf/blob/main/LICENSE +[Github-url]: https://github.com/al8n/serf/ +[CI-url]: https://github.com/al8n/serf/actions/workflows/ci-tokio.yml +[codecov-url]: https://app.codecov.io/gh/al8n/serf/ +[doc-url]: https://docs.rs/serf-reactor +[crates-url]: https://crates.io/crates/serf-reactor +[discord]: https://discord.gg/4JyVhKFcrt diff --git a/serf-reactor/src/lib.rs b/serf-reactor/src/lib.rs index 57154656..4d72db8e 100644 --- a/serf-reactor/src/lib.rs +++ b/serf-reactor/src/lib.rs @@ -1,46 +1,5 @@ -//! agnostic-runtime async driver for the Sans-I/O serf machine. -//! -//! `serf-reactor` binds the pure [`serf-proto`] Sans-I/O serf machine to a -//! readiness-based async runtime (tokio or smol, selected through the -//! [`agnostic`](https://docs.rs/agnostic) runtime abstraction). It owns the -//! driver task, the command queue, the observation delegate dispatch, and the -//! transport plumbing — everything that connects the stateless machine to real -//! I/O. The machine itself (`serf-proto`) carries no runtime reference and runs -//! identically regardless of which driver crate wraps it. -//! -//! It is the `Send`/`Arc`/`agnostic`-generic sibling of the `!Send` -//! [`serf-compio`](https://docs.rs/serf-compio) driver: every shared value is an -//! `Arc` with atomic / `arc_swap` interior mutability, the delegate hooks return -//! `Send` futures, and the driver is generic over `R: agnostic::Runtime`. -//! -//! # Feature flags -//! -//! Transport (at least one required): -//! - `tcp` (default) — plain-TCP reliable coordinator. -//! - `tls` — TLS-over-TCP reliable coordinator (implies `tcp`). -//! - `tls-rustls-ring` / `tls-rustls-aws-lc-rs` — TLS with a rustls provider. -//! - `quic` — QUIC coordinator. -//! - `quic-rustls-ring` — QUIC with the rustls+ring TLS backend (implies `quic`). -//! -//! Runtime selection (pick one for a runnable node): -//! - `tokio` — pull in the tokio agnostic runtime impl. -//! - `smol` — pull in the smol agnostic runtime impl. -//! -//! Optional protocol extensions: -//! - `coordinates` — Vivaldi network coordinate estimation. -//! - `aes-gcm` — AES-256-GCM encryption backend. -//! - `chacha20-poly1305` — ChaCha20-Poly1305 encryption backend. -//! - `tag-regex` (default) — regex-backed tag-filter matching. -//! -//! Optional config layering: -//! - `serde` — `Serialize`/`Deserialize` on `*Options` types. -//! - `clap` — `clap::Args` on `*Options` types (CLI flags + env vars). -//! - `tracing` — emit `tracing` spans around public driver operations. -//! -//! Optional resolver support: -//! - `dns` — `DnsResolver` backed by `hickory-proto`. -//! - `getifs` — auto-detect the advertise address from host interfaces. - +#![doc = include_str!("../README.md")] +#![doc(html_logo_url = "https://raw.githubusercontent.com/al8n/serf/main/art/logo_72x72.png")] #![deny(missing_docs)] #![cfg_attr(docsrs, feature(doc_cfg))] #![cfg_attr(docsrs, allow(unused_attributes))] diff --git a/serf-smoltcp/README.md b/serf-smoltcp/README.md index 26580f83..3d869c46 100644 --- a/serf-smoltcp/README.md +++ b/serf-smoltcp/README.md @@ -6,6 +6,18 @@ Executor-free `no_std` serf driver over the [smoltcp] TCP/IP stack — no OS and no async runtime required. +[github][Github-url] +LoC +[Build][CI-url] +[codecov][codecov-url] + +[docs.rs][doc-url] +[crates.io][crates-url] +[crates.io][crates-url] +license + +[Discord][discord] + ## Introduction @@ -106,3 +118,9 @@ Copyright (c) 2025 Al Liu. [`serf-embedded`]: https://crates.io/crates/serf-embedded [`memberlist-smoltcp`]: https://crates.io/crates/memberlist-smoltcp [LICENSE]: https://github.com/al8n/serf/blob/main/LICENSE +[Github-url]: https://github.com/al8n/serf/ +[CI-url]: https://github.com/al8n/serf/actions/workflows/embedded.yml +[codecov-url]: https://app.codecov.io/gh/al8n/serf/ +[doc-url]: https://docs.rs/serf-smoltcp +[crates-url]: https://crates.io/crates/serf-smoltcp +[discord]: https://discord.gg/4JyVhKFcrt diff --git a/serf-smoltcp/src/lib.rs b/serf-smoltcp/src/lib.rs index 54b101cf..f7f2376e 100644 --- a/serf-smoltcp/src/lib.rs +++ b/serf-smoltcp/src/lib.rs @@ -1,4 +1,5 @@ #![doc = include_str!("../README.md")] +#![doc(html_logo_url = "https://raw.githubusercontent.com/al8n/serf/main/art/logo_72x72.png")] #![cfg_attr(not(feature = "std"), no_std)] #![forbid(unsafe_code)] #![deny(missing_docs)] diff --git a/serf/Cargo.toml b/serf/Cargo.toml new file mode 100644 index 00000000..65c64051 --- /dev/null +++ b/serf/Cargo.toml @@ -0,0 +1,121 @@ +[package] +name = "serf" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "Sans-I/O serf: decentralized service discovery and orchestration — a gossip membership, user-event and query layer over memberlist." +repository.workspace = true +homepage.workspace = true +readme = "README.md" +rust-version.workspace = true +keywords.workspace = true +categories.workspace = true + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] + +[dependencies] +serf-proto = { path = "../serf-proto", version = "0.5.2", default-features = false } +serf-reactor = { path = "../serf-reactor", version = "0.5.2", optional = true, default-features = false } +serf-compio = { path = "../serf-compio", version = "0.5.2", optional = true, default-features = false } +serf-smoltcp = { path = "../serf-smoltcp", version = "0.5.2", optional = true, default-features = false } +serf-embassy = { path = "../serf-embassy", version = "0.5.2", optional = true, default-features = false } +serf-embedded = { path = "../serf-embedded", version = "0.5.2", optional = true, default-features = false } +agnostic = { workspace = true, optional = true } + +[dev-dependencies] +# The facade smoke test drives two loopback nodes on a concrete runtime: `#[tokio::test]` +# needs the tokio macros plus a multi-thread runtime with timers. The `TokioRuntime` +# itself arrives through the crate's own `tokio` feature. +tokio = { version = "1", features = ["rt-multi-thread", "macros", "time"] } +smol_str.workspace = true + +# The umbrella facade otherwise has no test target; this exercises the runtime-pinned +# handle end to end (build → join → converge → leave) through `serf::tokio`. +[[test]] +name = "smoke" +required-features = ["tokio", "tcp"] + +[features] +default = ["tokio", "tcp"] +alloc = ["serf-proto/alloc"] +std = ["alloc", "serf-proto/std"] + +# Driver selection. `tokio` / `smol` pin the agnostic reactor driver to a concrete +# runtime so callers never name `R`; `reactor` leaves it generic. +reactor = ["dep:serf-reactor", "std"] +tokio = ["reactor", "dep:agnostic", "agnostic/tokio", "serf-reactor/tokio"] +smol = ["reactor", "dep:agnostic", "agnostic/smol", "serf-reactor/smol"] +compio = ["dep:serf-compio", "std"] +# Bare-metal / no_std drivers: the `alloc` tier (they never pull `std` here). +smoltcp = ["dep:serf-smoltcp", "alloc", "serf-smoltcp/alloc"] +embassy = ["dep:serf-embassy", "alloc", "serf-embassy/alloc"] +embedded = ["dep:serf-embedded", "alloc", "serf-embedded/alloc"] + +# Transports. +# +# Bare `tls` / `quic` enable the record-layer types only; they do not pick a +# crypto provider, so a build with just `tls` (or `quic`) has no usable backend. +# Select a backend feature instead — `tls-rustls-ring` / `quic-rustls-ring` (or +# the `-aws-lc-rs` variant) — which turn on `tls` / `quic` transitively. +tcp = ["serf-proto/tcp", "serf-reactor?/tcp", "serf-compio?/tcp"] +tls = ["serf-proto/tls", "serf-reactor?/tls", "serf-compio?/tls"] +tls-rustls-ring = ["tls", "serf-reactor?/tls-rustls-ring"] +tls-rustls-aws-lc-rs = ["tls", "serf-reactor?/tls-rustls-aws-lc-rs"] +quic = ["serf-proto/quic", "serf-reactor?/quic", "serf-compio?/quic"] +quic-rustls-ring = [ + "quic", + "serf-proto/quic-rustls-ring", + "serf-reactor?/quic-rustls-ring", + "serf-compio?/quic-rustls-ring", +] + +# Encryption backends — the key-management messages plus the coordinator's gossip +# keyring. Every driver defines these. +aes-gcm = [ + "serf-proto/aes-gcm", + "serf-reactor?/aes-gcm", + "serf-compio?/aes-gcm", + "serf-smoltcp?/aes-gcm", + "serf-embassy?/aes-gcm", + "serf-embedded?/aes-gcm", +] +chacha20-poly1305 = [ + "serf-proto/chacha20-poly1305", + "serf-reactor?/chacha20-poly1305", + "serf-compio?/chacha20-poly1305", + "serf-smoltcp?/chacha20-poly1305", + "serf-embassy?/chacha20-poly1305", + "serf-embedded?/chacha20-poly1305", +] +# Convenience: both AEAD backends at once, matching the same feature on the +# underlying crates. +encryption = ["aes-gcm", "chacha20-poly1305"] + +# Vivaldi network coordinate estimation. +coordinates = [ + "serf-proto/coordinates", + "serf-reactor?/coordinates", + "serf-compio?/coordinates", +] +# Regex-backed tag-filter matching. +tag-regex = [ + "serf-proto/tag-regex", + "serf-reactor?/tag-regex", + "serf-compio?/tag-regex", +] +# CIDR peer-admission allow-list (the bare-metal drivers define it). +cidr = ["serf-smoltcp?/cidr", "serf-embassy?/cidr", "serf-embedded?/cidr"] + +# Config-options layering, propagated to whichever std drivers are enabled. std-only +# (clap is std-only, and the `Duration` fields use humantime-serde), so the no_std +# drivers are intentionally excluded. +serde = ["std", "serf-reactor?/serde", "serf-compio?/serde"] +clap = ["std", "serf-reactor?/clap", "serf-compio?/clap"] + +# Misc. +dns = ["serf-reactor?/dns", "serf-compio?/dns"] +# Auto-detect the advertise address from the host's own interfaces (std drivers only). +getifs = ["serf-reactor?/getifs", "serf-compio?/getifs"] +tracing = ["serf-reactor?/tracing", "serf-compio?/tracing"] diff --git a/serf/README.md b/serf/README.md new file mode 100644 index 00000000..0c27773d --- /dev/null +++ b/serf/README.md @@ -0,0 +1,209 @@ +
+ + + +

Serf

+ + +
+
+ +A highly customable, adaptable, runtime agnostic and WASM/WASI friendly decentralized solution for service discovery and orchestration that is lightweight, highly available, and fault tolerant. + +Port and improve [HashiCorp's serf](https://github.com/hashicorp/serf) to Rust. + +[github][Github-url] +LoC +[Build][CI-url] +[codecov][codecov-url] + +[docs.rs][doc-url] +[crates.io][crates-url] +[crates.io][crates-url] +license + +[github][discord] + +
+ +## Introduction + +`serf` is a facade over the serf crate family: depend on this one crate, pick a driver +through Cargo features, and get a ready-to-use node — instead of wiring the protocol +core, a transport, and a runtime together yourself. It mirrors the [`memberlist`] +umbrella crate. + +Its protocol logic composes serf's own event / query / tag super-machine +([`serf-proto`]) over [`memberlist-proto`]'s SWIM coordinator: memberlist supplies +gossip membership and failure detection, and serf layers per-node tags, custom user +events, and a gossip-relayed query/response protocol — including live key rotation +through the same query mechanism — on top of it. Thin async drivers adapt that pure +core to `tokio` / `smol`, `compio`, and bare-metal `no_std` targets, so the same +protocol logic runs on a server or a microcontroller. + +This is a Rust port of [HashiCorp's Serf], extended with a Sans-I/O architecture and +`no_std` / bare-metal support. + +For the full project overview — protocol background, cross-crate design rationale, +Q&A, and related projects — see the [project README]. + +## Highlights + +- **Sans-I/O core.** All protocol logic lives in [`serf-proto`] as a pure super-machine + composed over [`memberlist-proto`]'s SWIM coordinator — no sockets, threads, or + clocks — making it deterministic and exhaustively unit-tested. The drivers only + shuttle bytes and time in and out. +- **Runtime-agnostic.** Drive it from `tokio`, `smol`, or `compio` (thread-per-core) + with no change to protocol behavior. +- **`no_std` and bare-metal.** The core runs on `alloc`, and [`serf-smoltcp`] / + [`serf-embassy`] bring full serf membership, events, and queries to embedded targets. +- **Membership, events, and queries.** Inherits SWIM gossip membership and failure + detection from [`memberlist`], and adds per-node tags, custom user events, and a + gossip-relayed query/response protocol — including live key rotation through the + same query mechanism. +- **Pluggable transports.** Plain TCP, TLS-over-TCP (`rustls`), or QUIC (`quinn-proto`) + reliable planes, each with a UDP / datagram gossip plane carrying opt-in AEAD + encryption. +- **Customizable.** Bring your own `Id`, `Address`, `AddressResolver`, and delegates + (member / merge / query / user-event / reconnect). +- **Observable, à la carte.** Opt into `tracing` — compiled out when unused. +- **Config-file & CLI friendly.** Every `*Options` type optionally derives `serde` and + `clap`. + +## The family + +The crates split protocol logic from I/O, mirroring the `memberlist` layering: + +| Crate | Role | +|-------|------| +| [`serf`] | this crate — batteries-included facade (core + default `tokio` driver) | +| [`serf-proto`] | Sans-I/O protocol super-machine + wire codec (`no_std`-capable) | +| [`serf-driver`] | runtime-agnostic glue shared by the reactor and compio drivers | +| [`serf-reactor`] | runtime-agnostic async driver (`tokio` & `smol`), TCP/TLS/QUIC | +| [`serf-compio`] | `compio` (thread-per-core, io_uring / IOCP) async driver | +| [`serf-embedded`] | shared `no_std` driving core for the embedded drivers | +| [`serf-smoltcp`] | executor-free `no_std` driver over smoltcp (caller-poll) | +| [`serf-embassy`] | embassy-net async `no_std` driver, built on [`serf-embedded`] | + +## Installation + +> **Build requirement:** `serf-proto` and its [`memberlist-proto`] dependency invoke +> [`protoc`][protoc] at build time to generate the wire codec, so the Protocol Buffers +> compiler must be on `PATH` (e.g. `apt install protobuf-compiler`, `brew install +> protobuf`). + +```toml +[dependencies] +serf = "0.5" # tokio runtime + tcp transport by default +``` + +For `smol` instead of `tokio`: + +```toml +[dependencies] +serf = { version = "0.5", default-features = false, features = ["smol", "tcp"] } +``` + +For the `compio` (completion-based, thread-per-core) runtime: + +```toml +[dependencies] +serf = { version = "0.5", default-features = false, features = ["compio", "tcp"] } +``` + +For bare-metal (`no_std`) targets, enable `smoltcp` (the executor-free engine) or +`embassy` (the embassy-net async driver) — neither pulls in `std`: + +```toml +[dependencies] +serf = { version = "0.5", default-features = false, features = ["embassy", "tcp"] } +``` + +The minimum supported Rust version (MSRV) is **1.85.0** (edition 2024); the `smoltcp`, +`embassy`, and `embedded` drivers require **1.96.0**. + +## Example + +Common types (`Options` variants, delegates, resolvers, …) are re-exported from the +per-driver module, which also pins the runtime, so a `tokio` build reaches its handle +through `serf::tokio` and never names the runtime generic: + +```rust,ignore +use serf::tokio::{Serf, SerfOptions, SocketAddrResolver, TcpTransportOptions, VoidDelegate}; + +// `serf::tokio::Serf` is the reactor handle with its runtime already pinned to tokio, +// so the runtime generic never appears in your code. Build a node with the inherent +// constructors — `Serf::tcp`, `Serf::tls`, `Serf::quic`, and their `*_with_rng` +// variants — then drive the cluster through the returned handle's `join` / `leave` / +// `user_event` / `query` / `install_key` methods. +``` + +`serf::reactor` re-exports the same driver unpinned (generic over an [`agnostic`] +runtime); `serf::compio`, `serf::smoltcp`, `serf::embassy`, and `serf::embedded` expose +their respective driver crates the same way. The pure protocol core is always available +as `serf::proto` (a re-export of [`serf-proto`]), regardless of which driver feature is +enabled. + +## Feature flags + +Pick **one** driver, **one or more** transports, and any optional extensions you need. + +- **Drivers** — `tokio` *(default)*, `smol`, `compio` (thread-per-core), `reactor` + (generic over an [`agnostic`] runtime), `smoltcp` / `embassy` / `embedded` (`no_std`). +- **Transports** — `tcp` *(default)*; `tls` + a backend (`tls-rustls-ring`, + `tls-rustls-aws-lc-rs`); `quic` + a backend (`quic-rustls-ring`). +- **Capability tiers** — `std` and `alloc` are independent; the `no_std` drivers select + `alloc` instead of `std`. +- **Protocol extensions** — `coordinates` (Vivaldi network coordinate estimation), + `aes-gcm` / `chacha20-poly1305` (gossip + plain-TCP reliable AEAD encryption, plus + serf's gossip-driven key-management queries), `encryption` (umbrella for both + backends). +- **Config** — `serde` (config-file round-trips) and `clap` (CLI flags + env) on the + `*Options` types; std-only. +- **Other** — `tag-regex` (regex-backed tag-filter matching; falls back to exact-match + without it), `dns` (DNS address resolver), `getifs` (auto-detect the advertise + address from local interfaces), `cidr` (CIDR peer-admission allow-list, `no_std` + drivers), `tracing` (spans around the driver operations) — each forwarded to + whichever driver is selected. + +## Design + +Every driver wraps the same pure core: [`serf-proto`] runs entirely without sockets, +threads, or clocks of its own, so protocol behavior is identical no matter which driver +you pick. Drivers only shuttle bytes and time in and out and translate the machine's +outputs into delegate callbacks and an event stream. Swap `tokio` for `smoltcp` and the +membership, failure-detection, tag, event, and query semantics do not change — only the +I/O underneath does. + +## License + +`serf` is under the terms of the MPL-2.0 license. + +See [LICENSE] for details. + +Copyright (c) 2025 Al Liu. + +Copyright (c) 2013 HashiCorp, Inc. + +[HashiCorp's Serf]: https://github.com/hashicorp/serf +[project README]: https://github.com/al8n/serf/blob/main/README.md + +[`memberlist`]: https://crates.io/crates/memberlist +[`memberlist-proto`]: https://crates.io/crates/memberlist-proto +[`agnostic`]: https://github.com/al8n/agnostic +[`serf`]: https://crates.io/crates/serf +[`serf-proto`]: https://crates.io/crates/serf-proto +[`serf-driver`]: https://crates.io/crates/serf-driver +[`serf-reactor`]: https://crates.io/crates/serf-reactor +[`serf-compio`]: https://crates.io/crates/serf-compio +[`serf-embedded`]: https://crates.io/crates/serf-embedded +[`serf-smoltcp`]: https://crates.io/crates/serf-smoltcp +[`serf-embassy`]: https://crates.io/crates/serf-embassy +[protoc]: https://grpc.io/docs/protoc-installation/ +[LICENSE]: https://github.com/al8n/serf/blob/main/LICENSE +[Github-url]: https://github.com/al8n/serf/ +[CI-url]: https://github.com/al8n/serf/actions/workflows/coverage.yml +[doc-url]: https://docs.rs/serf +[crates-url]: https://crates.io/crates/serf +[codecov-url]: https://app.codecov.io/gh/al8n/serf/ +[discord]: https://discord.gg/4JyVhKFcrt diff --git a/serf/src/compio.rs b/serf/src/compio.rs new file mode 100644 index 00000000..00d68ef1 --- /dev/null +++ b/serf/src/compio.rs @@ -0,0 +1,2 @@ +//! The compio-native (completion-I/O) driver. +pub use serf_compio::*; diff --git a/serf/src/embassy.rs b/serf/src/embassy.rs new file mode 100644 index 00000000..125a131d --- /dev/null +++ b/serf/src/embassy.rs @@ -0,0 +1,2 @@ +//! The `embassy-net` async driver for bare-metal / `no_std` targets. +pub use serf_embassy::*; diff --git a/serf/src/embedded.rs b/serf/src/embedded.rs new file mode 100644 index 00000000..4a96a795 --- /dev/null +++ b/serf/src/embedded.rs @@ -0,0 +1,2 @@ +//! The neutral `no_std` embedded core shared by the bare-metal drivers. +pub use serf_embedded::*; diff --git a/serf/src/lib.rs b/serf/src/lib.rs new file mode 100644 index 00000000..598a088c --- /dev/null +++ b/serf/src/lib.rs @@ -0,0 +1,40 @@ +#![doc = include_str!("../README.md")] +#![doc(html_logo_url = "https://raw.githubusercontent.com/al8n/serf/main/art/logo_72x72.png")] +#![cfg_attr(not(feature = "std"), no_std)] +#![deny(missing_docs)] +#![cfg_attr(docsrs, feature(doc_cfg))] +#![cfg_attr(docsrs, allow(unused_attributes))] + +extern crate alloc; + +/// The runtime-agnostic protocol core: the serf state machine, the wire codec, +/// options, typed messages, events, and delegates. +pub use serf_proto as proto; + +#[cfg(feature = "tokio")] +#[cfg_attr(docsrs, doc(cfg(feature = "tokio")))] +pub mod tokio; + +#[cfg(feature = "smol")] +#[cfg_attr(docsrs, doc(cfg(feature = "smol")))] +pub mod smol; + +#[cfg(feature = "reactor")] +#[cfg_attr(docsrs, doc(cfg(feature = "reactor")))] +pub mod reactor; + +#[cfg(feature = "compio")] +#[cfg_attr(docsrs, doc(cfg(feature = "compio")))] +pub mod compio; + +#[cfg(feature = "smoltcp")] +#[cfg_attr(docsrs, doc(cfg(feature = "smoltcp")))] +pub mod smoltcp; + +#[cfg(feature = "embassy")] +#[cfg_attr(docsrs, doc(cfg(feature = "embassy")))] +pub mod embassy; + +#[cfg(feature = "embedded")] +#[cfg_attr(docsrs, doc(cfg(feature = "embedded")))] +pub mod embedded; diff --git a/serf/src/reactor.rs b/serf/src/reactor.rs new file mode 100644 index 00000000..3eacd804 --- /dev/null +++ b/serf/src/reactor.rs @@ -0,0 +1,2 @@ +//! The runtime-agnostic reactor driver (tokio / smol via `agnostic`). +pub use serf_reactor::*; diff --git a/serf/src/smol.rs b/serf/src/smol.rs new file mode 100644 index 00000000..b8d5f53b --- /dev/null +++ b/serf/src/smol.rs @@ -0,0 +1,17 @@ +//! serf on the smol runtime (via the runtime-agnostic reactor driver). +//! +//! Re-exports the full [`serf_reactor`] surface with the runtime pinned to smol, +//! so callers never name `R`. Build a node with the inherent constructors on the +//! runtime-pinned `Serf` alias — `tcp`, `tls`, `quic`, and their `*_with_rng` +//! variants. The unpinned three-parameter handle stays available as +//! [`crate::reactor`]. +pub use serf_reactor::*; + +/// The runtime these handles bind. +pub type Runtime = agnostic::smol::SmolRuntime; + +/// A smol-backed serf handle — [`serf_reactor::Serf`] with its runtime pinned to +/// smol, so callers never name `R`. +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub type Serf = serf_reactor::Serf; diff --git a/serf/src/smoltcp.rs b/serf/src/smoltcp.rs new file mode 100644 index 00000000..f5d4b737 --- /dev/null +++ b/serf/src/smoltcp.rs @@ -0,0 +1,2 @@ +//! The `smoltcp` caller-poll driver for bare-metal / `no_std` targets. +pub use serf_smoltcp::*; diff --git a/serf/src/tokio.rs b/serf/src/tokio.rs new file mode 100644 index 00000000..946176cd --- /dev/null +++ b/serf/src/tokio.rs @@ -0,0 +1,17 @@ +//! serf on the tokio runtime (via the runtime-agnostic reactor driver). +//! +//! Re-exports the full [`serf_reactor`] surface with the runtime pinned to tokio, +//! so callers never name `R`. Build a node with the inherent constructors on the +//! runtime-pinned `Serf` alias — `tcp`, `tls`, `quic`, and their `*_with_rng` +//! variants. The unpinned three-parameter handle stays available as +//! [`crate::reactor`]. +pub use serf_reactor::*; + +/// The runtime these handles bind. +pub type Runtime = agnostic::tokio::TokioRuntime; + +/// A tokio-backed serf handle — [`serf_reactor::Serf`] with its runtime pinned to +/// tokio, so callers never name `R`. +#[cfg(any(feature = "tcp", feature = "quic"))] +#[cfg_attr(docsrs, doc(cfg(any(feature = "tcp", feature = "quic"))))] +pub type Serf = serf_reactor::Serf; diff --git a/serf/tests/smoke.rs b/serf/tests/smoke.rs new file mode 100644 index 00000000..f567f721 --- /dev/null +++ b/serf/tests/smoke.rs @@ -0,0 +1,75 @@ +//! Umbrella facade smoke test: `serf::tokio` hides the runtime. +//! +//! `serf::tokio::Serf` is the reactor handle with `R = TokioRuntime` already +//! applied, so nothing below names a runtime type. The driver types come from +//! `serf::tokio` and the protocol types from `serf::proto`, which is the whole +//! point of the facade — two loopback nodes join over TCP and converge on a +//! two-member view without the caller depending on `serf-reactor` directly. + +#![cfg(all(feature = "tokio", feature = "tcp"))] + +use std::{net::SocketAddr, time::Duration}; + +use serf::{ + proto::options::Options as SerfOptions, + tokio::{ + FirstAddrResolver, MaybeResolved, RuntimeOptions, Serf, SocketAddrResolver, + TcpTransportOptions, VoidDelegate, + }, +}; +use smol_str::SmolStr; + +/// Fast SWIM timing, so the two nodes converge well inside the poll loop below. +fn transport_opts(id: &str) -> TcpTransportOptions { + TcpTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved( + "127.0.0.1:0".parse().expect("loopback addr"), + )) + .with_probe_interval(Duration::from_millis(50)) + .with_probe_timeout(Duration::from_millis(100)) + .with_gossip_interval(Duration::from_millis(20)) +} + +async fn node(id: &str) -> Serf { + Serf::::tcp( + transport_opts(id), + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + None, + None, + None, + #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] + std::sync::Arc::new(serf::tokio::VoidKeyringDelegate), + ) + .await + .expect("node builds on the tokio facade") +} + +#[tokio::test] +async fn umbrella_tokio_facade_joins_over_tcp() { + let a = node("a").await; + let b = node("b").await; + + let seed = a.advertise_address(); + b.join(&SocketAddrResolver, MaybeResolved::Resolved(seed), false) + .await + .expect("b joins a"); + + for _ in 0..100 { + if a.num_members() == 2 && b.num_members() == 2 { + break; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + + assert_eq!(a.num_members(), 2, "a converges on the two-member view"); + assert_eq!(b.num_members(), 2, "b converges on the two-member view"); + + b.leave().await.expect("b leaves gracefully"); + b.shutdown().await.expect("b shuts down"); + a.shutdown().await.expect("a shuts down"); +} From b57b960c7ef3efa6bbc437279603199127b27a62 Mon Sep 17 00:00:00 2001 From: Al Liu Date: Mon, 13 Jul 2026 19:36:04 +1200 Subject: [PATCH 128/128] test: raise coverage, and close three testing gaps (#96) --- .github/workflows/coverage.yml | 8 +- .github/workflows/embedded.yml | 13 +- serf-compio/Cargo.toml | 24 + serf-compio/src/delegate/mod.rs | 43 +- serf-compio/src/delegate/tests.rs | 47 + serf-compio/src/transport/mod.rs | 3 + serf-compio/src/transport/tests.rs | 142 ++ serf-compio/tests/cluster/mod.rs | 352 +++ serf-compio/tests/quic.rs | 1627 ++++++++++++++ serf-compio/tests/tcp.rs | 1571 +++++++++++++ serf-compio/tests/tls.rs | 508 +++++ serf-driver/src/keyring/tests.rs | 132 +- serf-driver/src/keyring_file/tests.rs | 160 ++ serf-driver/src/snapshot/tests.rs | 54 + serf-driver/src/snapshotter/tests.rs | 97 + serf-embassy/src/config/tests.rs | 30 + serf-embassy/src/error/mod.rs | 3 + serf-embassy/src/error/tests.rs | 414 ++++ serf-embassy/src/shared/tests.rs | 112 + serf-embassy/tests/handle.rs | 296 +++ serf-embassy/tests/key_refusals.rs | 348 +++ serf-embedded/src/engine/tests.rs | 1366 ++++++++++- serf-proto/src/coordinate_client/tests.rs | 73 + serf-proto/src/lib.rs | 3 + serf-proto/src/members/tests.rs | 25 + serf-proto/src/options/tests.rs | 79 + serf-proto/src/quic_endpoint/tests.rs | 2485 ++++++++++++++++++++- serf-proto/src/stream_endpoint/tests.rs | 407 ++++ serf-proto/src/tests.rs | 19 + serf-proto/src/typed/mod.rs | 3 + serf-proto/src/typed/tests.rs | 33 + serf-reactor/src/driver/options/tests.rs | 16 + serf-reactor/src/quic/tests.rs | 181 +- serf-reactor/src/resolver/getifs/tests.rs | 14 + serf-reactor/src/resolver/mod.rs | 3 + serf-reactor/src/resolver/tests.rs | 155 ++ serf-reactor/src/tcp/mod.rs | 3 + serf-reactor/src/tcp/tests.rs | 169 ++ serf-reactor/src/tls/tests.rs | 235 ++ serf-reactor/src/transport/mod.rs | 3 + serf-reactor/src/transport/tests.rs | 150 ++ serf-reactor/tests/cluster/mod.rs | 173 +- serf-reactor/tests/quic.rs | 2133 +++++++++++++++++- serf-reactor/tests/tcp.rs | 1937 ++++++++++++++-- serf-reactor/tests/tls.rs | 892 +++++++- serf-smoltcp/src/config/tests.rs | 34 + serf-smoltcp/src/error/tests.rs | 126 ++ serf-smoltcp/src/serf/tests.rs | 866 ++++++- serf-smoltcp/src/stream_io/tests.rs | 96 + serf-smoltcp/tests/inbound_admission.rs | 197 ++ 50 files changed, 17416 insertions(+), 444 deletions(-) create mode 100644 serf-compio/src/delegate/tests.rs create mode 100644 serf-compio/src/transport/tests.rs create mode 100644 serf-compio/tests/cluster/mod.rs create mode 100644 serf-compio/tests/quic.rs create mode 100644 serf-compio/tests/tcp.rs create mode 100644 serf-compio/tests/tls.rs create mode 100644 serf-embassy/src/error/tests.rs create mode 100644 serf-embassy/tests/handle.rs create mode 100644 serf-embassy/tests/key_refusals.rs create mode 100644 serf-proto/src/tests.rs create mode 100644 serf-proto/src/typed/tests.rs create mode 100644 serf-reactor/src/resolver/tests.rs create mode 100644 serf-reactor/src/tcp/tests.rs create mode 100644 serf-reactor/src/transport/tests.rs create mode 100644 serf-smoltcp/tests/inbound_admission.rs diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 39eddcdd..2bc29216 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -63,10 +63,8 @@ jobs: # and pass. A per-crate `--all-features` sweep is not usable: the rustls # backends (ring / aws-lc-rs) are alternatives, not additives. # - # serf-embedded carries no host test suite of its own — it is exercised - # through the smoltcp and embassy drivers' host suites, which is why it - # has no run of its own here. The `serf` umbrella is pure re-exports with - # no coverable logic; ci-tokio gates its smoke test instead. + # The `serf` umbrella is pure re-exports with no coverable logic; ci-tokio + # gates its smoke test instead. cargo llvm-cov --no-report -p serf-proto \ --features "tcp,tls,quic-rustls-ring,coordinates,aes-gcm,chacha20-poly1305,tag-regex" cargo llvm-cov --no-report -p serf-driver --all-features @@ -74,7 +72,7 @@ jobs: --features "tokio,tcp,tls-rustls-ring,quic-rustls-ring,aes-gcm,chacha20-poly1305,coordinates,tag-regex,tracing,serde,clap,dns,getifs,test" cargo llvm-cov --no-report -p serf-compio \ --features "tcp,tls,quic-rustls-ring,aes-gcm,chacha20-poly1305,coordinates,tag-regex,tracing,serde,clap,dns,getifs" - cargo llvm-cov --no-report -p serf-smoltcp -p serf-embassy \ + cargo llvm-cov --no-report -p serf-embedded -p serf-smoltcp -p serf-embassy \ --features "aes-gcm,cidr" cargo llvm-cov report --lcov --output-path lcov.info diff --git a/.github/workflows/embedded.yml b/.github/workflows/embedded.yml index f475b036..a303672e 100644 --- a/.github/workflows/embedded.yml +++ b/.github/workflows/embedded.yml @@ -55,9 +55,10 @@ jobs: --no-default-features --features alloc --target thumbv7em-none-eabihf - # Host test gate: the smoltcp and embassy drivers carry host-runnable test - # suites (block_on + bounded virtual time) that the bare-metal check above does - # not exercise. Run them on a std host, plus fmt + clippy for both libs. + # Host test gate: the embedded core and both its drivers carry host-runnable + # test suites (block_on + bounded virtual time) that the bare-metal check above + # does not exercise — a bare `cargo check` never builds a test target. Run them + # on a std host, plus fmt + clippy for all three libs. host-tests: runs-on: ubuntu-latest steps: @@ -72,11 +73,11 @@ jobs: sudo apt-get update sudo apt-get install -y protobuf-compiler - name: Format - run: cargo fmt -p serf-smoltcp -p serf-embassy --check + run: cargo fmt -p serf-embedded -p serf-smoltcp -p serf-embassy --check - name: Clippy - run: cargo clippy -p serf-smoltcp -p serf-embassy --all-targets --features aes-gcm,cidr --no-deps -- -D warnings + run: cargo clippy -p serf-embedded -p serf-smoltcp -p serf-embassy --all-targets --features aes-gcm,cidr --no-deps -- -D warnings - name: Test - run: cargo test -p serf-smoltcp -p serf-embassy --features aes-gcm,cidr + run: cargo test -p serf-embedded -p serf-smoltcp -p serf-embassy --features aes-gcm,cidr # Execution proof: build the `serf-embassy-qemu` firmware and run it under QEMU. # The crate's `.cargo/config.toml` supplies the target, the linker script, the diff --git a/serf-compio/Cargo.toml b/serf-compio/Cargo.toml index 98cc0d2f..a20dbc5a 100644 --- a/serf-compio/Cargo.toml +++ b/serf-compio/Cargo.toml @@ -98,6 +98,30 @@ ring = "0.17" # Used in the serde option-layer tests. serde_json = "1" +# The real-node TCP test suite: loopback nodes driving the compio stream driver +# end to end (join/converge, user event, query round-trip, set_tags, graceful +# leave, force-leave on a failed member, snapshot replay, observation-channel +# backpressure, bridge teardown, and the command-queue drain at shutdown). Shares +# the multi-node fixture in `tests/cluster/`. +[[test]] +name = "tcp" +required-features = ["tcp"] + +# The real-node TLS test suite: the same stream driver over a self-signed cert + +# accept-any verifier (options guards, unresolved-advertise resolution, merge +# delegate, snapshot persistence, user event, query round-trip, graceful leave). +[[test]] +name = "tls" +required-features = ["tls"] + +# The real-node QUIC test suite: loopback nodes over a quinn config bundle +# driving the QUIC pump's whole command surface (join/converge, user event, query +# round-trip, set_tags, graceful leave, not-running gate, key rotation, snapshot +# persistence, observation backpressure, and the command-queue drain at shutdown). +[[test]] +name = "quic" +required-features = ["quic-rustls-ring"] + [package.metadata.docs.rs] all-features = true rustdoc-args = ["--cfg", "docsrs"] diff --git a/serf-compio/src/delegate/mod.rs b/serf-compio/src/delegate/mod.rs index 33f31b59..e8fd0e63 100644 --- a/serf-compio/src/delegate/mod.rs +++ b/serf-compio/src/delegate/mod.rs @@ -212,45 +212,4 @@ pub trait KeyringDelegate: 'static { pub use memberlist_proto::delegate::MergeDelegate; #[cfg(test)] -mod tests { - use super::*; - use smol_str::SmolStr; - use std::net::SocketAddr; - - #[cfg(any(feature = "tcp", feature = "quic"))] - #[test] - fn void_delegate_satisfies_observation_composite() { - fn assert_delegate(_d: &D) - where - D: Delegate, - { - } - let v: VoidDelegate = VoidDelegate::default(); - assert_delegate(&v); - } - - /// The re-exported merge predicate is the machine's synchronous push/pull - /// filter — a type-level check that a plain permit-all impl satisfies it. - #[cfg(any(feature = "tcp", feature = "quic"))] - #[test] - fn a_sync_predicate_satisfies_the_merge_delegate() { - struct PermitAll; - impl MergeDelegate for PermitAll { - fn notify_merge( - &self, - _peers: memberlist_proto::MaybeOwned< - '_, - [memberlist_proto::typed::NodeState], - >, - ) -> bool { - true - } - } - fn assert_merge(_: &T) - where - T: MergeDelegate, - { - } - assert_merge(&PermitAll); - } -} +mod tests; diff --git a/serf-compio/src/delegate/tests.rs b/serf-compio/src/delegate/tests.rs new file mode 100644 index 00000000..2cf474d3 --- /dev/null +++ b/serf-compio/src/delegate/tests.rs @@ -0,0 +1,47 @@ +use super::*; +use smol_str::SmolStr; +use std::net::SocketAddr; + +/// The zero-cost default delegate satisfies the whole observation composite, so +/// a driver that needs no hooks constructs a node without boilerplate. +#[cfg(any(feature = "tcp", feature = "quic"))] +#[test] +fn void_delegate_satisfies_observation_composite() { + fn assert_delegate(_d: &D) + where + D: Delegate, + { + } + let v: VoidDelegate = VoidDelegate::default(); + assert_delegate(&v); +} + +/// The re-exported merge predicate is the machine's synchronous push/pull +/// filter: a plain permit-all impl satisfies it, and its verdict is the value +/// the machine acts on. +#[cfg(any(feature = "tcp", feature = "quic"))] +#[test] +fn a_sync_predicate_satisfies_the_merge_delegate() { + struct PermitAll; + impl MergeDelegate for PermitAll { + fn notify_merge( + &self, + _peers: memberlist_proto::MaybeOwned< + '_, + [memberlist_proto::typed::NodeState], + >, + ) -> bool { + true + } + } + fn assert_merge(t: &T) -> bool + where + T: MergeDelegate, + { + t.notify_merge(memberlist_proto::MaybeOwned::Borrowed(&[])) + } + assert!( + assert_merge(&PermitAll), + "a permit-all predicate admits the exchange" + ); +} diff --git a/serf-compio/src/transport/mod.rs b/serf-compio/src/transport/mod.rs index 6e492327..7475e37a 100644 --- a/serf-compio/src/transport/mod.rs +++ b/serf-compio/src/transport/mod.rs @@ -237,3 +237,6 @@ pub(crate) async fn close_stream_sockets( let _ = listener.close().await; let _ = gossip.close().await; } + +#[cfg(all(test, any(feature = "tcp", feature = "quic")))] +mod tests; diff --git a/serf-compio/src/transport/tests.rs b/serf-compio/src/transport/tests.rs new file mode 100644 index 00000000..50a39d9e --- /dev/null +++ b/serf-compio/src/transport/tests.rs @@ -0,0 +1,142 @@ +//! The construction-time advertise-address gate. +//! +//! Every transport reads its advertise `SocketAddr` back from the bound socket +//! and gossips it as the local node's contact identity, so an address peers +//! cannot dial — or cannot even decode off the compact `[16B IP][2B port]` wire +//! layout — must be refused at construction rather than published to the +//! cluster. The classes here are unreachable through a successful bind on a +//! normal host (a listener never hands back a multicast, broadcast, or +//! zero-port address), so they are pinned directly against the validator. + +use std::net::{Ipv4Addr, Ipv6Addr, SocketAddr, SocketAddrV4, SocketAddrV6}; + +use super::validate_advertise_addr; +use crate::SerfError; + +/// The reason string the validator attached, or a panic naming what it returned +/// instead of the expected `InvalidAdvertiseAddr` rejection. +fn rejection_reason(addr: SocketAddr) -> String { + match validate_advertise_addr(&addr) { + Err(SerfError::InvalidAdvertiseAddr(e)) => { + assert_eq!( + e.addr(), + addr, + "the rejection carries the address that was refused" + ); + e.to_string() + } + Err(other) => panic!("expected InvalidAdvertiseAddr for {addr}, got {other:?}"), + Ok(()) => panic!("{addr} must be refused as an advertise address, but it was accepted"), + } +} + +/// A routable unicast contact — loopback, private, or global — is accepted: the +/// gate refuses undialable classes, it does not narrow the deployment surface. +#[test] +fn a_routable_unicast_contact_is_accepted() { + for addr in [ + "127.0.0.1:7946", + "192.168.1.10:7946", + "8.8.8.8:7946", + "[::1]:7946", + "[2001:db8::1]:7946", + ] { + let addr: SocketAddr = addr.parse().expect("a valid socket address"); + validate_advertise_addr(&addr) + .unwrap_or_else(|e| panic!("{addr} is a routable unicast contact but was refused: {e}")); + } +} + +/// The wildcard bind address is the classic footgun: its `local_addr()` readback +/// keeps the unspecified IP, which peers cannot dial, so the node would join as a +/// member no peer can reach and be suspected and reaped. +#[test] +fn an_unspecified_ip_is_refused() { + assert!( + rejection_reason("0.0.0.0:7946".parse().expect("v4 wildcard")).contains("unspecified"), + "the IPv4 wildcard is refused as the wildcard-bind address" + ); + assert!( + rejection_reason("[::]:7946".parse().expect("v6 wildcard")).contains("unspecified"), + "the IPv6 wildcard is refused as the wildcard-bind address" + ); +} + +/// A multicast IP names a GROUP, not a single peer's unicast contact, so it is +/// not a usable identity to gossip. +#[test] +fn a_multicast_ip_is_refused() { + assert!( + rejection_reason("224.0.0.1:7946".parse().expect("v4 multicast")).contains("multicast"), + "an IPv4 multicast group is not a unicast contact" + ); + assert!( + rejection_reason("[ff02::1]:7946".parse().expect("v6 multicast")).contains("multicast"), + "an IPv6 multicast group is not a unicast contact" + ); +} + +/// The IPv4 broadcast address is a v4-only concept the multicast check does not +/// catch, so it has its own rejection. +#[test] +fn the_ipv4_broadcast_ip_is_refused() { + let addr = SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::BROADCAST, 7946)); + assert!( + rejection_reason(addr).contains("broadcast"), + "255.255.255.255 is not a unicast contact" + ); +} + +/// A zero port is undialable: the bound socket's `local_addr()` readback must +/// carry the concrete OS-assigned port, so a zero here means the readback was +/// skipped. +#[test] +fn a_zero_port_is_refused() { + assert!( + rejection_reason("127.0.0.1:0".parse().expect("zero-port v4")).contains("zero port"), + "a zero port is undialable" + ); + assert!( + rejection_reason("[::1]:0".parse().expect("zero-port v6")).contains("zero port"), + "a zero port is undialable" + ); +} + +/// The compact `[16B IP][2B port]` wire layout carries neither `scope_id` nor +/// `flowinfo`, so a scoped or flow-labelled IPv6 advertise address would decode +/// on a peer as a DIFFERENT, unroutable contact. Both fields are rejected +/// independently. +#[test] +fn a_scoped_or_flow_labelled_ipv6_is_refused() { + let scoped = SocketAddr::V6(SocketAddrV6::new( + "fe80::1".parse::().expect("link-local v6"), + 7946, + 0, + 3, + )); + assert!( + rejection_reason(scoped).contains("scope_id"), + "a nonzero scope_id is not representable on the wire layout" + ); + + let flow_labelled = SocketAddr::V6(SocketAddrV6::new( + "2001:db8::1".parse::().expect("global v6"), + 7946, + 0x1234, + 0, + )); + assert!( + rejection_reason(flow_labelled).contains("flowinfo"), + "a nonzero flowinfo is not representable on the wire layout" + ); + + // The same address with both fields zeroed is a perfectly good contact — the + // gate keys on the fields, not on the address family or its link-local scope. + let plain = SocketAddr::V6(SocketAddrV6::new( + "2001:db8::1".parse::().expect("global v6"), + 7946, + 0, + 0, + )); + validate_advertise_addr(&plain).expect("an unscoped IPv6 unicast contact is accepted"); +} diff --git a/serf-compio/tests/cluster/mod.rs b/serf-compio/tests/cluster/mod.rs new file mode 100644 index 00000000..6bb1287d --- /dev/null +++ b/serf-compio/tests/cluster/mod.rs @@ -0,0 +1,352 @@ +//! Reusable multi-node fixture for the compio real-node serf driver, shared by +//! the scenario bodies in the sibling test binaries. +//! +//! A [`Cluster`] spins up N ephemeral loopback TCP nodes, joins every non-seed +//! node to the first (a star), and waits for the whole cluster to converge. Each +//! node runs a detached collector that drains its event stream into a shared log; +//! the log is a separate `Rc`, so it survives an abrupt +//! [`kill_abrupt`](Cluster::kill_abrupt) (which drops the node's last handle) and +//! later assertions read an ordered per-node member-event history. +//! +//! compio is thread-per-core and `!Send`, so the fixture is `Rc`/`RefCell`-based +//! and every node, collector, and assertion runs on the one runtime thread. +//! +//! Unlike the reactor's transport options, serf-compio's `TcpTransportOptions` +//! exposes no memberlist SWIM knobs, so the nodes run the coordinator's default +//! probe / gossip / suspicion timing. [`ClusterTiming`] therefore tunes only the +//! serf-level reaper, and the poll ceiling is sized for a default-timing failure +//! detection. + +use core::time::Duration; +use std::{cell::RefCell, net::SocketAddr, rc::Rc}; + +use futures_util::StreamExt; +use memberlist_proto::MaybeResolved; +use serf_compio::{ + FirstAddrResolver, RuntimeOptions, Serf, SocketAddrResolver, TcpTransport, TcpTransportOptions, + VoidDelegate, gossip_rng, +}; +use serf_proto::{ + event::{Event, MemberEventKind}, + members::MemberStatus, + options::Options as SerfOptions, +}; +use smol_str::SmolStr; + +/// A compio TCP node handle. +pub type Node = Serf; + +/// Wall-clock ceiling for every fixture poll loop, so a convergence or detection +/// regression surfaces as a bounded timeout rather than a hang. Sized for the +/// coordinator's DEFAULT suspicion timing (a 1 s probe interval and a 4x +/// suspicion multiplier put an abrupt kill's Failed transition several seconds +/// out), which serf-compio's transport options cannot shorten. +const POLL_TIMEOUT: Duration = Duration::from_secs(45); +/// Poll granularity for the fixture's await loops. +const POLL_STEP: Duration = Duration::from_millis(20); + +/// An ephemeral loopback bind address (`127.0.0.1:0`). +pub fn loopback_ephemeral() -> SocketAddr { + "127.0.0.1:0".parse().expect("loopback addr") +} + +/// Serf-level reaper timing shared by every node in a fixture cluster. +/// +/// [`fast`](Self::fast) reaps a failed member almost immediately after the +/// coordinator declares it Failed (short reconnect timeout) and drops a +/// gracefully-left member almost immediately after its Leave (short tombstone +/// timeout). A test that wants to OBSERVE a member sitting in a Failed or Left +/// state raises the matching window with +/// [`with_reconnect_timeout`](Self::with_reconnect_timeout) / +/// [`with_tombstone_timeout`](Self::with_tombstone_timeout). +#[derive(Clone)] +pub struct ClusterTiming { + reap_interval: Duration, + reconnect_interval: Duration, + reconnect_timeout: Duration, + tombstone_timeout: Duration, + leave_propagate_delay: Duration, +} + +impl ClusterTiming { + /// CI-speed serf reaper timing: the reaper ticks every 100 ms and holds a + /// failed or left member for ~nothing, so an abrupt kill converges to a + /// reaped-out cluster as soon as the coordinator's default suspicion timing + /// declares the peer Failed. + pub fn fast() -> Self { + Self { + reap_interval: Duration::from_millis(100), + reconnect_interval: Duration::from_millis(100), + reconnect_timeout: Duration::from_millis(1), + tombstone_timeout: Duration::from_millis(1), + // Short enough to keep a graceful leave sub-second, long enough to give + // in-flight probes a gossip cycle to observe the leave intent. + leave_propagate_delay: Duration::from_millis(100), + } + } + + /// Override the failed-member retention window — the age at which the reaper + /// removes a Failed member. Raise it beyond the test window to HOLD a failed + /// peer in the Failed view instead of reaping it. + #[must_use] + pub fn with_reconnect_timeout(mut self, v: Duration) -> Self { + self.reconnect_timeout = v; + self + } + + /// Override the tombstone timeout — the age at which the reaper removes a + /// gracefully-Left member. Raise it beyond the test window to HOLD a left peer + /// in the tombstone view. + #[must_use] + pub fn with_tombstone_timeout(mut self, v: Duration) -> Self { + self.tombstone_timeout = v; + self + } + + /// The serf `Options` every fixture node is built with. + pub fn serf_opts(&self) -> SerfOptions { + SerfOptions::new() + .with_reap_interval(self.reap_interval) + .with_reconnect_interval(self.reconnect_interval) + .with_reconnect_timeout(self.reconnect_timeout) + .with_tombstone_timeout(self.tombstone_timeout) + .with_leave_propagate_delay(self.leave_propagate_delay) + } +} + +/// One observed member event: its kind and the member ids it names. +struct MemberRec { + kind: MemberEventKind, + ids: Vec, +} + +/// The shared, handle-independent event log a node's collector appends to. +type EventLog = Rc>>; + +/// One cluster node: its stable id, the live handle (taken while killed), and the +/// event log the collector drains into. +struct NodeSlot { + id: SmolStr, + serf: Option, + log: EventLog, +} + +/// A live multi-node loopback cluster with per-node member-event logs. +pub struct Cluster { + slots: Vec, +} + +impl Cluster { + /// Spawn `ids.len()` ephemeral loopback nodes with `timing`, attach a per-node + /// event collector, join every non-seed node to the first (a star), and wait + /// for the whole cluster to converge. + pub async fn spawn(ids: &[&str], timing: ClusterTiming) -> Self { + let mut slots = Vec::with_capacity(ids.len()); + for id in ids { + let serf = build_node(id, timing.serf_opts()) + .await + .expect("spawn serf tcp node"); + let log: EventLog = Rc::new(RefCell::new(Vec::new())); + // Attach the collector before the handle moves into the slot, so no member + // event can slip past between construction and the first join. + attach_collector(&serf, log.clone()); + slots.push(NodeSlot { + id: SmolStr::new(*id), + serf: Some(serf), + log, + }); + } + let cluster = Self { slots }; + let seed = cluster.node(0).advertise_address(); + for i in 1..cluster.slots.len() { + cluster + .node(i) + .join(&SocketAddrResolver, MaybeResolved::Resolved(seed), false) + .await + .expect("join reaches the seed node"); + } + cluster.converge(cluster.slots.len()).await; + cluster + } + + /// The live handle for node `i` (panics if the node is currently killed). + pub fn node(&self, i: usize) -> &Node { + self.slots[i].serf.as_ref().expect("node slot is live") + } + + /// The stable id of node `i`. + pub fn id(&self, i: usize) -> SmolStr { + self.slots[i].id.clone() + } + + /// Abruptly kill node `i`: shut its handle down without a leave. A shutdown + /// sends no farewell, so peers detect a probe-timeout Failed rather than a + /// graceful Leave. The slot's id and event log are retained for later + /// assertions. + pub async fn kill_abrupt(&mut self, i: usize) { + let serf = self.slots[i].serf.take().expect("node slot is live"); + serf.shutdown().await.expect("node shuts down"); + } + + /// Gracefully leave node `i`, then release its slot. `leave()` resolves only + /// once the machine's `LeftCluster` fires, so a successful return proves the + /// graceful-leave chain completed and the farewell reached the wire before the + /// teardown that follows. + pub async fn leave_graceful(&mut self, i: usize) { + let serf = self.slots[i].serf.take().expect("node slot is live"); + serf.leave().await.expect("node leaves gracefully"); + serf.shutdown().await.expect("node shuts down"); + } + + /// Poll every live node until each reports exactly `expect` members. + pub async fn converge(&self, expect: usize) { + compio::time::timeout(POLL_TIMEOUT, async { + loop { + if self + .slots + .iter() + .filter_map(|s| s.serf.as_ref()) + .all(|n| n.num_members() == expect) + { + break; + } + compio::time::sleep(POLL_STEP).await; + } + }) + .await + .expect("cluster converges to the expected member count"); + } + + /// Poll until `observer`'s membership view holds `subject` with `status`. + pub async fn await_member_status(&self, observer: usize, subject: &str, status: MemberStatus) { + compio::time::timeout(POLL_TIMEOUT, async { + loop { + if self + .node(observer) + .members() + .iter() + .any(|m| m.node().id_ref().as_str() == subject && m.status() == status) + { + break; + } + compio::time::sleep(POLL_STEP).await; + } + }) + .await + .unwrap_or_else(|_| panic!("node {observer} never holds {subject:?} as {status:?}")); + } + + /// Poll until `observer`'s log records a member event of `kind` naming + /// `subject`. + pub async fn await_member_event(&self, observer: usize, subject: &str, kind: MemberEventKind) { + compio::time::timeout(POLL_TIMEOUT, async { + loop { + if self.member_event_kinds(observer, subject).contains(&kind) { + break; + } + compio::time::sleep(POLL_STEP).await; + } + }) + .await + .unwrap_or_else(|_| { + panic!( + "node {observer} never records {kind:?} for {subject:?} (saw {:?})", + self.member_event_kinds(observer, subject) + ) + }); + } + + /// Poll until the ordered member-event kinds `observer` recorded about + /// `subject` are at least as long as `expected`, then assert exact equality. + /// Polling first lets a still-in-flight event land; the final assert then + /// catches a wrong, missing, or extra event. + pub async fn assert_member_events( + &self, + observer: usize, + subject: &str, + expected: &[MemberEventKind], + ) { + // Ignoring Err: a poll timeout here just means fewer events than expected + // arrived; the assert_eq below reports the precise sequence mismatch. + let _ = compio::time::timeout(POLL_TIMEOUT, async { + loop { + if self.member_event_kinds(observer, subject).len() >= expected.len() { + break; + } + compio::time::sleep(POLL_STEP).await; + } + }) + .await; + let actual = self.member_event_kinds(observer, subject); + assert_eq!( + actual, expected, + "member events for {subject:?} observed by node {observer}" + ); + } + + /// Shut down every live node, releasing all bound ports. + pub async fn shutdown_all(&mut self) { + for slot in &mut self.slots { + if let Some(serf) = slot.serf.take() { + serf.shutdown().await.expect("node shuts down"); + } + } + } + + /// The ordered member-event kinds `observer` recorded about `subject`. + pub fn member_event_kinds(&self, observer: usize, subject: &str) -> Vec { + self.slots[observer] + .log + .borrow() + .iter() + .filter(|rec| rec.ids.iter().any(|id| id.as_str() == subject)) + .map(|rec| rec.kind) + .collect() + } +} + +/// Spawn a fixture node on an ephemeral loopback port with `serf_opts`. +async fn build_node(id: &str, serf_opts: SerfOptions) -> serf_compio::Result { + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(loopback_ephemeral())); + Serf::new::, SocketAddrResolver, FirstAddrResolver, _, _>( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + serf_opts, + gossip_rng().expect("seed gossip rng"), + None, + None, + None, + #[cfg(encryption)] + Rc::new(serf_compio::VoidKeyringDelegate), + ) + .await +} + +/// Attach a detached collector that drains `serf`'s event stream into `log`, +/// recording every member event (kind + named ids). The collector holds only the +/// stream, so a later kill (last-handle drop) still tears the node down while the +/// log persists. +fn attach_collector(serf: &Node, log: EventLog) { + let mut stream = serf.events(); + compio::runtime::spawn(async move { + while let Some(ev) = stream.next().await { + if let Event::Member(me) = ev { + let ids = me + .members() + .iter() + .map(|m| m.node().id_ref().clone()) + .collect(); + log.borrow_mut().push(MemberRec { + kind: me.kind(), + ids, + }); + } + } + }) + .detach(); +} diff --git a/serf-compio/tests/quic.rs b/serf-compio/tests/quic.rs new file mode 100644 index 00000000..436b639d --- /dev/null +++ b/serf-compio/tests/quic.rs @@ -0,0 +1,1627 @@ +//! Real-node QUIC serf tests: loopback nodes exercising the compio QUIC driver +//! end-to-end over a quinn-proto config bundle. +//! +//! The QUIC pump owns exactly one UDP socket (quinn multiplexes the reliable +//! push/pull streams over it, and serf's datagram gossip rides the same socket), +//! so it has no bridge table and its own command dispatch, key-request handling, +//! snapshot persistence, and teardown drain. This suite drives that whole +//! surface: join/converge, user events, a query round-trip through `respond`, +//! `set_tags`, graceful leave, the not-running gate every command honours after a +//! leave, live-keyring rotation (install/use/remove/list), snapshot persistence +//! and replay, observation-channel backpressure, and the command queue the +//! teardown must answer rather than drop. + +#![cfg(feature = "quic")] + +use core::{future::Future, pin::Pin, time::Duration}; +use std::{cell::RefCell, net::SocketAddr, rc::Rc, sync::Arc}; + +use bytes::Bytes; +use futures_util::{StreamExt, future}; +use memberlist_proto::{MaybeResolved, UnreliableTransport}; +use rustls::{ + client::danger::{HandshakeSignatureValid, ServerCertVerified}, + version::TLS13, +}; +use rustls_pki_types::{CertificateDer, PrivateKeyDer}; +use serf_compio::{ + Channel, Delegate, FirstAddrResolver, Ipv4PreferringResolver, MemberDelegate, MergeDelegate, + QueryDelegate, QuicOptions, QuicTransport, QuicTransportOptions, Resolver, RuntimeOptions, Serf, + SerfError, SnapshotOptions, SocketAddrResolver, Transport, UserEventDelegate, VoidDelegate, + gossip_rng, +}; +use serf_proto::{ + Tags, UserEventMessage, + event::{Event, QueryEvent}, + members::SerfState, + options::Options as SerfOptions, +}; +use smol_str::SmolStr; + +#[cfg(encryption)] +use serf_compio::{EncryptionOptions, Keyring, KeyringDelegate, SecretKey, VoidKeyringDelegate}; + +/// Bound on every convergence / delivery poll in this file. +const WINDOW: Duration = Duration::from_secs(45); + +/// An ephemeral loopback bind address. +fn loopback_ephemeral() -> SocketAddr { + "127.0.0.1:0".parse().expect("loopback addr") +} + +// ── the quinn config bundle ─────────────────────────────────────────────────── + +/// A self-signed cert + key for `localhost`. +fn self_signed() -> (Vec>, PrivateKeyDer<'static>) { + let ck = rcgen::generate_simple_self_signed(vec!["localhost".into()]) + .expect("rcgen generate_simple_self_signed"); + let cert = CertificateDer::from(ck.cert.der().to_vec()); + let key = PrivateKeyDer::Pkcs8(ck.signing_key.serialize_der().into()); + (vec![cert], key) +} + +fn test_endpoint_config() -> quinn_proto::EndpointConfig { + let hmac = ring::hmac::Key::new(ring::hmac::HMAC_SHA256, &[0x5au8; 32]); + quinn_proto::EndpointConfig::new(Arc::new(hmac)) +} + +fn test_server() -> quinn_proto::ServerConfig { + let (chain, key) = self_signed(); + let provider = Arc::new(rustls::crypto::ring::default_provider()); + let rustls_server = rustls::ServerConfig::builder_with_provider(provider) + .with_protocol_versions(&[&TLS13]) + .expect("TLS 1.3 supported") + .with_no_client_auth() + .with_single_cert(chain, key) + .expect("valid self-signed cert"); + let qsc = quinn_proto::crypto::rustls::QuicServerConfig::try_from(Arc::new(rustls_server)) + .expect("a TLS 1.3 server config is a valid QUIC server config"); + quinn_proto::ServerConfig::with_crypto(Arc::new(qsc)) +} + +/// Accept-any server-cert verifier — test only. +#[derive(Debug)] +struct AnyServer; + +impl rustls::client::danger::ServerCertVerifier for AnyServer { + fn verify_server_cert( + &self, + _end_entity: &CertificateDer, + _intermediates: &[CertificateDer], + _server_name: &rustls_pki_types::ServerName, + _ocsp_response: &[u8], + _now: rustls_pki_types::UnixTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } + + fn verify_tls12_signature( + &self, + _message: &[u8], + _cert: &CertificateDer, + _dss: &rustls::DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + + fn verify_tls13_signature( + &self, + _message: &[u8], + _cert: &CertificateDer, + _dss: &rustls::DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + + fn supported_verify_schemes(&self) -> Vec { + rustls::crypto::ring::default_provider() + .signature_verification_algorithms + .supported_schemes() + } +} + +fn test_client() -> quinn_proto::ClientConfig { + let provider = Arc::new(rustls::crypto::ring::default_provider()); + let cfg = rustls::ClientConfig::builder_with_provider(provider) + .with_protocol_versions(&[&TLS13]) + .expect("TLS 1.3 supported") + .dangerous() + .with_custom_certificate_verifier(Arc::new(AnyServer)) + .with_no_client_auth(); + let qcc = quinn_proto::crypto::rustls::QuicClientConfig::try_from(Arc::new(cfg)) + .expect("a TLS 1.3 client config is a valid QUIC client config"); + quinn_proto::ClientConfig::new(Arc::new(qcc)) +} + +/// A QUIC bundle with an idle timeout well past a localhost handshake and +/// datagram-mode unreliable transport. A fresh bundle is built per node so each +/// owns its own cert and quinn endpoint config. +fn test_quic_options() -> QuicOptions { + let mut transport = quinn_proto::TransportConfig::default(); + transport.max_idle_timeout(Some( + quinn_proto::IdleTimeout::try_from(Duration::from_secs(20)).expect("a valid idle timeout"), + )); + QuicOptions::new( + test_endpoint_config(), + test_server(), + test_client(), + transport, + "localhost", + UnreliableTransport::Datagram, + ) +} + +// ── fixtures ────────────────────────────────────────────────────────────────── + +/// A resolver that answers with a dual-stack candidate set (IPv6 first, then +/// IPv4) on the port it was asked for — enough to drive the +/// `MaybeResolved::Unresolved` advertise path AND the advertise picker's +/// narrowing, without depending on the host's name resolution. +struct DualStackResolver; + +impl Resolver for DualStackResolver { + type Address = SocketAddr; + type Error = std::io::Error; + + async fn resolve(&self, addr: &SocketAddr) -> Result, std::io::Error> { + Ok(vec![ + SocketAddr::new("::1".parse().expect("v6 loopback"), addr.port()), + SocketAddr::new("127.0.0.1".parse().expect("v4 loopback"), addr.port()), + ]) + } +} + +/// Recorded observation-hook fan-out, shared between a [`RecordingDelegate`] +/// handed to the driver and the test that asserts on it. +#[derive(Default)] +struct Observed { + updated: RefCell>, + queries: RefCell>, + user_events: RefCell>, +} + +/// A [`Delegate`] that records which observation hooks the QUIC driver fired. +struct RecordingDelegate(Rc); + +impl MemberDelegate for RecordingDelegate { + type Id = SmolStr; + type Address = SocketAddr; + + async fn notify_update(&self, member: Arc>) { + self + .0 + .updated + .borrow_mut() + .push(member.node().id_ref().clone()); + } +} + +impl UserEventDelegate for RecordingDelegate { + async fn notify_user_event(&self, event: &UserEventMessage) { + self.0.user_events.borrow_mut().push(event.name.clone()); + } +} + +impl QueryDelegate for RecordingDelegate { + type Id = SmolStr; + type Address = SocketAddr; + + async fn notify_query(&self, event: &QueryEvent) { + self.0.queries.borrow_mut().push(SmolStr::new(event.name())); + } +} + +impl Delegate for RecordingDelegate { + type Id = SmolStr; + type Address = SocketAddr; +} + +/// A [`Delegate`] whose user-event hook parks for `stall`, so the driver's +/// observation task cannot drain its queue while the pump keeps enqueueing. +struct StallingDelegate { + stall: Duration, +} + +impl MemberDelegate for StallingDelegate { + type Id = SmolStr; + type Address = SocketAddr; +} + +impl UserEventDelegate for StallingDelegate { + async fn notify_user_event(&self, _event: &UserEventMessage) { + compio::time::sleep(self.stall).await; + } +} + +impl QueryDelegate for StallingDelegate { + type Id = SmolStr; + type Address = SocketAddr; +} + +impl Delegate for StallingDelegate { + type Id = SmolStr; + type Address = SocketAddr; +} + +/// A merge predicate that admits every exchange while recording the peers each +/// push/pull carried. `MergeDelegate` is the machine's `Send + Sync` predicate, +/// so the record is shared through an `Arc` even on the `!Send` compio driver. +type MergedPeers = Arc>>; + +struct RecordingMerge { + peers: MergedPeers, +} + +impl MergeDelegate for RecordingMerge { + fn notify_merge( + &self, + peers: memberlist_proto::MaybeOwned< + '_, + [memberlist_proto::typed::NodeState], + >, + ) -> bool { + let mut seen = self.peers.lock().expect("merge record lock"); + for p in peers.iter() { + seen.push(p.id_ref().clone()); + } + true + } +} + +/// Every knob a QUIC test node may vary, defaulted to the plain loopback node +/// most scenarios want. +struct NodeSpec +where + D: Delegate + 'static, +{ + delegate: D, + runtime: RuntimeOptions, + serf: SerfOptions, + merge: Option>>, + snapshot: Option, + #[cfg(encryption)] + encryption: EncryptionOptions, + #[cfg(encryption)] + keyring: Rc, +} + +impl NodeSpec> { + fn new() -> Self { + Self { + delegate: VoidDelegate::new(), + runtime: RuntimeOptions::new(), + serf: SerfOptions::new(), + merge: None, + snapshot: None, + #[cfg(encryption)] + encryption: EncryptionOptions::new(), + #[cfg(encryption)] + keyring: Rc::new(VoidKeyringDelegate), + } + } +} + +impl NodeSpec +where + D: Delegate + 'static, +{ + fn with_delegate(self, delegate: E) -> NodeSpec + where + E: Delegate + 'static, + { + NodeSpec { + delegate, + runtime: self.runtime, + serf: self.serf, + merge: self.merge, + snapshot: self.snapshot, + #[cfg(encryption)] + encryption: self.encryption, + #[cfg(encryption)] + keyring: self.keyring, + } + } + + fn with_runtime(mut self, runtime: RuntimeOptions) -> Self { + self.runtime = runtime; + self + } + + fn with_merge(mut self, merge: Box>) -> Self { + self.merge = Some(merge); + self + } + + fn with_snapshot(mut self, snapshot: SnapshotOptions) -> Self { + self.snapshot = Some(snapshot); + self + } + + #[cfg(encryption)] + fn with_encryption(mut self, encryption: EncryptionOptions) -> Self { + self.encryption = encryption; + self + } + + #[cfg(encryption)] + fn with_keyring(mut self, keyring: Rc) -> Self { + self.keyring = keyring; + self + } + + /// Build and spawn the node on an ephemeral loopback UDP port. + async fn spawn(self, id: &str) -> Serf { + #[allow(unused_mut)] + let mut opts = QuicTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(loopback_ephemeral())) + .with_quic_config(test_quic_options()); + #[cfg(encryption)] + { + opts = opts.with_encryption(self.encryption); + } + Serf::new::, SocketAddrResolver, FirstAddrResolver, _, _>( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + self.delegate, + self.runtime, + self.serf, + gossip_rng().expect("seed gossip rng"), + None, + self.merge, + self.snapshot, + #[cfg(encryption)] + self.keyring, + ) + .await + .expect("spawn serf quic node") + } +} + +/// Spawn a plain loopback QUIC node. +async fn spawn_node(id: &str) -> Serf { + NodeSpec::new().spawn(id).await +} + +/// Poll both nodes until each reports the full two-member cluster. +async fn converge(a: &Serf, b: &Serf) { + compio::time::timeout(WINDOW, async { + loop { + if a.num_members() == 2 && b.num_members() == 2 { + break; + } + compio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("both nodes converge to a 2-member cluster"); +} + +/// Join `joiner` to `seed` over QUIC and wait for both to converge. +async fn join_and_converge(joiner: &Serf, seed: &Serf) { + joiner + .join( + &SocketAddrResolver, + MaybeResolved::Resolved(seed.advertise_address()), + false, + ) + .await + .expect("join reaches the seed over QUIC"); + converge(joiner, seed).await; +} + +/// Drive `events` until the next inbound `Event::Query` named `name` arrives, +/// returning its response token. +async fn next_query_token(events: &mut S, name: &str) -> QueryEvent +where + S: futures_util::Stream> + Unpin, +{ + compio::time::timeout(WINDOW, async { + loop { + match events.next().await { + Some(Event::Query(qe)) if qe.name() == name => break qe, + Some(_) => {} + None => panic!("the event stream closed before the query arrived"), + } + } + }) + .await + .expect("the query reaches the responder within the window") +} + +/// Assert a command's reply is the teardown `Shutdown` error. +fn expect_shutdown(res: Result) +where + T: core::fmt::Debug, +{ + match res { + Err(SerfError::Shutdown) => {} + Err(other) => panic!("expected Shutdown, got {other:?}"), + Ok(v) => panic!("expected Shutdown, got Ok({v:?})"), + } +} + +/// A deterministic test secret key, selecting whichever AEAD cipher this build +/// compiled. +#[cfg(encryption)] +fn test_secret_key(fill: u8) -> SecretKey { + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes256([fill; 32]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = SecretKey::ChaCha20Poly1305([fill; 32]); + key +} + +// ── scenarios ───────────────────────────────────────────────────────────────── + +/// A user event broadcast by B over the QUIC datagram gossip plane reaches A's +/// event stream with the original name and payload, and fires A's +/// `notify_user_event` observation hook. +#[compio::test] +async fn a_quic_user_event_reaches_the_peer_stream_and_delegate() { + let seen = Rc::new(Observed::default()); + let a = NodeSpec::new() + .with_delegate(RecordingDelegate(seen.clone())) + .spawn("que-a") + .await; + let b = spawn_node("que-b").await; + + let mut a_events = a.events(); + join_and_converge(&a, &b).await; + + let payload = Bytes::from_static(b"quic-deploy"); + b.user_event("deploy", payload.clone(), false) + .await + .expect("B broadcasts a user event"); + + let got = compio::time::timeout(WINDOW, async { + loop { + match a_events.next().await { + Some(Event::User(u)) if u.name.as_str() == "deploy" => break u.payload.clone(), + Some(_) => {} + None => panic!("A's event stream closed before the user event arrived"), + } + } + }) + .await + .expect("A receives B's user event within the window"); + assert_eq!(got, payload, "the payload survives the QUIC broadcast"); + + assert!( + seen.user_events.borrow().iter().any(|n| n == "deploy"), + "the QUIC driver fired A's notify_user_event hook" + ); + + a.shutdown().await.expect("que-a shuts down"); + b.shutdown().await.expect("que-b shuts down"); +} + +/// A query issued by A over QUIC round-trips: B surfaces the inbound +/// `Event::Query` (firing its `notify_query` hook), answers through +/// `Serf::respond`, and A surfaces the matching `Event::QueryResponse`. +#[compio::test] +async fn a_quic_query_round_trips_through_respond() { + let seen = Rc::new(Observed::default()); + let b = NodeSpec::new() + .with_delegate(RecordingDelegate(seen.clone())) + .spawn("qq-b") + .await; + let a = spawn_node("qq-a").await; + + let mut b_events = b.events(); + let mut a_events = a.events(); + join_and_converge(&a, &b).await; + + let want = Bytes::from_static(b"quic-pong"); + a.query( + "ping", + Bytes::from_static(b"quic-ping"), + a.default_query_param(), + ) + .await + .expect("query issued"); + + let responder = async { + let token = next_query_token(&mut b_events, "ping").await; + assert_eq!( + token.payload(), + &Bytes::from_static(b"quic-ping"), + "the inbound query carries the originator's payload" + ); + b.respond(token, want.clone()) + .await + .expect("B responds to the query"); + }; + let collector = async { + loop { + match a_events.next().await { + Some(Event::QueryResponse(qr)) if qr.payload() == &want => break true, + Some(_) => {} + None => break false, + } + } + }; + + let got = compio::time::timeout(WINDOW, async { + let (_, got) = future::join(responder, collector).await; + got + }) + .await + .expect("the QUIC query round-trip completes within the window"); + assert!(got, "A must receive B's query response over QUIC"); + + assert_eq!( + seen.queries.borrow().as_slice(), + &[SmolStr::new("ping")], + "the QUIC driver fired B's notify_query hook exactly once" + ); + + a.shutdown().await.expect("qq-a shuts down"); + b.shutdown().await.expect("qq-b shuts down"); +} + +/// `set_tags` over QUIC re-tags the local node and propagates: the peer's view +/// carries the new tag and its `notify_update` hook fires. +#[compio::test] +async fn quic_set_tags_propagates_as_a_member_update() { + let seen = Rc::new(Observed::default()); + let a = NodeSpec::new() + .with_delegate(RecordingDelegate(seen.clone())) + .spawn("qt-a") + .await; + let b = spawn_node("qt-b").await; + + join_and_converge(&a, &b).await; + + let mut tags = Tags::new(); + tags.0.insert(SmolStr::new("role"), SmolStr::new("worker")); + b.set_tags(tags).await.expect("B re-tags itself"); + + let got = compio::time::timeout(WINDOW, async { + loop { + let seen_tag = a + .members() + .iter() + .find(|m| m.node().id_ref().as_str() == "qt-b") + .and_then(|m| m.tags().0.get("role").cloned()); + if let Some(v) = seen_tag { + break v; + } + compio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("A observes B's new tag within the window"); + assert_eq!(got.as_str(), "worker", "A's view of B carries the new tag"); + + assert!( + seen.updated.borrow().iter().any(|id| id == "qt-b"), + "the QUIC driver fired A's notify_update hook for the re-tagged peer" + ); + + a.shutdown().await.expect("qt-a shuts down"); + b.shutdown().await.expect("qt-b shuts down"); +} + +/// A graceful `leave()` over QUIC resolves only once the machine's `LeftCluster` +/// fires: the event surfaces on the leaver's own stream and its endpoint settles +/// at `Left`. Two racing `leave()` callers share ONE in-flight operation — the +/// second joins the first's waiter rather than re-invoking the machine's +/// terminal `leave()` (which emits no second `LeftCluster`). +#[compio::test] +async fn a_quic_graceful_leave_emits_left_cluster_once_for_every_caller() { + let b = spawn_node("qlv-b").await; + let a = spawn_node("qlv-a").await; + + let mut a_events = a.events(); + join_and_converge(&a, &b).await; + + let (first, second) = compio::time::timeout(WINDOW, future::join(a.leave(), a.leave())) + .await + .expect("both racing leaves resolve within the window"); + first.expect("the initiating leave resolves Ok"); + second.expect("the leave that joined the in-flight operation resolves Ok"); + + let saw = compio::time::timeout(WINDOW, async { + loop { + match a_events.next().await { + Some(Event::LeftCluster) => break true, + Some(_) => {} + None => break false, + } + } + }) + .await + .expect("A observes LeftCluster within the window"); + assert!(saw, "A must surface Event::LeftCluster after leave()"); + + compio::time::timeout(WINDOW, async { + loop { + if a.state() == SerfState::Left { + break; + } + compio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("A's endpoint state becomes Left"); + + a.shutdown().await.expect("qlv-a shuts down"); + b.shutdown().await.expect("qlv-b shuts down"); +} + +/// Once a QUIC node has left, every mutating command it is handed reports +/// [`SerfError::NotRunning`] rather than being applied to a non-participating +/// endpoint — while a repeat `leave()` stays idempotent and the read-only +/// coordinate probe still answers. +#[compio::test] +async fn quic_commands_after_leave_report_not_running() { + let a = spawn_node("qnr-a").await; + let b = spawn_node("qnr-b").await; + + let mut a_events = a.events(); + join_and_converge(&a, &b).await; + + b.query("probe", Bytes::new(), b.default_query_param()) + .await + .expect("B issues a query"); + let token = next_query_token(&mut a_events, "probe").await; + + a.leave().await.expect("A leaves the cluster"); + + assert!( + matches!( + a.join( + &SocketAddrResolver, + MaybeResolved::Resolved(b.advertise_address()), + false + ) + .await, + Err(SerfError::NotRunning) + ), + "join after leave must be refused" + ); + assert!( + matches!( + a.force_leave(SmolStr::new("qnr-b"), false).await, + Err(SerfError::NotRunning) + ), + "force_leave after leave must be refused" + ); + assert!( + matches!( + a.user_event("evt", Bytes::new(), false).await, + Err(SerfError::NotRunning) + ), + "user_event after leave must be refused" + ); + assert!( + matches!( + a.query("q", Bytes::new(), a.default_query_param()).await, + Err(SerfError::NotRunning) + ), + "query after leave must be refused" + ); + assert!( + matches!( + a.respond(token, Bytes::new()).await, + Err(SerfError::NotRunning) + ), + "respond after leave must be refused" + ); + assert!( + matches!(a.set_tags(Tags::new()).await, Err(SerfError::NotRunning)), + "set_tags after leave must be refused" + ); + #[cfg(encryption)] + { + let key = test_secret_key(0x5a); + assert!( + matches!(a.install_key(key).await, Err(SerfError::NotRunning)), + "install_key after leave must be refused" + ); + assert!( + matches!(a.use_key(key).await, Err(SerfError::NotRunning)), + "use_key after leave must be refused" + ); + assert!( + matches!(a.remove_key(key).await, Err(SerfError::NotRunning)), + "remove_key after leave must be refused" + ); + assert!( + matches!(a.list_keys().await, Err(SerfError::NotRunning)), + "list_keys after leave must be refused" + ); + } + + compio::time::timeout(WINDOW, a.leave()) + .await + .expect("the repeat leave resolves rather than parking") + .expect("a repeat leave is idempotent"); + + #[cfg(feature = "coordinates")] + a.cached_coordinate(SmolStr::new("qnr-b")) + .await + .expect("the coordinate cache answers after leave"); + + a.shutdown().await.expect("qnr-a shuts down"); + b.shutdown().await.expect("qnr-b shuts down"); +} + +/// Commands still queued behind a `Shutdown` when the QUIC pump breaks are +/// ANSWERED with [`SerfError::Shutdown`] at teardown, never dropped: a caller's +/// reply receiver must not hang forever because the driver exited between its +/// send and its dispatch. Every command variant is queued behind the shutdown in +/// one batch, so each teardown reply arm is exercised. +#[compio::test] +async fn quic_commands_queued_behind_a_shutdown_are_answered_not_dropped() { + let a = spawn_node("qtd-a").await; + let b = spawn_node("qtd-b").await; + + let mut b_events = b.events(); + join_and_converge(&a, &b).await; + + a.query("probe", Bytes::new(), a.default_query_param()) + .await + .expect("A issues a query"); + let token = next_query_token(&mut b_events, "probe").await; + + let a_addr = a.advertise_address(); + // Declared ahead of `queued` so it outlives the boxed futures that borrow it. + #[cfg(encryption)] + let key = test_secret_key(0x6b); + let mut queued: Vec>>> = Vec::new(); + // Polled first, so `Shutdown` is the head of the command queue and every + // command pushed after it lands behind it. + queued.push(Box::pin(async { + b.shutdown().await.expect("the shutdown itself is acked"); + })); + queued.push(Box::pin(async { + expect_shutdown(b.user_event("evt", Bytes::new(), false).await); + })); + queued.push(Box::pin(async { + expect_shutdown(b.query("q", Bytes::new(), b.default_query_param()).await); + })); + queued.push(Box::pin(async { + expect_shutdown(b.respond(token, Bytes::new()).await); + })); + queued.push(Box::pin(async { + expect_shutdown(b.set_tags(Tags::new()).await); + })); + queued.push(Box::pin(async { + expect_shutdown(b.force_leave(SmolStr::new("qtd-a"), false).await); + })); + queued.push(Box::pin(async { + expect_shutdown(b.leave().await); + })); + queued.push(Box::pin(async { + expect_shutdown( + b.join(&SocketAddrResolver, MaybeResolved::Resolved(a_addr), false) + .await, + ); + })); + queued.push(Box::pin(async { + expect_shutdown( + b.dispatch_join(&SocketAddrResolver, &[MaybeResolved::Resolved(a_addr)]) + .await, + ); + })); + #[cfg(encryption)] + { + queued.push(Box::pin(async { + expect_shutdown(b.install_key(key).await); + })); + queued.push(Box::pin(async { + expect_shutdown(b.use_key(key).await); + })); + queued.push(Box::pin(async { + expect_shutdown(b.remove_key(key).await); + })); + queued.push(Box::pin(async { + expect_shutdown(b.list_keys().await); + })); + } + #[cfg(feature = "coordinates")] + queued.push(Box::pin(async { + expect_shutdown(b.cached_coordinate(SmolStr::new("qtd-a")).await); + })); + + compio::time::timeout(WINDOW, future::join_all(queued)) + .await + .expect("every command queued behind the shutdown is answered, none hang"); + + // A command issued AFTER the driver has torn down fails fast on the handle's + // shutdown flag rather than queueing into a dead channel. + assert!( + matches!( + b.user_event("late", Bytes::new(), false).await, + Err(SerfError::Shutdown) + ), + "a post-teardown command fails fast with Shutdown" + ); + + a.shutdown().await.expect("qtd-a shuts down"); +} + +/// A burst of commands issued in one batch is all applied: the pump's iter-top +/// fairness drain picks up the commands queued behind the one that woke it and +/// flushes their outputs before it re-enters the select, so every event of the +/// burst still reaches a subscriber. +#[compio::test] +async fn a_quic_command_burst_is_drained_and_flushed_in_one_pass() { + const BURST: usize = 16; + let a = spawn_node("qb-a").await; + let mut events = a.events(); + + let mut batch: Vec>>> = Vec::with_capacity(BURST); + for i in 0..BURST { + // Every clone shares the one driver task; cloning per future keeps each an + // owned handle so the whole batch can be polled in a single pass. + let handle = a.clone(); + batch.push(Box::pin(async move { + handle + .user_event(format!("burst-{i}"), Bytes::new(), false) + .await + .expect("user event dispatched"); + })); + } + compio::time::timeout(WINDOW, future::join_all(batch)) + .await + .expect("every command of the burst is applied"); + + let delivered = compio::time::timeout(WINDOW, async { + let mut n = 0usize; + while n < BURST { + match events.next().await { + Some(Event::User(_)) => n += 1, + Some(_) => {} + None => panic!("the event stream closed mid-burst"), + } + } + n + }) + .await + .expect("every event of the command burst surfaces"); + assert_eq!(delivered, BURST, "no command of the burst is lost"); + + a.shutdown().await.expect("qb-a shuts down"); +} + +/// With an UNBOUNDED observation channel the QUIC driver opts out of shedding +/// entirely: a stalling delegate cannot make the pump drop a single event. +#[compio::test] +async fn a_quic_unbounded_observation_channel_sheds_nothing() { + const BURST: u32 = 24; + let a = NodeSpec::new() + .with_delegate(StallingDelegate { + stall: Duration::from_millis(2), + }) + .with_runtime(RuntimeOptions::new().with_observation_channel(Channel::Unbounded)) + .spawn("qunb-a") + .await; + + let mut events = a.events(); + for i in 0..BURST { + a.user_event(format!("burst-{i}"), Bytes::new(), false) + .await + .expect("user event dispatched"); + } + + let delivered = compio::time::timeout(WINDOW, async { + let mut n = 0u32; + while n < BURST { + match events.next().await { + Some(Event::User(_)) => n += 1, + Some(_) => {} + None => panic!("the event stream closed mid-burst"), + } + } + n + }) + .await + .expect("every event of the burst is delivered under an unbounded observation channel"); + + assert_eq!(delivered, BURST, "no event of the burst is shed"); + assert_eq!( + a.observation_dropped(), + 0, + "an unbounded observation channel never drops" + ); + + a.shutdown().await.expect("qunb-a shuts down"); +} + +/// A cap-1 observation channel behind a delegate that parks on every user event +/// makes the QUIC pump shed: the enqueue retries once (yielding to the +/// observation task) and then drops and counts, rather than blocking the FSM on +/// a full queue. +#[compio::test] +async fn a_stalled_delegate_makes_the_quic_pump_shed() { + const BURST: u32 = 32; + let a = NodeSpec::new() + .with_delegate(StallingDelegate { + stall: Duration::from_secs(30), + }) + .with_runtime(RuntimeOptions::new().with_observation_channel(Channel::Bounded(1))) + .spawn("qshed-a") + .await; + + for i in 0..BURST { + a.user_event(format!("shed-{i}"), Bytes::new(), false) + .await + .expect("user event dispatched"); + } + + let dropped = compio::time::timeout(WINDOW, async { + loop { + let n = a.observation_dropped(); + if n > 0 { + break n; + } + compio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("a stalled delegate on a cap-1 observation channel must make the QUIC pump shed"); + + assert!( + dropped > 0, + "the pump drops and counts events the stalled observation task cannot take (got {dropped})" + ); + + a.shutdown().await.expect("qshed-a shuts down"); +} + +/// With a cap-1 event queue and nobody draining the stream, the `events_dropped` +/// counter on the handle becomes non-zero: the observation task's forward to +/// subscribers is best-effort and counts what it sheds rather than blocking. +#[compio::test] +async fn a_full_quic_event_queue_counts_what_it_sheds() { + let a = NodeSpec::new() + .with_runtime(RuntimeOptions::new().with_event_queue_cap(1)) + .spawn("qdrop-a") + .await; + + // The stream is subscribed but never polled, so its cap-1 slot fills on the + // first event and every later delivery is shed and counted. + let _events = a.events(); + for i in 0..16u32 { + a.user_event(format!("drop-{i}"), Bytes::new(), false) + .await + .expect("user event dispatched"); + } + + let dropped = compio::time::timeout(WINDOW, async { + loop { + let n = a.events_dropped(); + if n > 0 { + break n; + } + compio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("an undrained cap-1 event queue must shed and count"); + assert!( + dropped > 0, + "events_dropped counts every event the full stream channel rejected (got {dropped})" + ); + + a.shutdown().await.expect("qdrop-a shuts down"); +} + +/// A fire-and-forget `dispatch_join` parks no await-result waiter: its exchange +/// still terminalizes cleanly through the pump's completion path (no waiter to +/// resolve), the membership converges from the dispatched push/pull alone, and +/// the constructor-supplied merge delegate is consulted on the seed side. +#[compio::test] +async fn a_quic_dispatch_join_converges_without_a_join_waiter() { + let peers: MergedPeers = Arc::new(std::sync::Mutex::new(Vec::new())); + let b = NodeSpec::new() + .with_merge(Box::new(RecordingMerge { + peers: peers.clone(), + })) + .spawn("qdj-b") + .await; + let a = spawn_node("qdj-a").await; + + let dispatched = a + .dispatch_join( + &SocketAddrResolver, + &[MaybeResolved::Resolved(b.advertise_address())], + ) + .await + .expect("the QUIC dial is dispatched"); + assert_eq!(dispatched, 1, "exactly one seed was dispatched"); + + converge(&a, &b).await; + + let merged = peers.lock().expect("merge record lock").clone(); + assert!( + merged.iter().any(|id| id == "qdj-a"), + "the QUIC push/pull consulted the installed merge delegate with the joining peer's state \ + (saw {merged:?})" + ); + + a.shutdown().await.expect("qdj-a shuts down"); + b.shutdown().await.expect("qdj-b shuts down"); +} + +/// Dropping the last handle of a quiet QUIC node tears the driver down through +/// the command-channel disconnect observed in the main select, releasing the +/// bound UDP port for an immediate rebind. +#[compio::test] +async fn dropping_the_last_quic_handle_releases_the_bound_port() { + let a = spawn_node("qdrp-a").await; + let addr = a.advertise_address(); + + // Let the pump settle into its select before the disconnect, so the drop is + // observed by the select's command arm rather than the iter-top drain. + compio::time::sleep(Duration::from_millis(200)).await; + drop(a); + + let gossip = compio::time::timeout(WINDOW, async { + loop { + if let Ok(sock) = compio::net::UdpSocket::bind(addr).await { + break sock; + } + compio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("the QUIC driver releases its bound UDP port after the last handle drops"); + // Ignoring Err: test cleanup of the rebind probe socket. + let _ = gossip.close().await; +} + +/// A QUIC node persisting to a snapshot writes its membership records, and a +/// fresh node replaying the SAME file rejoins the cluster with no explicit join +/// call — the constructor's snapshot argument is threaded through the QUIC `run` +/// into the endpoint's replay. +#[compio::test] +async fn a_quic_snapshot_replays_the_membership_on_restart() { + let mut path = std::env::temp_dir(); + path.push(format!("serf-compio-quic-snap-{}", std::process::id())); + // Ignoring Err: a leftover file from a previous run is fine to lose. + let _ = std::fs::remove_file(&path); + + let a = spawn_node("qsnap-a").await; + let b = NodeSpec::new() + .with_snapshot(SnapshotOptions::new(&path).with_compact_threshold(1)) + .spawn("qsnap-b") + .await; + + join_and_converge(&b, &a).await; + b.shutdown().await.expect("qsnap-b shuts down"); + drop(b); + + let bytes = std::fs::read(&path).expect("the QUIC run wrote the snapshot"); + assert!( + !bytes.is_empty(), + "compaction rewrites the live membership rather than truncating the file" + ); + + let b2 = NodeSpec::new() + .with_snapshot(SnapshotOptions::new(&path).with_compact_threshold(1)) + .spawn("qsnap-b") + .await; + converge(&a, &b2).await; + assert_eq!( + b2.num_members(), + 2, + "the restarted QUIC node recovers its membership from the snapshot" + ); + + a.shutdown().await.expect("qsnap-a shuts down"); + b2.shutdown().await.expect("qsnap-b2 shuts down"); + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} + +/// An inbound key-management request on a node with NO keyring configured is +/// answered with a failure result rather than silently ignored: the originator's +/// `KeyResponse` counts the refusal, so an operator sees the plaintext node +/// instead of a hung key query. +#[cfg(encryption)] +#[compio::test] +async fn a_key_request_on_a_plaintext_quic_node_is_refused() { + let b = spawn_node("qkp-b").await; + let a = spawn_node("qkp-a").await; + + let mut a_events = a.events(); + join_and_converge(&a, &b).await; + + a.install_key(test_secret_key(0x77)) + .await + .expect("install_key dispatched"); + let kr = next_key_response(&mut a_events).await; + + assert!( + kr.num_err >= 1, + "a node with no keyring must refuse the key op (num_err={}, messages={:?})", + kr.num_err, + kr.messages + ); + assert!( + kr.messages + .values() + .any(|m| m.contains("no keyring configured")), + "the refusal names the missing keyring (messages={:?})", + kr.messages + ); + + a.shutdown().await.expect("qkp-a shuts down"); + b.shutdown().await.expect("qkp-b shuts down"); +} + +/// Drive `events` until the originator's next `KeyResponse` surfaces (emitted +/// when the key query's deadline fires), draining any interleaved membership / +/// gossip events so a backlog cannot stall the stream. +#[cfg(encryption)] +async fn next_key_response(events: &mut S) -> serf_proto::event::KeyResponse +where + S: futures_util::Stream> + Unpin, +{ + compio::time::timeout(WINDOW, async { + loop { + match events.next().await { + Some(Event::KeyResponse(kr)) => break kr, + Some(_) => {} + None => panic!("the event stream closed before a KeyResponse"), + } + } + }) + .await + .expect("a KeyResponse within the window") +} + +/// A [`KeyringDelegate`] that records, in order, every live keyring the driver +/// publishes through `keyring_updated`. The driver fires it only after it has +/// pushed the rotated ring to the endpoint, so the recorded ring is exactly the +/// ring the gossip plane now encrypts under. +#[cfg(encryption)] +#[derive(Default)] +struct RecordingKeyring { + rings: RefCell>, +} + +#[cfg(encryption)] +impl RecordingKeyring { + fn rings(&self) -> Vec { + self.rings.borrow().clone() + } +} + +#[cfg(encryption)] +impl KeyringDelegate for RecordingKeyring { + fn keyring_updated(&self, keyring: &Keyring) -> serf_driver::KeyringPersistence { + self.rings.borrow_mut().push(keyring.clone()); + serf_driver::KeyringPersistence::Durable + } +} + +/// Two encrypted QUIC nodes share primary K1, then A rotates the cluster to K2 +/// via install -> use -> remove. Each op propagates over the encrypted gossip +/// plane and every node applies it to its LIVE wire keyring: both observers +/// record the exact ring sequence, a cluster-wide `list_keys` confirms K2 is the +/// primary everywhere and K1 is installed nowhere, the read-only `list` and the +/// refused repeat-remove fire no observer, and a user event still crosses the +/// wire afterwards — proving the datagram plane now runs under K2. +#[cfg(encryption)] +#[compio::test] +async fn quic_key_rotation_rotates_both_live_keyrings() { + let k1 = test_secret_key(0x11); + let k2 = test_secret_key(0x22); + + let rec_a = Rc::new(RecordingKeyring::default()); + let rec_b = Rc::new(RecordingKeyring::default()); + let enc = || EncryptionOptions::new().with_keyring(Keyring::new(k1)); + + let b = NodeSpec::new() + .with_encryption(enc()) + .with_keyring(rec_b.clone()) + .spawn("qrot-b") + .await; + let a = NodeSpec::new() + .with_encryption(enc()) + .with_keyring(rec_a.clone()) + .spawn("qrot-a") + .await; + + join_and_converge(&a, &b).await; + + // The published snapshot carries the coordinator's live encryption flag, so a + // keyring that never reached the QUIC coordinator would read back `false` here. + assert!( + a.encryption_enabled(), + "the keyring reaches the QUIC coordinator and surfaces on the handle" + ); + assert!(b.encryption_enabled(), "both nodes gossip under a keyring"); + + let mut a_events = a.events(); + let mut b_events = b.events(); + + a.install_key(k2).await.expect("install_key dispatched"); + let kr = next_key_response(&mut a_events).await; + assert!( + kr.num_resp >= 2, + "install_key must collect a response from BOTH nodes (num_resp={})", + kr.num_resp + ); + assert_eq!(kr.num_err, 0, "install_key must succeed on every node"); + + a.use_key(k2).await.expect("use_key dispatched"); + let kr = next_key_response(&mut a_events).await; + assert_eq!(kr.num_err, 0, "use_key must succeed on every node"); + + a.remove_key(k1).await.expect("remove_key dispatched"); + let kr = next_key_response(&mut a_events).await; + assert_eq!(kr.num_err, 0, "remove_key must succeed on every node"); + + for (name, rec) in [("qrot-a", &rec_a), ("qrot-b", &rec_b)] { + let rings = rec.rings(); + assert_eq!( + rings.len(), + 3, + "{name}: the observer fires exactly once per successful mutation" + ); + assert_eq!( + rings[0].primary_ref(), + &k1, + "{name}: install leaves K1 as primary" + ); + assert!( + rings[0].secondaries().contains(&k2), + "{name}: install adds K2 as a secondary" + ); + assert_eq!(rings[1].primary_ref(), &k2, "{name}: use promotes K2"); + assert_eq!( + rings[2].primary_ref(), + &k2, + "{name}: K2 stays primary after the remove" + ); + assert!( + !rings[2].secondaries().contains(&k1), + "{name}: the removed K1 is absent from the live keyring" + ); + } + + a.list_keys().await.expect("list_keys dispatched"); + let kr = next_key_response(&mut a_events).await; + assert_eq!( + kr.primary_keys.get(&k2).copied(), + Some(2), + "both nodes report K2 as their live primary" + ); + assert_eq!( + kr.keys.get(&k1), + None, + "the removed K1 is installed on no node" + ); + assert_eq!( + rec_a.rings().len(), + 3, + "list_keys is read-only and does not fire the keyring observer" + ); + + a.remove_key(k1).await.expect("remove_key dispatched"); + let kr = next_key_response(&mut a_events).await; + assert!( + kr.num_err >= 1, + "removing an already-absent key is refused (num_err={})", + kr.num_err + ); + assert_eq!( + rec_b.rings().len(), + 3, + "a refused op does not fire the keyring observer" + ); + + // Post-rotation traffic proof: a user event still crosses the encrypted + // datagram plane, which now runs under K2 on both nodes. + a.user_event("after-rotation", Bytes::from_static(b"payload"), false) + .await + .expect("user_event from a running node"); + let saw = compio::time::timeout(WINDOW, async { + loop { + match b_events.next().await { + Some(Event::User(u)) if u.name.as_str() == "after-rotation" => break true, + Some(_) => {} + None => break false, + } + } + }) + .await + .expect("B observes the post-rotation user event within the window"); + assert!( + saw, + "a user event must still propagate A -> B after the rotation (wire under K2)" + ); + + a.shutdown().await.expect("qrot-a shuts down"); + b.shutdown().await.expect("qrot-b shuts down"); +} + +/// The sending half of the driver's rotation-durability acknowledgement channel +/// ([`serf_driver::KeyringPersistRx`] is its receiver). +#[cfg(encryption)] +type PersistTx = std::sync::mpsc::Sender>; + +/// A keyring delegate whose persistence resolves OUT OF BAND: `keyring_updated` +/// hands back a pending receiver, and the test releases it after a delay. The +/// pump must park the key response until the acknowledgement lands and only then +/// route it — so the originator still collects BOTH nodes' successes. +#[cfg(encryption)] +#[derive(Default)] +struct DeferredKeyring { + /// Acknowledgement senders for every rotation this delegate parked, in order. + parked: RefCell>, +} + +#[cfg(encryption)] +impl DeferredKeyring { + /// Acknowledge every parked rotation as durable. + fn release_all(&self) { + for tx in self.parked.borrow_mut().drain(..) { + // Ignoring Err: the pump dropped the receiver (its key request already + // timed out); nothing to acknowledge. + let _ = tx.send(Ok(())); + } + } +} + +#[cfg(encryption)] +impl KeyringDelegate for DeferredKeyring { + fn keyring_updated(&self, _keyring: &Keyring) -> serf_driver::KeyringPersistence { + let (tx, rx) = std::sync::mpsc::channel(); + self.parked.borrow_mut().push(tx); + serf_driver::KeyringPersistence::Pending(rx) + } +} + +/// With node B parking its rotation on out-of-band persistence, an +/// `install_key` from A still collects BOTH nodes' successful responses: B's +/// response is held until the acknowledgement resolves and is then routed inside +/// the query window. A driver that answered ahead of the acknowledgement — or +/// dropped the parked response — would fail this. +#[cfg(encryption)] +#[compio::test] +async fn a_parked_key_response_is_routed_once_persistence_acknowledges() { + let k1 = test_secret_key(0x33); + let k2 = test_secret_key(0x44); + let enc = || EncryptionOptions::new().with_keyring(Keyring::new(k1)); + + let deferred = Rc::new(DeferredKeyring::default()); + let b = NodeSpec::new() + .with_encryption(enc()) + .with_keyring(deferred.clone()) + .spawn("qdef-b") + .await; + // A keeps the DEFAULT keyring delegate: its own live-ring rotation still + // applies, it simply persists nothing and answers durable-inline. + let a = NodeSpec::new().with_encryption(enc()).spawn("qdef-a").await; + + join_and_converge(&a, &b).await; + + let mut a_events = a.events(); + a.install_key(k2).await.expect("install_key dispatched"); + + // Release B's parked acknowledgement shortly after the rotation lands, well + // inside the key query's response window. + let releaser = deferred.clone(); + compio::runtime::spawn(async move { + compio::time::sleep(Duration::from_millis(300)).await; + releaser.release_all(); + }) + .detach(); + + let kr = next_key_response(&mut a_events).await; + assert!( + kr.num_resp >= 2, + "install_key must collect a response from BOTH nodes, including the one parked on \ + out-of-band persistence (num_resp={})", + kr.num_resp + ); + assert_eq!(kr.num_err, 0, "install_key must succeed on every node"); + + a.shutdown().await.expect("qdef-a shuts down"); + b.shutdown().await.expect("qdef-b shuts down"); +} + +/// The `QuicTransportOptions` accessors reflect exactly what the builders set, +/// and `Default` is the `new()` state: every required field the constructor's +/// guards check is unset. +#[test] +fn quic_transport_options_accessors_reflect_builders() { + let addr: SocketAddr = "127.0.0.1:7946".parse().expect("loopback addr"); + + let empty = QuicTransportOptions::::default(); + assert!(empty.local_id().is_none(), "Default leaves local_id unset"); + assert!( + empty.advertise_addr().is_none(), + "Default leaves advertise_addr unset" + ); + assert!( + empty.quic_config().is_none(), + "Default leaves the quic config unset" + ); + + let opts = QuicTransportOptions::::new() + .with_local_id(SmolStr::new("acc-node")) + .with_advertise_addr(MaybeResolved::Resolved(addr)) + .with_quic_config(test_quic_options()); + assert_eq!(opts.local_id().map(SmolStr::as_str), Some("acc-node")); + match opts.advertise_addr() { + Some(MaybeResolved::Resolved(s)) => assert_eq!(*s, addr), + other => panic!("expected a resolved advertise addr, got {other:?}"), + } + assert!( + opts.quic_config().is_some(), + "with_quic_config installs the caller's bundle" + ); + + #[cfg(encryption)] + { + assert!( + opts.encryption().keyring().is_none(), + "the default gossip-encryption policy carries no keyring" + ); + let key = test_secret_key(0x31); + let encrypted = QuicTransportOptions::::new() + .with_encryption(EncryptionOptions::new().with_keyring(Keyring::new(key))); + assert_eq!( + encrypted + .encryption() + .keyring() + .expect("the installed keyring is readable back") + .primary_ref(), + &key, + "with_encryption installs the caller's gossip keyring" + ); + } +} + +/// `QuicTransport::new` refuses each required field it cannot default, naming +/// the missing one, before binding a socket. +#[compio::test] +async fn quic_new_requires_a_local_id_an_advertise_addr_and_a_config() { + let no_id = QuicTransportOptions::::new() + .with_advertise_addr(MaybeResolved::Resolved(loopback_ephemeral())) + .with_quic_config(test_quic_options()); + assert_missing_field( + QuicTransport::::new(no_id, &SocketAddrResolver, &FirstAddrResolver).await, + "local_id", + ); + + let no_addr = QuicTransportOptions::::new() + .with_local_id(SmolStr::new("no-addr")) + .with_quic_config(test_quic_options()); + assert_missing_field( + QuicTransport::::new(no_addr, &SocketAddrResolver, &FirstAddrResolver) + .await, + "advertise_addr", + ); + + let no_cfg = QuicTransportOptions::::new() + .with_local_id(SmolStr::new("no-cfg")) + .with_advertise_addr(MaybeResolved::Resolved(loopback_ephemeral())); + assert_missing_field( + QuicTransport::::new(no_cfg, &SocketAddrResolver, &FirstAddrResolver) + .await, + "quic_config", + ); +} + +/// Assert a transport construction was refused with `InvalidInput` naming the +/// required field the caller left unset. +fn assert_missing_field(res: Result, field: &str) { + match res { + Err(SerfError::Io(e)) => { + assert_eq!( + e.kind(), + std::io::ErrorKind::InvalidInput, + "a missing required field is an InvalidInput refusal" + ); + assert!( + e.to_string().contains(field), + "the refusal names the missing field {field:?}, got {e}" + ); + } + Err(other) => panic!("expected InvalidInput({field}), got {other:?}"), + Ok(_) => panic!("a missing {field} must be refused, but construction succeeded"), + } +} + +/// An advertise address the caller supplied UNRESOLVED must not silently bind a +/// wrong contact when resolution cannot answer: a resolver outage surfaces as +/// `SerfError::Resolve`, and a resolution that yields ZERO candidates is refused +/// by the advertise picker rather than defaulted. +#[compio::test] +async fn quic_new_refuses_an_advertise_address_it_cannot_resolve() { + struct FailingResolver; + + impl Resolver for FailingResolver { + type Address = SocketAddr; + type Error = std::io::Error; + + async fn resolve(&self, _addr: &SocketAddr) -> Result, std::io::Error> { + Err(std::io::Error::other("discovery backend unavailable")) + } + } + + struct EmptyResolver; + + impl Resolver for EmptyResolver { + type Address = SocketAddr; + type Error = std::io::Error; + + async fn resolve(&self, _addr: &SocketAddr) -> Result, std::io::Error> { + Ok(Vec::new()) + } + } + + let opts = QuicTransportOptions::::new() + .with_local_id(SmolStr::new("quic-res-fail")) + .with_advertise_addr(MaybeResolved::Unresolved(loopback_ephemeral())) + .with_quic_config(test_quic_options()); + match QuicTransport::::new(opts, &FailingResolver, &FirstAddrResolver).await + { + Err(SerfError::Resolve(e)) => assert!( + e.to_string().contains("discovery backend unavailable"), + "the resolver's own error is surfaced, got {e}" + ), + Err(other) => panic!("expected Resolve, got {other:?}"), + Ok(_) => panic!("a resolver outage must refuse construction"), + } + + let opts = QuicTransportOptions::::new() + .with_local_id(SmolStr::new("quic-res-empty")) + .with_advertise_addr(MaybeResolved::Unresolved(loopback_ephemeral())) + .with_quic_config(test_quic_options()); + match QuicTransport::::new(opts, &EmptyResolver, &FirstAddrResolver).await { + Err(SerfError::Resolve(e)) => assert_eq!( + e.kind(), + std::io::ErrorKind::AddrNotAvailable, + "a zero-candidate resolution is an unavailable advertise address" + ), + Err(other) => panic!("expected Resolve(AddrNotAvailable), got {other:?}"), + Ok(_) => panic!("a zero-candidate resolution must refuse construction"), + } +} + +/// An UNRESOLVED advertise address is resolved through the caller's `Resolver` +/// and NARROWED by the `AdvertiseAddrResolver`: the resolver offers an IPv6 and +/// an IPv4 candidate, the IPv4-preferring picker chooses the IPv4 one, and the +/// QUIC transport binds THAT address and retains the unresolved input form. +#[compio::test] +async fn quic_new_resolves_and_narrows_an_unresolved_advertise_addr() { + let input: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let opts = QuicTransportOptions::::new() + .with_local_id(SmolStr::new("quic-unres")) + .with_advertise_addr(MaybeResolved::Unresolved(input)) + .with_quic_config(test_quic_options()); + let transport = + QuicTransport::::new(opts, &DualStackResolver, &Ipv4PreferringResolver) + .await + .expect("an unresolved advertise address resolves through the resolver"); + + assert_eq!(transport.local_id().as_str(), "quic-unres"); + let bound = *transport.advertise_address(); + assert!( + bound.is_ipv4(), + "the IPv4-preferring picker narrowed the dual-stack candidate set, got {bound}" + ); + assert!(bound.ip().is_loopback(), "the picked candidate was bound"); + assert_ne!( + bound.port(), + 0, + "the ephemeral port is read back concretely" + ); + match transport.local_address() { + MaybeResolved::Unresolved(a) => { + assert_eq!(*a, input, "the unresolved input form is retained") + } + other => panic!("expected the unresolved input form, got {other:?}"), + } +} diff --git a/serf-compio/tests/tcp.rs b/serf-compio/tests/tcp.rs new file mode 100644 index 00000000..f730eb19 --- /dev/null +++ b/serf-compio/tests/tcp.rs @@ -0,0 +1,1571 @@ +//! Real-node TCP serf tests: loopback nodes exercising the compio stream driver +//! end-to-end. Each test spins up ephemeral `127.0.0.1:0` nodes and drives the +//! full pump — join push/pull, coordinator merge, gossip, user events, queries, +//! graceful leave, force-leave, snapshot persistence, and teardown — on the +//! thread-per-core `!Send` compio runtime. +//! +//! The multi-node fault-injection scenarios run through the shared +//! [`cluster`] fixture; the single-purpose driver/handle contracts build their +//! nodes directly so they can vary one knob at a time. + +#![cfg(feature = "tcp")] + +use core::{future::Future, pin::Pin, time::Duration}; +use std::{cell::RefCell, net::SocketAddr, rc::Rc}; + +use bytes::Bytes; +use futures_util::{StreamExt, future}; +use memberlist_proto::MaybeResolved; +use serf_compio::{ + AdvertiseAddrResolver, AdvertiseResolutionError, Channel, Delegate, FirstAddrResolver, + Ipv4PreferringResolver, Ipv6PreferringResolver, MemberDelegate, OsResolver, QueryDelegate, + Resolver, RuntimeOptions, Serf, SerfError, SnapshotOptions, SocketAddrResolver, + StreamTransportOptions, TcpTransport, TcpTransportOptions, Transport, UserEventDelegate, + VoidDelegate, gossip_rng, +}; +use serf_proto::{ + Tags, UserEventMessage, + event::{Event, MemberEventKind, QueryEvent}, + members::{MemberStatus, SerfState}, + options::Options as SerfOptions, +}; +use smol_str::SmolStr; + +mod cluster; + +/// Bound on every convergence / delivery poll in this file, so a regression +/// surfaces as a timeout rather than a hang. +const WINDOW: Duration = Duration::from_secs(45); + +/// A loopback address with a port nothing listens on — `connect()` returns +/// `ECONNREFUSED` immediately. The port is below the OS ephemeral range, so a +/// `:0` test bind never collides with it. +fn blackhole_addr() -> SocketAddr { + "127.0.0.1:7215".parse().expect("loopback addr") +} + +/// A resolver whose every resolution fails — models a service-discovery backend +/// that is down. +struct FailingResolver; + +impl Resolver for FailingResolver { + type Address = String; + type Error = std::io::Error; + + async fn resolve(&self, _addr: &String) -> Result, std::io::Error> { + Err(std::io::Error::other("discovery backend unavailable")) + } +} + +/// A resolver that finds no live endpoints under the key it is given — models a +/// service-discovery backend whose service record is empty. +struct EmptyResolver; + +impl Resolver for EmptyResolver { + type Address = String; + type Error = std::io::Error; + + async fn resolve(&self, _addr: &String) -> Result, std::io::Error> { + Ok(Vec::new()) + } +} + +/// A resolver that answers with a dual-stack candidate set (IPv6 first, then +/// IPv4) on the port it was asked for — enough to drive the +/// `MaybeResolved::Unresolved` advertise path AND the advertise picker's +/// narrowing, without depending on the host's name resolution. +struct DualStackResolver; + +impl Resolver for DualStackResolver { + type Address = SocketAddr; + type Error = std::io::Error; + + async fn resolve(&self, addr: &SocketAddr) -> Result, std::io::Error> { + Ok(vec![ + SocketAddr::new("::1".parse().expect("v6 loopback"), addr.port()), + SocketAddr::new("127.0.0.1".parse().expect("v4 loopback"), addr.port()), + ]) + } +} + +/// Recorded observation-hook fan-out, shared between a [`RecordingDelegate`] +/// handed to the driver and the test that asserts on it. +#[derive(Default)] +struct Observed { + failed: RefCell>, + updated: RefCell>, + queries: RefCell>, + user_events: RefCell>, +} + +/// A [`Delegate`] that records which observation hooks the driver fired. +struct RecordingDelegate(Rc); + +impl MemberDelegate for RecordingDelegate { + type Id = SmolStr; + type Address = SocketAddr; + + async fn notify_failed( + &self, + member: std::sync::Arc>, + ) { + self + .0 + .failed + .borrow_mut() + .push(member.node().id_ref().clone()); + } + + async fn notify_update( + &self, + member: std::sync::Arc>, + ) { + self + .0 + .updated + .borrow_mut() + .push(member.node().id_ref().clone()); + } +} + +impl UserEventDelegate for RecordingDelegate { + async fn notify_user_event(&self, event: &UserEventMessage) { + self.0.user_events.borrow_mut().push(event.name.clone()); + } +} + +impl QueryDelegate for RecordingDelegate { + type Id = SmolStr; + type Address = SocketAddr; + + async fn notify_query(&self, event: &QueryEvent) { + self.0.queries.borrow_mut().push(SmolStr::new(event.name())); + } +} + +impl Delegate for RecordingDelegate { + type Id = SmolStr; + type Address = SocketAddr; +} + +/// A [`Delegate`] whose user-event hook parks for `stall`, so the driver's +/// observation task cannot drain its queue while the pump keeps enqueueing. +struct StallingDelegate { + stall: Duration, +} + +impl MemberDelegate for StallingDelegate { + type Id = SmolStr; + type Address = SocketAddr; +} + +impl UserEventDelegate for StallingDelegate { + async fn notify_user_event(&self, _event: &UserEventMessage) { + compio::time::sleep(self.stall).await; + } +} + +impl QueryDelegate for StallingDelegate { + type Id = SmolStr; + type Address = SocketAddr; +} + +impl Delegate for StallingDelegate { + type Id = SmolStr; + type Address = SocketAddr; +} + +/// Every knob a test node may vary, defaulted to the plain loopback node the +/// majority of scenarios want. +struct NodeSpec +where + D: Delegate + 'static, +{ + delegate: D, + runtime: RuntimeOptions, + serf: SerfOptions, + snapshot: Option, +} + +impl NodeSpec> { + fn new() -> Self { + Self { + delegate: VoidDelegate::new(), + runtime: RuntimeOptions::new(), + serf: SerfOptions::new(), + snapshot: None, + } + } +} + +impl NodeSpec +where + D: Delegate + 'static, +{ + fn with_delegate(self, delegate: E) -> NodeSpec + where + E: Delegate + 'static, + { + NodeSpec { + delegate, + runtime: self.runtime, + serf: self.serf, + snapshot: self.snapshot, + } + } + + fn with_runtime(mut self, runtime: RuntimeOptions) -> Self { + self.runtime = runtime; + self + } + + fn with_serf(mut self, serf: SerfOptions) -> Self { + self.serf = serf; + self + } + + fn with_snapshot(mut self, snapshot: SnapshotOptions) -> Self { + self.snapshot = Some(snapshot); + self + } + + /// Build and spawn the node on an ephemeral loopback port. + async fn spawn(self, id: &str) -> Serf { + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(cluster::loopback_ephemeral())); + Serf::new::, SocketAddrResolver, FirstAddrResolver, _, _>( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + self.delegate, + self.runtime, + self.serf, + gossip_rng().expect("seed gossip rng"), + None, + None, + self.snapshot, + #[cfg(encryption)] + Rc::new(serf_compio::VoidKeyringDelegate), + ) + .await + .expect("spawn serf tcp node") + } +} + +/// Spawn a plain loopback node. +async fn spawn_node(id: &str) -> Serf { + NodeSpec::new().spawn(id).await +} + +/// Poll both nodes until each reports the full two-member cluster. +async fn converge(a: &Serf, b: &Serf) { + compio::time::timeout(WINDOW, async { + loop { + if a.num_members() == 2 && b.num_members() == 2 { + break; + } + compio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("both nodes converge to a 2-member cluster"); +} + +/// Join `joiner` to `seed` and wait for both to converge. +async fn join_and_converge(joiner: &Serf, seed: &Serf) { + joiner + .join( + &SocketAddrResolver, + MaybeResolved::Resolved(seed.advertise_address()), + false, + ) + .await + .expect("join reaches the seed"); + converge(joiner, seed).await; +} + +/// Drive `events` until the next inbound `Event::Query` named `name` arrives, +/// returning its response token. +async fn next_query_token(events: &mut S, name: &str) -> QueryEvent +where + S: futures_util::Stream> + Unpin, +{ + compio::time::timeout(WINDOW, async { + loop { + match events.next().await { + Some(Event::Query(qe)) if qe.name() == name => break qe, + Some(_) => {} + None => panic!("the event stream closed before the query arrived"), + } + } + }) + .await + .expect("the query reaches the responder within the window") +} + +/// A unique temp path for a snapshot file. +fn snapshot_path(name: &str) -> std::path::PathBuf { + let mut p = std::env::temp_dir(); + p.push(format!("serf-compio-it-{name}-{}", std::process::id())); + // Ignoring Err: a leftover file from a previous run is fine to lose. + let _ = std::fs::remove_file(&p); + p +} + +/// A query issued by A round-trips: B surfaces the inbound `Event::Query` (and +/// fires its `notify_query` hook), answers through `Serf::respond`, and A +/// surfaces the matching `Event::QueryResponse` carrying B's payload. +#[compio::test] +async fn query_round_trips_through_respond() { + let seen = Rc::new(Observed::default()); + let b = NodeSpec::new() + .with_delegate(RecordingDelegate(seen.clone())) + .spawn("q-b") + .await; + let a = spawn_node("q-a").await; + + let mut b_events = b.events(); + let mut a_events = a.events(); + join_and_converge(&a, &b).await; + + let want = Bytes::from_static(b"pong"); + a.query( + "ping", + Bytes::from_static(b"ping-payload"), + a.default_query_param(), + ) + .await + .expect("query issued"); + + let responder = async { + let token = next_query_token(&mut b_events, "ping").await; + assert_eq!( + token.payload(), + &Bytes::from_static(b"ping-payload"), + "the inbound query carries the originator's payload" + ); + b.respond(token, want.clone()) + .await + .expect("B responds to the query"); + }; + let collector = async { + loop { + match a_events.next().await { + Some(Event::QueryResponse(qr)) if qr.payload() == &want => break true, + Some(_) => {} + None => break false, + } + } + }; + + let got = compio::time::timeout(WINDOW, async { + let (_, got) = future::join(responder, collector).await; + got + }) + .await + .expect("the query round-trip completes within the window"); + assert!(got, "A must receive B's query response"); + + assert_eq!( + seen.queries.borrow().as_slice(), + &[SmolStr::new("ping")], + "the driver fired B's notify_query hook exactly once for the inbound query" + ); + + a.shutdown().await.expect("q-a shuts down"); + b.shutdown().await.expect("q-b shuts down"); +} + +/// A user event broadcast by B reaches A's event stream with the original name +/// and payload, and fires A's `notify_user_event` observation hook. +#[compio::test] +async fn user_event_reaches_the_peer_stream_and_delegate() { + let seen = Rc::new(Observed::default()); + let a = NodeSpec::new() + .with_delegate(RecordingDelegate(seen.clone())) + .spawn("ue-a") + .await; + let b = spawn_node("ue-b").await; + + let mut a_events = a.events(); + join_and_converge(&a, &b).await; + + let payload = Bytes::from_static(b"deploy-42"); + b.user_event("deploy", payload.clone(), false) + .await + .expect("B broadcasts a user event"); + + let got = compio::time::timeout(WINDOW, async { + loop { + match a_events.next().await { + Some(Event::User(u)) if u.name.as_str() == "deploy" => break u.payload.clone(), + Some(_) => {} + None => panic!("A's event stream closed before the user event arrived"), + } + } + }) + .await + .expect("A receives B's user event within the window"); + assert_eq!(got, payload, "the payload survives the broadcast"); + + assert!( + seen.user_events.borrow().iter().any(|n| n == "deploy"), + "the driver fired A's notify_user_event hook for the broadcast" + ); + + a.shutdown().await.expect("ue-a shuts down"); + b.shutdown().await.expect("ue-b shuts down"); +} + +/// `set_tags` re-tags the local node and the change propagates: the peer records +/// a `Member(Update)` event, fires its `notify_update` hook, and its membership +/// view carries the new tag value. +#[compio::test] +async fn set_tags_propagates_as_a_member_update() { + let seen = Rc::new(Observed::default()); + let a = NodeSpec::new() + .with_delegate(RecordingDelegate(seen.clone())) + .spawn("tag-a") + .await; + let b = spawn_node("tag-b").await; + + join_and_converge(&a, &b).await; + + let mut tags = Tags::new(); + tags.0.insert(SmolStr::new("role"), SmolStr::new("worker")); + b.set_tags(tags).await.expect("B re-tags itself"); + + let got = compio::time::timeout(WINDOW, async { + loop { + let seen_tag = a + .members() + .iter() + .find(|m| m.node().id_ref().as_str() == "tag-b") + .and_then(|m| m.tags().0.get("role").cloned()); + if let Some(v) = seen_tag { + break v; + } + compio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("A observes B's new tag within the window"); + assert_eq!(got.as_str(), "worker", "A's view of B carries the new tag"); + + assert!( + seen.updated.borrow().iter().any(|id| id == "tag-b"), + "the driver fired A's notify_update hook for the re-tagged peer" + ); + + a.shutdown().await.expect("tag-a shuts down"); + b.shutdown().await.expect("tag-b shuts down"); +} + +/// A graceful `leave()` resolves only once the machine's `LeftCluster` fires, so +/// the event surfaces on the leaver's own stream and its endpoint settles at +/// `Left` — and the peer records a `Leave`, not a `Failed`. +#[compio::test] +async fn graceful_leave_emits_left_cluster_and_settles_left() { + let b = spawn_node("lv-b").await; + let a = spawn_node("lv-a").await; + + let mut a_events = a.events(); + join_and_converge(&a, &b).await; + + a.leave().await.expect("A leaves the cluster"); + + let saw = compio::time::timeout(WINDOW, async { + loop { + match a_events.next().await { + Some(Event::LeftCluster) => break true, + Some(_) => {} + None => break false, + } + } + }) + .await + .expect("A observes LeftCluster within the window"); + assert!(saw, "A must surface Event::LeftCluster after leave()"); + + compio::time::timeout(WINDOW, async { + loop { + if a.state() == SerfState::Left { + break; + } + compio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("A's endpoint state becomes Left"); + + a.shutdown().await.expect("lv-a shuts down"); + b.shutdown().await.expect("lv-b shuts down"); +} + +/// Two `leave()` callers racing on the same node share ONE in-flight leave: the +/// second joins the first's waiter rather than re-invoking the machine's +/// terminal `leave()` (which emits no second `LeftCluster` and would hang a fresh +/// waiter). Both must resolve `Ok`. +#[compio::test] +async fn concurrent_leave_calls_share_one_in_flight_operation() { + let b = spawn_node("cl-b").await; + let a = spawn_node("cl-a").await; + join_and_converge(&a, &b).await; + + let (first, second) = compio::time::timeout(WINDOW, future::join(a.leave(), a.leave())) + .await + .expect("both racing leaves resolve within the window"); + first.expect("the initiating leave resolves Ok"); + second.expect("the leave that joined the in-flight operation resolves Ok"); + + a.shutdown().await.expect("cl-a shuts down"); + b.shutdown().await.expect("cl-b shuts down"); +} + +/// Once a node has left, every mutating command it is handed reports +/// [`SerfError::NotRunning`] rather than being applied to a non-participating +/// endpoint — while a repeat `leave()` stays idempotent (`Ok`) and the read-only +/// coordinate probe still answers. +#[compio::test] +async fn commands_after_leave_report_not_running() { + let a = spawn_node("nr-a").await; + let b = spawn_node("nr-b").await; + + let mut a_events = a.events(); + join_and_converge(&a, &b).await; + + // A real inbound query token, captured while A is still running, so the + // post-leave `respond` below carries a valid token and can only be refused by + // the not-running gate. + b.query("probe", Bytes::new(), b.default_query_param()) + .await + .expect("B issues a query"); + let token = next_query_token(&mut a_events, "probe").await; + + a.leave().await.expect("A leaves the cluster"); + + assert!( + matches!( + a.join( + &SocketAddrResolver, + MaybeResolved::Resolved(b.advertise_address()), + false + ) + .await, + Err(SerfError::NotRunning) + ), + "join after leave must be refused" + ); + assert!( + matches!( + a.force_leave(SmolStr::new("nr-b"), false).await, + Err(SerfError::NotRunning) + ), + "force_leave after leave must be refused" + ); + assert!( + matches!( + a.user_event("evt", Bytes::new(), false).await, + Err(SerfError::NotRunning) + ), + "user_event after leave must be refused" + ); + assert!( + matches!( + a.query("q", Bytes::new(), a.default_query_param()).await, + Err(SerfError::NotRunning) + ), + "query after leave must be refused" + ); + assert!( + matches!( + a.respond(token, Bytes::new()).await, + Err(SerfError::NotRunning) + ), + "respond after leave must be refused" + ); + assert!( + matches!(a.set_tags(Tags::new()).await, Err(SerfError::NotRunning)), + "set_tags after leave must be refused" + ); + #[cfg(encryption)] + { + let key = test_secret_key(0x5a); + assert!( + matches!(a.install_key(key).await, Err(SerfError::NotRunning)), + "install_key after leave must be refused" + ); + assert!( + matches!(a.use_key(key).await, Err(SerfError::NotRunning)), + "use_key after leave must be refused" + ); + assert!( + matches!(a.remove_key(key).await, Err(SerfError::NotRunning)), + "remove_key after leave must be refused" + ); + assert!( + matches!(a.list_keys().await, Err(SerfError::NotRunning)), + "list_keys after leave must be refused" + ); + } + + // A repeat leave is a terminal no-op: it resolves immediately rather than + // parking a waiter for a `LeftCluster` that will never fire again. + compio::time::timeout(WINDOW, a.leave()) + .await + .expect("the repeat leave resolves rather than parking") + .expect("a repeat leave is idempotent"); + + // The coordinate cache stays readable after the node has left — it is a + // read-only probe, not a cluster mutation. + #[cfg(feature = "coordinates")] + a.cached_coordinate(SmolStr::new("nr-b")) + .await + .expect("the coordinate cache answers after leave"); + + a.shutdown().await.expect("nr-a shuts down"); + b.shutdown().await.expect("nr-b shuts down"); +} + +/// Commands still queued behind a `Shutdown` when the pump breaks are ANSWERED +/// with [`SerfError::Shutdown`] at teardown, never dropped: a caller's reply +/// receiver must not hang forever because the driver exited between its send and +/// its dispatch. Every command variant is queued behind the shutdown in one +/// batch, so each teardown reply arm is exercised. +#[compio::test] +async fn commands_queued_behind_a_shutdown_are_answered_not_dropped() { + let a = spawn_node("td-a").await; + let b = spawn_node("td-b").await; + + let mut b_events = b.events(); + join_and_converge(&a, &b).await; + + a.query("probe", Bytes::new(), a.default_query_param()) + .await + .expect("A issues a query"); + let token = next_query_token(&mut b_events, "probe").await; + + let a_addr = a.advertise_address(); + // Declared ahead of `queued` so it outlives the boxed futures that borrow it. + #[cfg(encryption)] + let key = test_secret_key(0x6b); + let mut queued: Vec>>> = Vec::new(); + // Polled first, so `Shutdown` is the head of the command queue and every + // command pushed after it lands behind it. + queued.push(Box::pin(async { + b.shutdown().await.expect("the shutdown itself is acked"); + })); + queued.push(Box::pin(async { + expect_shutdown(b.user_event("evt", Bytes::new(), false).await); + })); + queued.push(Box::pin(async { + expect_shutdown(b.query("q", Bytes::new(), b.default_query_param()).await); + })); + queued.push(Box::pin(async { + expect_shutdown(b.respond(token, Bytes::new()).await); + })); + queued.push(Box::pin(async { + expect_shutdown(b.set_tags(Tags::new()).await); + })); + queued.push(Box::pin(async { + expect_shutdown(b.force_leave(SmolStr::new("td-a"), false).await); + })); + queued.push(Box::pin(async { + expect_shutdown(b.leave().await); + })); + queued.push(Box::pin(async { + expect_shutdown( + b.join(&SocketAddrResolver, MaybeResolved::Resolved(a_addr), false) + .await, + ); + })); + queued.push(Box::pin(async { + expect_shutdown( + b.dispatch_join(&SocketAddrResolver, &[MaybeResolved::Resolved(a_addr)]) + .await, + ); + })); + #[cfg(encryption)] + { + queued.push(Box::pin(async { + expect_shutdown(b.install_key(key).await); + })); + queued.push(Box::pin(async { + expect_shutdown(b.use_key(key).await); + })); + queued.push(Box::pin(async { + expect_shutdown(b.remove_key(key).await); + })); + queued.push(Box::pin(async { + expect_shutdown(b.list_keys().await); + })); + } + #[cfg(feature = "coordinates")] + queued.push(Box::pin(async { + expect_shutdown(b.cached_coordinate(SmolStr::new("td-a")).await); + })); + + compio::time::timeout(WINDOW, future::join_all(queued)) + .await + .expect("every command queued behind the shutdown is answered, none hang"); + + // A command issued AFTER the driver has torn down fails fast on the handle's + // shutdown flag rather than queueing into a dead channel. + assert!( + matches!( + b.user_event("late", Bytes::new(), false).await, + Err(SerfError::Shutdown) + ), + "a post-teardown command fails fast with Shutdown" + ); + assert!( + matches!( + b.join(&SocketAddrResolver, MaybeResolved::Resolved(a_addr), false) + .await, + Err(SerfError::Shutdown) + ), + "a post-teardown join fails fast with Shutdown" + ); + + a.shutdown().await.expect("td-a shuts down"); +} + +/// Assert a command's reply is the teardown `Shutdown` error. +fn expect_shutdown(res: Result) +where + T: core::fmt::Debug, +{ + match res { + Err(SerfError::Shutdown) => {} + Err(other) => panic!("expected Shutdown, got {other:?}"), + Ok(v) => panic!("expected Shutdown, got Ok({v:?})"), + } +} + +/// With an UNBOUNDED observation channel the driver opts out of shedding +/// entirely: a stalling delegate cannot make the pump drop a single event, so a +/// burst of user events is delivered in full and the drop counters stay at zero. +#[compio::test] +async fn an_unbounded_observation_channel_sheds_nothing() { + const BURST: u32 = 32; + let a = NodeSpec::new() + .with_delegate(StallingDelegate { + stall: Duration::from_millis(2), + }) + .with_runtime(RuntimeOptions::new().with_observation_channel(Channel::Unbounded)) + .spawn("unb-a") + .await; + + let mut events = a.events(); + for i in 0..BURST { + a.user_event(format!("burst-{i}"), Bytes::new(), false) + .await + .expect("user event dispatched"); + } + + let delivered = compio::time::timeout(WINDOW, async { + let mut n = 0u32; + while n < BURST { + match events.next().await { + Some(Event::User(_)) => n += 1, + Some(_) => {} + None => panic!("the event stream closed mid-burst"), + } + } + n + }) + .await + .expect("every event of the burst is delivered under an unbounded observation channel"); + + assert_eq!(delivered, BURST, "no event of the burst is shed"); + assert_eq!( + a.observation_dropped(), + 0, + "an unbounded observation channel never drops" + ); + assert_eq!( + a.events_dropped(), + 0, + "the drained event stream never drops" + ); + + a.shutdown().await.expect("unb-a shuts down"); +} + +/// A cap-1 observation channel behind a delegate that parks on every user event +/// makes the pump shed: the enqueue retries once (yielding to the observation +/// task) and then drops and counts, so `observation_dropped` becomes non-zero. +/// A driver that blocked on the full queue instead would stall the FSM. +#[compio::test] +async fn a_stalled_delegate_makes_the_bounded_observation_channel_shed() { + const BURST: u32 = 32; + let a = NodeSpec::new() + .with_delegate(StallingDelegate { + stall: Duration::from_secs(30), + }) + .with_runtime(RuntimeOptions::new().with_observation_channel(Channel::Bounded(1))) + .spawn("shed-a") + .await; + + for i in 0..BURST { + a.user_event(format!("shed-{i}"), Bytes::new(), false) + .await + .expect("user event dispatched"); + } + + // Each `user_event().await` returned only after the pump processed that + // command, and the shed is counted on the same single-threaded executor, so a + // short settle is enough for the drain to surface every queued event. + let dropped = compio::time::timeout(WINDOW, async { + loop { + let n = a.observation_dropped(); + if n > 0 { + break n; + } + compio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("a stalled delegate on a cap-1 observation channel must make the pump shed"); + + assert!( + dropped > 0, + "the pump drops and counts events the stalled observation task cannot take (got {dropped})" + ); + + a.shutdown().await.expect("shed-a shuts down"); +} + +/// The user- and member-coalescer shed counters are distinct cells: a node whose +/// user coalescer is shedding must still report zero member-coalescer drops. A +/// construction typo aliasing the two readers onto one cell would leak the user +/// count into the member count and fail here. +#[compio::test] +async fn user_and_member_coalescer_drop_counters_are_distinct() { + let cap = core::num::NonZeroUsize::new(1).expect("1 is nonzero"); + let a = NodeSpec::new() + .with_serf( + SerfOptions::new() + .with_user_coalesce_period(Duration::from_secs(10)) + .with_user_quiescent_period(Duration::from_secs(2)) + .with_max_coalesced_user_events(Some(cap)), + ) + .spawn("dc-a") + .await; + + assert_eq!( + a.local_id().as_str(), + "dc-a", + "the handle carries its local id" + ); + + for i in 0..8u32 { + a.user_event(format!("cc-{i}"), Bytes::new(), true) + .await + .expect("user event dispatched"); + } + + assert!( + a.coalesced_user_events_dropped() > 0, + "the user coalescer sheds every distinct-named event past its cap" + ); + assert_eq!( + a.coalesced_member_events_dropped(), + 0, + "the member coalescer is a separate counter and has shed nothing" + ); + + a.shutdown().await.expect("dc-a shuts down"); +} + +/// An abruptly-killed peer is detected Failed (never Leave — the kill sends no +/// farewell), fires the observer's `notify_failed` hook, and is then reaped out +/// of the membership by the serf reaper. +#[compio::test] +async fn an_abrupt_kill_surfaces_failed_then_reap() { + let seen = Rc::new(Observed::default()); + let a = NodeSpec::new() + .with_delegate(RecordingDelegate(seen.clone())) + .with_serf(cluster::ClusterTiming::fast().serf_opts()) + .spawn("kill-a") + .await; + let b = NodeSpec::new() + .with_serf(cluster::ClusterTiming::fast().serf_opts()) + .spawn("kill-b") + .await; + + join_and_converge(&a, &b).await; + b.shutdown().await.expect("kill-b is torn down abruptly"); + drop(b); + + compio::time::timeout(WINDOW, async { + loop { + if a.num_members() == 1 { + break; + } + compio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("A detects the killed peer Failed and reaps it out of the membership"); + + assert!( + seen.failed.borrow().iter().any(|id| id == "kill-b"), + "the driver fired notify_failed for the abruptly-killed peer (saw {:?})", + seen.failed.borrow() + ); + + a.shutdown().await.expect("kill-a shuts down"); +} + +/// `force_leave` on a member the cluster has already declared Failed moves it to +/// a Left tombstone on every survivor, and the observer's ordered event log +/// records the full Join -> Failed -> Leave lifecycle. +#[compio::test] +async fn force_leave_moves_a_failed_member_to_a_left_tombstone() { + let mut cluster = cluster::Cluster::spawn( + &["fl-a", "fl-b"], + cluster::ClusterTiming::fast() + .with_reconnect_timeout(Duration::from_secs(120)) + .with_tombstone_timeout(Duration::from_secs(120)), + ) + .await; + let subject = cluster.id(1); + + cluster.kill_abrupt(1).await; + cluster + .await_member_event(0, subject.as_str(), MemberEventKind::Failed) + .await; + + cluster + .node(0) + .force_leave(subject.clone(), false) + .await + .expect("force_leave dispatches for a failed member"); + + cluster + .await_member_status(0, subject.as_str(), MemberStatus::Left) + .await; + cluster + .assert_member_events( + 0, + subject.as_str(), + &[ + MemberEventKind::Join, + MemberEventKind::Failed, + MemberEventKind::Leave, + ], + ) + .await; + + cluster.shutdown_all().await; +} + +/// A gracefully-left member is recorded as Leave (never Failed) by its peer and +/// lands in the peer's Left tombstone view. +#[compio::test] +async fn a_graceful_leave_is_recorded_as_leave_not_failed() { + let mut cluster = cluster::Cluster::spawn( + &["gl-a", "gl-b"], + cluster::ClusterTiming::fast().with_tombstone_timeout(Duration::from_secs(120)), + ) + .await; + let subject = cluster.id(1); + + cluster.leave_graceful(1).await; + cluster + .await_member_status(0, subject.as_str(), MemberStatus::Left) + .await; + cluster + .assert_member_events( + 0, + subject.as_str(), + &[MemberEventKind::Join, MemberEventKind::Leave], + ) + .await; + + cluster.shutdown_all().await; +} + +/// A snapshot with a 1-byte compaction threshold is rewritten to the live +/// membership on every flush, so the file stays a compact record of the current +/// cluster: after a restart the node replays it and rejoins the seed WITHOUT any +/// explicit join call. +#[compio::test] +async fn a_compacting_snapshot_replays_the_membership_on_restart() { + let path = snapshot_path("compact"); + let a = spawn_node("snap-a").await; + let b = NodeSpec::new() + .with_snapshot(SnapshotOptions::new(&path).with_compact_threshold(1)) + .spawn("snap-b") + .await; + assert_eq!( + SnapshotOptions::new(&path).with_compact_threshold(1).path(), + path.as_path(), + "the snapshot options carry the configured path" + ); + + join_and_converge(&b, &a).await; + b.shutdown().await.expect("snap-b shuts down"); + drop(b); + + // The compacted file still names the seed, so the restarted node re-dials it + // from the replayed membership alone. + let bytes = std::fs::read(&path).expect("the compacted snapshot survives the shutdown"); + assert!( + !bytes.is_empty(), + "compaction rewrites the live membership rather than truncating the file" + ); + + let b2 = NodeSpec::new() + .with_snapshot(SnapshotOptions::new(&path).with_compact_threshold(1)) + .spawn("snap-b") + .await; + converge(&a, &b2).await; + + a.shutdown().await.expect("snap-a shuts down"); + b2.shutdown().await.expect("snap-b2 shuts down"); + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} + +/// A reliable exchange whose peer accepts the connection and then walks away +/// still terminalizes: the driver signals the live bridge closed at teardown and +/// releases both bound ports, rather than leaking the bridge task and the socket. +#[compio::test] +async fn shutdown_closes_a_bridge_whose_peer_never_answers() { + let stall = compio::net::TcpListener::bind(cluster::loopback_ephemeral()) + .await + .expect("bind the stalling peer"); + let stall_addr = stall.local_addr().expect("stalling peer address"); + + // Accept and hold the connection open without ever answering, so the joiner's + // bridge is still live when the shutdown lands. + let held = Rc::new(RefCell::new(None)); + let held_slot = held.clone(); + compio::runtime::spawn(async move { + if let Ok((stream, _peer)) = stall.accept().await { + *held_slot.borrow_mut() = Some(stream); + } + // Ignoring Err: test cleanup of the stalling listener. + let _ = stall.close().await; + }) + .detach(); + + let a = spawn_node("stall-a").await; + let a_addr = a.advertise_address(); + let dispatched = a + .dispatch_join(&SocketAddrResolver, &[MaybeResolved::Resolved(stall_addr)]) + .await + .expect("the dial is dispatched"); + assert_eq!(dispatched, 1, "exactly one seed was dispatched"); + + // Wait until the stalling peer has actually accepted, so the bridge is live. + compio::time::timeout(WINDOW, async { + loop { + if held.borrow().is_some() { + break; + } + compio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("the stalling peer accepts the reliable dial"); + + a.shutdown() + .await + .expect("shutdown completes with a live bridge outstanding"); + + // The shutdown released both bound ports even with the bridge still open. + let listener = compio::net::TcpListener::bind(a_addr) + .await + .expect("the freed TCP port rebinds after a shutdown with a live bridge"); + let gossip = compio::net::UdpSocket::bind(a_addr) + .await + .expect("the freed UDP port rebinds after a shutdown with a live bridge"); + // Ignoring Err: test cleanup of the rebind probe sockets. + let _ = listener.close().await; + let _ = gossip.close().await; + // Take the stream OUT of the cell before awaiting: a `RefCell` borrow must not + // be held across an await point. + let held_stream = held.borrow_mut().take(); + if let Some(stream) = held_stream { + // Ignoring Err: test cleanup of the held peer connection. + let _ = stream.close().await; + } +} + +/// Dropping the last handle with a reliable exchange still in flight tears the +/// driver down through the command-channel disconnect: the live bridge is closed +/// and both bound ports are released, rather than the pump spinning on a bridge +/// that will never complete. +#[compio::test] +async fn dropping_the_last_handle_closes_a_live_bridge() { + let stall = compio::net::TcpListener::bind(cluster::loopback_ephemeral()) + .await + .expect("bind the stalling peer"); + let stall_addr = stall.local_addr().expect("stalling peer address"); + + let held = Rc::new(RefCell::new(None)); + let held_slot = held.clone(); + compio::runtime::spawn(async move { + if let Ok((stream, _peer)) = stall.accept().await { + *held_slot.borrow_mut() = Some(stream); + } + // Ignoring Err: test cleanup of the stalling listener. + let _ = stall.close().await; + }) + .detach(); + + let a = spawn_node("drop-bridge-a").await; + let a_addr = a.advertise_address(); + a.dispatch_join(&SocketAddrResolver, &[MaybeResolved::Resolved(stall_addr)]) + .await + .expect("the dial is dispatched"); + + compio::time::timeout(WINDOW, async { + loop { + if held.borrow().is_some() { + break; + } + compio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("the stalling peer accepts the reliable dial"); + + drop(a); + + let (listener, gossip) = compio::time::timeout(WINDOW, async { + loop { + if let Ok(listener) = compio::net::TcpListener::bind(a_addr).await { + if let Ok(gossip) = compio::net::UdpSocket::bind(a_addr).await { + break (listener, gossip); + } + // The teardown closes the listener before the UDP socket; release the + // probe listener and retry until the UDP port frees too. + // Ignoring Err: discarding the probe listener. + let _ = listener.close().await; + } + compio::time::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("the driver releases its bound ports after the handle drop, even with a live bridge"); + + // Ignoring Err: test cleanup of the rebind probe sockets. + let _ = listener.close().await; + let _ = gossip.close().await; + // Take the stream OUT of the cell before awaiting: a `RefCell` borrow must not + // be held across an await point. + let held_stream = held.borrow_mut().take(); + if let Some(stream) = held_stream { + // Ignoring Err: test cleanup of the held peer connection. + let _ = stream.close().await; + } +} + +/// A peer that accepts a reliable dial, never reads the push it is sent, and +/// then closes resets the connection. The bridge must terminalize that exchange +/// rather than leaking it: no member is admitted from the dead exchange, and the +/// pump still serves a subsequent REAL join to convergence — the discriminator +/// against a wedged pump or a leaked bridge. +#[compio::test] +async fn a_resetting_peer_fails_the_exchange_without_wedging_the_pump() { + let rude = compio::net::TcpListener::bind(cluster::loopback_ephemeral()) + .await + .expect("bind the resetting peer"); + let rude_addr = rude.local_addr().expect("resetting peer address"); + + compio::runtime::spawn(async move { + if let Ok((stream, _peer)) = rude.accept().await { + // Let the joiner's push bytes arrive and sit UNREAD in the receive buffer, + // then close: a close with unread data resets the connection rather than + // sending a clean FIN. + compio::time::sleep(Duration::from_millis(200)).await; + // Ignoring Err: the abrupt close is the point of the fixture. + let _ = stream.close().await; + } + // Ignoring Err: test cleanup of the resetting listener. + let _ = rude.close().await; + }) + .detach(); + + let a = spawn_node("reset-a").await; + let dispatched = a + .dispatch_join(&SocketAddrResolver, &[MaybeResolved::Resolved(rude_addr)]) + .await + .expect("the dial is dispatched"); + assert_eq!(dispatched, 1, "exactly one seed was dispatched"); + + // The dead exchange admits nothing, and the pump is still live: a real peer + // joined afterwards still converges. + let b = spawn_node("reset-b").await; + join_and_converge(&a, &b).await; + assert_eq!( + a.num_members(), + 2, + "only the real peer is admitted; the reset exchange contributed no member" + ); + + a.shutdown() + .await + .expect("the pump survives the failed exchange and shuts down"); + b.shutdown().await.expect("reset-b shuts down"); +} + +/// `join_many` whose resolver fails surfaces the resolver error rather than +/// silently reporting a zero-contact success, and an await-result `join` against +/// an unreachable seed reports the requested/contacted tally. +#[compio::test] +async fn join_surfaces_resolver_and_contact_failures() { + let a = spawn_node("jf-a").await; + + let (reached, err) = a + .join_many( + &FailingResolver, + core::iter::once(MaybeResolved::Unresolved("svc".to_string())), + false, + ) + .await + .expect_err("a failing resolver must not report a healthy join"); + assert!(reached.is_empty(), "a failed resolution contacts nothing"); + assert!( + matches!(err, SerfError::Resolve(_)), + "the resolver's error is surfaced, got {err:?}" + ); + + let err = a + .join( + &SocketAddrResolver, + MaybeResolved::Resolved(blackhole_addr()), + false, + ) + .await + .expect_err("a blackhole seed must fail the join"); + match err { + SerfError::JoinAllFailed(payload) => { + assert_eq!(payload.requested(), 1, "one seed requested"); + assert_eq!(payload.contacted(), 0, "no seed contacted"); + } + other => panic!("expected JoinAllFailed, got {other:?}"), + } + + a.shutdown().await.expect("jf-a shuts down"); +} + +/// The `TcpTransportOptions` accessors reflect exactly what the builders set, +/// and `Default` is the `new()` state: the required fields the constructor's +/// guards check are unset. +#[test] +fn tcp_transport_options_accessors_reflect_builders() { + let addr: SocketAddr = "127.0.0.1:7946".parse().expect("loopback addr"); + + let empty = TcpTransportOptions::::default(); + assert!(empty.local_id().is_none(), "Default leaves local_id unset"); + assert!( + empty.advertise_addr().is_none(), + "Default leaves advertise_addr unset" + ); + assert_eq!( + empty.stream().dial_timeout(), + StreamTransportOptions::default().dial_timeout(), + "Default carries the same stream knobs a fresh StreamTransportOptions does" + ); + assert_eq!( + StreamTransportOptions::default().dial_timeout(), + serf_compio::DEFAULT_DIAL_TIMEOUT, + "the documented dial-timeout default" + ); + + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new("acc-node")) + .with_advertise_addr(MaybeResolved::Resolved(addr)) + .with_stream(StreamTransportOptions::new().with_dial_timeout(Duration::from_millis(250))); + assert_eq!(opts.local_id().map(SmolStr::as_str), Some("acc-node")); + match opts.advertise_addr() { + Some(MaybeResolved::Resolved(s)) => assert_eq!(*s, addr), + other => panic!("expected a resolved advertise addr, got {other:?}"), + } + assert_eq!(opts.stream().dial_timeout(), Duration::from_millis(250)); + + #[cfg(encryption)] + assert!( + opts.encryption().keyring().is_none(), + "the default encryption policy carries no keyring" + ); +} + +/// `TcpTransport::new` refuses the two required fields it cannot default: +/// without a local id, and without an advertise address, construction fails with +/// `InvalidInput` naming the missing field — before binding a socket. +#[compio::test] +async fn tcp_new_requires_a_local_id_and_an_advertise_addr() { + let no_id = TcpTransportOptions::::new() + .with_advertise_addr(MaybeResolved::Resolved(cluster::loopback_ephemeral())); + assert_missing_field( + TcpTransport::::new(no_id, &SocketAddrResolver, &FirstAddrResolver).await, + "local_id", + ); + + let no_addr = + TcpTransportOptions::::new().with_local_id(SmolStr::new("no-addr")); + assert_missing_field( + TcpTransport::::new(no_addr, &SocketAddrResolver, &FirstAddrResolver) + .await, + "advertise_addr", + ); +} + +/// Assert a transport construction was refused with `InvalidInput` naming the +/// required field the caller left unset. +fn assert_missing_field(res: Result, field: &str) { + match res { + Err(SerfError::Io(e)) => { + assert_eq!( + e.kind(), + std::io::ErrorKind::InvalidInput, + "a missing required field is an InvalidInput refusal" + ); + assert!( + e.to_string().contains(field), + "the refusal names the missing field {field:?}, got {e}" + ); + } + Err(other) => panic!("expected InvalidInput({field}), got {other:?}"), + Ok(_) => panic!("a missing {field} must be refused, but construction succeeded"), + } +} + +/// An UNRESOLVED advertise address is resolved through the caller's `Resolver` +/// and NARROWED by the `AdvertiseAddrResolver`: the resolver offers an IPv6 and +/// an IPv4 candidate, the IPv4-preferring picker chooses the IPv4 one, and the +/// transport binds THAT address, reports it as its advertise contact, and still +/// remembers the unresolved input form it was constructed from. +#[compio::test] +async fn tcp_new_resolves_and_narrows_an_unresolved_advertise_addr() { + let input: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new("unres-node")) + .with_advertise_addr(MaybeResolved::Unresolved(input)); + let transport = + TcpTransport::::new(opts, &DualStackResolver, &Ipv4PreferringResolver) + .await + .expect("an unresolved advertise address resolves through the resolver"); + + assert_eq!(transport.local_id().as_str(), "unres-node"); + let bound = *transport.advertise_address(); + assert!( + bound.is_ipv4(), + "the IPv4-preferring picker narrowed the dual-stack candidate set, got {bound}" + ); + assert!(bound.ip().is_loopback(), "the picked candidate was bound"); + assert_ne!( + bound.port(), + 0, + "the ephemeral port is read back concretely" + ); + match transport.local_address() { + MaybeResolved::Unresolved(a) => { + assert_eq!(*a, input, "the unresolved input form is retained") + } + other => panic!("expected the unresolved input form, got {other:?}"), + } +} + +/// The built-in advertise resolvers implement their documented policies over a +/// candidate set, and every one of them reports `Empty` on an empty set rather +/// than silently picking nothing. +#[test] +fn advertise_resolvers_pick_by_their_documented_policy() { + let v4: SocketAddr = "127.0.0.1:7946".parse().expect("v4 addr"); + let v6: SocketAddr = "[::1]:7946".parse().expect("v6 addr"); + + assert_eq!( + FirstAddrResolver + .pick(vec![v6, v4]) + .expect("the first candidate"), + v6, + "FirstAddrResolver takes the head of the set" + ); + assert_eq!( + Ipv4PreferringResolver + .pick(vec![v6, v4]) + .expect("the v4 candidate"), + v4, + "Ipv4PreferringResolver prefers IPv4 over an earlier IPv6" + ); + assert_eq!( + Ipv4PreferringResolver + .pick(vec![v6]) + .expect("the only candidate"), + v6, + "Ipv4PreferringResolver falls back to the first when no IPv4 exists" + ); + assert_eq!( + Ipv6PreferringResolver + .pick(vec![v4, v6]) + .expect("the v6 candidate"), + v6, + "Ipv6PreferringResolver prefers IPv6 over an earlier IPv4" + ); + assert_eq!( + Ipv6PreferringResolver + .pick(vec![v4]) + .expect("the only candidate"), + v4, + "Ipv6PreferringResolver falls back to the first when no IPv6 exists" + ); + + for empty in [ + FirstAddrResolver.pick(Vec::new()), + Ipv4PreferringResolver.pick(Vec::new()), + Ipv6PreferringResolver.pick(Vec::new()), + ] { + assert!( + matches!(empty, Err(AdvertiseResolutionError::Empty)), + "an empty candidate set is an error, not a silent pick" + ); + } +} + +/// `SocketAddrResolver` is the identity pass-through, and `OsResolver` resolves +/// a literal-IP host without a name lookup. +#[compio::test] +async fn the_builtin_resolvers_resolve_concrete_addresses() { + let addr: SocketAddr = "127.0.0.1:7946".parse().expect("loopback addr"); + assert_eq!( + SocketAddrResolver + .resolve(&addr) + .await + .expect("identity resolution"), + vec![addr], + "SocketAddrResolver passes an already-resolved address straight through" + ); + + let host = hostaddr::HostAddr::::from(addr); + assert_eq!( + OsResolver.resolve(&host).await.expect("os resolution"), + vec![addr], + "OsResolver resolves a literal-IP host to that exact address" + ); +} + +/// `LocalAddrResolver::default()` is the PRIVATE scope — the LAN-cluster choice +/// the docs promise — and the public / all scopes are distinct from it. Every +/// scope passes a CONCRETE address straight through: the interface enumeration +/// runs only for a wildcard advertise address. +#[cfg(feature = "getifs")] +#[compio::test] +async fn local_addr_resolver_defaults_to_the_private_scope() { + use serf_compio::LocalAddrResolver; + + let default = format!("{:?}", LocalAddrResolver::default()); + assert_eq!( + default, + format!("{:?}", LocalAddrResolver::private()), + "the default scope is the private scope" + ); + assert_ne!( + format!("{:?}", LocalAddrResolver::public()), + default, + "the public scope is distinct from the default" + ); + assert_ne!( + format!("{:?}", LocalAddrResolver::all()), + default, + "the all scope is distinct from the default" + ); + + let concrete: SocketAddr = "127.0.0.1:7946".parse().expect("loopback addr"); + for resolver in [ + LocalAddrResolver::private(), + LocalAddrResolver::public(), + LocalAddrResolver::all(), + ] { + assert_eq!( + resolver + .resolve(&concrete) + .await + .expect("a concrete address needs no interface scan"), + vec![concrete], + "a concrete advertise address passes through unchanged in every scope" + ); + } +} + +/// An advertise address the caller supplied UNRESOLVED must not silently bind a +/// wrong contact when resolution cannot answer: a resolver outage surfaces as +/// `SerfError::Resolve`, and a resolution that yields ZERO candidates is refused +/// by the advertise picker rather than defaulted. +#[compio::test] +async fn tcp_new_refuses_an_advertise_address_it_cannot_resolve() { + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new("res-fail")) + .with_advertise_addr(MaybeResolved::Unresolved("self".to_string())); + match TcpTransport::::new(opts, &FailingResolver, &FirstAddrResolver).await { + Err(SerfError::Resolve(e)) => assert!( + e.to_string().contains("discovery backend unavailable"), + "the resolver's own error is surfaced, got {e}" + ), + Err(other) => panic!("expected Resolve, got {other:?}"), + Ok(_) => panic!("a resolver outage must refuse construction"), + } + + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new("res-empty")) + .with_advertise_addr(MaybeResolved::Unresolved("self".to_string())); + match TcpTransport::::new(opts, &EmptyResolver, &FirstAddrResolver).await { + Err(SerfError::Resolve(e)) => assert_eq!( + e.kind(), + std::io::ErrorKind::AddrNotAvailable, + "a zero-candidate resolution is an unavailable advertise address" + ), + Err(other) => panic!("expected Resolve(AddrNotAvailable), got {other:?}"), + Ok(_) => panic!("a zero-candidate resolution must refuse construction"), + } +} + +/// A seed keyring already carrying a cross-cipher byte twin (two keys sharing a +/// raw byte value across different cipher variants) makes every later +/// byte-keyed rotation op ambiguous — a `use` or `remove` could promote or drop +/// the WRONG cipher's key — so construction refuses it before binding a socket. +#[cfg(all(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[compio::test] +async fn tcp_new_refuses_a_cross_cipher_twin_keyring() { + use serf_compio::{EncryptionOptions, Keyring, SecretKey}; + + let twins = Keyring::with_secondaries( + SecretKey::Aes256([0x21; 32]), + [SecretKey::ChaCha20Poly1305([0x21; 32])], + ); + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new("twin-node")) + .with_advertise_addr(MaybeResolved::Resolved(cluster::loopback_ephemeral())) + .with_encryption(EncryptionOptions::new().with_keyring(twins)); + match TcpTransport::::new(opts, &SocketAddrResolver, &FirstAddrResolver) + .await + { + Err(SerfError::Io(e)) => { + assert_eq!(e.kind(), std::io::ErrorKind::InvalidInput); + assert!( + e.to_string().contains("cross-cipher"), + "the refusal names the collision, got {e}" + ); + } + Err(other) => panic!("expected InvalidInput(cross-cipher), got {other:?}"), + Ok(_) => panic!("a cross-cipher twin keyring must be refused at construction"), + } +} + +/// A deterministic test secret key, selecting whichever AEAD cipher this build +/// compiled. +#[cfg(encryption)] +fn test_secret_key(fill: u8) -> serf_compio::SecretKey { + #[cfg(feature = "aes-gcm")] + let key = serf_compio::SecretKey::Aes256([fill; 32]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = serf_compio::SecretKey::ChaCha20Poly1305([fill; 32]); + key +} diff --git a/serf-compio/tests/tls.rs b/serf-compio/tests/tls.rs new file mode 100644 index 00000000..3c0b7201 --- /dev/null +++ b/serf-compio/tests/tls.rs @@ -0,0 +1,508 @@ +//! Real-node TLS serf tests: loopback nodes exercising the compio stream driver +//! over a real rustls record layer (a self-signed localhost-SAN cert plus an +//! accept-any verifier). TLS rides the SAME stream driver as plain TCP and +//! differs only in the reliable record layer, so this suite pins the +//! TLS-specific construction surface — the options guards, the SNI provider, the +//! unresolved-advertise resolution, and the merge-delegate / snapshot wiring the +//! TLS `run` performs — plus an end-to-end cluster over the encrypted reliable +//! plane. +//! +//! serf-compio declares no `tls-rustls-*` backend feature of its own: the rustls +//! crypto provider is supplied by the CALLER inside the `TlsOptions` bundle +//! (`ServerConfig` / `ClientConfig`), which is exactly what these tests do. + +#![cfg(feature = "tls")] + +use core::time::Duration; +use std::{ + net::SocketAddr, + rc::Rc, + sync::{Arc, Mutex}, +}; + +use bytes::Bytes; +use futures_util::{StreamExt, future}; +use memberlist_proto::MaybeResolved; +use rustls::{ + client::danger::{HandshakeSignatureValid, ServerCertVerified}, + crypto::CryptoProvider, + pki_types::CertificateDer, + version::TLS13, +}; +use serf_compio::{ + FirstAddrResolver, Ipv4PreferringResolver, MergeDelegate, Resolver, RuntimeOptions, Serf, + SerfError, SnapshotOptions, SocketAddrResolver, TlsOptions, TlsTransport, TlsTransportOptions, + Transport, VoidDelegate, gossip_rng, +}; +use serf_proto::{ + event::{Event, MemberEventKind, QueryEvent}, + options::Options as SerfOptions, +}; +use smol_str::SmolStr; + +/// Bound on every convergence / delivery poll in this file. +const WINDOW: Duration = Duration::from_secs(45); + +/// An ephemeral loopback bind address. +fn loopback_ephemeral() -> SocketAddr { + "127.0.0.1:0".parse().expect("loopback addr") +} + +/// Accept-any server-cert verifier for the loopback suite. The nodes present +/// self-signed localhost-SAN certs; the client side accepts whatever the server +/// presents so the handshake completes without a real trust anchor. NEVER use +/// this outside a test. +#[derive(Debug)] +struct AcceptAnyServer(Arc); + +impl rustls::client::danger::ServerCertVerifier for AcceptAnyServer { + fn verify_server_cert( + &self, + _e: &CertificateDer<'_>, + _i: &[CertificateDer<'_>], + _n: &rustls::pki_types::ServerName<'_>, + _o: &[u8], + _t: rustls::pki_types::UnixTime, + ) -> Result { + Ok(ServerCertVerified::assertion()) + } + fn verify_tls12_signature( + &self, + _m: &[u8], + _c: &CertificateDer<'_>, + _d: &rustls::DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + fn verify_tls13_signature( + &self, + _m: &[u8], + _c: &CertificateDer<'_>, + _d: &rustls::DigitallySignedStruct, + ) -> Result { + Ok(HandshakeSignatureValid::assertion()) + } + fn supported_verify_schemes(&self) -> Vec { + self.0.signature_verification_algorithms.supported_schemes() + } +} + +/// The rustls crypto provider the caller supplies to the TLS bundle: the +/// process default if one is installed, otherwise `ring`. +fn crypto_provider() -> Arc { + CryptoProvider::get_default() + .cloned() + .unwrap_or_else(|| Arc::new(rustls::crypto::ring::default_provider())) +} + +/// A self-signed localhost-SAN `ServerConfig` + accept-any `ClientConfig` +/// bundle. A fresh bundle is built per node so each owns its own cert. +fn test_tls_options() -> TlsOptions { + let ck = rcgen::generate_simple_self_signed(vec!["localhost".into()]) + .expect("rcgen generate_simple_self_signed"); + let chain = vec![CertificateDer::from(ck.cert.der().to_vec())]; + let key = rustls::pki_types::PrivateKeyDer::Pkcs8(ck.signing_key.serialize_der().into()); + + let provider = crypto_provider(); + + let server_cfg = rustls::ServerConfig::builder_with_provider(provider.clone()) + .with_protocol_versions(&[&TLS13]) + .expect("TLS 1.3 supported") + .with_no_client_auth() + .with_single_cert(chain, key) + .expect("valid self-signed cert"); + + let client_cfg = rustls::ClientConfig::builder_with_provider(provider.clone()) + .with_protocol_versions(&[&TLS13]) + .expect("TLS 1.3 supported") + .dangerous() + .with_custom_certificate_verifier(Arc::new(AcceptAnyServer(provider))) + .with_no_client_auth(); + + TlsOptions::new(server_cfg, client_cfg) +} + +/// A resolver that answers with a dual-stack candidate set (IPv6 first, then +/// IPv4) on the port it was asked for — enough to drive the +/// `MaybeResolved::Unresolved` advertise path AND the advertise picker's +/// narrowing, without depending on the host's name resolution. +struct DualStackResolver; + +impl Resolver for DualStackResolver { + type Address = SocketAddr; + type Error = std::io::Error; + + async fn resolve(&self, addr: &SocketAddr) -> Result, std::io::Error> { + Ok(vec![ + SocketAddr::new("::1".parse().expect("v6 loopback"), addr.port()), + SocketAddr::new("127.0.0.1".parse().expect("v4 loopback"), addr.port()), + ]) + } +} + +/// The peers every consulted push/pull merge carried. `MergeDelegate` is the +/// machine's `Send + Sync` predicate, so the record is shared through an `Arc` +/// even on the `!Send` compio driver. +type MergedPeers = Arc>>; + +/// A merge predicate that admits every exchange while recording the peers each +/// push/pull carried, so a test can prove the constructor installed it. +struct RecordingMerge { + peers: MergedPeers, +} + +impl MergeDelegate for RecordingMerge { + fn notify_merge( + &self, + peers: memberlist_proto::MaybeOwned< + '_, + [memberlist_proto::typed::NodeState], + >, + ) -> bool { + let mut seen = self.peers.lock().expect("merge record lock"); + for p in peers.iter() { + seen.push(p.id_ref().clone()); + } + true + } +} + +/// Build a TLS node on an ephemeral loopback port with the fixture's cert bundle. +async fn spawn_node(id: &str) -> Serf { + spawn_node_with(id, None, None) + .await + .expect("spawn serf tls node") +} + +/// Build a TLS node with an optional merge delegate and snapshot file. +async fn spawn_node_with( + id: &str, + merge: Option>>, + snapshot: Option, +) -> Result, SerfError> { + let opts = TlsTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(loopback_ephemeral())) + .with_tls_options(test_tls_options()); + Serf::new::, SocketAddrResolver, FirstAddrResolver, _, _>( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + gossip_rng().expect("seed gossip rng"), + None, + merge, + snapshot, + #[cfg(encryption)] + Rc::new(serf_compio::VoidKeyringDelegate), + ) + .await +} + +/// Poll both nodes until each reports the full two-member cluster. +async fn converge(a: &Serf, b: &Serf) { + compio::time::timeout(WINDOW, async { + loop { + if a.num_members() == 2 && b.num_members() == 2 { + break; + } + compio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("both nodes converge to a 2-member cluster"); +} + +/// Two nodes over a real TLS handshake: A joins B, both converge, a user event +/// crosses the encrypted reliable/gossip planes, a query round-trips through +/// `respond`, and A leaves gracefully. +#[compio::test] +async fn a_tls_cluster_forms_and_carries_events_queries_and_a_leave() { + let b = spawn_node("tls-b").await; + let a = spawn_node("tls-a").await; + + let mut a_events = a.events(); + let mut b_events = b.events(); + + let reached = a + .join( + &SocketAddrResolver, + MaybeResolved::Resolved(b.advertise_address()), + false, + ) + .await + .expect("join reaches node B over TLS"); + assert_eq!( + reached, + b.advertise_address(), + "join returns the reached seed address" + ); + converge(&a, &b).await; + + // A `Member(Join)` for B surfaces on A's stream. + let joined = compio::time::timeout(WINDOW, async { + loop { + match a_events.next().await { + Some(Event::Member(me)) if me.kind() == MemberEventKind::Join => { + if me.members().iter().any(|m| m.node().id_ref() == "tls-b") { + break true; + } + } + Some(_) => {} + None => break false, + } + } + }) + .await + .expect("A observes B joining within the window"); + assert!(joined, "A must surface a Member(Join) naming B"); + + // A user event crosses the TLS cluster. + let payload = Bytes::from_static(b"tls-payload"); + b.user_event("tls-evt", payload.clone(), false) + .await + .expect("B broadcasts a user event"); + let got = compio::time::timeout(WINDOW, async { + loop { + match a_events.next().await { + Some(Event::User(u)) if u.name.as_str() == "tls-evt" => break u.payload.clone(), + Some(_) => {} + None => panic!("A's event stream closed before the user event arrived"), + } + } + }) + .await + .expect("A receives B's user event within the window"); + assert_eq!(got, payload, "the payload survives the TLS cluster"); + + // A query round-trips through B's `respond`. + let want = Bytes::from_static(b"tls-pong"); + a.query("tls-ping", Bytes::new(), a.default_query_param()) + .await + .expect("query issued"); + let responder = async { + let token: QueryEvent = loop { + match b_events.next().await { + Some(Event::Query(qe)) if qe.name() == "tls-ping" => break qe, + Some(_) => {} + None => panic!("B's event stream closed before the query arrived"), + } + }; + b.respond(token, want.clone()) + .await + .expect("B responds to the query"); + }; + let collector = async { + loop { + match a_events.next().await { + Some(Event::QueryResponse(qr)) if qr.payload() == &want => break true, + Some(_) => {} + None => break false, + } + } + }; + let answered = compio::time::timeout(WINDOW, async { + let (_, got) = future::join(responder, collector).await; + got + }) + .await + .expect("the TLS query round-trip completes within the window"); + assert!(answered, "A must receive B's query response over TLS"); + + a.leave().await.expect("A leaves the TLS cluster"); + a.shutdown().await.expect("tls-a shuts down"); + b.shutdown().await.expect("tls-b shuts down"); +} + +/// The constructor-supplied merge delegate reaches the TLS endpoint: A's join +/// push/pull drives at least one `notify_merge` on B carrying A's node state. +/// The snapshot file supplied alongside it is opened and written, proving both +/// constructor arguments are threaded through the TLS `run`. +#[compio::test] +async fn the_tls_run_installs_the_merge_delegate_and_the_snapshot() { + let mut path = std::env::temp_dir(); + path.push(format!("serf-compio-tls-snap-{}", std::process::id())); + // Ignoring Err: a leftover file from a previous run is fine to lose. + let _ = std::fs::remove_file(&path); + + let peers: MergedPeers = Arc::new(Mutex::new(Vec::new())); + let b = spawn_node_with( + "tlsm-b", + Some(Box::new(RecordingMerge { + peers: peers.clone(), + })), + Some(SnapshotOptions::new(&path)), + ) + .await + .expect("spawn a merge-recording, snapshot-backed TLS node"); + let a = spawn_node("tlsm-a").await; + + a.join( + &SocketAddrResolver, + MaybeResolved::Resolved(b.advertise_address()), + false, + ) + .await + .expect("join reaches node B"); + converge(&a, &b).await; + + let merged = peers.lock().expect("merge record lock").clone(); + assert!( + merged.iter().any(|id| id == "tlsm-a"), + "the TLS push/pull consulted the installed merge delegate with the joining peer's state \ + (saw {merged:?})" + ); + + a.shutdown().await.expect("tlsm-a shuts down"); + b.shutdown().await.expect("tlsm-b shuts down"); + + let bytes = std::fs::read(&path).expect("the snapshot the TLS run installed exists"); + assert!( + !bytes.is_empty(), + "the TLS run wrote membership records to the constructor-supplied snapshot" + ); + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} + +/// `TlsTransport::new` refuses each required field it cannot default, naming the +/// missing one, before binding a socket. +#[compio::test] +async fn tls_new_requires_a_local_id_and_an_advertise_addr() { + let no_id = TlsTransportOptions::::new() + .with_advertise_addr(MaybeResolved::Resolved(loopback_ephemeral())) + .with_tls_options(test_tls_options()); + assert_missing_field( + TlsTransport::::new(no_id, &SocketAddrResolver, &FirstAddrResolver).await, + "local_id", + ); + + let no_addr = TlsTransportOptions::::new() + .with_local_id(SmolStr::new("no-addr")) + .with_tls_options(test_tls_options()); + assert_missing_field( + TlsTransport::::new(no_addr, &SocketAddrResolver, &FirstAddrResolver) + .await, + "advertise_addr", + ); +} + +/// Assert a transport construction was refused with `InvalidInput` naming the +/// required field the caller left unset. +fn assert_missing_field(res: Result, field: &str) { + match res { + Err(SerfError::Io(e)) => { + assert_eq!( + e.kind(), + std::io::ErrorKind::InvalidInput, + "a missing required field is an InvalidInput refusal" + ); + assert!( + e.to_string().contains(field), + "the refusal names the missing field {field:?}, got {e}" + ); + } + Err(other) => panic!("expected InvalidInput({field}), got {other:?}"), + Ok(_) => panic!("a missing {field} must be refused, but construction succeeded"), + } +} + +/// An UNRESOLVED advertise address is resolved through the caller's `Resolver` +/// and NARROWED by the `AdvertiseAddrResolver`: the resolver offers an IPv6 and +/// an IPv4 candidate, the IPv4-preferring picker chooses the IPv4 one, and the +/// TLS transport binds THAT address and retains the unresolved input form. +#[compio::test] +async fn tls_new_resolves_and_narrows_an_unresolved_advertise_addr() { + let input: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let opts = TlsTransportOptions::::new() + .with_local_id(SmolStr::new("tls-unres")) + .with_advertise_addr(MaybeResolved::Unresolved(input)) + .with_tls_options(test_tls_options()); + let transport = + TlsTransport::::new(opts, &DualStackResolver, &Ipv4PreferringResolver) + .await + .expect("an unresolved advertise address resolves through the resolver"); + + assert_eq!(transport.local_id().as_str(), "tls-unres"); + let bound = *transport.advertise_address(); + assert!( + bound.is_ipv4(), + "the IPv4-preferring picker narrowed the dual-stack candidate set, got {bound}" + ); + assert!(bound.ip().is_loopback(), "the picked candidate was bound"); + assert_ne!( + bound.port(), + 0, + "the ephemeral port is read back concretely" + ); + match transport.local_address() { + MaybeResolved::Unresolved(a) => { + assert_eq!(*a, input, "the unresolved input form is retained") + } + other => panic!("expected the unresolved input form, got {other:?}"), + } +} + +/// An advertise address the caller supplied UNRESOLVED must not silently bind a +/// wrong contact when resolution cannot answer: a resolver outage surfaces as +/// `SerfError::Resolve` rather than a bound-but-undialable node. +#[compio::test] +async fn tls_new_refuses_an_advertise_address_it_cannot_resolve() { + struct FailingResolver; + + impl Resolver for FailingResolver { + type Address = SocketAddr; + type Error = std::io::Error; + + async fn resolve(&self, _addr: &SocketAddr) -> Result, std::io::Error> { + Err(std::io::Error::other("discovery backend unavailable")) + } + } + + let opts = TlsTransportOptions::::new() + .with_local_id(SmolStr::new("tls-res-fail")) + .with_advertise_addr(MaybeResolved::Unresolved(loopback_ephemeral())) + .with_tls_options(test_tls_options()); + match TlsTransport::::new(opts, &FailingResolver, &FirstAddrResolver).await { + Err(SerfError::Resolve(e)) => assert!( + e.to_string().contains("discovery backend unavailable"), + "the resolver's own error is surfaced, got {e}" + ), + Err(other) => panic!("expected Resolve, got {other:?}"), + Ok(_) => panic!("a resolver outage must refuse construction"), + } +} + +/// The gossip-encryption policy on the TLS options is a first-class accessor: +/// the default carries no keyring (plaintext gossip alongside the TLS-secured +/// reliable plane), and `with_encryption` installs one. +#[cfg(encryption)] +#[test] +fn tls_options_expose_the_gossip_encryption_policy() { + use serf_compio::{EncryptionOptions, Keyring, SecretKey}; + + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes256([0x31; 32]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = SecretKey::ChaCha20Poly1305([0x31; 32]); + + let plain = TlsTransportOptions::::new(); + assert!( + plain.encryption().keyring().is_none(), + "the default TLS options leave the gossip plane plaintext" + ); + + let encrypted = TlsTransportOptions::::new() + .with_encryption(EncryptionOptions::new().with_keyring(Keyring::new(key))); + assert_eq!( + encrypted + .encryption() + .keyring() + .expect("the installed keyring is readable back") + .primary_ref(), + &key, + "with_encryption installs the caller's keyring as the gossip policy" + ); +} diff --git a/serf-driver/src/keyring/tests.rs b/serf-driver/src/keyring/tests.rs index d4ba6975..8c493bc2 100644 --- a/serf-driver/src/keyring/tests.rs +++ b/serf-driver/src/keyring/tests.rs @@ -1,6 +1,9 @@ -use super::{apply_key_request, keyring_carries_cross_cipher_twin}; +use super::{ + KeyringPersistError, apply_key_request, keyring_carries_cross_cipher_twin, + settle_parked_key_response, +}; use memberlist_proto::{Keyring, SecretKey}; -use serf_proto::KeyRequestOperation; +use serf_proto::{KeyRequestOperation, KeyResponseArgs}; #[cfg(feature = "aes-gcm")] fn aes(b: u8) -> SecretKey { @@ -197,3 +200,128 @@ fn twin_detector_flags_cross_cipher_seed_ring() { let clean = Keyring::with_secondaries(aes(1), [aes(2), chacha(3)]); assert!(!keyring_carries_cross_cipher_twin(&clean)); } + +// ── the applied outcome's parts ─────────────────────────────────────────────── + +#[cfg(feature = "aes-gcm")] +#[test] +fn into_parts_yields_the_response_and_the_rotated_ring() { + let ring = Keyring::new(aes(1)); + + let (response, rotated) = + apply_key_request(&ring, KeyRequestOperation::Install, Some(&aes(2))).into_parts(); + assert!(response.result); + let rotated = rotated.expect("install mutates the ring"); + assert_eq!(rotated.primary_ref(), &aes(1)); + assert!(rotated.secondaries().contains(&aes(2))); + + // A read-only op splits into a response with no ring to publish, so the + // caller never re-keys the wire on a `list`. + let (response, rotated) = apply_key_request(&ring, KeyRequestOperation::List, None).into_parts(); + assert!(response.result); + assert_eq!(response.primary_key, Some(aes(1))); + assert!(rotated.is_none()); +} + +// ── parked key responses gated on persistence ───────────────────────────────── + +/// A `list` answer: the shape whose carried key material must survive a +/// persistence downgrade untouched. +#[cfg(feature = "aes-gcm")] +fn listed() -> KeyResponseArgs { + KeyResponseArgs { + result: true, + keys: vec![aes(1), aes(2)], + primary_key: Some(aes(1)), + ..Default::default() + } +} + +#[cfg(feature = "aes-gcm")] +#[test] +fn an_unacknowledged_rotation_leaves_the_response_parked() { + // The sender is alive and silent: persistence is still in flight, so the + // requester must hear nothing yet. + let (_worker, rx) = std::sync::mpsc::channel::>(); + assert!(settle_parked_key_response(&rx, &listed()).is_none()); +} + +#[cfg(feature = "aes-gcm")] +#[test] +fn a_persisted_rotation_releases_the_response_unchanged() { + let (worker, rx) = std::sync::mpsc::channel(); + worker.send(Ok(())).expect("the receiver is alive"); + + let parked = listed(); + let sent = settle_parked_key_response(&rx, &parked).expect("a durable rotation settles"); + assert!(sent.result); + assert!( + sent.message.is_empty(), + "a durable rotation carries no failure message, got {}", + sent.message + ); + assert_eq!(sent.keys, parked.keys); + assert_eq!(sent.primary_key, parked.primary_key); +} + +#[cfg(feature = "aes-gcm")] +#[test] +fn a_failed_persistence_downgrades_the_response_and_carries_the_cause() { + let (worker, rx) = std::sync::mpsc::channel(); + worker + .send(Err( + Box::new(std::io::Error::other("the volume is read-only")) as KeyringPersistError, + )) + .expect("the receiver is alive"); + + let parked = listed(); + let sent = settle_parked_key_response(&rx, &parked).expect("a failed rotation settles"); + assert!( + !sent.result, + "a rotation the delegate could not persist must not report success" + ); + assert!( + sent.message.contains("not persisted") && sent.message.contains("the volume is read-only"), + "the failure must name the cause, got {}", + sent.message + ); + // Only the verdict is downgraded: the state the apply reported is intact. + assert_eq!(sent.keys, parked.keys); + assert_eq!(sent.primary_key, parked.primary_key); +} + +#[cfg(feature = "aes-gcm")] +#[test] +fn a_persistence_worker_that_exits_without_acknowledging_is_a_failure() { + // A worker that vanished mid-rotation must never be read as a silent + // success, and must never leave the response parked forever. + let (worker, rx) = std::sync::mpsc::channel::>(); + drop(worker); + + let parked = listed(); + let sent = settle_parked_key_response(&rx, &parked).expect("a disconnected ack settles"); + assert!(!sent.result); + assert!( + sent.message.contains("without acknowledging"), + "the failure must name the vanished worker, got {}", + sent.message + ); + assert_eq!(sent.keys, parked.keys); +} + +/// A queued acknowledgement is read even after the worker hung up: the value is +/// buffered in the channel, so a rotation that WAS persisted before the worker +/// exited still releases its response as a success. +#[cfg(feature = "aes-gcm")] +#[test] +fn an_acknowledgement_queued_before_the_worker_exited_still_succeeds() { + let (worker, rx) = std::sync::mpsc::channel(); + worker.send(Ok(())).expect("the receiver is alive"); + drop(worker); + + let sent = settle_parked_key_response(&rx, &listed()).expect("the queued ack settles"); + assert!( + sent.result, + "a rotation persisted before the worker exited is durable" + ); +} diff --git a/serf-driver/src/keyring_file/tests.rs b/serf-driver/src/keyring_file/tests.rs index ac732b61..f26db164 100644 --- a/serf-driver/src/keyring_file/tests.rs +++ b/serf-driver/src/keyring_file/tests.rs @@ -535,3 +535,163 @@ fn a_nameless_destination_is_an_input_error() { .expect_err("a bare root has no file name to derive a temp from"); assert_eq!(err.kind(), io::ErrorKind::InvalidInput); } + +/// The engine reports the destination it persists to, so a caller can log or +/// re-open the very file the worker writes. +#[test] +fn the_engine_reports_its_persistence_path() { + let path = tmp_path("path-accessor"); + let engine = KeyringFilePersistence::new(&path); + assert_eq!(engine.path(), path.as_path()); +} + +/// A destination that exists but cannot be read is an I/O error, never the +/// `None` that means "first boot": a node whose key file is unreadable must +/// fail loudly rather than silently start with no keyring and gossip in the +/// clear. +#[test] +fn an_unreadable_destination_is_an_io_error_not_a_first_boot() { + let path = tmp_path("load-io-error"); + // Ignoring Err: a leftover directory from a previous run is fine to reuse. + let _ = std::fs::create_dir(&path); + let engine = KeyringFilePersistence::new(&path); + + assert!( + matches!(engine.load(), Err(KeyringFileError::Io(_))), + "an unreadable destination must not load as a missing keyring" + ); + + // Ignoring Err: best-effort test-tree cleanup. + let _ = std::fs::remove_dir_all(&path); +} + +/// A sibling that merely shares the temp prefix and suffix is not this engine's +/// to delete: only the exact `.{name}.{16 hex}.tmp` shape is swept, so an +/// operator's own backup survives construction. +#[test] +fn a_prefix_sharing_sibling_is_not_swept() { + let path = tmp_path("shape-guard"); + let name = path.file_name().and_then(|n| n.to_str()).expect("name"); + let dir = path.parent().expect("the temp dir is a parent"); + + // Right prefix and suffix, wrong middle: too short, and not all hex. + let backup = dir.join(format!(".{name}.backup.tmp")); + let short = dir.join(format!(".{name}.0123456789abcde.tmp")); + let nonhex = dir.join(format!(".{name}.0123456789abcdeg.tmp")); + for sibling in [&backup, &short, &nonhex] { + std::fs::write(sibling, "not this engine's file\n").expect("plant the sibling"); + } + + let _engine = KeyringFilePersistence::new(&path); + for sibling in [&backup, &short, &nonhex] { + assert!( + sibling.symlink_metadata().is_ok(), + "a sibling outside the exact temp shape must survive the sweep: {}", + sibling.display() + ); + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(sibling); + } +} + +/// The destination's own name is never sweepable — `remove_file` unlinks a +/// NAME, so a candidate that lexically IS the destination could only ever +/// delete the persisted keyring. +#[test] +fn the_destination_is_never_sweepable() { + let path = tmp_path("sweepable-self"); + std::fs::write(&path, "the persisted keyring\n").expect("seed the destination"); + + assert!(!sweepable(&path, &path)); + + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} + +/// A candidate whose name cannot be inspected is never swept: identity is not +/// even consulted, because a name that cannot be compared cannot be proven +/// distinct from the destination's. +#[test] +fn an_uninspectable_candidate_name_is_never_sweepable() { + use std::os::unix::ffi::OsStrExt as _; + + let path = tmp_path("sweepable-uninspectable"); + let dir = path.parent().expect("the temp dir is a parent"); + let candidate = dir.join(std::ffi::OsStr::from_bytes(b"\xFFnot-utf8")); + + assert!(!sweepable(&path, &candidate)); +} + +/// A dangling symlink at a temp-shaped name reaches no storage at all, so it +/// cannot be the destination and IS sweepable — the abandoned link is cleaned +/// up rather than left to fail a future exclusive create. +#[test] +fn a_dangling_symlink_candidate_is_sweepable() { + let path = tmp_path("sweepable-dangling"); + std::fs::write(&path, "the persisted keyring\n").expect("seed the destination"); + let name = path.file_name().and_then(|n| n.to_str()).expect("name"); + + let dangling = path.with_file_name(format!(".{name}.00000000cafebabe.tmp")); + // Ignoring Err: a leftover link from a previous run is about to be re-planted. + let _ = std::fs::remove_file(&dangling); + std::os::unix::fs::symlink(path.with_extension("no-such-target"), &dangling) + .expect("plant a dangling symlink"); + + assert!( + sweepable(&path, &dangling), + "a link resolving to nothing cannot be the destination" + ); + + // Construction acts on that verdict, and the destination is untouched. + let _engine = KeyringFilePersistence::new(&path); + assert!( + dangling.symlink_metadata().is_err(), + "the abandoned dangling temp must be swept" + ); + assert_eq!( + std::fs::read_to_string(&path).expect("the destination survives"), + "the persisted keyring\n" + ); + + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} + +/// A candidate whose storage identity cannot be established is never swept: +/// with a symlink loop the resolution fails for a reason that is NOT +/// "reaches nothing", so the sweep refuses it rather than unlink on uncertainty. +#[test] +fn a_candidate_of_unestablishable_identity_is_never_sweepable() { + let path = tmp_path("sweepable-loop"); + std::fs::write(&path, "the persisted keyring\n").expect("seed the destination"); + let name = path.file_name().and_then(|n| n.to_str()).expect("name"); + + let first = path.with_file_name(format!(".{name}.00000000feedface.tmp")); + let second = path.with_file_name(format!(".{name}.00000000feedfacf.tmp")); + // Ignoring Err: leftover links from a previous run are about to be re-planted. + let _ = std::fs::remove_file(&first); + let _ = std::fs::remove_file(&second); + std::os::unix::fs::symlink(&second, &first).expect("plant one half of the loop"); + std::os::unix::fs::symlink(&first, &second).expect("close the loop"); + + assert!( + !sweepable(&path, &first), + "a candidate that resolves to neither storage nor nothing is refused" + ); + + // Construction acts on that verdict: the loop is left alone, destination intact. + let _engine = KeyringFilePersistence::new(&path); + assert!( + first.symlink_metadata().is_ok(), + "an unresolvable candidate must survive the sweep" + ); + assert_eq!( + std::fs::read_to_string(&path).expect("the destination survives"), + "the persisted keyring\n" + ); + + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&first); + let _ = std::fs::remove_file(&second); + let _ = std::fs::remove_file(&path); +} diff --git a/serf-driver/src/snapshot/tests.rs b/serf-driver/src/snapshot/tests.rs index be75c80d..0495bdbe 100644 --- a/serf-driver/src/snapshot/tests.rs +++ b/serf-driver/src/snapshot/tests.rs @@ -282,3 +282,57 @@ fn coordinate_readings_ride_the_snapshot() { assert_eq!(snap.coordinate(), Some(&coordinate)); assert_eq!(snap.stats().coordinate_resets(), Some(3)); } + +/// The ops readings are legible straight off the snapshot, not only through the +/// aggregate: a handle reading `health_score` / `broadcast_queue_depth` / +/// `encrypted` sees exactly what the driver attached, and the zero posture +/// before it does. +#[test] +fn ops_readings_are_legible_on_the_snapshot_itself() { + let members = vec![make_member(1, "127.0.0.1:7946", MemberStatus::Alive)]; + let snap = SerfSnapshot::new( + members, + &1u32, + SerfState::Alive, + LamportTime::new(1), + LamportTime::new(2), + LamportTime::new(3), + ); + + // The zero posture before the driver attaches its live endpoint readings. + assert_eq!(snap.health_score(), 0); + assert_eq!(snap.broadcast_queue_depth(), 0); + assert!(!snap.encrypted()); + + let snap = snap.with_ops_stats(4, 11, true); + assert_eq!(snap.health_score(), 4); + assert_eq!(snap.broadcast_queue_depth(), 11); + assert!(snap.encrypted()); + + // The snapshot's own readings and the aggregate it assembles never disagree. + let stats = snap.stats(); + assert_eq!(snap.health_score(), stats.health_score()); + assert_eq!(snap.broadcast_queue_depth(), stats.broadcast_queue_depth()); + assert_eq!(snap.encrypted(), stats.encrypted()); +} + +/// The coordinate-reset counter is legible straight off the snapshot, and +/// agrees with the aggregate. +#[cfg(feature = "coordinates")] +#[test] +fn coordinate_resets_are_legible_on_the_snapshot_itself() { + let members = vec![make_member(1, "127.0.0.1:7952", MemberStatus::Alive)]; + let snap = SerfSnapshot::new( + members, + &1u32, + SerfState::Alive, + LamportTime::ZERO, + LamportTime::ZERO, + LamportTime::ZERO, + ); + assert_eq!(snap.coordinate_resets(), None, "absent until attached"); + + let snap = snap.with_coordinate_resets(Some(6)); + assert_eq!(snap.coordinate_resets(), Some(6)); + assert_eq!(snap.coordinate_resets(), snap.stats().coordinate_resets()); +} diff --git a/serf-driver/src/snapshotter/tests.rs b/serf-driver/src/snapshotter/tests.rs index 35335b14..e6e3dd3c 100644 --- a/serf-driver/src/snapshotter/tests.rs +++ b/serf-driver/src/snapshotter/tests.rs @@ -295,6 +295,103 @@ fn leave_tail_replays_identically_across_compaction() { cleanup(&compacted); } +/// A clock record trailing the leave marker does not mask the clean leave: the +/// clean-left state a reopen re-derives is set by the last MEMBERSHIP record, +/// looking past the clock high-water marks the pump appends after it. Were a +/// trailing clock read as membership activity, the next compaction would drop +/// the marker and a restart would silently rejoin a cluster the operator left. +#[test] +fn a_trailing_clock_record_does_not_mask_the_clean_leave() { + let o = opts("clock-after-leave"); + { + // The default threshold: this flush leaves the original records on disk. + let (mut snap, _) = o.open::().expect("open"); + snap.append_member(true, &node("peer", 7001)); + snap.append_leave(); + // A clock high-water mark advancing after the leave marker. + snap.append_clocks(LamportTime::new(4), LamportTime::ZERO, LamportTime::ZERO); + snap.flush_and_maybe_compact(Vec::new); + } + + // Reopening re-derives the clean-left state from the records on disk. + let (mut snap, records) = + Snapshotter::::open(o.path(), 1).expect("reopen with a compacting threshold"); + assert!( + matches!(records.last(), Some(SnapshotRecord::Clock(_))), + "the seeded file must end with the clock record" + ); + // Threshold 1: this flush compacts, and the rewrite must re-emit the marker. + snap.flush_and_maybe_compact(|| vec![node("peer", 7001)]); + drop(snap); + + let (_s, records) = o.open::().expect("reopen compacted"); + let fresh = ReplayResult::replay(records.clone(), false); + assert!( + fresh.alive_nodes.is_empty(), + "the compacted file must still gate the clean leave on the default posture" + ); + let rejoin = ReplayResult::replay(records, true); + assert_eq!( + rejoin.alive_nodes, + vec![node("peer", 7001)], + "the opt-in posture must still recover the pre-leave membership" + ); + cleanup(&o); +} + +/// A snapshot path that cannot be read at all fails the open loudly: a +/// directory standing in the file's place is an I/O error, never an empty +/// replay that would silently discard the recovered membership. +#[test] +fn an_unreadable_snapshot_path_is_an_io_error() { + let o = opts("unreadable"); + // Ignoring Err: a leftover directory from a previous run is fine to reuse. + let _ = fs::create_dir(o.path()); + assert!( + matches!(o.open::(), Err(SnapshotOpenError::Io(_))), + "an unreadable snapshot path must not open as an empty replay" + ); + // Ignoring Err: best-effort test-tree cleanup. + let _ = fs::remove_dir_all(o.path()); +} + +/// A compaction that cannot write its replacement leaves the GROWN file +/// authoritative: the records already appended survive untouched, so a failed +/// rewrite can never lose the membership the wire is carrying. +#[test] +fn a_failed_compaction_keeps_the_grown_file_authoritative() { + let o = opts("compact-blocked").with_compact_threshold(1); + // Occupy the compaction's temp path with a directory, so its create fails. + let blocked = o.path().with_extension("compact"); + // Ignoring Err: a leftover from a previous run is fine to reuse. + let _ = fs::remove_dir_all(&blocked); + fs::create_dir(&blocked).expect("block the compaction temp path"); + + { + let (mut snap, _) = o.open::().expect("open"); + snap.append_member(true, &node("peer", 7001)); + snap.append_clocks(LamportTime::new(6), LamportTime::ZERO, LamportTime::ZERO); + // A compaction that COMPLETED here would rewrite the file to this empty + // live set and drop the appended member; the blocked one must not. + snap.flush_and_maybe_compact(Vec::new); + } + + let (_s, records) = o + .open::() + .expect("reopen after the failed compaction"); + let replay = ReplayResult::replay(records, false); + assert_eq!( + replay.alive_nodes, + vec![node("peer", 7001)], + "a failed compaction must leave the grown file's records intact" + ); + assert_eq!(replay.last_clock, LamportTime::new(6)); + + // Ignoring Err: best-effort test-tree cleanup. + let _ = fs::remove_dir_all(&blocked); + cleanup(&o); +} + /// Membership activity after a leave clears the clean-left state: the next /// compaction does not re-emit a stale Leave marker over live members. #[test] diff --git a/serf-embassy/src/config/tests.rs b/serf-embassy/src/config/tests.rs index 314cd3c5..8c8d1bab 100644 --- a/serf-embassy/src/config/tests.rs +++ b/serf-embassy/src/config/tests.rs @@ -1,5 +1,35 @@ use super::*; +/// A CIDR policy installed through the builder is the one the engine will admit +/// peers against; the default posture installs none (every address admitted). +#[cfg(feature = "cidr")] +#[test] +fn a_cidr_policy_is_installable() { + use core::net::{IpAddr, Ipv4Addr}; + + let mut policy = serf_embedded::CidrPolicy::block_all(); + policy.add( + "10.0.0.0/8" + .parse::() + .expect("a well-formed CIDR parses"), + ); + + let installed = Options::new() + .with_cidr_policy(policy) + .cidr_policy + .expect("the policy is installed"); + assert!( + installed.is_allowed(&IpAddr::V4(Ipv4Addr::new(10, 1, 2, 3))), + "an address inside the allow-list is admitted" + ); + assert!( + installed.is_blocked(&IpAddr::V4(Ipv4Addr::new(192, 168, 1, 1))), + "an address outside the allow-list is refused" + ); + + assert!(Options::new().cidr_policy.is_none()); +} + #[test] fn defaults_are_sane_and_overridable() { let c = Options::new(); diff --git a/serf-embassy/src/error/mod.rs b/serf-embassy/src/error/mod.rs index 5a0d57ce..b0ed0742 100644 --- a/serf-embassy/src/error/mod.rs +++ b/serf-embassy/src/error/mod.rs @@ -1,6 +1,9 @@ //! Errors from constructing a [`Serf`](crate::Serf) and from an await-result //! [`join`](crate::Serf::join). +#[cfg(test)] +mod tests; + use alloc::boxed::Box; use core::{fmt, time::Duration}; diff --git a/serf-embassy/src/error/tests.rs b/serf-embassy/src/error/tests.rs new file mode 100644 index 00000000..090de06a --- /dev/null +++ b/serf-embassy/src/error/tests.rs @@ -0,0 +1,414 @@ +use super::*; + +use std::{format, string::String, vec, vec::Vec}; + +use memberlist_proto::EndpointInitError; +use serf_embedded::{InitError as EngineInitError, MemberlistInitError}; + +use crate::{SerfOptions, SerfState}; + +/// A resolver error with a recognisable rendering, so the boxed `Resolve` arms +/// can be checked for actually carrying their cause into the message. +#[derive(Debug)] +struct ResolverFault; + +impl fmt::Display for ResolverFault { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("the resolver gave up") + } +} + +impl core::error::Error for ResolverFault {} + +/// The typed cause an over-ceiling `max_user_event_size` produces. +fn invalid_serf_options() -> InvalidOptions { + SerfOptions::new() + .with_max_user_event_size(SerfOptions::DEFAULT_USER_EVENT_SIZE_LIMIT + 1) + .validate() + .expect_err("an over-ceiling max_user_event_size must fail validation") +} + +fn socket_timeout_payload() -> SocketTimeoutOutOfRange { + SocketTimeoutOutOfRange { + socket_timeout: Duration::from_millis(5), + close_timeout: Duration::from_millis(10), + stream_timeout: Duration::from_millis(20), + max: Duration::from_secs(3600), + tick_hz: 1_000, + } +} + +/// One representative value of every [`InitError`] variant. +fn all_init_errors() -> Vec { + vec![ + InitError::TcpPoolTooSmall(1), + InitError::ZeroBridgeRing, + InitError::SocketTimeoutOutOfRange(socket_timeout_payload()), + InitError::Engine(EngineInitError::Memberlist(MemberlistInitError::ZeroPort)), + InitError::Resolve(Box::new(ResolverFault)), + InitError::NoAddresses, + InitError::Entropy, + InitError::InvalidSerfOptions(invalid_serf_options()), + ] +} + +// ── InitError ───────────────────────────────────────────────────────────────── + +#[test] +fn init_error_renders_each_variant_with_its_cause() { + assert_eq!( + format!("{}", InitError::TcpPoolTooSmall(1)), + "the TCP socket pool needs at least 2 sockets (a listener plus one dial/accept socket); got 1" + ); + assert_eq!( + format!("{}", InitError::ZeroBridgeRing), + "tcp_socket_rx_bytes and tcp_socket_tx_bytes must both be non-zero" + ); + assert_eq!( + format!("{}", InitError::NoAddresses), + "advertise address resolution returned no addresses" + ); + assert_eq!( + format!("{}", InitError::Entropy), + "entropy source failed while seeding the RNGs" + ); + assert_eq!( + format!("{}", InitError::Resolve(Box::new(ResolverFault))), + "advertise address resolution failed: the resolver gave up" + ); + + // The engine arm is transparent: the shared engine's own rendering is what a + // caller reads, undecorated. + let engine = EngineInitError::Memberlist(MemberlistInitError::ZeroPort); + let engine_rendered = format!("{engine}"); + assert_eq!(format!("{}", InitError::Engine(engine)), engine_rendered); + assert!(!engine_rendered.is_empty()); + + let invalid = format!("{}", InitError::InvalidSerfOptions(invalid_serf_options())); + assert!( + invalid.starts_with("invalid serf options: "), + "the serf-options arm names its surface: {invalid}" + ); + assert!( + invalid.contains("max_user_event_size"), + "the typed cause is carried through: {invalid}" + ); +} + +/// The out-of-range rendering names every value an operator needs to fix the +/// configuration: the offending timeout, both deadlines it must exceed, the +/// maximum, and the tick rate the rounded comparison used. +#[test] +fn socket_timeout_out_of_range_renders_every_bound() { + let payload = socket_timeout_payload(); + let rendered = format!("{}", InitError::SocketTimeoutOutOfRange(payload)); + for needle in ["socket_timeout", "close_timeout", "stream_timeout", "1000"] { + assert!( + rendered.contains(needle), + "{needle} missing from {rendered}" + ); + } + // The four durations render through their `Debug` form. + for duration in [ + payload.socket_timeout, + payload.close_timeout, + payload.stream_timeout, + payload.max, + ] { + assert!( + rendered.contains(&format!("{duration:?}")), + "{duration:?} missing from {rendered}" + ); + } + assert_eq!(payload.tick_hz, 1_000); +} + +#[test] +fn init_error_predicates_select_exactly_one_variant() { + type Predicate = (&'static str, fn(&InitError) -> bool); + let predicates: [Predicate; 7] = [ + ("tcp_pool_too_small", InitError::is_tcp_pool_too_small), + ("zero_bridge_ring", InitError::is_zero_bridge_ring), + ( + "socket_timeout_out_of_range", + InitError::is_socket_timeout_out_of_range, + ), + ("engine", InitError::is_engine), + ("resolve", InitError::is_resolve), + ("no_addresses", InitError::is_no_addresses), + ("entropy", InitError::is_entropy), + ]; + + // `all_init_errors` lists the variants in the same order as the predicates, + // with the untested `InvalidSerfOptions` last: each error must satisfy its own + // predicate and no other. + for (i, err) in all_init_errors().iter().enumerate() { + for (j, (name, predicate)) in predicates.iter().enumerate() { + assert_eq!( + predicate(err), + i == j, + "is_{name} on {err:?} must be {}", + i == j + ); + } + } +} + +#[test] +fn init_error_converts_from_both_engine_halves() { + // The memberlist half nests through the engine's own error. + let from_memberlist: InitError = MemberlistInitError::ZeroPort.into(); + assert!(matches!( + from_memberlist, + InitError::Engine(EngineInitError::Memberlist(MemberlistInitError::ZeroPort)) + )); + + // And the engine's error passes through whole. + let from_engine: InitError = EngineInitError::InvalidSerfOptions(invalid_serf_options()).into(); + assert!(matches!( + from_engine, + InitError::Engine(EngineInitError::InvalidSerfOptions(_)) + )); +} + +/// `source` chains only for the variants that wrap another error, so a caller +/// walking the chain reaches the engine fault or the resolver's own error and +/// stops at every leaf. +#[test] +fn init_error_source_chains_only_for_wrapping_variants() { + use std::error::Error as _; + + let engine = InitError::Engine(EngineInitError::Memberlist( + MemberlistInitError::AdvertisePortMismatch, + )); + assert!(engine.source().is_some()); + + let resolve = InitError::Resolve(Box::new(ResolverFault)); + let source = resolve.source().expect("the boxed resolver error chains"); + assert_eq!(format!("{source}"), "the resolver gave up"); + + for leaf in [ + InitError::TcpPoolTooSmall(0), + InitError::ZeroBridgeRing, + InitError::SocketTimeoutOutOfRange(socket_timeout_payload()), + InitError::NoAddresses, + InitError::Entropy, + InitError::InvalidSerfOptions(invalid_serf_options()), + ] { + assert!( + leaf.source().is_none(), + "a leaf variant carries no source: {leaf:?}" + ); + } +} + +/// The shared engine's own error is the chained cause: its serf-options half +/// exposes the typed `InvalidOptions`, while its memberlist half carries its +/// cause through `Display` instead (the memberlist error is not an `Error` on +/// the minimal no_std build). +#[test] +fn the_engine_error_chains_its_serf_options_half_only() { + use std::error::Error as _; + + let options = EngineInitError::InvalidSerfOptions(invalid_serf_options()); + assert!(options.source().is_some()); + assert!( + format!("{options}").starts_with("invalid serf options: "), + "{options}" + ); + + let memberlist = EngineInitError::from(MemberlistInitError::Endpoint( + EndpointInitError::AwarenessMultiplierZero, + )); + assert!( + memberlist.source().is_none(), + "the memberlist half carries its cause through Display, not the source chain" + ); + assert!(!format!("{memberlist}").is_empty()); +} + +#[test] +fn init_error_debug_is_never_empty() { + for err in all_init_errors() { + assert!(!format!("{err:?}").is_empty()); + } +} + +// ── JoinError ───────────────────────────────────────────────────────────────── + +fn all_join_errors() -> Vec { + vec![ + JoinError::Resolve(Box::new(ResolverFault)), + JoinError::Control(SerfError::BadLeaveState(SerfState::Leaving)), + JoinError::NoAddresses, + JoinError::Failed(JoinFailed::new(3, 0)), + JoinError::Shutdown, + ] +} + +#[test] +fn join_error_renders_each_variant_with_its_cause() { + assert_eq!( + format!("{}", JoinError::Resolve(Box::new(ResolverFault))), + "seed address resolution failed: the resolver gave up" + ); + assert_eq!( + format!("{}", JoinError::NoAddresses), + "no wire address resolved for any seed" + ); + assert_eq!( + format!("{}", JoinError::Shutdown), + "the node shut down after losing an id-conflict vote before the join resolved" + ); + + // The control arm names the rejection and carries the engine's own message. + let control = SerfError::BadLeaveState(SerfState::Leaving); + let rendered = format!("{}", JoinError::Control(control)); + assert!(rendered.starts_with("join was rejected: "), "{rendered}"); + assert!( + rendered.contains(&format!("{}", SerfError::BadLeaveState(SerfState::Leaving))), + "{rendered}" + ); + + // The failed arm is transparent: the reached/requested counts are the message. + assert_eq!( + format!("{}", JoinError::Failed(JoinFailed::new(3, 0))), + "join reached 0 of 3 seed(s)" + ); +} + +#[test] +fn join_error_predicates_select_exactly_one_variant() { + type Predicate = (&'static str, fn(&JoinError) -> bool); + let predicates: [Predicate; 5] = [ + ("resolve", JoinError::is_resolve), + ("control", JoinError::is_control), + ("no_addresses", JoinError::is_no_addresses), + ("failed", JoinError::is_failed), + ("shutdown", JoinError::is_shutdown), + ]; + + for (i, err) in all_join_errors().iter().enumerate() { + for (j, (name, predicate)) in predicates.iter().enumerate() { + assert_eq!( + predicate(err), + i == j, + "is_{name} on {err:?} must be {}", + i == j + ); + } + } +} + +#[test] +fn join_error_converts_from_a_rejected_command() { + let err: JoinError = SerfError::BadLeaveState(SerfState::Left).into(); + assert!(matches!( + err, + JoinError::Control(SerfError::BadLeaveState(SerfState::Left)) + )); +} + +/// The failed-join payload chains as the cause, so a caller can recover the +/// requested-seed count from the source chain rather than re-parsing the message. +#[test] +fn join_error_source_chains_the_resolver_control_and_failure_causes() { + use std::error::Error as _; + + let resolve = JoinError::Resolve(Box::new(ResolverFault)); + assert_eq!( + format!("{}", resolve.source().expect("the resolver error chains")), + "the resolver gave up" + ); + + let control = JoinError::Control(SerfError::BadLeaveState(SerfState::Leaving)); + assert!(control.source().is_some()); + + let failed = JoinError::Failed(JoinFailed::new(4, 0)); + let source = failed.source().expect("the failure payload chains"); + assert_eq!(format!("{source}"), "join reached 0 of 4 seed(s)"); + + for leaf in [JoinError::NoAddresses, JoinError::Shutdown] { + assert!(leaf.source().is_none(), "{leaf:?} is a leaf"); + } +} + +/// The payload a fully-dispatched but unreachable join carries: the seed count +/// it asked for, and the zero it reached. +#[test] +fn join_failed_payload_reports_the_seed_counts() { + let failed = JoinFailed::new(5, 0); + assert_eq!(failed.requested(), 5); + assert_eq!(failed.contacted(), 0); + assert_eq!(format!("{failed}"), "join reached 0 of 5 seed(s)"); +} + +// ── OpError ─────────────────────────────────────────────────────────────────── + +#[test] +fn op_error_renders_each_variant_with_its_cause() { + assert_eq!( + format!("{}", OpError::Shutdown), + "the node has shut down after losing an id-conflict vote; it no longer accepts commands" + ); + + // The engine arm is transparent: the rejection reads exactly as the engine + // reported it. + let rejected = SerfError::BadLeaveState(SerfState::Leaving); + let expected = format!("{rejected}"); + assert_eq!(format!("{}", OpError::Serf(rejected)), expected); +} + +#[test] +fn op_error_predicates_and_accessor_agree_on_the_variant() { + let shutdown = OpError::Shutdown; + assert!(shutdown.is_shutdown()); + assert!(!shutdown.is_serf()); + assert!( + shutdown.as_serf().is_none(), + "a shutdown refusal carries no engine error" + ); + + let rejected = OpError::Serf(SerfError::BadLeaveState(SerfState::Left)); + assert!(rejected.is_serf()); + assert!(!rejected.is_shutdown()); + assert!(matches!( + rejected.as_serf(), + Some(SerfError::BadLeaveState(SerfState::Left)) + )); +} + +#[test] +fn op_error_converts_from_a_rejected_command() { + let err: OpError = SerfError::LeaveClockExhausted.into(); + assert!(matches!(err, OpError::Serf(SerfError::LeaveClockExhausted))); + assert!(err.as_serf().is_some()); +} + +#[test] +fn op_error_source_chains_only_the_engine_rejection() { + use std::error::Error as _; + + let rejected = OpError::Serf(SerfError::LeaveClockExhausted); + let source = rejected.source().expect("the engine rejection chains"); + assert_eq!( + format!("{source}"), + format!("{}", SerfError::LeaveClockExhausted) + ); + + assert!( + OpError::Shutdown.source().is_none(), + "a shutdown refusal is a leaf" + ); +} + +#[test] +fn op_error_debug_is_never_empty() { + for err in [ + OpError::Shutdown, + OpError::Serf(SerfError::LeaveClockExhausted), + ] { + let shown: String = format!("{err:?}"); + assert!(!shown.is_empty()); + } +} diff --git a/serf-embassy/src/shared/tests.rs b/serf-embassy/src/shared/tests.rs index e945f880..ecfaac79 100644 --- a/serf-embassy/src/shared/tests.rs +++ b/serf-embassy/src/shared/tests.rs @@ -33,6 +33,118 @@ fn shared() -> Shared { Shared::new(engine, advertise) } +/// Build an engine over an arbitrary port / advertise / transform triple. +fn try_engine( + port: u16, + advertise: SocketAddr, + transform: TransformOptions, +) -> Result, serf_embedded::InitError> { + SerfEngine::::try_new_at( + Options::new() + .with_port(port) + .with_close_timeout(Duration::from_secs(10)), + transform, + EndpointOptions::new(SmolStr::new("test"), advertise), + SerfOptions::new(), + at(), + SmallRng::seed_from_u64(42), + ) +} + +/// A node must advertise an address its peers can route a reply to: an unspecified +/// IP would be gossiped cluster-wide and then be useless to every peer that selected +/// it as an egress destination, so it is refused before the endpoint exists. +#[test] +fn a_non_routable_advertise_address_is_refused() { + // `SerfEngine` is not `Debug`, so the error is matched out with a `let-else`. + let Err(err) = try_engine( + 7946, + SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 7946), + TransformOptions::default(), + ) else { + panic!("an unspecified advertise address must be refused"); + }; + assert!( + matches!( + err, + serf_embedded::InitError::Memberlist( + serf_embedded::MemberlistInitError::NonRoutableAdvertiseAddr(_) + ) + ), + "{err:?}" + ); +} + +/// One port serves both the gossip and reliable planes, and an embedded interface +/// has no NAT — so a node advertising a port it does not bind would have every peer +/// routing to a port nothing listens on. +#[test] +fn an_advertise_port_that_does_not_match_the_bound_port_is_refused() { + let Err(err) = try_engine( + 7946, + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)), 7947), + TransformOptions::default(), + ) else { + panic!("an advertise port other than the bound port must be refused"); + }; + assert!( + matches!( + err, + serf_embedded::InitError::Memberlist( + serf_embedded::MemberlistInitError::AdvertisePortMismatch + ) + ), + "{err:?}" + ); +} + +/// A labelled cluster that opts out of the inbound label check still constructs: the +/// opt-out is a migration posture (accept unlabelled peers while the label rolls +/// out), not a rejected configuration. +#[test] +fn a_label_with_the_inbound_check_skipped_constructs() { + let advertise = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)), 7946); + let transform = TransformOptions::default() + .with_label(Some(alloc::vec![b'm', b'i', b'g'])) + .expect("a short label is valid") + .with_skip_inbound_label_check(true); + + let Ok(engine) = try_engine(7946, advertise, transform) else { + panic!("skipping the inbound label check is a valid migration posture"); + }; + assert_eq!(engine.port(), 7946); +} + +/// An app that never drains `poll_event` cannot grow the driver's buffer without +/// bound: at the cap the OLDEST buffered event is shed and counted, and the public +/// `events_dropped` total reports the loss rather than hiding it. +#[test] +fn the_app_event_buffer_sheds_the_oldest_at_the_cap() { + let shared = shared(); + assert_eq!(shared.events_dropped(), 0); + + let overflow = 5usize; + for _ in 0..DEFAULT_EVENT_BUFFER_CAP + overflow { + shared.push_app_event(Event::LeftCluster); + } + + assert_eq!( + shared.app_events.borrow().len(), + DEFAULT_EVENT_BUFFER_CAP, + "the buffer must stay bounded at the cap" + ); + assert_eq!( + shared.app_events_dropped.get(), + overflow as u64, + "every event past the cap is shed and counted" + ); + assert_eq!( + shared.events_dropped(), + overflow as u64, + "the public total must report the driver-side shedding" + ); +} + /// The production conflict pathway: a drained `Event::Shutdown` — the terminal event /// serf emits when the local node loses an id-conflict vote — routes through /// `route_drained_event` to `begin_shutdown` (latching the one-way shutdown state and diff --git a/serf-embassy/tests/handle.rs b/serf-embassy/tests/handle.rs new file mode 100644 index 00000000..b0f93fee --- /dev/null +++ b/serf-embassy/tests/handle.rs @@ -0,0 +1,296 @@ +//! The `Serf` handle's construction, cloning, read surface, and membership +//! commands over two real embassy-net stacks. +//! +//! The behavioral gates here: the default-RNG constructor stands a node up from +//! platform entropy (no injected seeds), a cloned handle is the SAME node rather +//! than a detached copy, every read accessor reports the engine's live view (not +//! a stale snapshot), and `set_tags` / `force_leave` actually reach the peer. + +// nested `if let X = ev { if cond }` kept for readability, as in the crate roots. +#![allow(clippy::collapsible_if)] + +mod support; + +use core::{net::SocketAddr, time::Duration as CoreDuration}; + +use embassy_net::StackResources; +use embassy_time::{Duration, Timer}; +use futures::executor::block_on; +use serf_embassy::{ + Bytes, EndpointOptions, Member, MemberStatus, Options, QueryParams, ReconnectDelegate, Serf, + SerfOptions, SerfState, SocketAddrResolver, Tags, TransformOptions, now, +}; +use smol_str::SmolStr; + +use support::cluster::{ + NodeBufs, POOL, addr, build_node, build_sockets, build_stack, devices, drive, join_and_converge, +}; + +/// A reconnect policy that pins every member to a fixed timeout, so installing it +/// is observable as "the delegate the engine now consults". +struct FixedReconnect(CoreDuration); + +impl ReconnectDelegate for FixedReconnect { + fn reconnect_timeout( + &self, + _member: &Member, + _timeout: CoreDuration, + ) -> CoreDuration { + self.0 + } +} + +/// The default-RNG constructor stands up a live node from platform entropy alone: +/// no injected seeds, and the resulting node is `Alive`, a member of its own +/// cluster, and advertising the address it was configured with. +#[test] +fn the_default_rng_constructor_stands_up_a_live_node() { + let (dev_a, _dev_b) = devices(); + let mut res_a = StackResources::<{ POOL + 2 }>::new(); + let (stack_a, mut net_a) = build_stack(dev_a, &mut res_a, 1, 0x1111_2222); + + let mut bufs_a = NodeBufs::new(); + let (udp_a, tcp_a) = build_sockets(stack_a, &mut bufs_a); + + let (serf, _runner) = block_on(Serf::::new::<_, POOL>( + Options::new(), + TransformOptions::default(), + EndpointOptions::new(SmolStr::new("solo"), addr(1, 7946)), + SerfOptions::new(), + &SocketAddrResolver, + udp_a, + tcp_a, + now(), + )) + .expect("entropy-seeded construction over a routable address"); + + assert_eq!(serf.state(), SerfState::Alive); + assert_eq!(serf.local_id(), SmolStr::new("solo")); + assert_eq!(serf.advertise_address(), addr(1, 7946)); + assert_eq!( + serf.num_members(), + 1, + "a fresh node is the only member of its own view" + ); + assert!(!serf.is_shutdown()); + + // Silence the unused stack runner without driving it: no I/O is required for + // construction, which is the property under test. + let _ = &mut net_a; +} + +/// A cloned handle is the SAME node, not a detached copy: both handles observe one +/// membership view, and a shutdown latched through the clone is visible on the +/// original. +#[test] +fn a_cloned_handle_shares_one_node() { + let (dev_a, _dev_b) = devices(); + let mut res_a = StackResources::<{ POOL + 2 }>::new(); + let (stack_a, mut net_a) = build_stack(dev_a, &mut res_a, 1, 0x1111_2222); + + let mut bufs_a = NodeBufs::new(); + let (udp_a, tcp_a) = build_sockets(stack_a, &mut bufs_a); + let (serf, _runner) = build_node(udp_a, tcp_a, "a", 1, now(), TransformOptions::default()); + + let clone = serf.clone(); + assert_eq!(clone.local_id(), serf.local_id()); + assert_eq!(clone.advertise_address(), serf.advertise_address()); + assert_eq!(clone.num_members(), serf.num_members()); + + // The latch is shared state, not per-handle state. + assert!(!serf.is_shutdown() && !clone.is_shutdown()); + clone.shutdown(); + assert!( + serf.is_shutdown(), + "a shutdown through the clone must stop the original handle's node" + ); + + let _ = &mut net_a; +} + +/// The read accessors report the engine's LIVE view: after a converged join the +/// clocks have advanced, the reliable plane holds a listener and has consumed pool +/// slots, and the member list agrees with the counts. +#[test] +fn the_read_surface_reports_the_live_engine_view() { + let (dev_a, dev_b) = devices(); + let mut res_a = StackResources::<{ POOL + 2 }>::new(); + let mut res_b = StackResources::<{ POOL + 2 }>::new(); + let (stack_a, mut net_a) = build_stack(dev_a, &mut res_a, 1, 0x1111_2222); + let (stack_b, mut net_b) = build_stack(dev_b, &mut res_b, 2, 0x3333_4444); + + let mut bufs_a = NodeBufs::new(); + let mut bufs_b = NodeBufs::new(); + let (udp_a, tcp_a) = build_sockets(stack_a, &mut bufs_a); + let (udp_b, tcp_b) = build_sockets(stack_b, &mut bufs_b); + + let now_ = now(); + let (a, run_a) = build_node(udp_a, tcp_a, "a", 1, now_, TransformOptions::default()); + let (b, run_b) = build_node(udp_b, tcp_b, "b", 2, now_, TransformOptions::default()); + + // Before the run loop turns, nothing has been shed and no clock has moved. + assert_eq!(a.events_dropped(), 0); + assert_eq!(a.coalesced_user_events_dropped(), 0); + assert_eq!(a.coalesced_member_events_dropped(), 0); + assert_eq!(a.event_time(), 0); + assert_eq!(a.query_time(), 0); + + block_on(async { + let op = async { + join_and_converge(&a, &b).await; + + // A user event and a query advance the two application clocks; the member + // clock advanced with the join itself. + a.user_event("evt", Bytes::from_static(b"p"), false) + .expect("user_event from a running node"); + a.query( + "q", + Bytes::from_static(b"p"), + QueryParams { + timeout: CoreDuration::from_secs(1), + ..Default::default() + }, + ) + .expect("query from a running node"); + + // The member view is the source of truth for the counts. + let members = a.members(); + assert_eq!(members.len(), a.num_members()); + assert_eq!(members.len(), 2); + let mut ids: std::vec::Vec = + members.iter().map(|m| m.node().id_ref().clone()).collect(); + ids.sort(); + assert_eq!(ids, ["a", "b"].map(SmolStr::new)); + + assert!( + a.member_time() > 0, + "the join advanced the membership clock" + ); + assert!( + a.event_time() > 0, + "the user event advanced the event clock" + ); + assert!(a.query_time() > 0, "the query advanced the query clock"); + + // The reliable plane: a dedicated listener, and the join consumed slots from + // the pool it was seeded with. + assert!(a.listener_present(), "the listener slot must be armed"); + assert!( + a.pool_free_count() < POOL, + "the listener plus the exchange must hold pooled slots" + ); + assert!(a.closing_count() <= POOL); + assert!(a.half_closed_count() <= POOL); + assert_eq!( + a.pending_join_count(), + 0, + "the converged join left no pending seed" + ); + assert_eq!(a.pending_dial_count(), 0); + // A accepted B's inbound push/pull exchange. + assert!( + a.accepted_inbound_count() >= 1, + "A must have accepted B's join exchange" + ); + }; + drive(op, run_a, run_b, &mut net_a, &mut net_b).await; + }); +} + +/// New tags set on A reach B's membership view: `set_tags` re-advertises the local +/// member rather than only mutating a local copy. +#[test] +fn set_tags_reaches_the_peers_member_view() { + let (dev_a, dev_b) = devices(); + let mut res_a = StackResources::<{ POOL + 2 }>::new(); + let mut res_b = StackResources::<{ POOL + 2 }>::new(); + let (stack_a, mut net_a) = build_stack(dev_a, &mut res_a, 1, 0x1111_2222); + let (stack_b, mut net_b) = build_stack(dev_b, &mut res_b, 2, 0x3333_4444); + + let mut bufs_a = NodeBufs::new(); + let mut bufs_b = NodeBufs::new(); + let (udp_a, tcp_a) = build_sockets(stack_a, &mut bufs_a); + let (udp_b, tcp_b) = build_sockets(stack_b, &mut bufs_b); + + let now_ = now(); + let (a, run_a) = build_node(udp_a, tcp_a, "a", 1, now_, TransformOptions::default()); + let (b, run_b) = build_node(udp_b, tcp_b, "b", 2, now_, TransformOptions::default()); + + block_on(async { + let op = async { + join_and_converge(&a, &b).await; + + let tags: Tags = [("role", "leader"), ("dc", "eu-west")] + .into_iter() + .collect(); + a.set_tags(tags).expect("set_tags from a running node"); + + // B's view of A must carry the new tags. + loop { + let seen = b.members().into_iter().find(|m| m.node().id_ref() == "a"); + if let Some(m) = seen { + if m.tags().0.get("role").map(SmolStr::as_str) == Some("leader") { + assert_eq!( + m.tags().0.get("dc").map(SmolStr::as_str), + Some("eu-west"), + "every tag in the set must propagate, not just the first" + ); + return; + } + } + Timer::after(Duration::from_millis(10)).await; + } + }; + drive(op, run_a, run_b, &mut net_a, &mut net_b).await; + }); +} + +/// `force_leave` on A removes B from A's own view: the operator-driven removal +/// marks the target as no longer alive rather than being a silent no-op. +#[test] +fn force_leave_removes_the_target_from_the_local_view() { + let (dev_a, dev_b) = devices(); + let mut res_a = StackResources::<{ POOL + 2 }>::new(); + let mut res_b = StackResources::<{ POOL + 2 }>::new(); + let (stack_a, mut net_a) = build_stack(dev_a, &mut res_a, 1, 0x1111_2222); + let (stack_b, mut net_b) = build_stack(dev_b, &mut res_b, 2, 0x3333_4444); + + let mut bufs_a = NodeBufs::new(); + let mut bufs_b = NodeBufs::new(); + let (udp_a, tcp_a) = build_sockets(stack_a, &mut bufs_a); + let (udp_b, tcp_b) = build_sockets(stack_b, &mut bufs_b); + + let now_ = now(); + let (a, run_a) = build_node(udp_a, tcp_a, "a", 1, now_, TransformOptions::default()); + let (b, run_b) = build_node(udp_b, tcp_b, "b", 2, now_, TransformOptions::default()); + + block_on(async { + let op = async { + join_and_converge(&a, &b).await; + + // Installing a reconnect policy is accepted on a running node and does not + // disturb the converged view. + a.set_reconnect_delegate(Some(std::boxed::Box::new(FixedReconnect( + CoreDuration::from_secs(30), + )))); + assert_eq!(a.num_members(), 2); + + a.force_leave(SmolStr::new("b"), false) + .expect("force_leave from a running node"); + + // A's view of B must stop being Alive. + loop { + let seen = a.members().into_iter().find(|m| m.node().id_ref() == "b"); + match seen { + None => return, + Some(m) if m.status() != MemberStatus::Alive => return, + _ => Timer::after(Duration::from_millis(10)).await, + } + } + }; + drive(op, run_a, run_b, &mut net_a, &mut net_b).await; + }); + + // And clearing the override afterwards is equally accepted. + a.set_reconnect_delegate(None); +} diff --git a/serf-embassy/tests/key_refusals.rs b/serf-embassy/tests/key_refusals.rs new file mode 100644 index 00000000..e12270f7 --- /dev/null +++ b/serf-embassy/tests/key_refusals.rs @@ -0,0 +1,348 @@ +//! Key-management REFUSALS end-to-end: the ops a node must decline, and the one it +//! must accept as a no-op, with the LIVE wire keyring left untouched either way. +//! +//! The happy-path rotation is covered by the `key_rotation` suite. What is pinned +//! here is the other half of the contract — every refusal reaches the originator as +//! a counted, message-carrying error rather than a silent success, and no refused +//! op re-keys the wire: +//! +//! - promoting the CURRENT primary is a trivial success that installs nothing, +//! - promoting or removing a key that is not installed is refused, +//! - removing the primary is refused (an operator promotes a secondary first), +//! - `list_keys` reports the live census without changing it, +//! - a key op on an UNENCRYPTED cluster is refused, never silently accepted. +//! +//! Each test issues at most two key-management queries, because every one of them +//! runs to its own multi-second deadline and the suite shares one wall-clock cap. + +// The whole suite exercises key management, so without an AEAD backend the file +// compiles to nothing — gating item-by-item would leave the shared harness +// helpers dead in a plaintext build. +#![cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#![allow(clippy::collapsible_if)] + +mod support; + +use core::net::SocketAddr; + +use embassy_net::StackResources; +use embassy_time::{Duration, Timer}; +use futures::executor::block_on; +use serf_embassy::{EncryptionOptions, Event, Keyring, SecretKey, Serf, TransformOptions, now}; +use smol_str::SmolStr; + +use support::cluster::{ + NodeBufs, POOL, build_node, build_sockets, build_stack, devices, drive, join_and_converge, +}; + +/// A fixed AEAD key filled with `fill`, in whichever backend is compiled. +fn secret_key(fill: u8) -> SecretKey { + #[cfg(feature = "aes-gcm")] + { + SecretKey::Aes256([fill; 32]) + } + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + { + SecretKey::ChaCha20Poly1305([fill; 32]) + } +} + +/// The keyring every encrypted node in this suite starts from: K1 as the sole +/// primary, no secondaries. +fn keyed(k1: SecretKey) -> TransformOptions { + TransformOptions::default() + .with_encryption(EncryptionOptions::new().with_keyring(Keyring::new(k1))) +} + +/// One closed key-management query: the aggregate counts, every responding node's +/// failure message, and the key census the responses reported. +struct KeyOutcome { + num_resp: usize, + num_err: usize, + messages: std::vec::Vec, + keys: std::vec::Vec, + primary_keys: std::vec::Vec, +} + +/// Poll A until its in-flight key query closes, draining B so neither node's +/// backlog stalls the gossip. +async fn drive_to_key_response( + a: &Serf, + b: &Serf, +) -> KeyOutcome { + loop { + let mut seen = None; + while let Some(ev) = a.poll_event() { + if let Event::KeyResponse(kr) = ev { + seen = Some(KeyOutcome { + num_resp: kr.num_resp, + num_err: kr.num_err, + messages: kr.messages.values().cloned().collect(), + keys: kr.keys.keys().copied().collect(), + primary_keys: kr.primary_keys.keys().copied().collect(), + }); + } + } + while b.poll_event().is_some() {} + if let Some(outcome) = seen { + return outcome; + } + Timer::after(Duration::from_millis(5)).await; + } +} + +/// Both nodes' live keyrings still hold `primary` alone — no refused or trivial op +/// may re-key the wire or install a secondary behind it. +fn assert_ring_untouched( + a: &Serf, + b: &Serf, + primary: &SecretKey, +) { + for (name, node) in [("a", a), ("b", b)] { + let kr = node + .keyring() + .unwrap_or_else(|| panic!("{name} must be encrypted")); + assert_eq!( + kr.primary_ref(), + primary, + "{name}: the live primary must be unchanged" + ); + assert!( + kr.secondaries().is_empty(), + "{name}: nothing may be installed on the live keyring" + ); + } +} + +/// Promoting the key that is ALREADY primary is a trivial success on every node: +/// reported as a success, and the ring is not re-keyed behind it. +#[test] +fn promoting_the_current_primary_installs_nothing() { + let k1 = secret_key(0x11); + + let (dev_a, dev_b) = devices(); + let mut res_a = StackResources::<{ POOL + 2 }>::new(); + let mut res_b = StackResources::<{ POOL + 2 }>::new(); + let (stack_a, mut net_a) = build_stack(dev_a, &mut res_a, 1, 0x1111_2222); + let (stack_b, mut net_b) = build_stack(dev_b, &mut res_b, 2, 0x3333_4444); + + let mut bufs_a = NodeBufs::new(); + let mut bufs_b = NodeBufs::new(); + let (udp_a, tcp_a) = build_sockets(stack_a, &mut bufs_a); + let (udp_b, tcp_b) = build_sockets(stack_b, &mut bufs_b); + + let now = now(); + let (a, run_a) = build_node(udp_a, tcp_a, "a", 1, now, keyed(k1)); + let (b, run_b) = build_node(udp_b, tcp_b, "b", 2, now, keyed(k1)); + + block_on(async { + let op = async { + join_and_converge(&a, &b).await; + + a.use_key(k1).expect("use_key from a running node"); + let out = drive_to_key_response(&a, &b).await; + + assert!( + out.num_resp >= 2, + "both nodes must answer (num_resp={})", + out.num_resp + ); + assert_eq!( + out.num_err, 0, + "promoting the current primary is a no-op success, not a failure: {:?}", + out.messages + ); + assert_ring_untouched(&a, &b, &k1); + }; + drive(op, run_a, run_b, &mut net_a, &mut net_b).await; + }); +} + +/// A `use` or `remove` of a key NO node has installed is refused everywhere, with +/// the cause named, and leaves the live keyring untouched — so a key op can never +/// silently resolve to some other installed key. +#[test] +fn uninstalled_keys_are_refused_by_every_node() { + let k1 = secret_key(0x11); + let absent = secret_key(0x99); + + let (dev_a, dev_b) = devices(); + let mut res_a = StackResources::<{ POOL + 2 }>::new(); + let mut res_b = StackResources::<{ POOL + 2 }>::new(); + let (stack_a, mut net_a) = build_stack(dev_a, &mut res_a, 1, 0x1111_2222); + let (stack_b, mut net_b) = build_stack(dev_b, &mut res_b, 2, 0x3333_4444); + + let mut bufs_a = NodeBufs::new(); + let mut bufs_b = NodeBufs::new(); + let (udp_a, tcp_a) = build_sockets(stack_a, &mut bufs_a); + let (udp_b, tcp_b) = build_sockets(stack_b, &mut bufs_b); + + let now = now(); + let (a, run_a) = build_node(udp_a, tcp_a, "a", 1, now, keyed(k1)); + let (b, run_b) = build_node(udp_b, tcp_b, "b", 2, now, keyed(k1)); + + block_on(async { + let op = async { + join_and_converge(&a, &b).await; + + a.use_key(absent).expect("use_key from a running node"); + let out = drive_to_key_response(&a, &b).await; + assert_eq!( + out.num_err, 2, + "a `use` of an uninstalled key must be refused on BOTH nodes" + ); + assert!( + out + .messages + .iter() + .all(|m| m.as_str() == "requested key is not installed"), + "each refusal must name its cause: {:?}", + out.messages + ); + + a.remove_key(absent) + .expect("remove_key from a running node"); + let out = drive_to_key_response(&a, &b).await; + assert_eq!( + out.num_err, 2, + "a `remove` of an uninstalled key must be refused on BOTH nodes" + ); + assert!( + out + .messages + .iter() + .all(|m| m.as_str() == "requested key is not installed"), + "each refusal must name its cause: {:?}", + out.messages + ); + + assert_ring_untouched(&a, &b, &k1); + }; + drive(op, run_a, run_b, &mut net_a, &mut net_b).await; + }); +} + +/// Removing the PRIMARY is refused on every node — an operator promotes a secondary +/// first, or the cluster would be left with no key to encrypt under — and the +/// read-only `list_keys` census then still reports that primary as installed. +#[test] +fn the_primary_cannot_be_removed_and_the_census_still_reports_it() { + let k1 = secret_key(0x11); + + let (dev_a, dev_b) = devices(); + let mut res_a = StackResources::<{ POOL + 2 }>::new(); + let mut res_b = StackResources::<{ POOL + 2 }>::new(); + let (stack_a, mut net_a) = build_stack(dev_a, &mut res_a, 1, 0x1111_2222); + let (stack_b, mut net_b) = build_stack(dev_b, &mut res_b, 2, 0x3333_4444); + + let mut bufs_a = NodeBufs::new(); + let mut bufs_b = NodeBufs::new(); + let (udp_a, tcp_a) = build_sockets(stack_a, &mut bufs_a); + let (udp_b, tcp_b) = build_sockets(stack_b, &mut bufs_b); + + let now = now(); + let (a, run_a) = build_node(udp_a, tcp_a, "a", 1, now, keyed(k1)); + let (b, run_b) = build_node(udp_b, tcp_b, "b", 2, now, keyed(k1)); + + block_on(async { + let op = async { + join_and_converge(&a, &b).await; + + a.remove_key(k1).expect("remove_key from a running node"); + let out = drive_to_key_response(&a, &b).await; + assert_eq!( + out.num_err, 2, + "removing the primary must be refused on BOTH nodes" + ); + assert!( + out + .messages + .iter() + .all(|m| m.as_str() == "cannot remove the primary key; promote a secondary first"), + "each refusal must direct the operator: {:?}", + out.messages + ); + assert_ring_untouched(&a, &b, &k1); + + // The census is read-only and still sees the key the refused remove targeted. + a.list_keys().expect("list_keys from a running node"); + let out = drive_to_key_response(&a, &b).await; + assert_eq!( + out.num_err, 0, + "listing keys never fails on an encrypted node: {:?}", + out.messages + ); + assert!( + out.keys.contains(&k1), + "the census must report the installed key: {:?}", + out.keys + ); + assert_eq!( + out.primary_keys, + std::vec![k1], + "the census must report K1 as the only primary" + ); + assert_ring_untouched(&a, &b, &k1); + }; + drive(op, run_a, run_b, &mut net_a, &mut net_b).await; + }); +} + +/// A key op issued on an UNENCRYPTED cluster is refused by every node with a message +/// naming the missing keyring — never silently reported as an applied rotation, +/// which would leave an operator believing the cluster is keyed. +#[test] +fn an_unencrypted_cluster_refuses_key_management() { + let (dev_a, dev_b) = devices(); + let mut res_a = StackResources::<{ POOL + 2 }>::new(); + let mut res_b = StackResources::<{ POOL + 2 }>::new(); + let (stack_a, mut net_a) = build_stack(dev_a, &mut res_a, 1, 0x1111_2222); + let (stack_b, mut net_b) = build_stack(dev_b, &mut res_b, 2, 0x3333_4444); + + let mut bufs_a = NodeBufs::new(); + let mut bufs_b = NodeBufs::new(); + let (udp_a, tcp_a) = build_sockets(stack_a, &mut bufs_a); + let (udp_b, tcp_b) = build_sockets(stack_b, &mut bufs_b); + + let now = now(); + let (a, run_a) = build_node(udp_a, tcp_a, "a", 1, now, TransformOptions::default()); + let (b, run_b) = build_node(udp_b, tcp_b, "b", 2, now, TransformOptions::default()); + + assert!( + a.keyring().is_none(), + "a plaintext node carries no wire keyring" + ); + + block_on(async { + let op = async { + join_and_converge(&a, &b).await; + + a.install_key(secret_key(0x33)) + .expect("the query is accepted; the nodes refuse the op"); + let out = drive_to_key_response(&a, &b).await; + + assert!( + out.num_resp >= 2, + "both nodes must answer (num_resp={})", + out.num_resp + ); + assert_eq!( + out.num_err, out.num_resp, + "every unencrypted node must refuse the op" + ); + assert!( + out + .messages + .iter() + .all(|m| m.as_str() == "no keyring configured on this node"), + "each refusal must name the missing keyring: {:?}", + out.messages + ); + assert!( + a.keyring().is_none() && b.keyring().is_none(), + "a refused install must not conjure a keyring onto the wire" + ); + }; + drive(op, run_a, run_b, &mut net_a, &mut net_b).await; + }); +} diff --git a/serf-embedded/src/engine/tests.rs b/serf-embedded/src/engine/tests.rs index 92c970c3..0fce574f 100644 --- a/serf-embedded/src/engine/tests.rs +++ b/serf-embedded/src/engine/tests.rs @@ -7,7 +7,10 @@ use core::{ use std::{cell::RefCell, collections::BTreeMap, rc::Rc}; -use memberlist_proto::{SeedableRng, SmallRng}; +use memberlist_proto::{ + Node, PushPullKind, SeedableRng, SmallRng, + typed::{Alive, Message, Ping}, +}; use smol_str::SmolStr; #[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] @@ -483,8 +486,10 @@ fn failed_ignore_old_join_resolves_err_and_clears_ignore_streams() { // engine's core-owned join folds into its reached set. One end's `connect` // registers a pending SYN on the shared fabric; the destination's listener // completes the passive open when its `accepted_peer` is polled, after which bytes -// ferry both ways over the matched pipe. The link acks instantly (`send_queue` -// is always 0), so a graceful close FINs without a `Closing` drain. +// ferry both ways over the matched pipe. The link acks instantly by default +// (`send_queue` is 0), so a graceful close FINs without a `Closing` drain; +// [`LinkPair::hold_b_tx`] withholds B's acknowledgements so the drain-before-close +// path can be driven. /// The two directional byte streams of one established pipe plus its FIN/reset /// flags. The dialer writes `d2a` (the acceptor reads it) and vice versa. @@ -496,6 +501,14 @@ struct Pipe { a_fin: bool, reset: bool, established: bool, + /// Bytes the dialer end handed to `send` that the peer has not yet acknowledged — + /// what `send_queue` reports for that end. The bytes are DELIVERED to the peer's rx + /// immediately (so the FSM still completes); only the acknowledgement lingers, so a + /// graceful close over this end parks in `Closing` until a test acks it. Zero unless + /// the end's `hold_tx` is set. + d_unacked: usize, + /// As `d_unacked`, for the acceptor end. + a_unacked: usize, } /// A SYN parked on the fabric by a dialer's `connect`, awaiting the destination's @@ -548,6 +561,11 @@ struct LinkRel { me: SocketAddr, free: std::vec::Vec, role: RefCell>, + /// When set, a `send` on this end ALSO accrues unacked tx (`send_queue` > 0), + /// modelling a peer slow to acknowledge. A graceful close over such a connection + /// parks in `Closing` until the tx drains, exercising the drain-before-close path. + /// Off by default (the link acks instantly). + hold_tx: bool, } impl LinkRel { @@ -561,6 +579,7 @@ impl LinkRel { me, free: handles.to_vec(), role: RefCell::new(role), + hold_tx: false, } } @@ -737,6 +756,7 @@ impl StreamIo for LinkRel { Some(SlotRole::Bound(pipe, end)) => (pipe, end), _ => return 0, }; + let hold = self.hold_tx; let mut fab = self.fabric.borrow_mut(); let Some(p) = fab.pipes.get_mut(&pipe) else { return 0; @@ -745,16 +765,39 @@ impl StreamIo for LinkRel { return 0; } // Deliver to the peer's rx immediately (the FSM sees the bytes); the link acks - // instantly so nothing lingers as unacked tx. + // instantly unless this end is holding, in which case the acknowledgement lingers + // as unacked tx so a later graceful close parks in `Closing`. match end { - End::Dialer => p.d2a.extend(bytes.iter().copied()), - End::Acceptor => p.a2d.extend(bytes.iter().copied()), + End::Dialer => { + p.d2a.extend(bytes.iter().copied()); + if hold { + p.d_unacked += bytes.len(); + } + } + End::Acceptor => { + p.a2d.extend(bytes.iter().copied()); + if hold { + p.a_unacked += bytes.len(); + } + } } bytes.len() } - fn send_queue(&self, _c: u32) -> usize { - 0 + fn send_queue(&self, c: u32) -> usize { + match self.role_of(c) { + Some(SlotRole::Bound(pipe, end)) => { + let fab = self.fabric.borrow(); + match fab.pipes.get(&pipe) { + Some(p) => match end { + End::Dialer => p.d_unacked, + End::Acceptor => p.a_unacked, + }, + None => 0, + } + } + _ => 0, + } } fn close(&mut self, c: u32) { @@ -897,6 +940,29 @@ impl LinkPair { } } + /// Make B (the acceptor) accrue unacked tx on every `send`, so when B's bridge + /// gracefully closes with its push/pull reply still unacknowledged the connection + /// parks in `Closing` rather than FIN-ing at once — the drain-before-close path. + fn hold_b_tx(&mut self) { + self.b_rel.hold_tx = true; + } + + /// Acknowledge up to `amount` of B's accrued unacked tx across its pipes, modelling + /// the peer draining B's reply. Steps the `Closing` drain through its progress + /// (partial ack) and terminal-FIN (fully drained) branches. + fn ack_b(&mut self, amount: usize) { + let mut fab = self.b_rel.fabric.borrow_mut(); + let mut left = amount; + for p in fab.pipes.values_mut() { + let take = p.a_unacked.min(left); + p.a_unacked -= take; + left -= take; + if left == 0 { + break; + } + } + } + /// Pump both engines once at `now`, then ferry each side's emitted gossip into /// the peer's inbound queue. fn step(&mut self, now: Instant) { @@ -2310,3 +2376,1287 @@ fn construction_rejects_a_keyring_with_cross_cipher_byte_twins() { Ok(_) => panic!("construction must reject a keyring carrying cross-cipher byte twins"), } } + +// ── panicking convenience constructors ─────────────────────────────────────── + +/// The first query id a running engine issues — a `u32` drawn straight from serf's +/// core RNG, so with every other input held fixed it is a pure function of that +/// RNG's seed. +fn first_query_id_of(engine: &mut SerfEngine, now: Instant) -> u32 { + engine + .query( + "probe", + Bytes::from_static(b"payload"), + QueryParams::default(), + now, + ) + .expect("a query is issued while running") + .id +} + +/// `new_at_with_rng` — the panicking form of `try_new_at_with_rng` — builds on a +/// valid configuration AND injects serf's core RNG: with the gossip RNG held fixed, +/// distinct serf seeds yield distinct first query ids and the same seed reproduces +/// one. So the production constructor's entropy actually reaches query-id +/// generation through the panicking wrapper too. +#[test] +fn new_at_with_rng_builds_and_injects_the_serf_rng() { + let now = Instant::from_origin(Duration::from_secs(86_400)); + let build = |serf_seed: u64| -> SerfEngine { + let cfg = Options::new() + .with_port(7946) + .with_close_timeout(Duration::from_secs(10)); + let ep_cfg = EndpointOptions::new(SmolStr::new("q"), node_addr(7946)); + let mut engine = SerfEngine::new_at_with_rng( + cfg, + TransformOptions::default(), + ep_cfg, + SerfOptions::new(), + now, + SmallRng::seed_from_u64(1), + SmallRng::seed_from_u64(serf_seed), + ); + engine.start(now); + engine + }; + + let mut engine = build(100); + assert_eq!( + engine.port(), + 7946, + "port() reports the configured bind port" + ); + assert_eq!( + engine.state(), + SerfState::Alive, + "an engine built by the panicking constructor is running" + ); + assert_eq!(engine.num_members(), 1, "a fresh engine tracks only itself"); + assert_eq!(engine.local_id(), &SmolStr::new("q")); + + let seeded_100 = first_query_id_of(&mut engine, now); + assert_ne!( + seeded_100, + first_query_id_of(&mut build(200), now), + "distinct serf RNG seeds must produce distinct first query ids" + ); + assert_eq!( + seeded_100, + first_query_id_of(&mut build(100), now), + "the same serf RNG seed must reproduce the same first query id" + ); +} + +/// `new_at` — the panicking form of `try_new_at` — builds on a valid configuration +/// with serf's core RNG ZERO-SEEDED. That is the documented determinism caveat: two +/// fresh engines built this way emit the SAME first query id, which is exactly why a +/// production driver must construct via `try_new_at_with_rng` instead. +#[test] +fn new_at_builds_with_a_zero_seeded_serf_rng() { + let now = Instant::from_origin(Duration::from_secs(86_400)); + let build = || -> SerfEngine { + let cfg = Options::new() + .with_port(7946) + .with_close_timeout(Duration::from_secs(10)); + let ep_cfg = EndpointOptions::new(SmolStr::new("z"), node_addr(7946)); + let mut engine = SerfEngine::new_at( + cfg, + TransformOptions::default(), + ep_cfg, + SerfOptions::new(), + now, + test_rng(), + ); + engine.start(now); + engine + }; + + let mut one = build(); + assert_eq!(one.port(), 7946, "port() reports the configured bind port"); + assert_eq!( + one.state(), + SerfState::Alive, + "an engine built by the panicking constructor is running" + ); + assert_eq!( + first_query_id_of(&mut one, now), + first_query_id_of(&mut build(), now), + "a zero-seeded serf RNG makes two fresh engines share one query-id sequence" + ); +} + +/// `new_at_with_rng` panics on an invalid configuration rather than building a node +/// past the construction-time checks — the documented contract steering a fallible +/// caller to `try_new_at_with_rng`. +#[test] +#[should_panic(expected = "invalid configuration")] +fn new_at_with_rng_panics_on_an_invalid_configuration() { + let cfg = Options::new() + .with_port(7946) + .with_close_timeout(Duration::from_secs(10)); + let ep_cfg = EndpointOptions::new(SmolStr::new("bad"), node_addr(7946)); + let serf_opts = + SerfOptions::new().with_max_user_event_size(SerfOptions::DEFAULT_USER_EVENT_SIZE_LIMIT + 1); + let now = Instant::from_origin(Duration::from_secs(86_400)); + let _engine: SerfEngine = SerfEngine::new_at_with_rng( + cfg, + TransformOptions::default(), + ep_cfg, + serf_opts, + now, + test_rng(), + SmallRng::seed_from_u64(7), + ); +} + +/// `new_at` panics on an invalid configuration rather than building a node past the +/// construction-time checks — the documented contract steering a fallible caller to +/// `try_new_at`. +#[test] +#[should_panic(expected = "invalid configuration")] +fn new_at_panics_on_an_invalid_configuration() { + // A non-routable advertise address: a node must advertise an address its peers can + // route a reply to. + let cfg = Options::new() + .with_port(7946) + .with_close_timeout(Duration::from_secs(10)); + let bad = SocketAddr::new(IpAddr::V4(Ipv4Addr::UNSPECIFIED), 7946); + let ep_cfg = EndpointOptions::new(SmolStr::new("bad"), bad); + let now = Instant::from_origin(Duration::from_secs(86_400)); + let _engine: SerfEngine = SerfEngine::new_at( + cfg, + TransformOptions::default(), + ep_cfg, + SerfOptions::new(), + now, + test_rng(), + ); +} + +// ── running-state gate ─────────────────────────────────────────────────────── + +/// `ensure_running` — the gate every operation that would queue work no peer could +/// observe consults — passes while the node is `Alive` and, once it has left, fails +/// with `BadJoinState` CARRYING the current serf state (so a caller can report why +/// it was refused rather than guessing). +#[test] +fn ensure_running_rejects_once_the_node_has_left() { + let mut engine = make_engine(); + let now = Instant::from_origin(Duration::from_secs(86_400)); + engine.start(now); + // Let the construction self-join sieve settle before leaving. + let mut gossip = NoGossip; + let mut stream = NoStream::with_pool(2); + engine.pump(now, &mut gossip, &mut stream); + assert!( + engine.ensure_running().is_ok(), + "a running node must pass the gate" + ); + + engine + .leave(now) + .expect("leave from a running node succeeds"); + match engine.ensure_running() { + Err(SerfError::BadJoinState(state)) => assert_eq!( + state, + engine.state(), + "the rejection must carry the node's current serf state" + ), + other => panic!("expected Err(BadJoinState) once the node has left, got {other:?}"), + } +} + +// ── join-handle allocation and per-join exchange binding ───────────────────── + +/// Two concurrent joins mint DISTINCT handles from a monotonic sequence, and each +/// dispatched `Connect` binds its exchange to the join that actually started its +/// stream — never to the other in-flight join. Once a slot frees, each parked dial is +/// serviced and each join terminalizes on its OWN exchange, so their outcomes never +/// cross-resolve. A binding that credited the first waiter it found (rather than +/// matching the START `StreamId`) would leave both exchanges in one join's pending set. +#[test] +fn concurrent_joins_mint_distinct_handles_and_bind_their_own_exchanges() { + let mut engine = make_engine(); + let now = Instant::from_origin(Duration::from_secs(86_400)); + engine.start(now); + // A listener but an EMPTY dial pool: each `Connect` parks as `PendingDial`, so both + // exchanges stay in flight while their bindings are inspected. + engine.set_listener(9); + + let first = engine + .join(&[node_addr(7002)], false, now) + .expect("join announces intent and mints a handle"); + let second = engine + .join(&[node_addr(7003)], false, now) + .expect("a second concurrent join mints its own handle"); + assert_ne!( + first, second, + "two concurrent joins must mint distinct handles" + ); + assert_eq!( + second.get(), + first.get() + 1, + "handles are minted from a monotonic sequence" + ); + + let mut gossip = NoGossip; + let mut stream = NoStream::with_pool(0); + engine.pump(now, &mut gossip, &mut stream); + assert_eq!( + engine.pending_dial_count(), + 2, + "both seeds dispatched a push/pull that parked on the exhausted pool" + ); + + { + let one = &engine.pending_joins[&first]; + let two = &engine.pending_joins[&second]; + assert_eq!( + one.started.len(), + 1, + "each join records only its own stream" + ); + assert_eq!( + two.started.len(), + 1, + "each join records only its own stream" + ); + assert!( + one.started.is_disjoint(&two.started), + "two joins never share a start stream" + ); + assert_eq!( + one.pending.len(), + 1, + "exactly the exchange this join started is bound to it" + ); + assert_eq!( + two.pending.len(), + 1, + "exactly the exchange this join started is bound to it" + ); + assert!( + one.pending.is_disjoint(&two.pending), + "a Connect binds only the join whose start stream it carries" + ); + } + + // Free two reuse-ready slots: the rebalance assigns them to the parked dials, which + // `NoStream::connect` rejects — so each join terminalizes on its OWN exchange. + engine.plane_mut().pool.push(5); + engine.plane_mut().pool.push(6); + + let mut outcomes = (None, None); + for _ in 0..8 { + engine.pump(now, &mut gossip, &mut stream); + if outcomes.0.is_none() { + outcomes.0 = engine.poll_join(first); + } + if outcomes.1.is_none() { + outcomes.1 = engine.poll_join(second); + } + if outcomes.0.is_some() && outcomes.1.is_some() { + break; + } + } + assert_eq!( + engine.pending_dial_count(), + 0, + "each deferred dial must be assigned a freed slot and leave PendingDial" + ); + + for outcome in [outcomes.0, outcomes.1] { + match outcome { + Some(Err(jf)) => { + assert_eq!( + jf.requested(), + 1, + "each join dispatched exactly its own seed" + ); + assert_eq!(jf.contacted(), 0, "its failed dial contacted no seed"); + } + other => panic!("each join must resolve independently as Err(JoinFailed), got {other:?}"), + } + } + assert_eq!( + engine.pending_join_count(), + 0, + "both resolved joins are reaped once delivered" + ); +} + +// ── programmable single-engine reliable mock ───────────────────────────────── +// +// The reliable-plane lifecycle (dial → promote → flush → half-close → teardown → +// reap) is link-layer-independent engine code: the machine emits the `StreamAction`s +// and the engine pumps them over `StreamIo`. `ProgRel` stands in for a driver's +// socket pool so a single engine can be walked through those paths — a test flips a +// slot's state between pumps and reads back exactly what reached the wire. + +/// The simulated TCP state of one mock reliable socket, as the engine observes it +/// through [`StreamIo`]. +#[derive(Clone)] +struct SockState { + /// The handshake is modelled complete: `may_send` is true and writes are accepted. + /// A test flips this to promote a dial. + established: bool, + /// The socket has not reached `Closed`/`TimeWait` — `is_open` is true. `connect` + /// opens it, a `close` leaves it open (our FIN in flight), and an `abort` drops it. + open: bool, + /// Cap on the bytes one `send` accepts, modelling partial-write backpressure: the + /// remainder stays parked in the connection's `out` queue. `usize::MAX` accepts + /// the whole buffer. + send_cap: usize, +} + +impl SockState { + fn idle() -> Self { + Self { + established: false, + open: false, + send_cap: usize::MAX, + } + } +} + +/// A directly-programmable single-engine reliable mock: a test mutates each slot's +/// [`SockState`] between pumps and asserts on the recorded sends / closes / aborts. +struct ProgRel { + free: std::vec::Vec, + socks: BTreeMap, + /// Every `(handle, bytes)` a `send` accepted, in order. + sent: std::vec::Vec<(u32, std::vec::Vec)>, + /// Handles `close` (a graceful FIN) was called on. + closed: std::vec::Vec, + /// Handles `abort` (an RST) was called on. + aborted: std::vec::Vec, +} + +impl ProgRel { + /// A mock realizing sockets for `handles`. The engine's own `plane_mut().pool` is + /// the authority its reliable handlers consult, so a test pushes the same handles + /// there (or installs one via `set_listener`). + fn new(handles: &[u32]) -> Self { + let mut socks = BTreeMap::new(); + for &h in handles { + socks.insert(h, SockState::idle()); + } + Self { + free: handles.to_vec(), + socks, + sent: std::vec::Vec::new(), + closed: std::vec::Vec::new(), + aborted: std::vec::Vec::new(), + } + } + + fn sock(&self, c: u32) -> &SockState { + self.socks.get(&c).expect("handle exists") + } + + fn sock_mut(&mut self, c: u32) -> &mut SockState { + self.socks.get_mut(&c).expect("handle exists") + } +} + +impl StreamIo for ProgRel { + type Conn = u32; + + fn take_free(&mut self) -> Option { + self.free.pop() + } + + fn give(&mut self, c: u32) { + self.free.push(c); + } + + fn free_count(&self) -> usize { + self.free.len() + } + + fn listen(&mut self, c: u32, _port: u16) -> Result<(), crate::StreamIoError> { + // A listening socket is open and awaiting a passive open; clear any per-slot + // residue so a reclaimed-then-relistened handle starts clean. + *self.sock_mut(c) = SockState::idle(); + self.sock_mut(c).open = true; + Ok(()) + } + + fn accepted_peer(&self, _c: u32) -> Option { + None + } + + fn connect( + &mut self, + c: u32, + _remote: SocketAddr, + _local_port: u16, + ) -> Result<(), crate::StreamIoError> { + // A dial opens the socket; the test flips `established` to model the handshake + // completing on a later tick. + self.sock_mut(c).open = true; + Ok(()) + } + + fn may_send(&self, c: u32) -> bool { + let s = self.sock(c); + s.established && s.open + } + + fn may_recv(&self, _c: u32) -> bool { + false + } + + fn is_open(&self, c: u32) -> bool { + self.sock(c).open + } + + fn is_established(&self, c: u32) -> bool { + self.sock(c).established + } + + fn recv(&mut self, _c: u32, _buf: &mut [u8]) -> Option { + None + } + + fn recv_finished(&self, _c: u32) -> bool { + false + } + + fn send(&mut self, c: u32, bytes: &[u8]) -> usize { + let n = bytes.len().min(self.sock(c).send_cap); + self.sent.push((c, bytes[..n].to_vec())); + n + } + + fn send_queue(&self, _c: u32) -> usize { + 0 + } + + fn close(&mut self, c: u32) { + // A graceful close leaves the socket open (our FIN in flight) until the peer + // FINs back or the reap backstop forces it closed. + self.closed.push(c); + } + + fn abort(&mut self, c: u32) { + self.aborted.push(c); + let s = self.sock_mut(c); + s.open = false; + s.established = false; + } +} + +/// A running single-node engine with the given reliable-exchange (`stream_timeout`) +/// deadline, plus the clock it was started at. +fn engine_with_stream_timeout(stream_timeout: Duration) -> (SerfEngine, Instant) { + let now = Instant::from_origin(Duration::from_secs(86_400)); + let cfg = Options::new() + .with_port(7946) + .with_close_timeout(Duration::from_secs(10)); + let ep_cfg = + EndpointOptions::new(SmolStr::new("test"), node_addr(7946)).with_stream_timeout(stream_timeout); + let mut engine: SerfEngine = SerfEngine::try_new_at( + cfg, + TransformOptions::default(), + ep_cfg, + SerfOptions::new(), + now, + test_rng(), + ) + .expect("valid configuration must construct without error"); + engine.start(now); + (engine, now) +} + +/// The sole reliable exchange currently mapped on the engine's plane. +fn sole_exchange(engine: &mut SerfEngine) -> ExchangeId { + let mut ids = engine.plane_mut().connections.keys().copied(); + let eid = ids.next().expect("exactly one exchange is mapped"); + assert!(ids.next().is_none(), "exactly one exchange is mapped"); + eid +} + +// ── reliable-plane reap / teardown by socket state ─────────────────────────── + +/// The reap pass reclaims a gracefully-closing handle the moment its socket reaches a +/// reusable (`!is_open`) state, and FORCE-ABORTS one whose close has outlived +/// `close_timeout` — so a peer that vanished mid-FIN can never permanently shrink the +/// pool. Without the force-abort backstop the stalled handle stays parked forever and +/// the pool loses a slot for good. +#[test] +fn reap_closing_reclaims_a_finished_close_and_force_aborts_a_vanished_peer() { + let (mut engine, now) = engine_with_stream_timeout(Duration::from_secs(30)); + let mut stream = ProgRel::new(&[0, 1]); + // Slot 0: a clean close that already reached `Closed`. Slot 1: a peer that vanished + // mid-FIN — still open, and past its close deadline. + stream.sock_mut(0).open = false; + stream.sock_mut(1).open = true; + engine + .plane_mut() + .closing + .insert(0, now + Duration::from_secs(10)); + engine.plane_mut().closing.insert(1, now); + assert_eq!( + engine.closing_count(), + 2, + "two handles are parked mid-close" + ); + + let mut gossip = NoGossip; + engine.pump(now, &mut gossip, &mut stream); + + assert_eq!( + engine.closing_count(), + 0, + "both parked handles must be reaped — one cleanly, one force-aborted" + ); + assert!( + stream.aborted.contains(&1), + "the handle past its close deadline must be force-aborted so its slot is reclaimable" + ); + assert!( + !stream.aborted.contains(&0), + "an already-closed handle must be reclaimed without an abort" + ); + assert_eq!( + engine.pool_free_count() + engine.listener_present() as usize, + 2, + "both reaped handles must return to the pool, never leak" + ); +} + +/// Tearing down a `PendingDial` — an exchange whose dial was deferred and which +/// therefore holds NO socket — removes it outright, so the deferred dial is never +/// later issued for a retired exchange, and reclaims nothing (there is no slot to +/// reclaim). Tearing down an exchange that is no longer mapped is inert: it must not +/// panic, reclaim a phantom slot, or touch the link. +#[test] +fn teardown_of_a_socketless_pending_dial_removes_it_and_reclaims_nothing() { + let mut engine = make_engine(); + let now = Instant::from_origin(Duration::from_secs(86_400)); + engine.start(now); + // A listener but an empty dial pool: the join's Connect parks as PendingDial. + engine.set_listener(9); + let mut stream = ProgRel::new(&[9]); + + engine + .join(&[node_addr(7002)], false, now) + .expect("join announces intent and queues the routable seed"); + let mut gossip = NoGossip; + engine.pump(now, &mut gossip, &mut stream); + assert_eq!( + engine.pending_dial_count(), + 1, + "the exhausted-pool seed parked as a socketless PendingDial" + ); + + let eid = sole_exchange(&mut engine); + let free_before = engine.pool_free_count(); + engine.teardown(eid, now, &mut stream); + + assert_eq!( + engine.pending_dial_count(), + 0, + "the retired exchange must be removed so a later rebalance never dials it" + ); + assert_eq!( + engine.pool_free_count(), + free_before, + "a socketless exchange reclaims no slot" + ); + assert_eq!(engine.closing_count(), 0, "nothing is parked mid-close"); + assert!( + stream.aborted.is_empty() && stream.closed.is_empty(), + "a socketless teardown must not touch the link" + ); + + // The same exchange is now unknown: a repeat teardown is a no-op. + engine.teardown(eid, now, &mut stream); + assert_eq!( + engine.pool_free_count(), + free_before, + "tearing down an unknown exchange must reclaim no slot" + ); + assert!( + stream.aborted.is_empty() && stream.closed.is_empty(), + "tearing down an unknown exchange must not touch the link" + ); +} + +/// Tearing down a HALF-CLOSED connection — our FIN already emitted, the peer's not yet +/// received — parks its handle for the reap backstop instead of returning it to the +/// pool: the socket is still open, so reusing the slot now would clobber a connection +/// that is still finishing its close. The handle comes back only once the reap sees it +/// closed (or its close deadline elapses). +#[test] +fn teardown_of_a_half_closed_connection_parks_it_for_the_reap() { + let (mut engine, now) = engine_with_stream_timeout(Duration::from_secs(30)); + engine.plane_mut().pool.push(0); + engine.set_listener(1); + let mut stream = ProgRel::new(&[0, 1]); + + engine + .join(&[node_addr(7002)], false, now) + .expect("join announces intent and queues the routable seed"); + let mut gossip = NoGossip; + // Tick 1: the Connect dials slot 0; the mock leaves the handshake incomplete. + engine.pump(now, &mut gossip, &mut stream); + // Complete the handshake: the request flushes and the machine's deferred FIN is + // emitted once the write queue has drained, half-closing the connection. + stream.sock_mut(0).established = true; + let mut half_closed = false; + for _ in 0..8 { + engine.pump(now, &mut gossip, &mut stream); + if engine.half_closed_count() == 1 { + half_closed = true; + break; + } + } + assert!( + half_closed, + "the push/pull initiator half-closes its write half once the request is written" + ); + assert!( + stream.closed.contains(&0), + "the deferred FIN reached the socket" + ); + + let eid = sole_exchange(&mut engine); + let free_before = engine.pool_free_count(); + engine.teardown(eid, now, &mut stream); + + assert_eq!( + engine.closing_count(), + 1, + "a half-closed connection's handle must be parked for the reap backstop" + ); + assert_eq!( + engine.pool_free_count(), + free_before, + "its handle must NOT return to the pool while the socket is still closing" + ); + assert!( + !stream.aborted.contains(&0), + "a half-closed teardown must not RST a socket whose FIN is already in flight" + ); +} + +/// Tearing down a dial the peer NEVER established — the socket is open but not +/// send-capable — RST-aborts it and returns the slot straight to the pool. FIN-ing a +/// connection the peer never opened would strand the slot in the close backstop for a +/// close that can never complete. +#[test] +fn teardown_of_a_never_established_dial_aborts_and_reclaims_the_slot() { + let (mut engine, now) = engine_with_stream_timeout(Duration::from_secs(30)); + engine.plane_mut().pool.push(0); + engine.set_listener(1); + let mut stream = ProgRel::new(&[0, 1]); + + engine + .join(&[node_addr(7002)], false, now) + .expect("join announces intent and queues the routable seed"); + let mut gossip = NoGossip; + engine.pump(now, &mut gossip, &mut stream); + assert!( + StreamIo::is_open(&stream, 0), + "the dial issued connect on the pooled slot" + ); + assert!( + !StreamIo::may_send(&stream, 0), + "the mock leaves the handshake incomplete, so the socket is not send-capable" + ); + + let eid = sole_exchange(&mut engine); + let free_before = engine.pool_free_count(); + engine.teardown(eid, now, &mut stream); + + assert!( + stream.aborted.contains(&0), + "a socket the peer never established must be RST, not FIN'd" + ); + assert_eq!( + engine.pool_free_count(), + free_before + 1, + "its slot must return straight to the pool" + ); + assert_eq!( + engine.closing_count(), + 0, + "nothing is parked mid-close for a connection that never opened" + ); + assert!( + engine.plane_mut().connections.is_empty(), + "the torn-down exchange is unmapped" + ); +} + +// ── reliable egress under partial-write backpressure ───────────────────────── + +/// Flush a join push/pull's request bytes over a link whose `send` accepts at most +/// `send_cap` bytes at a time, returning the byte stream that reached the socket plus +/// the length of each accepted write. +fn push_pull_bytes_written(send_cap: usize) -> (std::vec::Vec, std::vec::Vec) { + let (mut engine, now) = engine_with_stream_timeout(Duration::from_secs(30)); + engine.plane_mut().pool.push(0); + engine.set_listener(1); + let mut stream = ProgRel::new(&[0, 1]); + stream.sock_mut(0).send_cap = send_cap; + + engine + .join(&[node_addr(7004)], false, now) + .expect("join announces intent and queues the routable seed"); + + let mut gossip = NoGossip; + // Tick 1: the Connect dials slot 0. The mock leaves it un-established, so the egress + // pump skips the `!may_send` socket and nothing flushes yet. + engine.pump(now, &mut gossip, &mut stream); + assert!( + stream.sent.is_empty(), + "a still-handshaking socket must not be written to" + ); + // Establish it and pump until the queue has drained. The clock is held, so no + // deadline elapses and only the capped writes limit progress. + stream.sock_mut(0).established = true; + for _ in 0..400 { + engine.pump(now, &mut gossip, &mut stream); + } + + let writes: std::vec::Vec = stream + .sent + .iter() + .filter(|(c, _)| *c == 0) + .map(|(_, b)| b.len()) + .collect(); + let bytes: std::vec::Vec = stream + .sent + .iter() + .filter(|(c, _)| *c == 0) + .flat_map(|(_, b)| b.iter().copied()) + .collect(); + (bytes, writes) +} + +/// A `send` that accepts fewer bytes than offered leaves the UNSENT TAIL at the front +/// of the connection's out queue, so a later tick delivers exactly the remainder: the +/// reassembled stream is byte-identical to the one an uncapped link receives — nothing +/// dropped, duplicated, or reordered. Popping the front on a partial write would +/// truncate the request; re-sending the whole front would duplicate its prefix. Either +/// corrupts the push/pull framing. +#[test] +fn partial_writes_park_the_remainder_and_preserve_byte_order() { + let (whole, _) = push_pull_bytes_written(usize::MAX); + assert!( + !whole.is_empty(), + "the join push/pull must write its request to the dialed socket" + ); + + let (chunked, writes) = push_pull_bytes_written(4); + assert!( + writes.len() >= 2, + "the capped link must force the request across multiple writes, got {writes:?}" + ); + assert!( + writes.iter().all(|&n| n <= 4), + "no write may exceed the link's per-send cap, got {writes:?}" + ); + assert_eq!( + chunked, whole, + "the partial-write remainder must reassemble to exactly the bytes an uncapped link \ + receives — no byte dropped, duplicated, or reordered" + ); +} + +// ── gossip ingress screens and the egress destination screen ───────────────── + +/// A [`GossipIo`] with a programmable inbound queue and a capture of every emitted +/// datagram, so a test can feed one exact datagram from one exact source and assert +/// precisely what — if anything — went back on the wire. +struct QueueGossip { + inbound: std::vec::Vec<(SocketAddr, std::vec::Vec)>, + outbound: std::vec::Vec<(std::vec::Vec, SocketAddr)>, +} + +impl QueueGossip { + fn new() -> Self { + Self { + inbound: std::vec::Vec::new(), + outbound: std::vec::Vec::new(), + } + } + + fn push(&mut self, src: SocketAddr, bytes: std::vec::Vec) { + self.inbound.push((src, bytes)); + } +} + +impl GossipIo for QueueGossip { + fn recv(&mut self, buf: &mut [u8]) -> Option<(SocketAddr, usize)> { + if self.inbound.is_empty() { + return None; + } + let (src, bytes) = self.inbound.remove(0); + let n = bytes.len().min(buf.len()); + buf[..n].copy_from_slice(&bytes[..n]); + Some((src, n)) + } + + fn send(&mut self, bytes: &[u8], dest: SocketAddr) { + self.outbound.push((bytes.to_vec(), dest)); + } +} + +/// A peer address on the test subnet. +fn peer_addr(host: u8, port: u16) -> SocketAddr { + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, host)), port) +} + +/// A plaintext gossip datagram carrying an `Alive` for `ghost`, encoded with `label`. +/// Incarnation 1 passes SWIM's freshness check for a node the receiver has never seen. +fn alive_datagram(ghost: &Node, label: Option) -> Bytes { + encode_outgoing::( + &Message::Alive(Alive::new(1, ghost.clone())), + &EncodeOptions::new(label), + ) + .expect("a well-formed Alive encodes") +} + +/// A malformed inbound gossip datagram is dropped at the decode step: bad network +/// input must never panic the node or mutate membership. The SAME source then lands a +/// well-formed `Alive` and IS admitted, so the drop is the codec rejecting the garbage +/// rather than the ingress path being inert. +#[test] +fn malformed_gossip_datagram_is_dropped_without_membership_change() { + let mut engine = make_engine(); + let now = Instant::from_origin(Duration::from_secs(86_400)); + engine.start(now); + + let src = peer_addr(3, 7946); + let mut gossip = QueueGossip::new(); + let mut stream = NoStream::with_pool(2); + // No byte of this is a valid message frame. + gossip.push(src, std::vec![0xffu8; 32]); + engine.pump(now, &mut gossip, &mut stream); + assert_eq!( + engine.num_members(), + 1, + "a malformed datagram must be dropped — no member may be learned from it" + ); + + let ghost = Node::new(SmolStr::new("ghost"), peer_addr(2, 7946)); + gossip.push(src, alive_datagram(&ghost, None).to_vec()); + engine.pump(now, &mut gossip, &mut stream); + assert_eq!( + engine.num_members(), + 2, + "a well-formed Alive from the same source must still be admitted" + ); +} + +/// A node with a cluster label REJECTS an inbound gossip datagram that does not carry +/// it — the label check drops the frame before the machine sees it, so a neighbouring +/// cluster's chatter can never inject a member. The same `Alive` stamped with the +/// node's own label is admitted. +#[test] +fn a_labeled_node_rejects_gossip_without_its_cluster_label() { + let cfg = Options::new() + .with_port(7946) + .with_close_timeout(Duration::from_secs(10)); + let ep_cfg = EndpointOptions::new(SmolStr::new("alpha"), node_addr(7946)); + let transform = TransformOptions::default() + .with_label(Some(b"alpha".to_vec())) + .expect("a valid cluster label"); + let now = Instant::from_origin(Duration::from_secs(86_400)); + let mut engine: SerfEngine = + SerfEngine::try_new_at(cfg, transform, ep_cfg, SerfOptions::new(), now, test_rng()) + .expect("valid configuration must construct without error"); + engine.start(now); + + let ghost = Node::new(SmolStr::new("ghost"), peer_addr(2, 7946)); + let src = peer_addr(3, 7946); + let mut gossip = QueueGossip::new(); + let mut stream = NoStream::with_pool(2); + + gossip.push(src, alive_datagram(&ghost, None).to_vec()); + engine.pump(now, &mut gossip, &mut stream); + assert_eq!( + engine.num_members(), + 1, + "an unlabeled datagram must be rejected by a labeled node" + ); + + gossip.push( + src, + alive_datagram(&ghost, Some(Bytes::from_static(b"beta"))).to_vec(), + ); + engine.pump(now, &mut gossip, &mut stream); + assert_eq!( + engine.num_members(), + 1, + "a wrong-label datagram must be rejected by a labeled node" + ); + + gossip.push( + src, + alive_datagram(&ghost, Some(Bytes::from_static(b"alpha"))).to_vec(), + ); + engine.pump(now, &mut gossip, &mut stream); + assert_eq!( + engine.num_members(), + 2, + "the same Alive stamped with the node's own label must be admitted" + ); +} + +/// The last-line egress screen: the engine never writes a gossip datagram to a +/// destination no packet could reach. A Ping addressed to this node but arriving from +/// a NON-ROUTABLE source would have its ack reflected straight back to that source, so +/// the ack is screened at egress and nothing goes on the wire. The identical Ping from +/// a routable source IS acked, so the drop is the destination screen and not a rejected +/// Ping. +#[test] +fn no_gossip_datagram_is_emitted_to_a_non_routable_destination() { + let mut engine = make_engine(); + let now = Instant::from_origin(Duration::from_secs(86_400)); + engine.start(now); + + // A Ping must name this node as its target to be answered at all. + let me = Node::new(SmolStr::new("test"), node_addr(7946)); + let prober = Node::new(SmolStr::new("prober"), peer_addr(9, 7002)); + let ping = encode_outgoing::( + &Message::Ping(Ping::new(7, prober, me)), + &EncodeOptions::new(None), + ) + .expect("a well-formed Ping encodes"); + + let mut gossip = QueueGossip::new(); + let mut stream = NoStream::with_pool(2); + + // Port 0 is non-routable: an ack sent there could never arrive. + gossip.push(peer_addr(9, 0), ping.to_vec()); + engine.pump(now, &mut gossip, &mut stream); + assert!( + gossip.outbound.is_empty(), + "an ack to a non-routable source must be screened at egress, not written to the wire" + ); + + let routable = peer_addr(9, 7002); + gossip.push(routable, ping.to_vec()); + engine.pump(now, &mut gossip, &mut stream); + assert!( + gossip.outbound.iter().any(|(_, dest)| *dest == routable), + "the identical Ping from a routable source must be acked" + ); +} + +/// An encrypted node DROPS an inbound plaintext gossip datagram at the decrypt step: a +/// node on an encrypted cluster must never admit an unauthenticated frame. The same +/// `Alive`, sealed under the node's live keyring, IS admitted — so the drop is the +/// decrypt guard rejecting an unauthenticated frame, not a malformed one. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn an_encrypted_node_drops_plaintext_inbound_gossip() { + let mut engine = make_encrypted_engine("enc", 7946, Keyring::new(secret_key(0x11))); + let now = Instant::from_origin(Duration::from_secs(86_400)); + engine.start(now); + + let ghost = Node::new(SmolStr::new("ghost"), peer_addr(2, 7946)); + let plaintext = alive_datagram(&ghost, None); + let src = peer_addr(3, 7946); + let mut gossip = QueueGossip::new(); + let mut stream = NoStream::with_pool(2); + + gossip.push(src, plaintext.to_vec()); + engine.pump(now, &mut gossip, &mut stream); + assert_eq!( + engine.num_members(), + 1, + "an unauthenticated plaintext datagram must be dropped by an encrypted node" + ); + + let sealed = engine + .endpoint + .encrypt_gossip(&plaintext) + .expect("the live keyring seals the frame"); + gossip.push(src, sealed); + engine.pump(now, &mut gossip, &mut stream); + assert_eq!( + engine.num_members(), + 2, + "the same Alive, sealed under the node's keyring, must be admitted" + ); +} + +// ── key-management requests missing their key ──────────────────────────────── + +/// A keyed key-management op (`install` / `use` / `remove`) that arrives WITHOUT its +/// key is refused with a message naming the omission, and the live keyring is left +/// exactly as it was — a malformed request can neither mutate the wire keyring nor be +/// reported as a success. `list`, which needs no key, is unaffected: it still reports +/// the live state. +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn a_keyed_op_without_its_key_is_refused_and_leaves_the_ring_untouched() { + let k1 = secret_key(0x11); + let k2 = secret_key(0x22); + let mut engine = make_encrypted_engine("a", 7946, Keyring::with_secondaries(k1, [k2])); + let primary_before = *engine.keyring().expect("encrypted").primary_ref(); + let secondaries_before = engine.keyring().expect("encrypted").secondaries().to_vec(); + + for op in [ + KeyRequestOperation::Install, + KeyRequestOperation::Use, + KeyRequestOperation::Remove, + ] { + let resp = engine.apply_key_request(&key_op(op, None)); + assert!( + !resp.result, + "a keyed op carrying no key must be refused, never reported as applied" + ); + assert!( + resp.message.contains("missing"), + "the refusal must name the omitted key, got {:?}", + resp.message + ); + let kr = engine.keyring().expect("encrypted"); + assert_eq!( + *kr.primary_ref(), + primary_before, + "a refused op must not move the primary" + ); + assert_eq!( + kr.secondaries(), + secondaries_before.as_slice(), + "a refused op must not change the ring" + ); + } + + // `list` needs no key, so the keyless refusal must not swallow it. + let resp = engine.apply_key_request(&key_op(KeyRequestOperation::List, None)); + assert!(resp.result, "list carries no key and must still succeed"); + assert_eq!( + resp.primary_key, + Some(primary_before), + "list reports the live primary" + ); +} + +// ── the Closing drain: progress re-arms, a stall force-aborts ──────────────── + +/// Whether the engine holds a reliable connection in the `Closing` drain state. +/// `closing_count` counts the DETACHED handles, not the still-mapped draining +/// connections, so this scans the live connections for the drain state. +fn has_closing_connection(engine: &mut SerfEngine) -> bool { + engine + .plane_mut() + .connections + .values() + .any(|c| c.state == ConnState::Closing) +} + +/// The undelivered-byte mark of the engine's sole draining (`Closing`) connection. +fn closing_drain_mark(engine: &mut SerfEngine) -> Option { + engine + .plane_mut() + .connections + .values() + .find(|c| c.state == ConnState::Closing) + .map(|c| c.close_drain_mark) +} + +/// A graceful close whose reply is still unacknowledged must NOT truncate it: the +/// connection parks in `Closing` (keeping its slot) while the egress pump keeps +/// draining. Each acknowledged byte is progress, which re-arms the drain rather than +/// letting the close deadline fire — `close_timeout` bounds a STALL, not the total +/// drain — and once the reply is fully acked the terminal FIN goes out and the slot is +/// reclaimed. +#[test] +fn closing_drain_re_arms_on_progress_then_fins_once_the_reply_is_acked() { + let mut link = LinkPair::new(&[10, 11], &[20, 21]); + let now = Instant::from_origin(Duration::from_secs(86_400)); + // B withholds its acknowledgements, so its push/pull reply is still unacked when its + // bridge gracefully closes. + link.hold_b_tx(); + + link + .a + .join(&[link.b_addr], false, now) + .expect("join announces intent and mints a handle"); + + let mut parked = false; + for _ in 0..40 { + link.step(now); + while link.a.poll_event().is_some() {} + while link.b.poll_event().is_some() {} + if has_closing_connection(&mut link.b) { + parked = true; + break; + } + } + assert!( + parked, + "a graceful close with an unacked reply must park in Closing, not FIN at once" + ); + assert!( + (link.b.pool_free_count() + link.b.listener_present() as usize) < 2, + "the draining connection still pins its slot until the reply is acked" + ); + + // Acknowledge part of the reply: the undelivered count shrinks, so the drain re-arms + // and the connection stays mapped rather than being force-closed. + let mark_before = closing_drain_mark(&mut link.b).expect("the draining connection is mapped"); + link.ack_b(1); + link.step(now); + while link.b.poll_event().is_some() {} + let mark_after = + closing_drain_mark(&mut link.b).expect("progress keeps the draining connection mapped"); + assert!( + mark_after < mark_before, + "an acknowledged byte is progress: the drain mark must shrink ({mark_before} -> {mark_after})" + ); + + // Acknowledge the remainder: the drain completes, the terminal FIN goes out, and the + // reap returns the slot. + link.ack_b(usize::MAX); + let mut t = now; + for _ in 0..20 { + link.step(t); + while link.b.poll_event().is_some() {} + if link.b.pool_free_count() + link.b.listener_present() as usize == 2 { + break; + } + t += Duration::from_millis(200); + } + assert!( + !has_closing_connection(&mut link.b), + "the fully-drained connection must FIN and leave the Closing state" + ); + assert_eq!( + link.b.pool_free_count() + link.b.listener_present() as usize, + 2, + "once the reply is fully acked, the drained connection's slot is reclaimed" + ); +} + +/// The `Closing` drain's force-abort backstop: a peer that never acknowledges the reply +/// makes no progress, so at the close deadline the engine gives up on the remainder, +/// RSTs the socket, and reclaims the slot — a stalled peer can never permanently wedge +/// a pooled slot mid-drain. +#[test] +fn closing_drain_force_aborts_a_stalled_peer_at_the_deadline() { + let mut link = LinkPair::new(&[10, 11], &[20, 21]); + let now = Instant::from_origin(Duration::from_secs(86_400)); + // B's reply is never acknowledged, so its drain makes no progress at all. + link.hold_b_tx(); + + link + .a + .join(&[link.b_addr], false, now) + .expect("join announces intent and mints a handle"); + + let mut parked = false; + for _ in 0..40 { + link.step(now); + while link.a.poll_event().is_some() {} + while link.b.poll_event().is_some() {} + if has_closing_connection(&mut link.b) { + parked = true; + break; + } + } + assert!(parked, "the unacked reply parked the connection in Closing"); + + // Never acknowledge. Advance past the 10 s close timeout: with no progress, the + // no-progress bound elapses and the drain is force-aborted. + let mut t = now + Duration::from_secs(15); + for _ in 0..10 { + link.step(t); + while link.b.poll_event().is_some() {} + if !has_closing_connection(&mut link.b) + && link.b.pool_free_count() + link.b.listener_present() as usize == 2 + { + break; + } + t += Duration::from_secs(15); + } + assert!( + !has_closing_connection(&mut link.b), + "a stalled drain must be force-aborted off the connection map at its deadline" + ); + assert_eq!( + link.b.pool_free_count() + link.b.listener_present() as usize, + 2, + "the force-aborted slot must be reclaimed so the pool cannot wedge" + ); +} + +// ── push/pull completions the engine did not start for a join ──────────────── + +/// An anti-entropy push/pull — one the engine started for a state REFRESH, not for an +/// await-result join — is never credited to a join: its terminal `ExchangeCompleted` +/// folds into no waiter, so an already-delivered join is not re-resolved and no waiter +/// is resurrected. A fold keyed on the peer ADDRESS rather than the exchange id would +/// re-resolve the join that reached the very same peer. +#[test] +fn a_refresh_push_pull_completion_is_credited_to_no_join() { + let mut link = LinkPair::new(&[10, 11], &[20, 21]); + let now = Instant::from_origin(Duration::from_secs(86_400)); + + let handle = link + .a + .join(&[link.b_addr], false, now) + .expect("join announces intent and mints a handle"); + + let mut outcome = None; + for _ in 0..40 { + link.step(now); + while link.a.poll_event().is_some() {} + while link.b.poll_event().is_some() {} + if let Some(res) = link.a.poll_join(handle) { + outcome = Some(res); + break; + } + } + assert!( + matches!(outcome, Some(Ok(_))), + "the join must resolve Ok before the refresh, got {outcome:?}" + ); + assert_eq!( + link.a.pending_join_count(), + 0, + "the delivered join is reaped" + ); + + // A push/pull started for anti-entropy, owned by no join. + link + .a + .endpoint + .start_push_pull(link.b_addr, PushPullKind::Refresh, now); + + let mut completed = false; + for _ in 0..40 { + link.step(now); + while link.b.poll_event().is_some() {} + while let Some(ev) = link.a.poll_event() { + if matches!(&ev, Event::ExchangeCompleted(ec) if ec.kind() == ExchangeKind::PushPull) { + completed = true; + } + } + if completed { + break; + } + } + assert!( + completed, + "the refresh push/pull must reach its terminal ExchangeCompleted" + ); + assert_eq!( + link.a.pending_join_count(), + 0, + "a completion owned by no join must not resurrect a waiter" + ); + assert!( + link.a.poll_join(handle).is_none(), + "the already-delivered join must not be re-resolved by an unrelated completion" + ); +} diff --git a/serf-proto/src/coordinate_client/tests.rs b/serf-proto/src/coordinate_client/tests.rs index 00fc0aa0..6b66b66a 100644 --- a/serf-proto/src/coordinate_client/tests.rs +++ b/serf-proto/src/coordinate_client/tests.rs @@ -584,3 +584,76 @@ fn dimensionality_zero_is_clamped_to_one_and_update_does_not_spin() { "update with dimensionality clamped to 1 must not spin or error: {result:?}" ); } + +// ── Options builders ───────────────────────────────────────────────────────── + +/// Every tuning builder writes the field it names. A transposed setter would be +/// invisible to a defaults test yet silently mis-tune the Vivaldi model, so each +/// is pinned to its own getter with a value distinct from every default. +#[test] +fn each_tuning_builder_writes_its_own_field() { + let o = CoordinateOptions::new() + .with_vivaldi_error_max(0.75) + .with_vivaldi_ce(0.11) + .with_vivaldi_cc(0.22) + .with_adjustment_window_size(33) + .with_height_min(0.44) + .with_gravity_rho(0.55); + + assert_float_eq(o.vivaldi_error_max(), 0.75); + assert_float_eq(o.vivaldi_ce(), 0.11); + assert_float_eq(o.vivaldi_cc(), 0.22); + assert_eq!(o.adjustment_window_size(), 33); + assert_float_eq(o.height_min(), 0.44); + assert_float_eq(o.gravity_rho(), 0.55); +} + +/// Zero is clamped to one on both size knobs. A zero dimensionality leaves the +/// coincident-point retry loop with no component to randomise — it spins forever; +/// a zero latency-filter window indexes an emptied median buffer — it panics. +/// Both are CPU/panic hazards, not merely odd configuration. +#[test] +fn zero_sized_knobs_are_clamped_to_one() { + assert_eq!( + CoordinateOptions::new() + .with_dimensionality(0) + .dimensionality(), + 1, + "a zero dimensionality would hang the coincident-point retry loop" + ); + assert_eq!( + CoordinateOptions::new() + .with_latency_filter_size(0) + .latency_filter_size(), + 1, + "a zero latency-filter window would panic the median lookup" + ); +} + +// ── Coincident points ──────────────────────────────────────────────────────── + +/// Two nodes whose coordinates are exactly coincident have no direction to push +/// apart along. Rather than emit a zero vector (which would stall the model at +/// the origin forever), the client draws a random unit direction, so the update +/// still moves the local coordinate. +#[test] +fn a_coincident_peer_still_moves_the_local_coordinate() { + let mut c = CoordinateClient::::new(opts_dim(3)); + let mut rng = test_rng(); + + // The peer sits exactly where we do: the origin. + let coincident = zero_coord(3); + assert!( + c.get_coordinate().vec.iter().all(|x| *x == 0.0), + "the local coordinate starts at the origin" + ); + + let updated = c + .update(&1u32, &coincident, Duration::from_millis(50), &mut rng) + .expect("a coincident peer is a valid update"); + + assert!( + updated.vec.iter().any(|x| *x != 0.0), + "a coincident peer must be pushed apart along a random direction, not left at zero" + ); +} diff --git a/serf-proto/src/lib.rs b/serf-proto/src/lib.rs index 5bcc24b5..7a71dbea 100644 --- a/serf-proto/src/lib.rs +++ b/serf-proto/src/lib.rs @@ -182,3 +182,6 @@ pub use memberlist_proto::event::{ExchangeCompleted, ExchangeId, ExchangeKind, E /// Fx hasher — rustc-hash's own `Fx*` map aliases are std-only. pub(crate) type FxHashMap = hashbrown::HashMap; pub(crate) type FxHashSet = hashbrown::HashSet; + +#[cfg(test)] +mod tests; diff --git a/serf-proto/src/members/tests.rs b/serf-proto/src/members/tests.rs index a4e2bdac..059c9689 100644 --- a/serf-proto/src/members/tests.rs +++ b/serf-proto/src/members/tests.rs @@ -208,3 +208,28 @@ fn remove_old_member_drops_only_the_named_id() { "other ids must be retained" ); } + +/// A recent intent carries the four fields the eviction tie-break and the +/// join/leave reconciliation read back: kind, Lamport time, the driver-threaded +/// arrival instant, and the monotonic insertion sequence. +#[cfg(any(feature = "tcp", feature = "quic"))] +#[test] +fn recent_intent_reports_the_fields_it_was_built_with() { + use memberlist_proto::Instant; + + let at = Instant::ORIGIN + core::time::Duration::from_secs(7); + let intent = NodeIntent::new(IntentKind::Leave, LamportTime::new(9), at, 3); + + assert_eq!(intent.kind(), IntentKind::Leave); + assert_eq!(intent.ltime(), LamportTime::new(9)); + assert_eq!( + intent.wall_time(), + at, + "the arrival instant is what the reaper ages the intent against" + ); + assert_eq!( + intent.sequence(), + 3, + "the insertion sequence is the deterministic eviction tie-break" + ); +} diff --git a/serf-proto/src/options/tests.rs b/serf-proto/src/options/tests.rs index 21b4b4e9..17c33ff8 100644 --- a/serf-proto/src/options/tests.rs +++ b/serf-proto/src/options/tests.rs @@ -1,4 +1,5 @@ use core::time::Duration; +use std::string::ToString; use super::{InvalidOptions, Options}; @@ -316,3 +317,81 @@ fn user_event_size_limit_builder_pair_round_trips() { set.set_user_event_size_limit(10 * 1024); assert_eq!(set.user_event_size_limit(), 10 * 1024); } + +/// Every builder writes the field it names. A transposed setter (a copy-paste +/// slip in a long builder chain) is invisible to a defaults test and silently +/// mis-configures the machine, so each setter is pinned to its own getter with a +/// value distinct from every default. +#[test] +fn each_builder_writes_its_own_field() { + let o = Options::new() + .with_recent_intent_timeout(Duration::from_secs(11)) + .with_leave_propagate_delay(Duration::from_secs(12)) + .with_query_buffer_size(13) + .with_queue_depth_warning(14) + .with_flap_timeout(Duration::from_secs(15)); + + assert_eq!(o.recent_intent_timeout(), Duration::from_secs(11)); + assert_eq!(o.leave_propagate_delay(), Duration::from_secs(12)); + assert_eq!(o.query_buffer_size(), 13); + assert_eq!(o.queue_depth_warning(), 14); + assert_eq!(o.flap_timeout(), Duration::from_secs(15)); + + // The untouched neighbours keep their defaults — no setter bleeds into another. + let d = Options::new(); + assert_eq!(o.reap_interval(), d.reap_interval()); + assert_eq!(o.event_buffer_size(), d.event_buffer_size()); + assert_eq!(o.max_queue_depth(), d.max_queue_depth()); +} + +/// A rejected coalescing window reports BOTH periods, so an operator can see the +/// exact pair that violates `quiescent < coalesce` without re-reading the config. +#[test] +fn invalid_coalesce_window_names_both_periods() { + let err = Options::new() + .with_coalesce_period(Duration::from_secs(1)) + .with_quiescent_period(Duration::from_secs(5)) + .validate() + .expect_err("a quiescent period past the coalesce period is invalid"); + + let msg = err.to_string(); + assert!( + matches!(err, InvalidOptions::MemberCoalesce(_)), + "expected the member-coalescing-window variant, got {err:?}" + ); + assert!( + msg.contains("5s") && msg.contains("1s"), + "the message must name both periods, got: {msg}" + ); +} + +/// A `max_user_event_size` over the ceiling reports both numbers; a zero ceiling +/// is reported as its own distinct message, since "must not exceed 0" would read +/// as nonsense. +#[test] +fn invalid_user_event_size_names_the_size_and_the_ceiling() { + let err = Options::new() + .with_user_event_size_limit(100) + .with_max_user_event_size(200) + .validate() + .expect_err("a max_user_event_size over the ceiling is invalid"); + + let msg = err.to_string(); + assert!( + matches!(err, InvalidOptions::UserEventSize(_)), + "expected the user-event-size variant, got {err:?}" + ); + assert!( + msg.contains("200") && msg.contains("100"), + "the message must name the size and the ceiling, got: {msg}" + ); + + let zero = Options::new() + .with_user_event_size_limit(0) + .validate() + .expect_err("a zero ceiling is invalid"); + assert!( + zero.to_string().contains("nonzero"), + "a zero ceiling is reported on its own terms, got: {zero}" + ); +} diff --git a/serf-proto/src/quic_endpoint/tests.rs b/serf-proto/src/quic_endpoint/tests.rs index 5a7ba603..de372eb7 100644 --- a/serf-proto/src/quic_endpoint/tests.rs +++ b/serf-proto/src/quic_endpoint/tests.rs @@ -19,7 +19,9 @@ use core::{net::SocketAddr, time::Duration}; use std::sync::Arc; use memberlist_proto::{ - EndpointOptions, Instant, PushPullKind, QuicOptions, SeedableRng, SmallRng, UnreliableTransport, + DatagramSendStatus, EncodeOptions, EndpointOptions, Instant, Node, PushPullKind, QuicOptions, + SeedableRng, SmallRng, UnreliableTransport, encode_outgoing, + typed::{Alive, Message}, }; use rustls::{ client::danger::{HandshakeSignatureValid, ServerCertVerified}, @@ -28,16 +30,24 @@ use rustls::{ use rustls_pki_types::{CertificateDer, PrivateKeyDer}; use crate::{ - QuicEndpoint, + AnyMessage, LamportTime, QuicEndpoint, + endpoint::{QueryId, QueryParams}, event::{Event, MemberEventKind}, - members::MemberStatus, + members::{IntentKind, MemberStatus}, options::Options, + typed::{QueryFlag, QueryMessage, QueryResponseMessage, RelayMessage, Tags, UserEventMessage}, }; fn sa(port: u16) -> SocketAddr { format!("127.0.0.1:{port}").parse().unwrap() } +/// `Instant::ORIGIN + s` seconds — the wall clock the tests thread through the +/// machine. +fn t_secs(s: u64) -> Instant { + Instant::ORIGIN + Duration::from_secs(s) +} + /// A self-signed cert + key for `localhost`, for the test TLS bundle. fn self_signed() -> (Vec>, PrivateKeyDer<'static>) { let ck = rcgen::generate_simple_self_signed(vec!["localhost".into()]).unwrap(); @@ -119,8 +129,8 @@ fn test_client() -> quinn_proto::ClientConfig { } /// A QUIC config bundle with a 20s idle timeout (well past a single-instant -/// localhost handshake) and datagram-mode unreliable transport. -fn test_quic_options() -> QuicOptions { +/// localhost handshake) and the given unreliable-transport wire. +fn quic_options_on(wire: UnreliableTransport) -> QuicOptions { let mut transport = quinn_proto::TransportConfig::default(); transport.max_idle_timeout(Some( quinn_proto::IdleTimeout::try_from(Duration::from_secs(20)).unwrap(), @@ -131,30 +141,51 @@ fn test_quic_options() -> QuicOptions { test_client(), transport, "localhost", - UnreliableTransport::Datagram, + wire, ) } -/// Build a serf `QuicEndpoint` rooted at `id` / `port`, seeded -/// deterministically. The quinn rng seed is derived from the port so the two -/// loopback endpoints draw distinct connection IDs. -fn ep(id: u32, port: u16) -> QuicEndpoint { - let inner_opts = EndpointOptions::new(id, sa(port)) - .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()); +/// The default bundle: datagram-mode unreliable transport. +fn test_quic_options() -> QuicOptions { + quic_options_on(UnreliableTransport::Datagram) +} + +/// Wrap `inner_opts` into the memberlist QUIC coordinator, seeding quinn's rng +/// from the bound port so two loopback coordinators draw distinct connection ids. +fn coord( + inner_opts: EndpointOptions, + quic: QuicOptions, + port: u16, +) -> memberlist_proto::QuicEndpoint { let inner = memberlist_proto::Endpoint::new_at(inner_opts, Instant::ORIGIN, SmallRng::seed_from_u64(0)); let mut seed = [0u8; 32]; seed[..2].copy_from_slice(&port.to_le_bytes()); - let coord = memberlist_proto::QuicEndpoint::::with_quinn_rng_seed( - inner, - test_quic_options(), - Some(seed), - ); - let mut e = QuicEndpoint::new(coord, Options::new()); + memberlist_proto::QuicEndpoint::::with_quinn_rng_seed(inner, quic, Some(seed)) +} + +/// The inner memberlist options every fixture roots at. +fn inner_opts(id: u32, port: u16) -> EndpointOptions { + EndpointOptions::new(id, sa(port)) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()) +} + +/// Build a serf `QuicEndpoint` rooted at `id` / `port` with caller-supplied +/// serf `opts`, seeded deterministically. The inner memberlist emits +/// `NodeJoined(self)` on construction; drain it so every test starts from the +/// post-self-join-drained state (self is a member). +fn ep_with_options(id: u32, port: u16, opts: Options) -> QuicEndpoint { + let mut e = QuicEndpoint::new(coord(inner_opts(id, port), test_quic_options(), port), opts); let _ = e.poll_event(); e } +/// Build a serf `QuicEndpoint` rooted at `id` / `port` with default serf +/// options. +fn ep(id: u32, port: u16) -> QuicEndpoint { + ep_with_options(id, port, Options::new()) +} + #[test] fn constructs_alive_with_zero_clocks() { let e = ep(1, 7946); @@ -414,3 +445,2421 @@ fn operator_forwarders_reach_the_inner_machine() { "a fresh coordinate client has reset nothing" ); } + +// ── construction ────────────────────────────────────────────────────────────── + +/// The two coalescer shed counters are INJECTED, not created: `new_with_rng_in` +/// must thread `user_drop` into the user coalescer's slot and `member_drop` into +/// the member coalescer's, without transposing them. A driver that shares these +/// counters with a detached handle reads the wrong metric if they cross. +#[test] +fn new_with_rng_in_threads_each_injected_drop_counter_to_its_own_slot() { + let e: QuicEndpoint = QuicEndpoint::new_with_rng_in( + coord(inner_opts(1, 7946), test_quic_options(), 7946), + Options::new(), + SmallRng::seed_from_u64(0), + 7u64, + 9u64, + ); + assert_eq!( + e.coalesced_user_events_dropped(), + 7, + "the user shed count reads back the injected user_drop" + ); + assert_eq!( + e.coalesced_member_events_dropped(), + 9, + "the member shed count reads back the injected member_drop" + ); +} + +/// The super-machine roots serf at the coordinator's local id: `new` reads the +/// id off the membership endpoint rather than taking it as a separate parameter, +/// so the two can never disagree. +#[test] +fn local_id_is_the_coordinators_membership_id() { + let e = ep(42, 7946); + assert_eq!(*e.local_id(), 42u32); + assert_eq!( + e.num_members(), + 1, + "the construction self-join leaves the local node as the sole member" + ); + let snapshot = e.members_snapshot(); + assert_eq!(snapshot.len(), 1, "the snapshot carries the local member"); + assert_eq!(*snapshot[0].node().id_ref(), 42u32); + assert_eq!(snapshot[0].status(), MemberStatus::Alive); +} + +/// `members_snapshot` publishes every tracked member — alive, leaving, left, and +/// failed-within-the-reap-window alike — so a driver's observable view does not +/// silently drop tombstones. +#[test] +fn members_snapshot_carries_every_tracked_status() { + let mut e = ep(1, 7946); + e.test_seed_member(2, MemberStatus::Leaving, 1.into()); + e.test_seed_failed_member_by_status(3, 1.into(), Instant::ORIGIN); + e.test_seed_left_member_by_status(4, 1.into(), Instant::ORIGIN); + + let mut got: Vec<(u32, MemberStatus)> = e + .members_snapshot() + .iter() + .map(|m| (*m.node().id_ref(), m.status())) + .collect(); + got.sort_by_key(|(id, _)| *id); + + assert_eq!( + got, + vec![ + (1, MemberStatus::Alive), + (2, MemberStatus::Leaving), + (3, MemberStatus::Failed), + (4, MemberStatus::Left), + ], + "the snapshot must publish every tracked member with its live status" + ); +} + +// ── reconnect delegate ──────────────────────────────────────────────────────── + +/// A per-member reconnect-timeout override installed through the builder +/// `with_reconnect_delegate` shortens the reaper's failed-member window: the +/// delegate's 10 s timeout reaps at t+11 s, where the flat 24 h default would +/// still be holding the member. +#[test] +fn builder_reconnect_delegate_overrides_the_failed_reap_timeout() { + struct TenSeconds; + impl crate::ReconnectDelegate for TenSeconds { + fn reconnect_timeout( + &self, + _member: &crate::members::Member, + _default: Duration, + ) -> Duration { + Duration::from_secs(10) + } + } + + let e = QuicEndpoint::::new( + coord(inner_opts(1, 7946), test_quic_options(), 7946), + Options::new(), + ); + let mut e = e.with_reconnect_delegate(Some(Box::new(TenSeconds))); + let _ = e.poll_event(); + e.test_seed_failed_member(2, sa(7000), Instant::ORIGIN); + + e.test_fire_reap(t_secs(11)); + assert_eq!( + e.test_member_status(2), + None, + "the delegate's 10s timeout reaps the failed member at t+11s" + ); +} + +/// Clearing the delegate restores the flat configured timeouts: the same +/// failed member that the 10 s delegate would have reaped at t+11 s survives once +/// the delegate is set back to `None`. +#[test] +fn clearing_the_reconnect_delegate_restores_the_flat_timeout() { + struct TenSeconds; + impl crate::ReconnectDelegate for TenSeconds { + fn reconnect_timeout( + &self, + _member: &crate::members::Member, + _default: Duration, + ) -> Duration { + Duration::from_secs(10) + } + } + + let mut e = ep(1, 7946); + e.set_reconnect_delegate(Some(Box::new(TenSeconds))); + e.set_reconnect_delegate(None); + e.test_seed_failed_member(2, sa(7000), Instant::ORIGIN); + + e.test_fire_reap(t_secs(11)); + assert_eq!( + e.test_member_status(2), + Some(MemberStatus::Failed), + "with no delegate the flat 24h reconnect_timeout still holds the member at t+11s" + ); +} + +// ── transport-level driver surface ──────────────────────────────────────────── + +/// Encode one memberlist gossip `Message` exactly as the wire carries it (no +/// cluster label), for the ingress paths that take raw frame bytes. +fn gossip_frame(msg: &Message) -> Bytes { + encode_outgoing(msg, &EncodeOptions::new(None)).expect("encode memberlist gossip frame") +} + +/// An `Alive` for `id` at `addr`, the gossip message that admits a peer into the +/// inner membership. +fn alive(id: u32, addr: SocketAddr) -> Message { + Message::Alive(Alive::new(1, Node::new(id, addr))) +} + +/// A well-formed gossip frame fed to `handle_packet` reaches the coordinator: +/// the inner machine admits the peer and the resulting `NodeJoined` is sieved +/// into serf on the same call, so the member and its `Member(Join)` event are +/// observable without a further tick. +#[test] +fn handle_packet_with_an_alive_frame_admits_the_peer_into_serf() { + let mut e = ep(1, 7946); + e.handle_packet(sa(7000), gossip_frame(&alive(2, sa(7000))), Instant::ORIGIN); + + assert_eq!( + e.test_member_status(2), + Some(MemberStatus::Alive), + "the decoded Alive must reach the coordinator and surface as a serf member" + ); + assert_eq!(e.num_members(), 2, "self plus the admitted peer"); + let ev = e + .poll_event() + .expect("the admitted peer emits a serf event"); + assert!( + matches!(ev, Event::Member(ref me) if me.kind() == MemberEventKind::Join), + "expected Member(Join) for the admitted peer, got {ev:?}" + ); +} + +/// `handle_message` is the compound-aware ingress: it feeds an ALREADY-decoded +/// message straight to the coordinator, skipping the per-call `parse_message` +/// that `handle_packet` performs, and sieves the result identically. +#[test] +fn handle_message_admits_a_typed_alive_without_reparsing() { + let mut e = ep(1, 7946); + e.handle_message(sa(7000), alive(2, sa(7000)), Instant::ORIGIN); + + assert_eq!( + e.test_member_status(2), + Some(MemberStatus::Alive), + "a typed Alive fed to handle_message must land in serf membership" + ); + assert!( + matches!(e.poll_event(), Some(Event::Member(ref me)) if me.kind() == MemberEventKind::Join), + "handle_message must sieve the inner NodeJoined into serf" + ); +} + +/// The single conceptual socket carries both quinn packets and plain-UDP gossip. +/// A datagram whose first byte is a memberlist tag is NOT handed to quinn: the +/// coordinator buffers it for the codec-owning driver to drain via +/// `poll_memberlist_ingress`, decode, and feed back through `handle_packet`. +#[test] +fn handle_udp_buffers_a_gossip_datagram_for_the_codec_owning_driver() { + let mut e = ep(1, 7946); + let frame = gossip_frame(&alive(2, sa(7000))); + e.handle_udp(sa(7000), &frame, Instant::ORIGIN); + + assert_eq!( + e.test_member_status(2), + None, + "handle_udp must not decode the gossip frame itself — the codec layer owns that" + ); + let (from, bytes) = e + .poll_memberlist_ingress() + .expect("the gossip datagram is buffered for the driver to drain"); + assert_eq!(from, sa(7000), "the ingress carries the sender address"); + assert_eq!(bytes, frame, "the buffered bytes are the datagram verbatim"); + assert!( + e.poll_memberlist_ingress().is_none(), + "the ingress queue drains once" + ); + + // The full round: decode-and-feed the drained bytes back through handle_packet. + e.handle_packet(from, bytes, Instant::ORIGIN); + assert_eq!( + e.test_member_status(2), + Some(MemberStatus::Alive), + "handle_udp -> poll_memberlist_ingress -> handle_packet admits the peer" + ); +} + +/// `poll_timeout` folds the coordinator's deadline with serf's own. Each of the +/// four fold cases is pinned: an un-scheduled coordinator under an alive serf +/// yields serf's earliest periodic deadline (the reap interval); once the +/// coordinator's schedulers are armed its sooner probe/gossip deadline wins. +#[test] +fn poll_timeout_folds_the_coordinator_and_serf_deadlines() { + // (None, Some): the coordinator's schedulers are unset until `start_scheduling`, + // so only serf's periodic deadlines are in play. reap_interval (15s) is the + // earliest of reap / reconnect (30s) / queue-check (30s). + let mut e = ep(1, 7946); + assert_eq!( + e.poll_timeout(), + Some(t_secs(15)), + "an un-scheduled coordinator leaves serf's reap deadline as the wake" + ); + + // (Some, Some): the armed coordinator's probe/gossip deadline is sooner than + // serf's 15s reap, so the fold must surrender to the coordinator. + e.start_scheduling(Instant::ORIGIN); + let inner = e + .transport_mut() + .poll_timeout() + .expect("start_scheduling arms the coordinator's probe / gossip / push-pull timers"); + assert!( + inner < t_secs(15), + "the coordinator's first scheduled deadline is sooner than serf's reap" + ); + assert_eq!( + e.poll_timeout(), + Some(inner), + "the fold takes the minimum of the coordinator's and serf's deadlines" + ); +} + +/// A machine that lost an id-conflict vote schedules no serf wakeups; with the +/// coordinator un-scheduled too, `poll_timeout` is `None` — the driver must not +/// spin on a dead machine. Arming the coordinator then re-supplies the only +/// remaining deadline. +#[test] +fn poll_timeout_is_none_on_a_shutdown_machine_with_no_coordinator_schedule() { + let mut e = ep(1, 7946); + shut_down_via_lost_conflict(&mut e); + + assert_eq!( + e.poll_timeout(), + None, + "a shut-down serf with an un-scheduled coordinator requests no wakeup" + ); + + // (Some, None): the coordinator's schedule is the only live deadline left. + e.start_scheduling(Instant::ORIGIN); + let inner = e.transport_mut().poll_timeout(); + assert!(inner.is_some(), "the coordinator schedules its own timers"); + assert_eq!( + e.poll_timeout(), + inner, + "with serf shut down the coordinator's deadline is the fold" + ); +} + +/// The gossip MTU and the max reliable-stream frame size are the coordinator's +/// configured limits, not constants: the driver sizes its recv buffer and its +/// observation budget from them, so the forwarders must report what was +/// configured. +#[test] +fn wire_size_forwarders_report_the_configured_limits() { + let opts = inner_opts(1, 7946) + .with_gossip_mtu(1234) + .with_max_stream_frame_size(4096); + let mut e = QuicEndpoint::::new(coord(opts, test_quic_options(), 7946), Options::new()); + let _ = e.poll_event(); + + assert_eq!(e.gossip_mtu(), 1234, "the configured gossip MTU reads back"); + assert_eq!( + e.max_stream_frame_size(), + 4096, + "the configured max reliable-stream frame size reads back" + ); +} + +/// The unreliable (gossip + probe) wire is chosen at construction and the driver +/// reads it back to route each outbound gossip transmit: a `Datagram`-mode +/// coordinator queues QUIC datagrams, a `Udp`-mode one uses the shared socket. +#[test] +fn unreliable_transport_reports_the_configured_wire() { + let mut datagram = QuicEndpoint::::new( + coord( + inner_opts(1, 7946), + quic_options_on(UnreliableTransport::Datagram), + 7946, + ), + Options::new(), + ); + let _ = datagram.poll_event(); + assert_eq!( + datagram.unreliable_transport(), + UnreliableTransport::Datagram + ); + + let mut udp = QuicEndpoint::::new( + coord( + inner_opts(2, 7000), + quic_options_on(UnreliableTransport::Udp), + 7000, + ), + Options::new(), + ); + let _ = udp.poll_event(); + assert_eq!(udp.unreliable_transport(), UnreliableTransport::Udp); +} + +/// A datagram offered for a peer with no established QUIC connection is +/// `NotReady`: the driver must fall back to the plain-UDP path. Connection +/// liveness is never a membership signal, so this is a routing answer and not a +/// failure the machine records. +#[test] +fn queue_unreliable_datagram_is_not_ready_without_an_established_connection() { + let mut e = ep(1, 7946); + let status = + e.queue_unreliable_datagram(sa(7000), Bytes::from_static(b"gossip"), Instant::ORIGIN); + assert_eq!( + status, + DatagramSendStatus::NotReady, + "with no pooled connection the driver must be told to fall back, not silently drop" + ); + assert!( + e.poll_event().is_none(), + "a not-ready datagram is never a membership event" + ); +} + +/// `flush_outbound_transmits` drains quinn's queued outbound at `now` WITHOUT +/// advancing any membership timer, so a packet leaves on the tick it was +/// queued. A dial started with `start_push_pull` must therefore surface its +/// handshake datagram after a bare flush — no `handle_timeout` needed. +#[test] +fn flush_outbound_transmits_releases_the_dial_without_a_membership_tick() { + let mut e = ep(1, 7946); + let before = e.member_time(); + e.start_push_pull(sa(7000), PushPullKind::Join, Instant::ORIGIN); + + e.flush_outbound_transmits(Instant::ORIGIN); + let (to, bytes) = e + .poll_transmit() + .expect("the dial's handshake datagram is flushed on the same instant"); + assert_eq!( + to, + sa(7000), + "the handshake is addressed to the dialed peer" + ); + assert!( + !bytes.is_empty(), + "the flushed datagram carries the Initial" + ); + assert_eq!( + e.member_time(), + before, + "a flush must not advance any membership clock" + ); +} + +/// `start_join_push_pull(ignore_old = true)` records the returned exchange id as +/// a per-EXCHANGE ignore-join target, so the merge it produces suppresses replay +/// of the peer's pre-join user events. A plain (non-`ignore_old`) join records +/// nothing, and a join that terminates without merging is cleared by the driver. +#[test] +fn ignore_old_join_records_its_exchange_and_the_driver_can_clear_it() { + let mut e = ep(1, 7946); + + let plain = e.start_join_push_pull(sa(7000), false, Instant::ORIGIN); + assert!( + !e.test_has_ignore_join_stream(plain), + "a join without ignore_old records no ignore-join target" + ); + + let ignoring = e.start_join_push_pull(sa(7001), true, Instant::ORIGIN); + assert!( + e.test_has_ignore_join_stream(ignoring), + "an ignore_old join records its exchange id" + ); + assert_ne!( + plain, ignoring, + "each dial gets its own exchange id, so the ignore set is per-exchange" + ); + + // A join that terminates without a merge is cleared by the driver; the call is + // idempotent for an id the success-path merge already consumed. + e.clear_ignore_join_stream(ignoring); + assert!( + !e.test_has_ignore_join_stream(ignoring), + "clear_ignore_join_stream removes the terminated join's entry" + ); + e.clear_ignore_join_stream(ignoring); + assert!( + !e.test_has_ignore_join_stream(ignoring), + "clearing an already-absent id is a no-op" + ); +} + +/// A remote push-pull body carrying one buffered user event `name` at `ltime`, +/// with the peer's event clock one past it. +fn push_pull_with_event(name: &str, ltime: u64) -> Bytes { + use crate::typed::{PushPullMessage, UserEvent, UserEvents}; + + let pp: PushPullMessage = PushPullMessage::new( + 1.into(), + Vec::new(), + Vec::new(), + (ltime + 1).into(), + vec![UserEvents { + ltime: ltime.into(), + events: vec![UserEvent { + name: name.into(), + payload: Bytes::from_static(b"x"), + }], + }], + 1.into(), + ); + AnyMessage::::PushPull(pp) + .encode() + .expect("encode push-pull body") +} + +/// The ignore-join entry is consumed by the merge it was recorded for: the join +/// merge carrying the recorded exchange id raises the event floor to the peer's +/// event clock, so the peer's pre-join user events never replay — and the entry +/// is one-shot. +#[test] +fn ignore_old_join_merge_suppresses_the_peers_pre_join_user_events() { + let mut e = ep(1, 7946); + let sid = e.start_join_push_pull(sa(7000), true, Instant::ORIGIN); + + e.test_merge_remote_state_with_stream(push_pull_with_event("pre-join", 3), true, sid); + + assert!( + e.poll_event().is_none(), + "the ignore_old join's merge must not replay the peer's pre-join user events" + ); + assert!( + !e.test_has_ignore_join_stream(sid), + "the merge consumes the ignore-join entry (one-shot)" + ); + assert_eq!( + e.test_event_min_time(), + 4, + "the event floor is raised to the peer's event clock, fencing off its history" + ); +} + +/// Without the ignore-join entry, the SAME join merge REPLAYS the peer's buffered +/// user events into the local event stream — the behaviour the `ignore_old` flag +/// exists to suppress. +#[test] +fn plain_join_merge_replays_the_peers_user_events() { + let mut e = ep(1, 7946); + let sid = e.start_join_push_pull(sa(7000), false, Instant::ORIGIN); + + e.test_merge_remote_state_with_stream(push_pull_with_event("replayed", 3), true, sid); + + let ev = e.poll_event().expect("the peer's user event replays"); + assert!( + matches!(ev, Event::User(ref u) if u.name == "replayed"), + "a merge with no ignore-join entry replays the peer's buffered user events, got {ev:?}" + ); + assert_eq!( + e.test_event_min_time(), + 0, + "a plain join leaves the event floor at zero" + ); +} + +/// The `suppress_pre_join_events` merge path drops the peer's buffered user +/// events without consulting any exchange id — the direct form, used when the +/// caller has already resolved that this join ignores old events. +#[test] +fn suppressed_merge_drops_the_peers_user_events() { + let mut e = ep(1, 7946); + e.test_merge_remote_state_suppressed(push_pull_with_event("suppressed", 3)); + assert!( + e.poll_event().is_none(), + "a suppressed merge must not replay the peer's user events" + ); + assert_eq!(e.test_event_min_time(), 4, "the event floor is raised"); +} + +// ── serf commands over the QUIC coordinator ─────────────────────────────────── + +/// `join` announces the local join intent on the coordinator's gossip plane: it +/// stamps the member clock and enqueues the intent broadcast. The driver owns +/// the seed dials separately (`start_join_push_pull`); `join` itself is the +/// announcement alone. +#[test] +fn join_announces_the_local_intent_on_the_broadcast_queue() { + let mut e = ep(1, 7946); + assert!(e.join().is_ok(), "join on an alive endpoint must succeed"); + assert!( + e.user_broadcast_queue_len() > 0, + "join must enqueue the local join intent for gossip" + ); + assert_eq!( + e.test_intent_ltime(1, IntentKind::Join), + None, + "the local node is already a member, so its own join is applied, not buffered" + ); +} + +/// `leave` drives the lifecycle Alive -> Leaving, witnesses the member clock, and +/// hands the farewell to the coordinator, which arms the leave-complete deadline +/// once the inner machine reports it has left the cluster. +#[test] +fn leave_transitions_to_leaving_and_arms_the_complete_deadline_on_inner_left() { + let mut e = ep(1, 7946); + e.leave(Instant::ORIGIN).expect("leave from Alive succeeds"); + assert!( + e.state().is_leaving(), + "leave must set the state to Leaving" + ); + assert!( + e.member_time() >= 1, + "leave stamps and witnesses the member clock" + ); + assert_eq!( + e.leave_complete_deadline(), + None, + "the deadline is armed by the inner LeftCluster, not by leave() itself" + ); + + e.test_inner_left_cluster(); + assert_eq!( + e.leave_complete_deadline(), + Some(Instant::ORIGIN + Options::new().leave_propagate_delay()), + "the inner LeftCluster arms the leave-complete deadline at the propagate delay" + ); +} + +/// A second `leave` on an already-Leaving machine is rejected with the typed +/// state error rather than restarting the chain. +#[test] +fn double_leave_is_rejected() { + let mut e = ep(1, 7946); + e.leave(Instant::ORIGIN).expect("first leave succeeds"); + assert!( + matches!( + e.leave(Instant::ORIGIN), + Err(crate::endpoint::Error::BadLeaveState(_)) + ), + "a second leave must be refused with BadLeaveState" + ); +} + +/// `force_leave(prune = true)` forgets the member outright instead of leaving a +/// tombstone for the reaper: the state entry, the failed/left index lists and +/// the recent-intent entry all go, and a `Member(Reap)` event is emitted now. +#[test] +fn force_leave_with_prune_forgets_the_member_immediately() { + let mut e = ep(1, 7946); + // status_time 0 keeps the force-leave's stamped ltime ahead of the member's, + // so the intent is fresh rather than stale. + e.test_seed_failed_member_by_status(2, 0.into(), Instant::ORIGIN); + assert!( + e.test_in_failed_members(2), + "the peer starts in failed_members" + ); + + e.force_leave(2, true, Instant::ORIGIN) + .expect("force_leave on a known member succeeds"); + + assert_eq!( + e.test_member_status(2), + None, + "a pruned force-leave forgets the member outright" + ); + assert!( + !e.test_in_failed_members(2), + "the pruned member is scrubbed from failed_members" + ); + assert!( + !e.test_in_left_members(2), + "the pruned member is not left behind in left_members either" + ); + let reaped = core::iter::from_fn(|| e.poll_event()) + .any(|ev| matches!(ev, Event::Member(ref me) if me.kind() == MemberEventKind::Reap)); + assert!(reaped, "the prune emits Member(Reap) immediately"); +} + +/// `force_leave(prune = false)` leaves an alive member as a `Leaving` tombstone +/// for the reaper — the contrast that proves the prune flag is load-bearing. +#[test] +fn force_leave_without_prune_leaves_a_tombstone() { + let mut e = ep(1, 7946); + e.test_seed_member(2, MemberStatus::Alive, 0.into()); + + e.force_leave(2, false, Instant::ORIGIN) + .expect("force_leave on a known member succeeds"); + + assert_eq!( + e.test_member_status(2), + Some(MemberStatus::Leaving), + "an unpruned force-leave tombstones the member as Leaving" + ); +} + +/// `set_tags` is a two-sided write: the serf-side member's tags AND the +/// coordinator's advertised node meta must both carry the new value, or the +/// local view and the wire diverge. +#[test] +fn set_tags_updates_both_the_serf_member_and_the_advertised_meta() { + let mut e = ep(1, 7946); + let tags: Tags = [("role", "web"), ("dc", "eu")].into_iter().collect(); + + e.set_tags(tags.clone(), Instant::ORIGIN) + .expect("set_tags within the meta cap succeeds"); + + assert_eq!( + e.test_local_tags(), + Some(tags), + "the local serf member carries the new tags" + ); + let meta = e + .test_local_meta() + .expect("the coordinator tracks the local node"); + assert!( + !meta.as_bytes().is_empty(), + "the coordinator re-advertises the encoded tags as node meta" + ); +} + +/// Tags that do not fit the memberlist meta cap are refused, and the refusal is +/// pre-mutation: the previous tags survive. +#[test] +fn oversized_tags_are_refused_and_leave_the_previous_tags_intact() { + let mut e = ep(1, 7946); + let small: Tags = [("role", "web")].into_iter().collect(); + e.set_tags(small.clone(), Instant::ORIGIN) + .expect("a small tag map fits the meta cap"); + + let huge: Tags = (0..64) + .map(|i| (format!("key{i}"), "x".repeat(64))) + .collect::>() + .into_iter() + .map(|(k, v)| (smol_str::SmolStr::from(k), smol_str::SmolStr::from(v))) + .collect(); + assert!( + matches!( + e.set_tags(huge, Instant::ORIGIN), + Err(crate::endpoint::Error::SetTagsMeta(_)) + ), + "tags over the meta cap must be refused" + ); + assert_eq!( + e.test_local_tags(), + Some(small), + "a refused set_tags must not clobber the previous tags" + ); +} + +/// A local `user_event` stamps the event clock, surfaces locally, and rides the +/// coordinator's event-tier broadcast queue. +#[test] +fn user_event_stamps_the_clock_emits_locally_and_queues_the_broadcast() { + let mut e = ep(1, 7946); + e.user_event("deploy", Bytes::from_static(b"v2"), false, Instant::ORIGIN) + .expect("user_event on an alive endpoint"); + + assert_eq!( + e.event_time(), + 1, + "the local event stamps ltime 0, clock -> 1" + ); + assert!( + e.user_broadcast_queue_len() > 0, + "the user event is queued on the coordinator's broadcast plane" + ); + match e.poll_event().expect("the local user event surfaces") { + Event::User(u) => { + assert_eq!(u.name.as_str(), "deploy"); + assert_eq!(u.payload.as_ref(), b"v2"); + } + other => panic!("expected Event::User, got {other:?}"), + } +} + +/// A user event over the meta size cap is refused before it can reach the wire. +#[test] +fn oversized_user_event_is_rejected() { + let mut e = ep(1, 7946); + let big = Bytes::from(vec![0u8; 1024]); + assert!( + e.user_event("big", big, false, Instant::ORIGIN).is_err(), + "a payload over max_user_event_size (512) must be refused" + ); + assert_eq!( + e.event_time(), + 0, + "a refused user event must not advance the event clock" + ); +} + +/// `resync_local_state` rebuilds the coordinator's push-pull snapshot from serf's +/// live clocks and clears the dirty flag, so the next anti-entropy exchange ships +/// current state. +#[test] +fn resync_local_state_rebuilds_the_push_pull_snapshot_and_clears_dirty() { + let mut e = ep(1, 7946); + e.test_set_clocks(11, 22, 33); + e.resync_local_state(); + + assert!(!e.test_is_dirty(), "resync clears the dirty flag"); + let snapshot = e.test_inner_local_state_snapshot(); + let pp = e.test_decode_pushpull(&snapshot); + assert_eq!( + u64::from(pp.ltime), + 11, + "the snapshot carries the member clock" + ); + assert_eq!( + u64::from(pp.event_ltime), + 22, + "the snapshot carries the event clock" + ); + assert_eq!( + u64::from(pp.query_ltime), + 33, + "the snapshot carries the query clock" + ); +} + +/// `load_snapshot` restores the three clocks from a replayed on-disk snapshot and +/// re-dials every alive peer it names, skipping the local node. +#[test] +fn load_snapshot_restores_the_clocks_and_rejoins_the_snapshot_peers() { + let mut e = ep(1, 7946); + let replay = crate::snapshot::ReplayResult { + alive_nodes: vec![ + Node::new(1u32, sa(7946)), // self — must be skipped + Node::new(2u32, sa(7000)), + ], + last_clock: 5.into(), + last_event_clock: 7.into(), + last_query_clock: 9.into(), + }; + e.load_snapshot(replay, Instant::ORIGIN) + .expect("load_snapshot on an alive machine succeeds"); + + assert!(e.member_time() >= 5, "the member clock recovers last_clock"); + assert_eq!( + e.test_event_min_time(), + 8, + "the event floor is last_event_clock + 1" + ); + assert_eq!( + e.test_query_min_time(), + 10, + "the query floor is last_query_clock + 1" + ); + assert_eq!( + e.test_rejoin_dials(), + vec![sa(7000)], + "exactly the non-self alive peers are re-dialed" + ); +} + +// ── membership FSM driven through the QUIC coordinator ──────────────────────── + +/// A fresh leave intent for an Alive peer moves it to Leaving and is rebroadcast; +/// the member clock witnesses the intent's Lamport time. +#[test] +fn live_leave_intent_transitions_alive_to_leaving() { + let mut e = ep(1, 7946); + e.test_seed_member(2, MemberStatus::Alive, 5.into()); + assert!( + e.test_handle_leave_intent(2, 8.into(), Instant::ORIGIN), + "a fresh leave intent for an Alive member is rebroadcast" + ); + assert_eq!(e.test_member_status(2), Some(MemberStatus::Leaving)); + assert_eq!( + e.test_member_status_time(2), + Some(LamportTime::new(8)), + "the member's status_time advances to the intent's ltime" + ); + assert!(e.member_time() >= 9, "the member clock witnessed ltime 8"); +} + +/// A leave intent for a Failed peer completes its departure: Failed -> Left, moved +/// from the failed list to the left list, with a `Member(Leave)` event. +#[test] +fn leave_intent_for_a_failed_peer_transitions_to_left() { + let mut e = ep(1, 7946); + e.test_seed_failed_member_by_status(2, 5.into(), Instant::ORIGIN); + assert!(e.test_handle_leave_intent(2, 9.into(), Instant::ORIGIN)); + + assert_eq!(e.test_member_status(2), Some(MemberStatus::Left)); + assert!(!e.test_in_failed_members(2), "moved out of failed_members"); + assert!(e.test_in_left_members(2), "moved into left_members"); + assert!( + matches!(e.poll_event(), Some(Event::Member(ref me)) if me.kind() == MemberEventKind::Leave), + "the completed departure emits Member(Leave)" + ); +} + +/// A fresh leave intent for the LOCAL node while Alive is refuted, not applied: +/// the node re-announces its own join and suppresses the rebroadcast. +#[test] +fn fresh_self_leave_intent_is_refuted() { + let mut e = ep(1, 7946); + e.test_seed_member(1, MemberStatus::Alive, 0.into()); + assert!( + !e.handle_node_leave_intent(5.into(), &1, false, Instant::ORIGIN), + "a self-leave must be refuted, never rebroadcast" + ); + assert_eq!( + e.test_member_status(1), + Some(MemberStatus::Alive), + "the refuted self-leave leaves the local node Alive" + ); + assert!(e.member_time() >= 6, "the member clock witnessed ltime 5"); +} + +/// A join intent for an unknown node is buffered as a recent intent rather than +/// creating a phantom member; the buffered ltime is what a later `NodeJoined` +/// reconciles against. +#[test] +fn join_intent_for_an_unknown_node_is_buffered() { + let mut e = ep(1, 7946); + assert!( + e.handle_node_join_intent(7.into(), &3, Instant::ORIGIN), + "the first join intent for an unknown node is buffered" + ); + assert_eq!( + e.test_member_status(3), + None, + "buffering an intent must not create a member" + ); + assert_eq!( + e.test_intent_ltime(3, IntentKind::Join), + Some(LamportTime::new(7)), + "the buffered intent carries the announced ltime" + ); + assert_eq!(e.test_recent_intents_len(), 1); +} + +/// A buffered leave intent that predates the peer's `NodeJoined` is applied at +/// join time: the member materialises as Leaving, not Alive. +#[test] +fn inner_node_joined_applies_a_pending_leave_intent() { + let mut e = ep(1, 7946); + e.test_handle_leave_intent(2, 7.into(), Instant::ORIGIN); + e.test_inner_node_joined(2, Instant::ORIGIN); + + assert_eq!( + e.test_member_status(2), + Some(MemberStatus::Leaving), + "the pending leave intent is reconciled at join time" + ); + assert!( + matches!(e.poll_event(), Some(Event::Member(ref me)) if me.kind() == MemberEventKind::Join), + "the join event is emitted regardless of the reconciled status" + ); +} + +/// The inner SWIM machine declaring a node dead means Failed (recoverable) for an +/// Alive member, but Left (final) for one that was already Leaving. +#[test] +fn inner_node_left_maps_alive_to_failed_and_leaving_to_left() { + let mut e = ep(1, 7946); + e.test_seed_member(2, MemberStatus::Alive, 5.into()); + e.test_seed_member(3, MemberStatus::Leaving, 5.into()); + + e.test_inner_node_left(2, Instant::ORIGIN); + assert_eq!(e.test_member_status(2), Some(MemberStatus::Failed)); + assert!( + matches!(e.poll_event(), Some(Event::Member(ref me)) if me.kind() == MemberEventKind::Failed) + ); + + e.test_inner_node_left(3, Instant::ORIGIN); + assert_eq!(e.test_member_status(3), Some(MemberStatus::Left)); + assert!( + matches!(e.poll_event(), Some(Event::Member(ref me)) if me.kind() == MemberEventKind::Leave) + ); +} + +/// A `NodeUpdated` refreshes the member and emits `Member(Update)`, but must not +/// dirty the push-pull snapshot: tags and address are not in that body, so +/// dirtying would force a wasted resync on every tag change. +#[test] +fn inner_node_updated_emits_update_without_dirtying_local_state() { + let mut e = ep(1, 7946); + e.test_seed_member(2, MemberStatus::Alive, 3.into()); + e.resync_local_state(); + assert!(!e.test_is_dirty()); + + e.test_inner_node_updated(2, Instant::ORIGIN); + + assert!( + !e.test_is_dirty(), + "a tag-only NodeUpdated must not mark the local state dirty" + ); + assert!( + matches!(e.poll_event(), Some(Event::Member(ref me)) if me.kind() == MemberEventKind::Update), + "the Update event is still emitted" + ); +} + +/// The reaper forgets a failed member once `reconnect_timeout` has elapsed and a +/// left member once `tombstone_timeout` has, emitting `Member(Reap)` for each; +/// before the timeout the member is held. +#[test] +fn reaper_forgets_failed_and_left_members_past_their_timeouts() { + let mut e = ep(1, 7946); + e.test_seed_failed_member(2, sa(7000), Instant::ORIGIN); + e.test_seed_left_member(3, 3.into()); + + e.test_fire_reap(t_secs(3600)); + assert_eq!( + e.test_member_status(2), + Some(MemberStatus::Failed), + "one hour is well inside the 24h reconnect_timeout" + ); + + let past = t_secs(3600 * 25); + e.test_fire_reap(past); + assert_eq!(e.test_member_status(2), None, "the failed member is reaped"); + assert_eq!(e.test_member_status(3), None, "the left member is reaped"); + let reaps = core::iter::from_fn(|| e.poll_event()) + .filter(|ev| matches!(ev, Event::Member(me) if me.kind() == MemberEventKind::Reap)) + .count(); + assert_eq!(reaps, 2, "both reaped members surface a Member(Reap)"); +} + +/// The reconnector picks a failed peer and dials it through the coordinator; with +/// one failed and one alive member the probability gate always fires. +#[test] +fn reconnect_dials_a_failed_peer_through_the_coordinator() { + let mut e = ep(1, 7946); + e.test_seed_member(1, MemberStatus::Alive, 1.into()); + e.test_seed_failed_member(2, sa(7000), Instant::ORIGIN); + + e.test_fire_reconnect(Instant::ORIGIN); + + assert_eq!( + e.test_last_dial_addr(), + Some(sa(7000)), + "the reconnect dial targets the failed peer's address" + ); +} + +/// The whole periodic pass runs off `handle_timeout`: the coordinator's SWIM timer +/// fires between serf's pre-tick snapshot resync and serf's post-tick drain, and +/// serf's own reap deadline fires in that post-tick pass. +#[test] +fn handle_timeout_fires_the_serf_reap_deadline() { + let mut e = ep(1, 7946); + e.test_seed_failed_member(2, sa(7000), Instant::ORIGIN); + + e.handle_timeout(t_secs(3600 * 25 + 16)); + + assert_eq!( + e.test_member_status(2), + None, + "handle_timeout must fire serf's reap deadline, not just the coordinator's" + ); +} + +/// An inner `NodeJoined` folded through the tick path materialises the member at +/// the tick's instant. +#[test] +fn inner_joined_injected_at_a_tick_creates_the_member() { + let mut e = ep(1, 7946); + e.test_inject_inner_joined(2, t_secs(5)); + assert_eq!(e.test_member_status(2), Some(MemberStatus::Alive)); + assert!( + matches!(e.poll_event(), Some(Event::Member(ref me)) if me.kind() == MemberEventKind::Join) + ); +} + +// ── user-event ring: dedup, floors, and the gossip packet path ──────────────── + +/// The event ring dedups by `(ltime, name, payload)`: the first sight of an event +/// is emitted and rebroadcast, an exact repeat is dropped silently. +#[test] +fn duplicate_user_event_is_deduped() { + let mut e = ep(1, 7946); + let msg = UserEventMessage { + ltime: 4.into(), + cc: false, + name: "x".into(), + payload: Bytes::from_static(b"p"), + }; + + assert!( + e.handle_user_event(msg.clone()), + "first sight rebroadcasts the event" + ); + assert!(e.poll_event().is_some(), "first sight emits Event::User"); + assert!( + !e.test_handle_user_event(msg), + "an exact duplicate is dropped, not rebroadcast" + ); + assert!( + e.poll_event().is_none(), + "a duplicate emits no second event" + ); + assert_eq!( + e.test_event_slot_len(4), + 1, + "the ring slot holds exactly one copy of the deduped event" + ); +} + +/// An event below the replay floor (`min_time`, set by a snapshot restore or an +/// `ignore_old` join) is dropped without emission. +#[test] +fn user_event_below_the_event_floor_is_dropped() { + let mut e = ep(1, 7946); + e.test_set_event_min_time(10); + assert_eq!(e.test_event_min_time(), 10); + + let stale = UserEventMessage { + ltime: 3.into(), + cc: false, + name: "old".into(), + payload: Bytes::new(), + }; + assert!( + !e.test_handle_user_event(stale), + "an event below the floor must not rebroadcast" + ); + assert!( + e.poll_event().is_none(), + "no event surfaces below the floor" + ); +} + +/// An event whose ltime has fallen out of the ring's window (clock has advanced +/// more than the buffer size past it) is dropped: the ring can no longer prove it +/// is not a duplicate. +#[test] +fn user_event_older_than_the_ring_window_is_dropped() { + let mut e = ep(1, 7946); + e.test_set_event_clock(600); // ring size 512; ltime 0 is 88 slots behind + let stale = UserEventMessage { + ltime: 0.into(), + cc: false, + name: "stale".into(), + payload: Bytes::new(), + }; + assert!( + !e.test_handle_user_event(stale), + "an event older than the ring window must be dropped" + ); + assert!(e.poll_event().is_none()); +} + +/// A gossiped user event arriving on the coordinator's user-packet plane is +/// decoded, deduped, surfaced, and re-queued for onward gossip — the packet path +/// end to end. +#[test] +fn user_event_over_the_gossip_packet_path_surfaces_and_rebroadcasts() { + let mut e = ep(1, 7946); + let frame = AnyMessage::::UserEvent(UserEventMessage { + ltime: 1.into(), + cc: false, + name: "deploy".into(), + payload: Bytes::from_static(b"v3"), + }) + .encode() + .expect("encode the gossiped user event"); + + e.test_inject_user_packet(sa(7000), frame, Instant::ORIGIN); + + let ev = e.poll_event().expect("the gossiped user event surfaces"); + assert!( + matches!(ev, Event::User(ref u) if u.name == "deploy" && u.payload.as_ref() == b"v3"), + "expected the decoded Event::User, got {ev:?}" + ); + assert!( + e.user_broadcast_queue_len() > 0, + "the event is re-queued on the coordinator's broadcast plane" + ); + assert!( + e.event_time() >= 2, + "the inbound event witnesses the local event clock" + ); +} + +/// Undecodable bytes on the gossip user-packet plane are dropped silently — the +/// machine must never panic on hostile input from the network. +#[test] +fn malformed_bytes_on_the_gossip_packet_path_are_dropped() { + let mut e = ep(1, 7946); + e.test_inject_user_packet( + sa(7000), + Bytes::from_static(b"\xff\xfe\xfd"), + Instant::ORIGIN, + ); + assert!( + e.poll_event().is_none(), + "a malformed frame yields no event" + ); + assert_eq!(e.event_time(), 0, "and advances no clock"); +} + +/// A join intent arriving over the gossip packet path is buffered and requeued for +/// onward gossip on the intent tier. +#[test] +fn join_intent_over_the_gossip_packet_path_is_buffered_and_requeued() { + let mut e = ep(1, 7946); + let frame = AnyMessage::::Join(crate::JoinMessage::new(8u64.into(), 2u32)) + .encode() + .expect("encode the gossiped join intent"); + + e.test_inject_user_packet(sa(7000), frame, Instant::ORIGIN); + + assert_eq!( + e.test_intent_ltime(2, IntentKind::Join), + Some(LamportTime::new(8)), + "the gossiped join intent is buffered for the unknown node" + ); + assert!( + e.member_time() >= 9, + "the gossiped intent witnesses the member clock" + ); + assert!( + e.user_broadcast_queue_len() > 0, + "the intent is requeued for onward gossip" + ); +} + +/// The broadcast enqueue helpers ride the coordinator's ranked user-broadcast +/// plane: intents go out at the membership rank, queries at the application rank. +#[test] +fn intent_and_query_broadcasts_reach_the_coordinator_queue() { + let mut e = ep(1, 7946); + assert_eq!(e.user_broadcast_queue_len(), 0, "the queue starts empty"); + + e.test_enqueue_intent_broadcast(Bytes::from_static(b"intent")); + assert_eq!(e.user_broadcast_queue_len(), 1); + + e.test_enqueue_query_broadcast(Bytes::from_static(b"query")); + assert_eq!( + e.user_broadcast_queue_len(), + 2, + "both tiers land on the same coordinator queue" + ); +} + +/// An installed `MessageDropper` drops the selected inbound message class before +/// the machine witnesses its clock — the fault-injection seam a driver test uses +/// to force an out-of-order delivery. +#[test] +fn message_dropper_drops_an_inbound_join_intent_before_it_witnesses_the_clock() { + struct DropJoins; + impl crate::MessageDropper for DropJoins { + fn should_drop(&self, kind: crate::DropKind) -> bool { + matches!(kind, crate::DropKind::Join) + } + } + + let frame = AnyMessage::::Join(crate::JoinMessage::new(8u64.into(), 2u32)) + .encode() + .expect("encode the gossiped join intent"); + + let mut e = ep(1, 7946); + e.core_mut().set_message_dropper(Arc::new(DropJoins)); + e.test_inject_user_packet(sa(7000), frame, Instant::ORIGIN); + + assert_eq!( + e.test_intent_ltime(2, IntentKind::Join), + None, + "the dropped join intent must not be buffered" + ); + assert_eq!( + e.member_time(), + 0, + "a dropped intent must not witness the member clock" + ); +} + +// ── queries: issue, ingress, respond, relay ─────────────────────────────────── + +/// Build a minimal inbound `QueryMessage` from node 99. +fn test_query(ltime: LamportTime, id: u32) -> QueryMessage { + QueryMessage { + ltime, + id, + from: Node::new(99u32, sa(9999)), + filters: vec![], + flags: QueryFlag::empty(), + relay_factor: 0, + timeout: Duration::from_secs(5), + name: "ping".into(), + payload: Bytes::from_static(b"payload"), + } +} + +/// A locally-issued query registers a pending entry, processes locally (surfacing +/// `Event::Query` with the issued name and payload), and rides the coordinator's +/// query broadcast tier. +#[test] +fn query_registers_a_pending_entry_and_surfaces_the_local_query_event() { + let mut e = ep(1, 7946); + let id = e + .query( + "status", + Bytes::from_static(b"probe"), + QueryParams::default(), + Instant::ORIGIN, + ) + .expect("query on an alive endpoint"); + + assert_eq!(e.test_pending_query_count(), 1); + assert_eq!(e.test_last_query_id(), Some(id)); + assert!( + e.user_broadcast_queue_len() > 0, + "the query rides the coordinator's broadcast plane" + ); + + match e + .poll_event() + .expect("the local node processes its own query") + { + Event::Query(q) => { + assert_eq!( + q.name(), + "status", + "the query event carries the issued name" + ); + assert_eq!( + q.payload().as_ref(), + b"probe", + "the query event carries the issued payload" + ); + assert_eq!(q.ltime(), id.ltime); + } + other => panic!("expected Event::Query, got {other:?}"), + } +} + +/// An inbound query witnesses the query clock, registers an answerable token, and +/// surfaces as `Event::Query`; a repeat of the same `(ltime, id)` is deduped. +#[test] +fn inbound_query_witnesses_the_clock_and_dedups_by_id() { + let mut e = ep(1, 7946); + + assert!( + e.test_handle_query(test_query(3.into(), 7)), + "a first-sight query is rebroadcast" + ); + assert!(e.query_time() >= 4, "the query clock witnessed ltime 3"); + assert_eq!(e.test_received_queries_len(), 1, "the token is answerable"); + assert_eq!(e.test_query_slot_len(3), 1); + assert!(matches!(e.poll_event(), Some(Event::Query(_)))); + + assert!( + !e.test_handle_query(test_query(3.into(), 7)), + "the same (ltime, id) is deduped and not rebroadcast" + ); + assert!(e.poll_event().is_none(), "a duplicate query emits no event"); +} + +/// An answerable token is removed once responded, and the second `respond` on the +/// same token is refused — a query is answered exactly once. +#[test] +fn respond_directed_sends_once_then_refuses() { + let mut e = ep(1, 7946); + let qid = QueryId { + ltime: LamportTime::new(2), + id: 9, + }; + let token = e.test_register_received_query(qid, sa(1002), t_secs(10)); + assert!(!e.test_is_responded(qid), "the token starts unanswered"); + + e.respond(&token, Bytes::from_static(b"ok"), Instant::ORIGIN) + .expect("the first respond succeeds"); + + let (dest, bytes) = e + .test_last_directed_send() + .expect("respond directed-sends the response to the querier"); + assert_eq!(dest, sa(1002), "the response goes to the querier's address"); + assert!(!bytes.is_empty(), "the response frame is non-empty"); + assert_eq!( + e.test_received_queries_len(), + 0, + "a successful respond consumes the token" + ); + + assert!( + matches!( + e.respond(&token, Bytes::new(), Instant::ORIGIN), + Err(crate::endpoint::Error::AlreadyResponded) + ), + "a second respond on the same token is refused" + ); +} + +/// A response past the query's deadline is refused: the querier has already closed +/// the query, so the answer would be wasted wire. +#[test] +fn respond_after_the_deadline_is_refused() { + let mut e = ep(1, 7946); + let token = e.test_register_received_query( + QueryId { + ltime: LamportTime::new(1), + id: 5, + }, + sa(1002), + t_secs(10), + ); + assert!( + e.respond(&token, Bytes::new(), t_secs(11)).is_err(), + "a respond past the deadline must be refused" + ); +} + +/// Expired answerable tokens are pruned by the periodic pass, and the pruned +/// deadlines are exactly the ones that elapsed. +#[test] +fn expired_received_queries_are_pruned_on_the_tick() { + let mut e = ep(1, 7946); + e.test_register_received_query( + QueryId { + ltime: LamportTime::new(1), + id: 1, + }, + sa(1002), + t_secs(5), + ); + e.test_register_received_query( + QueryId { + ltime: LamportTime::new(1), + id: 2, + }, + sa(1002), + t_secs(100), + ); + assert_eq!(e.test_received_queries_len(), 2); + assert_eq!(e.test_peek_received_query_deadlines().len(), 2); + + e.handle_timeout(t_secs(6)); + + assert_eq!( + e.test_received_queries_len(), + 1, + "only the elapsed token is pruned" + ); + assert_eq!( + e.test_peek_received_query_deadlines(), + vec![t_secs(100)], + "the surviving token keeps its deadline" + ); +} + +/// A response to a query this node originated surfaces as `Event::QueryResponse` +/// carrying the responder and its payload; a second response from the same node is +/// deduped. +#[test] +fn query_response_surfaces_with_the_responder_and_payload() { + let mut e = ep(1, 7946); + let id = e + .query( + "ping", + Bytes::new(), + QueryParams::default(), + Instant::ORIGIN, + ) + .expect("query on an alive endpoint"); + let _ = e.poll_event(); // drain the local Event::Query + + let resp = QueryResponseMessage { + ltime: id.ltime, + id: id.id, + from: Node::new(2u32, sa(1002)), + flags: QueryFlag::empty(), + payload: Bytes::from_static(b"pong"), + }; + e.test_handle_query_response(resp.clone()); + + match e.poll_event().expect("the response surfaces") { + Event::QueryResponse(qr) => { + assert_eq!(qr.id(), id.id, "the response carries the query id"); + assert_eq!(*qr.from().id_ref(), 2u32, "and the responder node"); + assert_eq!(qr.payload().as_ref(), b"pong", "and its payload"); + } + other => panic!("expected Event::QueryResponse, got {other:?}"), + } + assert_eq!( + e.test_pending_query_response_count(id), + 1, + "the responder is tallied once" + ); + + e.test_handle_query_response(resp); + assert!( + e.poll_event().is_none(), + "a second response from the same node is deduped" + ); + assert_eq!( + e.test_pending_query_response_count(id), + 1, + "the duplicate does not inflate the tally" + ); +} + +/// A query issued with `request_ack` sets the ACK flag on the wire, and a peer's +/// ack surfaces as `Event::QueryAck` naming the acking node. +#[test] +fn ack_requested_query_surfaces_the_peer_ack() { + let mut e = ep(1, 7946); + let params = QueryParams:: { + request_ack: true, + ..Default::default() + }; + let id = e + .query("ping", Bytes::new(), params, Instant::ORIGIN) + .expect("query on an alive endpoint"); + let _ = e.poll_event(); // drain the local Event::Query + + let ack = QueryResponseMessage { + ltime: id.ltime, + id: id.id, + from: Node::new(2u32, sa(1002)), + flags: QueryFlag::ACK, + payload: Bytes::new(), + }; + e.test_handle_query_response(ack.clone()); + + match e.poll_event().expect("the ack surfaces") { + Event::QueryAck(a) => { + assert_eq!(a.id(), id.id, "the ack carries the query id"); + assert_eq!(*a.from().id_ref(), 2u32, "and the acking node"); + } + other => panic!("expected Event::QueryAck, got {other:?}"), + } + + e.test_handle_query_response(ack); + assert!( + e.poll_event().is_none(), + "a duplicate ack from the same node is deduped" + ); +} + +/// A responder-side relay picks alive peers and directed-sends the wrapped frame +/// to each; the local node is never a relay hop. +#[test] +fn relay_response_directed_sends_through_alive_peers() { + let mut e = ep(1, 7946); + e.test_seed_member_at(10u32, sa(1010), MemberStatus::Alive, 1.into()); + e.test_seed_member_at(11u32, sa(1011), MemberStatus::Alive, 1.into()); + + e.test_relay_response( + Node::new(2000u32, sa(2000)), + Bytes::from_static(b"\x06relay-payload"), + 1, + ); + + let sends = e.test_relay_all_directed_sends(); + assert_eq!(sends.len(), 1, "relay_factor 1 picks exactly one hop"); + let (dest, bytes) = &sends[0]; + assert!( + *dest == sa(1010) || *dest == sa(1011), + "the relay hop must be one of the alive peers, got {dest}" + ); + assert!(!bytes.is_empty(), "the relay-wrapped frame is non-empty"); + assert!(e.poll_event().is_none(), "a delivered relay drops nothing"); +} + +/// A relay with fewer members than hops requested is a silent no-op — not a +/// `RelayDropped`, which is reserved for a real delivery failure. +#[test] +fn relay_response_with_too_few_members_is_a_silent_noop() { + let mut e = ep(1, 7946); + e.test_relay_response(Node::new(2000u32, sa(2000)), Bytes::from_static(b"f"), 2); + assert!( + e.poll_event().is_none(), + "too few members must not emit RelayDropped" + ); + assert!( + e.test_last_directed_send().is_none(), + "and must not send anything" + ); +} + +/// A relayed frame arriving at an intermediary is forwarded VERBATIM to the named +/// destination — the relay must not re-encode the inner response. +#[test] +fn handle_relay_forwards_the_inner_frame_verbatim() { + let mut e = ep(1, 7946); + let inner = Bytes::from_static(b"\x06inner-qresp"); + e.test_handle_relay(RelayMessage::new(Node::new(2u32, sa(1002)), inner.clone())); + + let (dest, sent) = e + .test_last_directed_send() + .expect("the relay forwards to the destination"); + assert_eq!(dest, sa(1002), "forwarded to the wrapped destination"); + assert_eq!(sent, inner, "the payload is forwarded verbatim"); + assert!(e.poll_event().is_none(), "a delivered relay emits no event"); +} + +/// A relay addressed at the local node cannot be forwarded; the machine surfaces +/// the undeliverable hop as `Event::RelayDropped` rather than discarding it. +#[test] +fn relay_to_self_surfaces_relay_dropped() { + let mut e = ep(1, 7946); + e.test_handle_relay(RelayMessage::new( + Node::new(1u32, sa(7946)), + Bytes::from_static(b"x"), + )); + + match e.poll_event().expect("a self-relay must surface") { + Event::RelayDropped(d) => assert_eq!( + *d.destination(), + sa(7946), + "the dropped relay names the destination it could not reach" + ), + other => panic!("expected Event::RelayDropped, got {other:?}"), + } +} + +// ── id-conflict resolution ──────────────────────────────────────────────────── + +/// Drive `e` into the Shutdown state by losing an id-conflict vote (1 agreeing +/// response against 2 disagreeing), and drain the resulting `Event::Shutdown`. +fn shut_down_via_lost_conflict(e: &mut QuicEndpoint) { + let deadline = t_secs(3600); + let qid = e.test_register_conflict_query(deadline); + e.test_fold_conflict_response(qid, 200u32, true); + e.test_fold_conflict_response(qid, 201u32, false); + e.test_fold_conflict_response(qid, 202u32, false); + e.test_fire_due_query_closes(t_secs(3601)); + assert!(matches!(e.poll_event(), Some(Event::Shutdown))); + assert!(e.state().is_shutdown()); +} + +/// A conflict vote the local node WINS (a majority of responders agree the id is +/// ours) leaves the machine Alive with its command surface open. +#[test] +fn winning_the_conflict_vote_leaves_the_machine_alive() { + let mut e = ep(1, 7946); + let qid = e.test_register_conflict_query(t_secs(3600)); + e.test_fold_conflict_response(qid, 100u32, true); + e.test_fold_conflict_response(qid, 101u32, true); + e.test_fold_conflict_response(qid, 102u32, false); + assert_eq!( + e.test_pending_query_conflict_matching(qid), + Some(2), + "two of the three responders agree the id is ours" + ); + + e.test_fire_due_query_closes(t_secs(3601)); + + assert!(e.poll_event().is_none(), "a won vote emits no Shutdown"); + assert!(e.state().is_alive(), "and leaves the machine Alive"); + assert!( + e.user_event("post-win", Bytes::new(), false, Instant::ORIGIN) + .is_ok(), + "a won vote does not gate the command surface" + ); +} + +/// A conflict vote the local node LOSES forces the documented Alive -> Shutdown +/// transition, and the buffered `Event::Shutdown` still drains from the dead +/// machine. +#[test] +fn losing_the_conflict_vote_shuts_the_machine_down() { + let mut e = ep(1, 7946); + assert!(e.state().is_alive()); + shut_down_via_lost_conflict(&mut e); + assert!( + e.state().is_shutdown(), + "the machine stays Shutdown after the event drains" + ); +} + +/// Every command that originates cluster work is refused on a shut-down machine, +/// and the lifecycle commands keep their own typed state errors. +#[test] +fn shutdown_refuses_the_originating_commands() { + use crate::endpoint::Error; + + let mut e = ep(1, 7946); + // A live token registered BEFORE the shutdown, so respond's refusal is proven + // to precede its token lookup. + let token = e.test_register_received_query( + QueryId { + ltime: LamportTime::new(1), + id: 5, + }, + sa(1002), + t_secs(3600), + ); + + shut_down_via_lost_conflict(&mut e); + + let now = Instant::ORIGIN; + let tags: Tags = [("role", "web")].into_iter().collect(); + + assert!(matches!( + e.user_event("x", Bytes::new(), false, now), + Err(Error::Shutdown) + )); + assert!(matches!( + e.query("q", Bytes::new(), QueryParams::default(), now), + Err(Error::Shutdown) + )); + assert!(matches!(e.set_tags(tags, now), Err(Error::Shutdown))); + assert!(matches!( + e.respond(&token, Bytes::new(), now), + Err(Error::Shutdown) + )); + assert!(matches!( + e.load_snapshot( + crate::snapshot::ReplayResult { + alive_nodes: vec![], + last_clock: 1.into(), + last_event_clock: 1.into(), + last_query_clock: 1.into(), + }, + now + ), + Err(Error::Shutdown) + )); + assert!(matches!(e.join(), Err(Error::BadJoinState(_)))); + assert!(matches!(e.leave(now), Err(Error::BadLeaveState(_)))); + assert!(matches!( + e.force_leave(2, false, now), + Err(Error::BadLeaveState(_)) + )); +} + +/// Build a `_serf_conflict` query naming `conflict_id` — the internal query a +/// node broadcasts when the inner machine reports two peers claiming one id. +fn conflict_query(conflict_id: u32) -> QueryMessage { + use memberlist_proto::Data; + + let mut q = test_query(3.into(), 11); + q.name = "_serf_conflict".into(); + q.payload = conflict_id + .encode_to_bytes() + .expect("encode the conflicting id"); + q +} + +/// A `_serf_conflict` query naming a member this node knows is answered +/// autonomously: the machine sends its view of the conflicting node straight back +/// to the originator on the gossip plane, the answerable token is consumed (the +/// driver is never asked to respond), and the query never surfaces to the +/// application. +#[test] +fn conflict_query_is_answered_autonomously_and_never_surfaces() { + let mut e = ep(1, 7946); + e.test_seed_member_at(2u32, sa(1002), MemberStatus::Alive, 1.into()); + + e.test_handle_query(conflict_query(2)); + + let transmit = e + .poll_memberlist_transmit() + .expect("the conflict query is answered directly to the originator"); + match transmit { + memberlist_proto::Transmit::Packet(p) => { + assert_eq!( + *p.to_ref(), + sa(9999), + "the conflict response goes back to the querier" + ); + assert!( + matches!(p.message_ref(), Message::UserData(b) if !b.is_empty()), + "the conflict response rides the serf user-data plane" + ); + } + other => panic!("expected a packet transmit, got {other:?}"), + } + assert_eq!( + e.test_received_queries_len(), + 0, + "an autonomously-answered query leaves no token for the driver" + ); + assert!( + e.poll_event().is_none(), + "an internal conflict query must never surface as Event::Query" + ); + assert!( + e.query_time() > 3, + "the conflict query still witnesses the query clock" + ); +} + +/// A `_serf_conflict` query naming a node this machine does not track is left +/// unanswered — the machine has no view to report. +#[test] +fn conflict_query_for_an_unknown_node_is_unanswered() { + let mut e = ep(1, 7946); + e.test_handle_query(conflict_query(77)); + + assert!( + e.poll_memberlist_transmit().is_none(), + "an unknown conflicting id yields no response" + ); + assert!(e.poll_event().is_none()); +} + +/// The originator of a conflict query does not vote in its own conflict: a +/// `_serf_conflict` naming the LOCAL id is processed but never answered. +#[test] +fn conflict_query_naming_the_local_node_is_not_answered() { + let mut e = ep(1, 7946); + e.test_handle_query(conflict_query(1)); + + assert!( + e.poll_memberlist_transmit().is_none(), + "a node must not respond to a conflict query about itself" + ); + assert!(e.poll_event().is_none()); +} + +/// A due query closes on its deadline while a query whose deadline has not yet +/// elapsed is left pending — the close pass must not sweep the whole list. +#[test] +fn only_due_queries_close_on_the_deadline_pass() { + let mut e = ep(1, 7946); + // The synthetic registration stamps the query id from the live query clock, so + // advance it between registrations to get two distinct pending queries. + e.test_set_clocks(0, 0, 1); + let due = e.test_register_conflict_query(t_secs(10)); + e.test_set_clocks(0, 0, 2); + let not_due = e.test_register_conflict_query(t_secs(1000)); + assert_ne!(due, not_due, "the two registrations are distinct queries"); + assert_eq!(e.test_pending_query_count(), 2); + + e.test_fire_due_query_closes(t_secs(11)); + + assert_eq!( + e.test_pending_query_count(), + 1, + "only the elapsed query is closed" + ); + assert_eq!( + e.test_pending_query_conflict_matching(due), + None, + "the closed query is gone" + ); + assert_eq!( + e.test_pending_query_conflict_matching(not_due), + Some(0), + "the un-elapsed query is still pending" + ); +} + +// ── key management ──────────────────────────────────────────────────────────── + +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +mod keys { + use super::*; + use crate::{ + KeyRequestMessage, + event::{KeyRequestOperation, KeyResponseArgs}, + }; + use memberlist_proto::SecretKey; + + #[cfg(feature = "aes-gcm")] + fn test_key() -> SecretKey { + SecretKey::Aes128([7u8; 16]) + } + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + fn test_key() -> SecretKey { + SecretKey::ChaCha20Poly1305([7u8; 32]) + } + + /// Each key-management command issues an internal query under its own reserved + /// name and registers a pending entry whose `num_nodes` denominator is the + /// member count captured at issue time. + #[test] + fn key_commands_issue_internal_queries() { + let mut e = ep(1, 7946); + e.test_seed_member(2, MemberStatus::Alive, 1.into()); + + let install = e + .install_key(test_key(), 0, Instant::ORIGIN) + .expect("install_key issues a query"); + assert_eq!(e.test_pending_query_count(), 1); + assert_eq!( + e.test_last_pending_query_num_nodes(), + Some(2), + "the denominator is the member count at issue time" + ); + assert!( + e.user_broadcast_queue_len() > 0, + "the key query rides the broadcast plane" + ); + + let use_id = e.use_key(test_key(), 0, Instant::ORIGIN).expect("use_key"); + let remove_id = e + .remove_key(test_key(), 0, Instant::ORIGIN) + .expect("remove_key"); + let list_id = e.list_keys(0, Instant::ORIGIN).expect("list_keys"); + + assert_eq!( + e.test_pending_query_count(), + 4, + "each key command registers its own pending query" + ); + let mut ids = vec![install.id, use_id.id, remove_id.id, list_id.id]; + ids.sort_unstable(); + ids.dedup(); + assert_eq!(ids.len(), 4, "each key query gets a distinct id"); + } + + /// An inbound key query surfaces as `Event::KeyRequest` — never as + /// `Event::Query` — carrying the operation and the key it names, and leaves an + /// answerable token behind for the driver's `respond_key`. + #[test] + fn inbound_key_query_surfaces_as_a_key_request() { + let mut e = ep(1, 7946); + let payload = + AnyMessage::::KeyRequest(KeyRequestMessage::new(Some(test_key()))) + .encode() + .expect("encode the key request"); + + let mut q = test_query(1.into(), 55); + q.name = "_serf_install_key".into(); + q.payload = payload; + e.test_handle_query(q); + + assert_eq!( + e.test_received_queries_len(), + 1, + "the key query leaves an answerable token" + ); + match e.poll_event().expect("the key query surfaces") { + Event::KeyRequest(kr) => { + assert_eq!(kr.op(), KeyRequestOperation::Install); + assert_eq!( + kr.key(), + Some(&test_key()), + "the install request carries the key to install" + ); + assert_eq!(*kr.from().id_ref(), 99u32, "and names the originator"); + assert_eq!( + kr.id(), + 55, + "the request carries the query id respond_key routes the answer on" + ); + assert_eq!( + kr.deadline(), + Instant::ORIGIN + Duration::from_secs(5), + "and the deadline the answer must beat" + ); + } + other => panic!("expected Event::KeyRequest, got {other:?}"), + } + assert!( + e.poll_event().is_none(), + "a key query must never also surface as Event::Query" + ); + } + + /// `list_keys` carries no key; the surfaced request reflects that, and the + /// operation's own metadata agrees. + #[test] + fn list_keys_query_surfaces_without_a_key() { + let mut e = ep(1, 7946); + let payload = AnyMessage::::KeyRequest(KeyRequestMessage::new(None)) + .encode() + .expect("encode the list-keys request"); + + let mut q = test_query(1.into(), 56); + q.name = "_serf_list_keys".into(); + q.payload = payload; + e.test_handle_query(q); + + match e.poll_event().expect("the list-keys query surfaces") { + Event::KeyRequest(kr) => { + assert_eq!(kr.op(), KeyRequestOperation::List); + assert!(kr.key().is_none(), "a list request carries no key"); + assert!( + !kr.op().has_key(), + "the List operation declares that it carries no key" + ); + } + other => panic!("expected Event::KeyRequest, got {other:?}"), + } + } + + /// `respond_key` answers the surfaced request: the token is consumed and the + /// key response is directed-sent back to the originator. + #[test] + fn respond_key_answers_the_originator_and_consumes_the_token() { + let mut e = ep(1, 7946); + let payload = + AnyMessage::::KeyRequest(KeyRequestMessage::new(Some(test_key()))) + .encode() + .expect("encode the key request"); + let mut q = test_query(1.into(), 55); + q.name = "_serf_install_key".into(); + q.payload = payload; + e.test_handle_query(q); + + let req = match e.poll_event().expect("the key query surfaces") { + Event::KeyRequest(kr) => kr, + other => panic!("expected Event::KeyRequest, got {other:?}"), + }; + + e.respond_key( + &req, + KeyResponseArgs { + result: true, + message: smol_str::SmolStr::default(), + keys: vec![test_key()], + primary_key: Some(test_key()), + }, + Instant::ORIGIN, + ) + .expect("respond_key succeeds within the deadline"); + + assert_eq!( + e.test_received_queries_len(), + 0, + "respond_key consumes the answerable token" + ); + let (dest, bytes) = e + .test_last_directed_send() + .expect("respond_key directed-sends the key response"); + assert_eq!(dest, sa(9999), "the response goes back to the originator"); + assert!(!bytes.is_empty()); + } + + /// The responses to a key query are folded into a tally and reported once, at + /// the query's close: the installed-key counts, the primary-key report, and each + /// failing node's message all reach `Event::KeyResponse`. + #[test] + fn key_query_responses_fold_into_the_closing_tally() { + use crate::KeyResponseMessage; + + let mut e = ep(1, 7946); + e.test_seed_member(2, MemberStatus::Alive, 1.into()); + e.test_seed_member(3, MemberStatus::Alive, 1.into()); + + let qid = e.test_register_key_query(t_secs(10)); + + // Node 2 succeeded and reports the key ring; node 3 failed with a message. + let ok = AnyMessage::::KeyResponse(KeyResponseMessage { + result: true, + message: smol_str::SmolStr::default(), + keys: vec![test_key()], + primary_key: Some(test_key()), + }) + .encode() + .expect("encode the successful key response"); + let failed = AnyMessage::::KeyResponse(KeyResponseMessage { + result: false, + message: "no such key".into(), + keys: vec![], + primary_key: None, + }) + .encode() + .expect("encode the failing key response"); + + e.test_handle_query_response(QueryResponseMessage { + ltime: qid.ltime, + id: qid.id, + from: Node::new(2u32, sa(1002)), + flags: QueryFlag::empty(), + payload: ok, + }); + e.test_handle_query_response(QueryResponseMessage { + ltime: qid.ltime, + id: qid.id, + from: Node::new(3u32, sa(1003)), + flags: QueryFlag::empty(), + payload: failed, + }); + + e.test_fire_due_query_closes(t_secs(11)); + + match e.poll_event().expect("the closing key query reports") { + Event::KeyResponse(kr) => { + assert_eq!(kr.num_resp, 2, "both responders are counted"); + assert_eq!(kr.num_err, 1, "one responder reported a failure"); + assert_eq!( + kr.keys.get(&test_key()).copied(), + Some(1), + "one node reported holding the key" + ); + assert_eq!( + kr.primary_keys.get(&test_key()).copied(), + Some(1), + "one node reported the key as primary" + ); + assert_eq!( + kr.messages.get(&3u32).map(|m| m.as_str()), + Some("no such key"), + "the failing node's message is carried under its id" + ); + } + other => panic!("expected Event::KeyResponse, got {other:?}"), + } + } + + /// A key operation issued with a relay factor rides that factor on the wire, so + /// responders relay their answers back through intermediaries. + #[test] + fn key_op_relay_factor_rides_the_issued_query() { + let mut e = ep(1, 7946); + e.install_key(test_key(), 3, Instant::ORIGIN) + .expect("install_key issues a query"); + + match e + .poll_event() + .expect("the local node processes its own key query") + { + Event::KeyRequest(kr) => assert_eq!( + kr.relay_factor(), + 3, + "the issued relay factor rides the key query" + ), + other => panic!("expected Event::KeyRequest, got {other:?}"), + } + } + + /// The key-request operation names are stable wire vocabulary; each maps to its + /// own string and only `List` carries no key. + #[test] + fn key_request_operations_map_to_their_wire_names() { + assert_eq!(KeyRequestOperation::Install.as_str(), "install"); + assert_eq!(KeyRequestOperation::Use.as_str(), "use"); + assert_eq!(KeyRequestOperation::Remove.as_str(), "remove"); + assert_eq!(KeyRequestOperation::List.as_str(), "list"); + + assert!(KeyRequestOperation::Install.has_key()); + assert!(KeyRequestOperation::Use.has_key()); + assert!(KeyRequestOperation::Remove.has_key()); + assert!(!KeyRequestOperation::List.has_key()); + } +} + +// ── gossip-plane encryption ─────────────────────────────────────────────────── + +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +mod encryption { + use super::*; + use memberlist_proto::{EncryptionOptions, Keyring, SecretKey}; + + #[cfg(feature = "aes-gcm")] + fn key(b: u8) -> SecretKey { + SecretKey::Aes128([b; 16]) + } + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + fn key(b: u8) -> SecretKey { + SecretKey::ChaCha20Poly1305([b; 32]) + } + + /// With no keyring configured the gossip transforms are the identity: a datagram + /// goes out as plaintext and comes back unchanged. + #[test] + fn gossip_transforms_are_the_identity_without_a_keyring() { + let e = ep(1, 7946); + assert!( + !e.encryption_options().is_enabled(), + "a fresh coordinator has no keyring" + ); + + let plain = b"\x05gossip-frame"; + let out = e.encrypt_gossip(plain).expect("encrypt without a keyring"); + assert_eq!( + out.as_slice(), + plain, + "with no keyring the datagram is left unencrypted" + ); + assert_eq!( + e.decrypt_gossip(&out).expect("decrypt without a keyring"), + plain, + "and the inbound transform is the identity too" + ); + } + + /// Installing a keyring re-keys the live gossip plane: the reported key state and + /// the bytes on the wire move together, and an encrypted datagram round-trips + /// through the same keyring. + #[test] + fn setting_the_encryption_options_re_keys_the_live_gossip_plane() { + let mut e = ep(1, 7946); + e.set_encryption_options(EncryptionOptions::new().with_keyring(Keyring::new(key(1)))); + + assert!( + e.encryption_options().is_enabled(), + "the coordinator reports the installed keyring" + ); + assert_eq!( + e.encryption_options().keyring().map(|kr| *kr.primary_ref()), + Some(key(1)), + "and reports the primary key that outbound datagrams seal under" + ); + + let plain = b"\x05gossip-frame"; + let sealed = e.encrypt_gossip(plain).expect("encrypt under the keyring"); + assert_ne!( + sealed.as_slice(), + plain, + "an encrypted-cluster datagram must not leave as plaintext" + ); + assert_eq!( + e.decrypt_gossip(&sealed) + .expect("decrypt under the keyring"), + plain, + "the datagram round-trips through the same keyring" + ); + } + + /// A datagram sealed under one keyring is NOT readable by a node holding a + /// different key: the driver must drop the frame rather than admit it. + #[test] + fn a_datagram_sealed_under_a_foreign_key_fails_to_decrypt() { + let mut sender = ep(1, 7946); + sender.set_encryption_options(EncryptionOptions::new().with_keyring(Keyring::new(key(1)))); + let sealed = sender + .encrypt_gossip(b"\x05gossip-frame") + .expect("encrypt under the sender's keyring"); + + let mut receiver = ep(2, 7000); + receiver.set_encryption_options(EncryptionOptions::new().with_keyring(Keyring::new(key(2)))); + + assert!( + receiver.decrypt_gossip(&sealed).is_err(), + "a frame the keyring cannot open must be an error, never silently admitted" + ); + } +} + +// ── coordinates ─────────────────────────────────────────────────────────────── + +/// A completed probe round-trip feeds the RTT and the peer's piggybacked +/// coordinate into the local Vivaldi model and caches the remote coordinate. +#[cfg(feature = "coordinates")] +#[test] +fn ping_completed_updates_the_local_model_and_caches_the_remote_coordinate() { + use crate::bridge::coordinate_to_pb; + use buffa::Message as _; + + let mut e = ep(1, 7946); + let peer = crate::typed::Coordinate { + vec: vec![5.0; 8], + error: 1.0, + adjustment: 0.0, + height: 0.0, + }; + let encoded = coordinate_to_pb(&peer).encode_to_vec(); + let mut payload = Vec::with_capacity(1 + encoded.len()); + payload.push(1u8); // PING_VERSION + payload.extend_from_slice(&encoded); + + e.test_ping_completed(2u32, Duration::from_millis(40), Bytes::from(payload)); + + assert!( + e.cached_coordinate(&2u32).is_some(), + "the peer's coordinate is cached under its id" + ); + assert!( + e.get_coordinate().is_some(), + "the local Vivaldi model is updated by the round-trip" + ); +} + +/// A probe payload with the wrong version byte, or an empty one, is dropped: the +/// coordinate plane never trusts an unrecognised wire format. +#[cfg(feature = "coordinates")] +#[test] +fn ping_completed_with_an_unusable_payload_is_a_noop() { + let mut e = ep(1, 7946); + e.test_ping_completed( + 2u32, + Duration::from_millis(10), + Bytes::from_static(b"\x02junk"), + ); + assert!( + e.cached_coordinate(&2u32).is_none(), + "a bad version byte must not update the cache" + ); + + e.test_ping_completed(3u32, Duration::from_millis(10), Bytes::new()); + assert!( + e.cached_coordinate(&3u32).is_none(), + "an empty payload must not update the cache" + ); +} + +/// Coordinates disabled at construction means no Vivaldi client at all: the local +/// coordinate, the cache and the reset counter are all absent, and a completed +/// probe round-trip changes nothing. +#[cfg(feature = "coordinates")] +#[test] +fn coordinates_disabled_at_construction_leaves_no_vivaldi_client() { + let mut e = ep_with_options(1, 7946, Options::new().with_disable_coordinates(true)); + + assert_eq!( + e.get_coordinate(), + None, + "a coordinates-disabled machine reports no local coordinate" + ); + assert_eq!( + e.coordinate_resets(), + None, + "and no reset counter to read through to" + ); + + e.test_ping_completed( + 2u32, + Duration::from_millis(40), + Bytes::from_static(b"\x01x"), + ); + assert_eq!( + e.cached_coordinate(&2u32), + None, + "a probe round-trip caches nothing when coordinates are disabled" + ); + assert!(e.poll_event().is_none(), "and emits no event"); +} + +/// A reaped member's coordinate is purged from the cache — a forgotten node must +/// not keep influencing the coordinate plane. +#[cfg(feature = "coordinates")] +#[test] +fn reaping_a_member_forgets_its_coordinate() { + use crate::bridge::coordinate_to_pb; + use buffa::Message as _; + + let mut e = ep(1, 7946); + let peer = crate::typed::Coordinate { + vec: vec![1.0; 8], + error: 0.5, + adjustment: 0.0, + height: 0.0, + }; + let encoded = coordinate_to_pb(&peer).encode_to_vec(); + let mut payload = Vec::with_capacity(1 + encoded.len()); + payload.push(1u8); + payload.extend_from_slice(&encoded); + e.test_ping_completed(42u32, Duration::from_millis(20), Bytes::from(payload)); + assert!(e.cached_coordinate(&42u32).is_some()); + + e.test_seed_failed_member(42u32, sa(7947), Instant::ORIGIN); + e.test_fire_reap(t_secs(90_000)); + + assert!( + e.cached_coordinate(&42u32).is_none(), + "the reaped member's coordinate must be purged from the cache" + ); +} + +// ── coalescing ──────────────────────────────────────────────────────────────── + +/// With member coalescing enabled the membership changes are buffered in the +/// window rather than delivered one by one: `pending_events_len` reports the +/// buffered depth, and the batch flushes as ONE `Member(Join)` at the quiescent +/// deadline. +#[test] +fn coalesced_member_changes_batch_into_one_event_at_the_flush_deadline() { + let mut e = ep_with_options( + 1, + 7946, + Options::new() + .with_coalesce_period(Duration::from_secs(10)) + .with_quiescent_period(Duration::from_secs(2)), + ); + e.handle_timeout(t_secs(100)); + while e.poll_event().is_some() {} + + e.test_inject_inner_joined(2, t_secs(100)); + e.test_inject_inner_joined(3, t_secs(100)); + assert_eq!( + e.pending_events_len(), + 0, + "coalesced joins are held in the window, not queued for delivery" + ); + assert!( + e.poll_event().is_none(), + "nothing is delivered before the flush deadline" + ); + + e.handle_timeout(t_secs(102)); + match e.poll_event().expect("the window flushes at its deadline") { + Event::Member(me) => { + assert_eq!(me.kind(), MemberEventKind::Join); + let mut ids: Vec = me.members().iter().map(|m| *m.node().id_ref()).collect(); + ids.sort_unstable(); + assert_eq!(ids, vec![2, 3], "both joins flush as one batched event"); + } + other => panic!("expected the coalesced Member(Join) batch, got {other:?}"), + } + assert!( + e.poll_event().is_none(), + "the batch is delivered exactly once" + ); +} + +/// A join intent whose Lamport time is newer than the member's recorded status +/// time clears a `Leaving` member back to Alive; a stale one is ignored. +#[test] +fn join_intent_clears_a_leaving_member_and_a_stale_one_is_ignored() { + let mut e = ep(1, 7946); + e.test_seed_member(2, MemberStatus::Leaving, 5.into()); + + assert!( + e.test_handle_join_intent(2, 8.into(), Instant::ORIGIN), + "a fresh join intent for a Leaving member is rebroadcast" + ); + assert_eq!( + e.test_member_status(2), + Some(MemberStatus::Alive), + "the fresh join intent clears the member back to Alive" + ); + + assert!( + !e.test_handle_join_intent(2, 3.into(), Instant::ORIGIN), + "a stale join intent is not rebroadcast" + ); + assert_eq!( + e.test_member_status_time(2), + Some(LamportTime::new(8)), + "and does not regress the member's status time" + ); +} + +/// A tag filter scopes a query to the members whose tag value matches: a node +/// whose tags do not satisfy the filter must not surface the query locally. +#[cfg(feature = "tag-regex")] +#[test] +fn tag_filtered_query_is_not_surfaced_by_a_non_matching_node() { + use crate::typed::{Filter, TagFilter}; + + let mut e = ep(1, 7946); + let tags: Tags = [("role", "db")].into_iter().collect(); + e.test_seed_member_with_tags(1, tags, MemberStatus::Alive, 1.into()); + + let mut q = test_query(3.into(), 21); + q.filters = vec![Filter::Tag(TagFilter { + tag: "role".into(), + expr: Some("web".into()), + })]; + + assert!( + e.test_handle_query(q), + "a filtered-out query is still rebroadcast so matching peers see it" + ); + assert!( + e.poll_event().is_none(), + "a node whose tags do not match the filter must not surface the query" + ); +} + +/// The same tag filter DOES surface on a node whose tag value matches — the +/// contrast that proves the filter is evaluated, not merely dropped. +#[cfg(feature = "tag-regex")] +#[test] +fn tag_filtered_query_is_surfaced_by_a_matching_node() { + use crate::typed::{Filter, TagFilter}; + + let mut e = ep(1, 7946); + let tags: Tags = [("role", "web")].into_iter().collect(); + e.test_seed_member_with_tags(1, tags, MemberStatus::Alive, 1.into()); + + let mut q = test_query(3.into(), 22); + q.filters = vec![Filter::Tag(TagFilter { + tag: "role".into(), + expr: Some("web".into()), + })]; + + assert!(e.test_handle_query(q)); + assert!( + matches!(e.poll_event(), Some(Event::Query(_))), + "a node whose tag matches the filter surfaces the query" + ); +} + +/// The ignore-join set is a plain per-exchange set: an id can be recorded and +/// cleared directly, independent of a live dial. +#[test] +fn the_ignore_join_set_records_and_clears_an_exchange_id() { + let mut e = ep(1, 7946); + let id = e.start_push_pull(sa(7000), PushPullKind::Join, Instant::ORIGIN); + + assert!(!e.test_has_ignore_join_stream(id)); + e.test_note_ignore_join_stream(id); + assert!( + e.test_has_ignore_join_stream(id), + "the exchange id is recorded" + ); + e.test_clear_ignore_join_stream(id); + assert!(!e.test_has_ignore_join_stream(id), "and cleared again"); +} diff --git a/serf-proto/src/stream_endpoint/tests.rs b/serf-proto/src/stream_endpoint/tests.rs index 95451c1a..5d6e12bf 100644 --- a/serf-proto/src/stream_endpoint/tests.rs +++ b/serf-proto/src/stream_endpoint/tests.rs @@ -367,3 +367,410 @@ fn completed_join_push_pull_surfaces_exchange_completed_event() { completed loopback Join push/pull" ); } + +// ── composition seam: the driver-facing forwarders ──────────────────────────── +// +// The stream and QUIC super-machines expose the same serf surface over different +// transports; these pin the stream side of that surface so the two cannot drift. + +use core::time::Duration; +use memberlist_proto::{EncodeOptions, Node, encode_outgoing, typed::Alive}; + +use crate::{ + event::{Event, MemberEventKind}, + options::Options as SerfOptions, +}; + +/// `Instant::ORIGIN + s` seconds. +fn t_secs(s: u64) -> Instant { + Instant::ORIGIN + Duration::from_secs(s) +} + +/// Wrap a raw membership endpoint into the plain-TCP reliable coordinator. +fn coord( + inner: memberlist_proto::Endpoint, +) -> memberlist_proto::streams::StreamEndpoint { + memberlist_proto::streams::StreamEndpoint::<_, _, RawRecords>::new( + inner, + LabelOptions::new_in(Some(CLUSTER.to_vec()), ()), + Box::new(|_addr: &SocketAddr| None), + Box::new(|addr: &SocketAddr| *addr), + ) +} + +/// The inner memberlist options every fixture roots at. +fn inner_opts(id: u32, port: u16) -> EndpointOptions { + EndpointOptions::new(id, sa(port)) + .with_user_broadcast_tiers(core::num::NonZeroU8::new(3).unwrap()) +} + +/// A raw membership endpoint at `id` / `port`, deterministically seeded. +fn inner(id: u32, port: u16) -> memberlist_proto::Endpoint { + memberlist_proto::Endpoint::new_at( + inner_opts(id, port), + Instant::ORIGIN, + SmallRng::seed_from_u64(0), + ) +} + +/// Encode one memberlist gossip `Message` exactly as the wire carries it, under +/// the loopback cluster label. +fn gossip_frame(msg: &memberlist_proto::typed::Message) -> Bytes { + encode_outgoing(msg, &EncodeOptions::new(None)).expect("encode memberlist gossip frame") +} + +/// An `Alive` for `id` at `addr` — the gossip message that admits a peer. +fn alive(id: u32, addr: SocketAddr) -> memberlist_proto::typed::Message { + memberlist_proto::typed::Message::Alive(Alive::new(1, Node::new(id, addr))) +} + +/// The two coalescer shed counters are INJECTED: `new_with_rng_in` must thread +/// `user_drop` into the user coalescer's slot and `member_drop` into the member +/// coalescer's, without transposing them. +#[test] +fn new_with_rng_in_threads_each_injected_drop_counter_to_its_own_slot() { + let e: StreamEndpoint = StreamEndpoint::new_with_rng_in( + coord(inner(1, 7946)), + SerfOptions::new(), + SmallRng::seed_from_u64(0), + 7u64, + 9u64, + ); + assert_eq!(e.coalesced_user_events_dropped(), 7); + assert_eq!(e.coalesced_member_events_dropped(), 9); +} + +/// The super-machine roots serf at the coordinator's local id, and publishes the +/// membership view the driver observes. +#[test] +fn local_id_and_members_snapshot_come_from_the_coordinator() { + let mut e = ep(42, 7946); + assert_eq!(*e.local_id(), 42u32); + assert_eq!(e.health_score(), 0, "a fresh node is healthy"); + + e.test_seed_member(2, MemberStatus::Leaving, 1.into()); + let mut got: Vec<(u32, MemberStatus)> = e + .members_snapshot() + .iter() + .map(|m| (*m.node().id_ref(), m.status())) + .collect(); + got.sort_by_key(|(id, _)| *id); + assert_eq!( + got, + vec![(2, MemberStatus::Leaving), (42, MemberStatus::Alive)], + "the snapshot publishes every tracked member with its live status" + ); +} + +/// A per-member reconnect-timeout override installed through the builder shortens +/// the reaper's failed-member window. +#[test] +fn builder_reconnect_delegate_overrides_the_failed_reap_timeout() { + struct TenSeconds; + impl crate::ReconnectDelegate for TenSeconds { + fn reconnect_timeout( + &self, + _member: &crate::members::Member, + _default: Duration, + ) -> Duration { + Duration::from_secs(10) + } + } + + let mut e = + StreamEndpoint::::new(coord(inner(1, 7946)), SerfOptions::new()) + .with_reconnect_delegate(Some(Box::new(TenSeconds))); + let _ = e.poll_event(); + e.test_seed_failed_member(2, sa(7000), Instant::ORIGIN); + + e.test_fire_reap(t_secs(11)); + assert_eq!( + e.test_member_status(2), + None, + "the delegate's 10s timeout reaps the failed member at t+11s" + ); +} + +/// A well-formed gossip frame fed to `handle_packet` reaches the coordinator and +/// the resulting `NodeJoined` is sieved into serf on the same call; `handle_message` +/// is the same path for an already-decoded message. +#[test] +fn a_decoded_alive_admits_the_peer_through_either_ingress() { + let mut packet_side = ep(1, 7946); + packet_side.handle_packet(sa(7000), gossip_frame(&alive(2, sa(7000))), Instant::ORIGIN); + assert_eq!( + packet_side.test_member_status(2), + Some(MemberStatus::Alive), + "handle_packet decodes the frame and admits the peer" + ); + assert!( + matches!(packet_side.poll_event(), Some(Event::Member(ref me)) if me.kind() == MemberEventKind::Join) + ); + + let mut typed_side = ep(1, 7946); + typed_side.handle_message(sa(7000), alive(2, sa(7000)), Instant::ORIGIN); + assert_eq!( + typed_side.test_member_status(2), + Some(MemberStatus::Alive), + "handle_message feeds the already-decoded message to the same path" + ); +} + +/// `handle_gossip` buffers a raw inbound datagram for the codec-owning driver to +/// drain via `poll_memberlist_ingress`, decode, and feed back through +/// `handle_packet` — the machine never decodes it in place. +#[test] +fn handle_gossip_buffers_the_datagram_for_the_codec_owning_driver() { + let mut e = ep(1, 7946); + let frame = gossip_frame(&alive(2, sa(7000))); + e.handle_gossip(sa(7000), &frame, Instant::ORIGIN); + + assert_eq!( + e.test_member_status(2), + None, + "handle_gossip must not decode the frame itself" + ); + let (from, bytes) = e + .poll_memberlist_ingress() + .expect("the datagram is buffered for the driver"); + assert_eq!(from, sa(7000)); + + e.handle_packet(from, bytes, Instant::ORIGIN); + assert_eq!( + e.test_member_status(2), + Some(MemberStatus::Alive), + "handle_gossip -> poll_memberlist_ingress -> handle_packet admits the peer" + ); +} + +/// `poll_timeout` folds the coordinator's deadline with serf's own: an +/// un-scheduled coordinator leaves serf's reap deadline as the wake, and a +/// shut-down machine with an un-scheduled coordinator requests no wake at all. +#[test] +fn poll_timeout_folds_the_coordinator_and_serf_deadlines() { + let mut e = ep(1, 7946); + assert_eq!( + e.poll_timeout(), + Some(t_secs(15)), + "an un-scheduled coordinator leaves serf's reap deadline as the wake" + ); + + e.start_scheduling(Instant::ORIGIN); + let inner_deadline = e + .transport_mut() + .poll_timeout() + .expect("start_scheduling arms the coordinator's timers"); + assert!(inner_deadline < t_secs(15)); + assert_eq!( + e.poll_timeout(), + Some(inner_deadline), + "the fold takes the minimum of the two deadlines" + ); + + // A machine that lost an id-conflict vote schedules no serf wakeups. + let mut dead = ep(1, 7946); + let qid = dead.test_register_conflict_query(t_secs(3600)); + dead.test_fold_conflict_response(qid, 200u32, true); + dead.test_fold_conflict_response(qid, 201u32, false); + dead.test_fold_conflict_response(qid, 202u32, false); + dead.test_fire_due_query_closes(t_secs(3601)); + assert!(matches!(dead.poll_event(), Some(Event::Shutdown))); + assert_eq!( + dead.poll_timeout(), + None, + "a shut-down serf with an un-scheduled coordinator requests no wakeup" + ); + + // Arming the coordinator re-supplies the only deadline left on a dead machine. + dead.start_scheduling(Instant::ORIGIN); + let only = dead.transport_mut().poll_timeout(); + assert!(only.is_some(), "the coordinator schedules its own timers"); + assert_eq!( + dead.poll_timeout(), + only, + "with serf shut down the coordinator's deadline is the whole fold" + ); +} + +/// The wire-size forwarders report the coordinator's CONFIGURED limits — the +/// driver sizes its recv buffer and observation budget from them. +#[test] +fn wire_size_forwarders_report_the_configured_limits() { + let opts = inner_opts(1, 7946) + .with_gossip_mtu(1234) + .with_max_stream_frame_size(4096); + let raw = memberlist_proto::Endpoint::new_at(opts, Instant::ORIGIN, SmallRng::seed_from_u64(0)); + let mut e = StreamEndpoint::::new(coord(raw), SerfOptions::new()); + let _ = e.poll_event(); + + assert_eq!(e.gossip_mtu(), 1234); + assert_eq!(e.max_stream_frame_size(), 4096); +} + +/// A dial that never connects, and a live exchange that errors, both terminate +/// their exchange without producing a serf-level membership signal: transport +/// liveness is never a membership signal, so neither may invent or evict a member. +/// +/// An `ignore_old` join that dies this way leaves its ignore-join entry behind for +/// the driver to clear, since no merge ever consumed it. +#[test] +fn a_failed_dial_and_an_errored_exchange_are_not_membership_signals() { + let mut e = ep(1, 7946); + + let dial_stream = e.start_join_push_pull(sa(7000), true, Instant::ORIGIN); + let dial_exchange = match e.poll_action() { + Some(StreamAction::Connect(c)) => c.id(), + other => panic!("the dialer must surface a Connect, got {other:?}"), + }; + while e.poll_action().is_some() {} + assert!( + e.test_has_ignore_join_stream(dial_stream), + "the ignore_old join records its exchange" + ); + + e.handle_dial_failed(dial_exchange, Instant::ORIGIN); + assert!( + e.test_member_status(2).is_none(), + "a failed dial must not invent membership" + ); + assert_failed_exchange( + &mut e, + "a failed dial resolves the exchange as Failed, never as a membership change", + ); + assert!( + e.test_has_ignore_join_stream(dial_stream), + "no merge consumed the entry, so the driver must still clear it" + ); + e.clear_ignore_join_stream(dial_stream); + assert!(!e.test_has_ignore_join_stream(dial_stream)); + + let second_stream = e.start_push_pull(sa(7001), PushPullKind::Join, Instant::ORIGIN); + let second_exchange = match e.poll_action() { + Some(StreamAction::Connect(c)) => c.id(), + other => panic!("the dialer must surface a Connect, got {other:?}"), + }; + while e.poll_action().is_some() {} + + e.handle_transport_error(second_exchange, Instant::ORIGIN); + assert!( + e.test_member_status(2).is_none(), + "a transport error must not invent membership" + ); + assert_failed_exchange( + &mut e, + "a transport error resolves the exchange as Failed, never as a membership change", + ); + assert!( + !e.test_has_ignore_join_stream(second_stream), + "a plain join recorded no ignore-join entry to begin with" + ); +} + +/// Drain `e`'s event queue, asserting the ONLY events it yields are terminal +/// push/pull `ExchangeCompleted(Failed)` resolutions — never a membership change. +fn assert_failed_exchange(e: &mut StreamEndpoint, why: &str) { + use crate::{ExchangeKind, ExchangeStatus}; + + let mut saw = false; + while let Some(ev) = e.poll_event() { + match ev { + Event::ExchangeCompleted(c) => { + assert_eq!(c.kind(), ExchangeKind::PushPull); + assert_eq!(c.outcome(), ExchangeStatus::Failed, "{why}"); + saw = true; + } + other => panic!("{why}, got {other:?}"), + } + } + assert!(saw, "{why}"); +} + +/// A custom join-merge predicate installs through the coordinator, and the +/// coordinate-reset counter reads through to the Vivaldi client. +#[test] +fn operator_forwarders_reach_the_inner_machine() { + let mut e = ep(1, 7946); + + struct RejectAll; + impl memberlist_proto::delegate::MergeDelegate for RejectAll { + fn notify_merge( + &self, + _peers: memberlist_proto::MaybeOwned< + '_, + [memberlist_proto::typed::NodeState], + >, + ) -> bool { + false + } + } + e.set_merge_delegate(RejectAll); + + #[cfg(feature = "coordinates")] + assert_eq!( + e.coordinate_resets(), + Some(0), + "a fresh coordinate client has reset nothing" + ); +} + +// ── gossip-plane encryption ─────────────────────────────────────────────────── + +#[cfg(any(feature = "aes-gcm", feature = "chacha20-poly1305"))] +mod encryption { + use super::*; + use memberlist_proto::{EncryptionOptions, Keyring, SecretKey}; + + #[cfg(feature = "aes-gcm")] + fn key(b: u8) -> SecretKey { + SecretKey::Aes128([b; 16]) + } + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + fn key(b: u8) -> SecretKey { + SecretKey::ChaCha20Poly1305([b; 32]) + } + + /// With no keyring the gossip transforms are the identity; installing one + /// re-keys the live plane so an encrypted datagram round-trips and no longer + /// leaves as plaintext. + #[test] + fn setting_the_encryption_options_re_keys_the_live_gossip_plane() { + let mut e = ep(1, 7946); + assert!(!e.encryption_options().is_enabled()); + + let plain = b"\x05gossip-frame"; + assert_eq!( + e.encrypt_gossip(plain) + .expect("encrypt without a keyring") + .as_slice(), + plain, + "with no keyring the datagram is left unencrypted" + ); + + e.set_encryption_options(EncryptionOptions::new().with_keyring(Keyring::new(key(1)))); + assert_eq!( + e.encryption_options().keyring().map(|kr| *kr.primary_ref()), + Some(key(1)), + "the coordinator reports the primary key outbound datagrams seal under" + ); + + let sealed = e.encrypt_gossip(plain).expect("encrypt under the keyring"); + assert_ne!( + sealed.as_slice(), + plain, + "an encrypted-cluster datagram must not leave as plaintext" + ); + assert_eq!( + e.decrypt_gossip(&sealed) + .expect("decrypt under the keyring"), + plain, + "the datagram round-trips through the same keyring" + ); + + let mut foreign = ep(2, 7000); + foreign.set_encryption_options(EncryptionOptions::new().with_keyring(Keyring::new(key(2)))); + assert!( + foreign.decrypt_gossip(&sealed).is_err(), + "a frame the keyring cannot open must be an error, never silently admitted" + ); + } +} diff --git a/serf-proto/src/tests.rs b/serf-proto/src/tests.rs new file mode 100644 index 00000000..5d5871e2 --- /dev/null +++ b/serf-proto/src/tests.rs @@ -0,0 +1,19 @@ +//! Unit tests for the crate-root vocabulary types. + +use std::string::ToString; + +use super::LamportTime; + +/// A Lamport time renders as its bare counter — the form every log line, error +/// message and operator-facing diagnostic in the crate interpolates. A wrapper +/// like `LamportTime(7)` leaking into that output would make clock values +/// unreadable next to the plain integers the wire protocol reports. +#[test] +fn lamport_time_displays_as_its_bare_counter() { + assert_eq!(LamportTime::new(7).to_string(), "7"); + assert_eq!(LamportTime::ZERO.to_string(), "0"); + assert_eq!( + LamportTime::from(u64::MAX).to_string(), + u64::MAX.to_string() + ); +} diff --git a/serf-proto/src/typed/mod.rs b/serf-proto/src/typed/mod.rs index c5559aac..b31368e2 100644 --- a/serf-proto/src/typed/mod.rs +++ b/serf-proto/src/typed/mod.rs @@ -468,3 +468,6 @@ impl ConflictResponseMessage { Self { member } } } + +#[cfg(test)] +mod tests; diff --git a/serf-proto/src/typed/tests.rs b/serf-proto/src/typed/tests.rs new file mode 100644 index 00000000..c5dee887 --- /dev/null +++ b/serf-proto/src/typed/tests.rs @@ -0,0 +1,33 @@ +//! Unit tests for the serf wire vocabulary types. + +use super::*; + +/// A default `Coordinate` is the EMPTY coordinate — no dimensions at all — not a +/// zeroed point in some assumed dimensionality. The distinction matters: an +/// empty vector carries no dimensionality claim, so a peer's first observed +/// coordinate establishes it rather than silently mismatching an assumed one. +#[cfg(feature = "coordinates")] +#[test] +fn a_default_coordinate_is_dimensionless_and_at_rest() { + let c = Coordinate::default(); + assert!( + c.vec.is_empty(), + "the default coordinate claims no dimensionality" + ); + assert_eq!(c.error, 0.0); + assert_eq!(c.adjustment, 0.0); + assert_eq!(c.height, 0.0); +} + +/// `Tags::with_capacity` reserves without inserting: the map is still empty, and +/// `is_empty` agrees with `len` as entries arrive. +#[test] +fn tags_with_capacity_starts_empty_and_tracks_its_length() { + let mut tags = Tags::with_capacity(8); + assert!(tags.is_empty(), "a reserved-but-unfilled map is empty"); + assert_eq!(tags.len(), 0); + + tags.0.insert("role".into(), "web".into()); + assert!(!tags.is_empty(), "an inserted tag makes the map non-empty"); + assert_eq!(tags.len(), 1); +} diff --git a/serf-reactor/src/driver/options/tests.rs b/serf-reactor/src/driver/options/tests.rs index 7a48654c..29e64407 100644 --- a/serf-reactor/src/driver/options/tests.rs +++ b/serf-reactor/src/driver/options/tests.rs @@ -429,3 +429,19 @@ fn tracing_forwards_to_the_shared_engines() { ) }; } + +/// `StreamTransportOptions::default()` is the `new()` state — the tuned defaults a +/// caller who sets no stream knob at all is handed, and which `validate` admits. +#[test] +fn stream_transport_options_default_matches_new() { + let d = StreamTransportOptions::default(); + let n = StreamTransportOptions::new(); + assert_eq!(d.dial_timeout(), n.dial_timeout()); + assert_eq!(d.close_timeout(), n.close_timeout()); + assert_eq!(d.bridge_inbound_cap(), n.bridge_inbound_cap()); + assert_eq!(d.bridge_recv_buf_len(), n.bridge_recv_buf_len()); + assert!( + d.validate().is_ok(), + "the default stream knobs are admissible" + ); +} diff --git a/serf-reactor/src/quic/tests.rs b/serf-reactor/src/quic/tests.rs index 84c50011..74ea6a80 100644 --- a/serf-reactor/src/quic/tests.rs +++ b/serf-reactor/src/quic/tests.rs @@ -4,18 +4,105 @@ //! the tokio suite in `tests/quic.rs`. use super::QuicTransportOptions; +use core::time::Duration; use memberlist_proto::MaybeResolved; use smol_str::SmolStr; use std::net::SocketAddr; /// A fresh `QuicTransportOptions` carries no id / advertise / config until the -/// builder sets them. +/// builder sets them, and no SWIM override — every knob is `None`, which is what +/// makes `Transport::run` keep the coordinator's own defaults. #[test] fn new_starts_empty() { let opts = QuicTransportOptions::::new(); assert!(opts.local_id().is_none()); assert!(opts.advertise_addr().is_none()); assert!(opts.quic_config().is_none()); + assert!(opts.push_pull_interval().is_none()); + assert!(opts.probe_interval().is_none()); + assert!(opts.probe_timeout().is_none()); + assert!(opts.gossip_interval().is_none()); + assert!(opts.suspicion_mult().is_none()); + assert!(opts.dead_node_reclaim_time().is_none()); + assert!(opts.suspicion_max_timeout_mult().is_none()); + #[cfg(encryption)] + assert!( + opts.encryption().keyring().is_none(), + "the default policy leaves the gossip datagrams plaintext" + ); +} + +/// Every builder writes its OWN field: the accessors read back exactly what was +/// set, with distinct values per knob so a crossed assignment surfaces. +#[test] +fn builders_round_trip_each_knob() { + let opts = QuicTransportOptions::::new() + .with_push_pull_interval(Duration::from_millis(1)) + .with_probe_interval(Duration::from_millis(2)) + .with_probe_timeout(Duration::from_millis(3)) + .with_gossip_interval(Duration::from_millis(4)) + .with_suspicion_mult(5) + .with_dead_node_reclaim_time(Duration::from_millis(6)) + .with_suspicion_max_timeout_mult(7); + + assert_eq!(opts.push_pull_interval(), Some(Duration::from_millis(1))); + assert_eq!(opts.probe_interval(), Some(Duration::from_millis(2))); + assert_eq!(opts.probe_timeout(), Some(Duration::from_millis(3))); + assert_eq!(opts.gossip_interval(), Some(Duration::from_millis(4))); + assert_eq!(opts.suspicion_mult(), Some(5)); + assert_eq!( + opts.dead_node_reclaim_time(), + Some(Duration::from_millis(6)) + ); + assert_eq!(opts.suspicion_max_timeout_mult(), Some(7)); +} + +/// A zero push/pull interval is a MEANINGFUL setting (it disables periodic +/// anti-entropy, isolating the gossip datagram plane), so it must round-trip as +/// `Some(ZERO)` — never collapse back to the `None` that means "keep the +/// coordinator default". +#[test] +fn zero_push_pull_interval_is_set_not_unset() { + let opts = + QuicTransportOptions::::new().with_push_pull_interval(Duration::ZERO); + assert_eq!(opts.push_pull_interval(), Some(Duration::ZERO)); +} + +/// An unresolved advertise address is retained in its unresolved form — the +/// transport constructor is what resolves it, so the block must not resolve early. +#[test] +fn unresolved_advertise_addr_round_trips_unresolved() { + let host: hostaddr::HostAddr = "example.com:7946".parse().expect("host addr"); + let opts = QuicTransportOptions::>::new() + .with_advertise_addr(MaybeResolved::Unresolved(host.clone())); + match opts.advertise_addr() { + Some(MaybeResolved::Unresolved(h)) => assert_eq!(*h, host), + other => panic!("expected an unresolved advertise addr, got {other:?}"), + } +} + +/// The gossip keyring reaches the block through the builder. (On QUIC it seals +/// only the datagram plane — the reliable plane rides quinn's own TLS.) +#[cfg(encryption)] +#[test] +fn encryption_policy_round_trips() { + use memberlist_proto::{EncryptionOptions, Keyring, SecretKey}; + + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes256([0x41; 32]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = SecretKey::ChaCha20Poly1305([0x41; 32]); + + let opts = QuicTransportOptions::::new() + .with_encryption(EncryptionOptions::new().with_keyring(Keyring::new(key))); + assert_eq!( + opts + .encryption() + .keyring() + .expect("the configured keyring reaches the options block") + .primary_ref(), + &key + ); } /// The builder setters round-trip through the accessors (the id and advertise the @@ -41,3 +128,95 @@ fn default_matches_new() { assert!(opts.local_id().is_none()); assert!(opts.quic_config().is_none()); } + +/// A constructed transport reports the identity it was built with: the local id, +/// the advertise input in the ORIGINAL form the caller supplied (an unresolved +/// input stays unresolved — resolution happens for the bind, not for this +/// accessor), and the concrete bound contact the node will gossip. +#[cfg(feature = "tokio")] +#[tokio::test] +async fn transport_reports_its_identity_and_bound_contact() { + use core::time::Duration; + use std::sync::Arc; + + use agnostic::tokio::TokioRuntime; + use memberlist_proto::UnreliableTransport; + use rustls::version::TLS13; + use rustls_pki_types::{CertificateDer, PrivateKeyDer}; + + use super::{QuicOptions, QuicTransport}; + use crate::{FirstAddrResolver, SocketAddrResolver, transport::Transport}; + + /// A minimal quinn bundle: a fresh self-signed localhost cert, an accept-any + /// client, and datagram-mode gossip. Nothing dials in this test — the bundle + /// only has to be well-formed enough for the transport to bind. + fn test_quic_options() -> QuicOptions { + let ck = + rcgen::generate_simple_self_signed(vec!["localhost".into()]).expect("self-signed cert"); + let chain = vec![CertificateDer::from(ck.cert.der().to_vec())]; + let key = PrivateKeyDer::Pkcs8(ck.signing_key.serialize_der().into()); + + let provider = Arc::new(rustls::crypto::ring::default_provider()); + let rustls_server = rustls::ServerConfig::builder_with_provider(provider.clone()) + .with_protocol_versions(&[&TLS13]) + .expect("tls13 server") + .with_no_client_auth() + .with_single_cert(chain, key) + .expect("single cert"); + let server = quinn_proto::ServerConfig::with_crypto(Arc::new( + quinn_proto::crypto::rustls::QuicServerConfig::try_from(Arc::new(rustls_server)) + .expect("quic server config"), + )); + + let roots = rustls::RootCertStore::empty(); + let rustls_client = rustls::ClientConfig::builder_with_provider(provider) + .with_protocol_versions(&[&TLS13]) + .expect("tls13 client") + .with_root_certificates(roots) + .with_no_client_auth(); + let client = quinn_proto::ClientConfig::new(Arc::new( + quinn_proto::crypto::rustls::QuicClientConfig::try_from(Arc::new(rustls_client)) + .expect("quic client config"), + )); + + let hmac = ring::hmac::Key::new(ring::hmac::HMAC_SHA256, &[0x5au8; 32]); + let mut transport = quinn_proto::TransportConfig::default(); + transport.max_idle_timeout(Some( + quinn_proto::IdleTimeout::try_from(Duration::from_secs(20)).expect("idle timeout"), + )); + QuicOptions::new( + quinn_proto::EndpointConfig::new(Arc::new(hmac)), + server, + client, + transport, + "localhost", + UnreliableTransport::Datagram, + ) + } + + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let transport = + as Transport>::new( + QuicTransportOptions::new() + .with_local_id(SmolStr::new("ident")) + .with_advertise_addr(MaybeResolved::Unresolved(bind)) + .with_quic_config(test_quic_options()), + &SocketAddrResolver, + &FirstAddrResolver, + ) + .await + .expect("the transport binds an ephemeral loopback port"); + + assert_eq!(transport.local_id(), &SmolStr::new("ident")); + match transport.local_address() { + MaybeResolved::Unresolved(a) => assert_eq!(*a, bind), + other => panic!("the advertise INPUT form must be retained, got {other:?}"), + } + let advertise = *transport.advertise_address(); + assert!(advertise.ip().is_loopback()); + assert_ne!( + advertise.port(), + 0, + "the bound contact carries the OS-assigned port, not the ephemeral `:0`" + ); +} diff --git a/serf-reactor/src/resolver/getifs/tests.rs b/serf-reactor/src/resolver/getifs/tests.rs index c5e29076..4f4264c2 100644 --- a/serf-reactor/src/resolver/getifs/tests.rs +++ b/serf-reactor/src/resolver/getifs/tests.rs @@ -36,6 +36,20 @@ async fn wildcard_respects_address_family() { assert!(v6.iter().all(|s| s.is_ipv6()), "[::] must yield only IPv6"); } +/// Each named constructor carries its own scope, and `Default` is the private +/// scope — the LAN-cluster posture a resolver built with no explicit scope gets. +#[test] +fn named_constructors_carry_their_scope() { + assert_eq!(LocalAddrResolver::private().scope, LocalAddrScope::Private); + assert_eq!(LocalAddrResolver::public().scope, LocalAddrScope::Public); + assert_eq!(LocalAddrResolver::all().scope, LocalAddrScope::All); + assert_eq!( + LocalAddrResolver::default().scope, + LocalAddrScope::Private, + "the default scope is private" + ); +} + #[test] fn local_advertise_attaches_port() { // Best-effort: a host may have no address in a given scope, but when one is found diff --git a/serf-reactor/src/resolver/mod.rs b/serf-reactor/src/resolver/mod.rs index 5be2d27b..9fedb49b 100644 --- a/serf-reactor/src/resolver/mod.rs +++ b/serf-reactor/src/resolver/mod.rs @@ -67,3 +67,6 @@ pub trait Resolver: Send + Sync + 'static { addr: &Self::Address, ) -> impl Future, Self::Error>> + Send + '_; } + +#[cfg(test)] +mod tests; diff --git a/serf-reactor/src/resolver/tests.rs b/serf-reactor/src/resolver/tests.rs new file mode 100644 index 00000000..17dc8b70 --- /dev/null +++ b/serf-reactor/src/resolver/tests.rs @@ -0,0 +1,155 @@ +//! Unit tests for the built-in resolvers: the advertise-candidate pickers (the +//! policy `Transport::new` applies when an unresolved advertise address resolves +//! to more than one candidate) and the address resolvers themselves. + +use std::net::SocketAddr; + +use super::{ + AdvertiseAddrResolver, AdvertiseResolutionError, FirstAddrResolver, Ipv4PreferringResolver, + Ipv6PreferringResolver, +}; + +fn addr(s: &str) -> SocketAddr { + s.parse().expect("socket addr") +} + +/// The default picker takes the candidate set's FIRST address, whatever its +/// family — the order the resolver returned is the policy. +#[test] +fn first_addr_resolver_takes_the_head_of_the_candidate_set() { + let picked = FirstAddrResolver + .pick(vec![addr("[::1]:7946"), addr("127.0.0.1:7946")]) + .expect("a non-empty candidate set resolves"); + assert_eq!(picked, addr("[::1]:7946"), "the head candidate is picked"); + + let picked = FirstAddrResolver + .pick(vec![addr("10.0.0.1:1"), addr("10.0.0.2:2")]) + .expect("a non-empty candidate set resolves"); + assert_eq!(picked, addr("10.0.0.1:1")); +} + +/// The IPv4-preferring picker skips past IPv6 candidates to the first IPv4 one, +/// and falls back to the head of the set when the resolution returned no IPv4 +/// address at all (an IPv6-only host still gets a contact). +#[test] +fn ipv4_preferring_resolver_prefers_v4_then_falls_back() { + let picked = Ipv4PreferringResolver + .pick(vec![ + addr("[::1]:7946"), + addr("[2001:db8::1]:7946"), + addr("127.0.0.1:7946"), + addr("10.0.0.1:7946"), + ]) + .expect("a non-empty candidate set resolves"); + assert_eq!( + picked, + addr("127.0.0.1:7946"), + "the FIRST IPv4 candidate wins over any IPv6 candidate ahead of it" + ); + + let picked = Ipv4PreferringResolver + .pick(vec![addr("[::1]:7946"), addr("[2001:db8::1]:7946")]) + .expect("an IPv6-only candidate set still resolves"); + assert_eq!( + picked, + addr("[::1]:7946"), + "with no IPv4 candidate the preference falls back to the head of the set" + ); +} + +/// The IPv6-preferring picker is the mirror image: the first IPv6 candidate +/// wins, and an IPv4-only set falls back to the head. +#[test] +fn ipv6_preferring_resolver_prefers_v6_then_falls_back() { + let picked = Ipv6PreferringResolver + .pick(vec![ + addr("127.0.0.1:7946"), + addr("10.0.0.1:7946"), + addr("[2001:db8::1]:7946"), + addr("[::1]:7946"), + ]) + .expect("a non-empty candidate set resolves"); + assert_eq!( + picked, + addr("[2001:db8::1]:7946"), + "the FIRST IPv6 candidate wins over any IPv4 candidate ahead of it" + ); + + let picked = Ipv6PreferringResolver + .pick(vec![addr("127.0.0.1:7946"), addr("10.0.0.1:7946")]) + .expect("an IPv4-only candidate set still resolves"); + assert_eq!( + picked, + addr("127.0.0.1:7946"), + "with no IPv6 candidate the preference falls back to the head of the set" + ); +} + +/// An empty candidate set is a resolution FAILURE on every picker, never a +/// silent default: a node with no resolvable advertise address must not boot. +#[test] +fn every_picker_rejects_an_empty_candidate_set() { + assert!(matches!( + FirstAddrResolver.pick(Vec::new()), + Err(AdvertiseResolutionError::Empty) + )); + assert!(matches!( + Ipv4PreferringResolver.pick(Vec::new()), + Err(AdvertiseResolutionError::Empty) + )); + assert!(matches!( + Ipv6PreferringResolver.pick(Vec::new()), + Err(AdvertiseResolutionError::Empty) + )); +} + +/// The identity resolver returns its already-concrete input verbatim as the sole +/// candidate — the pass-through the `SocketAddr`-addressed node type relies on. +#[cfg(feature = "tokio")] +#[tokio::test] +async fn socket_addr_resolver_passes_its_input_through() { + use super::{Resolver, SocketAddrResolver}; + + let input = addr("192.0.2.7:7946"); + let out = SocketAddrResolver + .resolve(&input) + .await + .expect("the identity resolver cannot fail"); + assert_eq!( + out, + vec![input], + "the identity resolver yields exactly its input" + ); +} + +/// The OS resolver resolves a literal-IP host without a DNS round-trip, keeps +/// the port, and reports a lookup failure for an unresolvable name rather than +/// yielding an empty candidate set. +#[cfg(feature = "tokio")] +#[tokio::test] +async fn os_resolver_resolves_a_literal_ip_host() { + use agnostic::tokio::TokioRuntime; + use hostaddr::HostAddr; + + use super::{OsResolver, Resolver}; + + let resolver = OsResolver::::default(); + let host: HostAddr = "127.0.0.1:7946".parse().expect("literal-IP host addr"); + let out = resolver + .resolve(&host) + .await + .expect("a literal IP resolves without DNS"); + assert_eq!( + out, + vec![addr("127.0.0.1:7946")], + "the literal IP and its port pass through" + ); + + let bad: HostAddr = "no-such-host.invalid:7946" + .parse() + .expect("domain host addr"); + assert!( + resolver.resolve(&bad).await.is_err(), + "an unresolvable name is a resolution error, never an empty candidate set" + ); +} diff --git a/serf-reactor/src/tcp/mod.rs b/serf-reactor/src/tcp/mod.rs index 0040e125..844abbe4 100644 --- a/serf-reactor/src/tcp/mod.rs +++ b/serf-reactor/src/tcp/mod.rs @@ -606,3 +606,6 @@ where driver.await; } } + +#[cfg(test)] +mod tests; diff --git a/serf-reactor/src/tcp/tests.rs b/serf-reactor/src/tcp/tests.rs new file mode 100644 index 00000000..f60e2ecd --- /dev/null +++ b/serf-reactor/src/tcp/tests.rs @@ -0,0 +1,169 @@ +//! Unit tests for the TCP transport options builder — the pure accessor / +//! builder wiring that feeds `TcpTransport::new`. Real-node construction (bind, +//! join/converge, the SWIM overrides taking effect on the wire) is exercised +//! end-to-end by the suite in `tests/tcp.rs`. + +use core::time::Duration; +use std::net::SocketAddr; + +use memberlist_proto::MaybeResolved; +use smol_str::SmolStr; + +use super::TcpTransportOptions; +use crate::driver::options::StreamTransportOptions; + +/// A fresh block carries none of the required fields and no SWIM override — every +/// knob is `None`, which is what makes `Transport::run` keep the coordinator's own +/// defaults and `Transport::new` refuse a half-built block. +#[test] +fn new_starts_empty() { + let opts = TcpTransportOptions::::new(); + assert!(opts.local_id().is_none()); + assert!(opts.advertise_addr().is_none()); + assert!(opts.push_pull_interval().is_none()); + assert!(opts.probe_interval().is_none()); + assert!(opts.probe_timeout().is_none()); + assert!(opts.gossip_interval().is_none()); + assert!(opts.suspicion_mult().is_none()); + assert!(opts.dead_node_reclaim_time().is_none()); + assert!(opts.suspicion_max_timeout_mult().is_none()); + assert!(opts.stream().validate().is_ok()); + #[cfg(encryption)] + assert!( + opts.encryption().keyring().is_none(), + "the default policy leaves both planes plaintext" + ); +} + +/// Every builder writes its OWN field: the accessors read back exactly what was +/// set, with distinct values per knob so a crossed assignment surfaces. +#[test] +fn builders_round_trip_each_knob() { + let addr: SocketAddr = "127.0.0.1:7946".parse().expect("advertise addr"); + let opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new("acc-node")) + .with_advertise_addr(MaybeResolved::Resolved(addr)) + .with_stream(StreamTransportOptions::new().with_dial_timeout(Duration::from_millis(11))) + .with_push_pull_interval(Duration::from_millis(1)) + .with_probe_interval(Duration::from_millis(2)) + .with_probe_timeout(Duration::from_millis(3)) + .with_gossip_interval(Duration::from_millis(4)) + .with_suspicion_mult(5) + .with_dead_node_reclaim_time(Duration::from_millis(6)) + .with_suspicion_max_timeout_mult(7); + + assert_eq!(opts.local_id(), Some(&SmolStr::new("acc-node"))); + match opts.advertise_addr() { + Some(MaybeResolved::Resolved(s)) => assert_eq!(*s, addr), + other => panic!("expected a resolved advertise addr, got {other:?}"), + } + assert_eq!(opts.stream().dial_timeout(), Duration::from_millis(11)); + assert_eq!(opts.push_pull_interval(), Some(Duration::from_millis(1))); + assert_eq!(opts.probe_interval(), Some(Duration::from_millis(2))); + assert_eq!(opts.probe_timeout(), Some(Duration::from_millis(3))); + assert_eq!(opts.gossip_interval(), Some(Duration::from_millis(4))); + assert_eq!(opts.suspicion_mult(), Some(5)); + assert_eq!( + opts.dead_node_reclaim_time(), + Some(Duration::from_millis(6)) + ); + assert_eq!(opts.suspicion_max_timeout_mult(), Some(7)); +} + +/// A zero push/pull interval is a MEANINGFUL setting (it disables periodic +/// anti-entropy), so it must round-trip as `Some(ZERO)` — never collapse back to +/// the `None` that means "keep the coordinator default". +#[test] +fn zero_push_pull_interval_is_set_not_unset() { + let opts = + TcpTransportOptions::::new().with_push_pull_interval(Duration::ZERO); + assert_eq!( + opts.push_pull_interval(), + Some(Duration::ZERO), + "a zero interval disables periodic push/pull; it is not the absent default" + ); +} + +/// An unresolved advertise address is retained in its unresolved form — the +/// transport constructor is what resolves it, so the block must not resolve early. +#[test] +fn unresolved_advertise_addr_round_trips_unresolved() { + let host: hostaddr::HostAddr = "example.com:7946".parse().expect("host addr"); + let opts = TcpTransportOptions::>::new() + .with_advertise_addr(MaybeResolved::Unresolved(host.clone())); + match opts.advertise_addr() { + Some(MaybeResolved::Unresolved(h)) => assert_eq!(*h, host), + other => panic!("expected an unresolved advertise addr, got {other:?}"), + } +} + +/// The gossip-and-reliable keyring reaches the block through the builder. +#[cfg(encryption)] +#[test] +fn encryption_policy_round_trips() { + use memberlist_proto::{EncryptionOptions, Keyring, SecretKey}; + + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes256([0x21; 32]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = SecretKey::ChaCha20Poly1305([0x21; 32]); + + let opts = TcpTransportOptions::::new() + .with_encryption(EncryptionOptions::new().with_keyring(Keyring::new(key))); + let keyring = opts + .encryption() + .keyring() + .expect("the configured keyring reaches the options block"); + assert_eq!( + keyring.primary_ref(), + &key, + "the primary key is the one that was configured" + ); +} + +/// `Default` is the `new()` state. +#[test] +fn default_matches_new() { + let d = TcpTransportOptions::::default(); + assert!(d.local_id().is_none()); + assert!(d.advertise_addr().is_none()); + assert!(d.probe_interval().is_none()); +} + +/// A constructed transport reports the identity it was built with: the local id, +/// the advertise input in the ORIGINAL form the caller supplied (an unresolved +/// input stays unresolved — resolution happens for the bind, not for this +/// accessor), and the concrete bound contact the node will gossip. +#[cfg(feature = "tokio")] +#[tokio::test] +async fn transport_reports_its_identity_and_bound_contact() { + use agnostic::tokio::TokioRuntime; + + use super::TcpTransport; + use crate::{FirstAddrResolver, SocketAddrResolver, transport::Transport}; + + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let transport = + as Transport>::new( + TcpTransportOptions::new() + .with_local_id(SmolStr::new("ident")) + .with_advertise_addr(MaybeResolved::Unresolved(bind)), + &SocketAddrResolver, + &FirstAddrResolver, + ) + .await + .expect("the transport binds an ephemeral loopback port"); + + assert_eq!(transport.local_id(), &SmolStr::new("ident")); + match transport.local_address() { + MaybeResolved::Unresolved(a) => assert_eq!(*a, bind), + other => panic!("the advertise INPUT form must be retained, got {other:?}"), + } + let advertise = *transport.advertise_address(); + assert!(advertise.ip().is_loopback()); + assert_ne!( + advertise.port(), + 0, + "the bound contact carries the OS-assigned port, not the ephemeral `:0`" + ); +} diff --git a/serf-reactor/src/tls/tests.rs b/serf-reactor/src/tls/tests.rs index f76ccf59..90db8d4d 100644 --- a/serf-reactor/src/tls/tests.rs +++ b/serf-reactor/src/tls/tests.rs @@ -3,6 +3,7 @@ //! record-layer handshake, join/converge, freed-port rebind) is exercised //! end-to-end by the tokio suite in `tests/tls.rs`. +use core::time::Duration; use std::{net::SocketAddr, sync::Arc}; use memberlist_proto::MaybeResolved; @@ -11,6 +12,7 @@ use rustls_pki_types::{CertificateDer, PrivateKeyDer}; use smol_str::SmolStr; use super::{TlsOptions, TlsTransportOptions}; +use crate::driver::options::StreamTransportOptions; /// The process-default crypto provider, falling back to ring for the dev build. fn crypto_provider() -> Arc { @@ -81,6 +83,109 @@ fn options_accessors_reflect_builders() { ); } +/// A fresh block carries no SWIM override — every knob is `None`, which is what +/// makes `Transport::run` keep the coordinator's own defaults. +#[test] +fn new_starts_with_no_swim_override() { + let opts = TlsTransportOptions::::new(); + assert!(opts.push_pull_interval().is_none()); + assert!(opts.probe_interval().is_none()); + assert!(opts.probe_timeout().is_none()); + assert!(opts.gossip_interval().is_none()); + assert!(opts.suspicion_mult().is_none()); + assert!(opts.dead_node_reclaim_time().is_none()); + assert!(opts.suspicion_max_timeout_mult().is_none()); + #[cfg(encryption)] + assert!( + opts.encryption().keyring().is_none(), + "the default policy leaves the gossip datagrams plaintext" + ); +} + +/// Every builder writes its OWN field: the accessors read back exactly what was +/// set, with distinct values per knob so a crossed assignment surfaces. +#[test] +fn builders_round_trip_each_knob() { + let opts = TlsTransportOptions::::new() + .with_stream(StreamTransportOptions::new().with_close_timeout(Duration::from_millis(12))) + .with_push_pull_interval(Duration::from_millis(1)) + .with_probe_interval(Duration::from_millis(2)) + .with_probe_timeout(Duration::from_millis(3)) + .with_gossip_interval(Duration::from_millis(4)) + .with_suspicion_mult(5) + .with_dead_node_reclaim_time(Duration::from_millis(6)) + .with_suspicion_max_timeout_mult(7); + + assert_eq!(opts.stream().close_timeout(), Duration::from_millis(12)); + assert_eq!(opts.push_pull_interval(), Some(Duration::from_millis(1))); + assert_eq!(opts.probe_interval(), Some(Duration::from_millis(2))); + assert_eq!(opts.probe_timeout(), Some(Duration::from_millis(3))); + assert_eq!(opts.gossip_interval(), Some(Duration::from_millis(4))); + assert_eq!(opts.suspicion_mult(), Some(5)); + assert_eq!( + opts.dead_node_reclaim_time(), + Some(Duration::from_millis(6)) + ); + assert_eq!(opts.suspicion_max_timeout_mult(), Some(7)); +} + +/// A zero push/pull interval is a MEANINGFUL setting (it disables periodic +/// anti-entropy), so it must round-trip as `Some(ZERO)` — never collapse back to +/// the `None` that means "keep the coordinator default". +#[test] +fn zero_push_pull_interval_is_set_not_unset() { + let opts = + TlsTransportOptions::::new().with_push_pull_interval(Duration::ZERO); + assert_eq!(opts.push_pull_interval(), Some(Duration::ZERO)); +} + +/// The SNI provider is consulted PER PEER, so a provider that maps each dialed +/// address to its own name must be carried unflattened. +#[test] +fn sni_provider_is_consulted_per_peer() { + let opts = TlsTransportOptions::::new().with_sni_provider(Box::new(|a| { + (a.port() != 9).then(|| format!("peer-{}.example", a.port())) + })); + let sni = opts.sni_provider(); + assert_eq!( + sni(&"127.0.0.1:1".parse().unwrap()), + Some("peer-1.example".to_string()) + ); + assert_eq!( + sni(&"127.0.0.1:2".parse().unwrap()), + Some("peer-2.example".to_string()) + ); + assert_eq!( + sni(&"127.0.0.1:9".parse().unwrap()), + None, + "a provider may refuse a peer, which aborts the dial before the handshake" + ); +} + +/// The gossip keyring reaches the block through the builder. (On TLS it seals +/// only the gossip datagrams — the reliable plane rides the TLS session.) +#[cfg(encryption)] +#[test] +fn encryption_policy_round_trips() { + use memberlist_proto::{EncryptionOptions, Keyring, SecretKey}; + + #[cfg(feature = "aes-gcm")] + let key = SecretKey::Aes256([0x31; 32]); + #[cfg(all(not(feature = "aes-gcm"), feature = "chacha20-poly1305"))] + let key = SecretKey::ChaCha20Poly1305([0x31; 32]); + + let opts = TlsTransportOptions::::new() + .with_encryption(EncryptionOptions::new().with_keyring(Keyring::new(key))); + assert_eq!( + opts + .encryption() + .keyring() + .expect("the configured keyring reaches the options block") + .primary_ref(), + &key + ); +} + /// `Default` is the `new()` state: required fields `None`, default SNI provider /// installed. #[test] @@ -90,5 +195,135 @@ fn default_matches_new() { assert!(d.local_id().is_none()); assert!(d.advertise_addr().is_none()); assert!(d.tls_options().is_none()); + assert!(d.probe_interval().is_none()); assert_eq!((d.sni_provider())(&addr), Some("localhost".to_string())); } + +/// A constructed transport reports the identity it was built with: the local id, +/// the advertise input in the ORIGINAL form the caller supplied (an unresolved +/// input stays unresolved — resolution happens for the bind, not for this +/// accessor), and the concrete bound contact the node will gossip. +#[cfg(feature = "tokio")] +#[tokio::test] +async fn transport_reports_its_identity_and_bound_contact() { + use agnostic::tokio::TokioRuntime; + + use super::TlsTransport; + use crate::{FirstAddrResolver, SocketAddrResolver, transport::Transport}; + + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let transport = + as Transport>::new( + TlsTransportOptions::new() + .with_local_id(SmolStr::new("ident")) + .with_advertise_addr(MaybeResolved::Unresolved(bind)) + .with_tls_options(test_tls_options()), + &SocketAddrResolver, + &FirstAddrResolver, + ) + .await + .expect("the transport binds an ephemeral loopback port"); + + assert_eq!(transport.local_id(), &SmolStr::new("ident")); + match transport.local_address() { + MaybeResolved::Unresolved(a) => assert_eq!(*a, bind), + other => panic!("the advertise INPUT form must be retained, got {other:?}"), + } + let advertise = *transport.advertise_address(); + assert!(advertise.ip().is_loopback()); + assert_ne!( + advertise.port(), + 0, + "the bound contact carries the OS-assigned port, not the ephemeral `:0`" + ); +} + +/// The TLS transport needs BOTH planes: the reliable TLS-over-TCP listener and the +/// plain-UDP gossip socket on the same port. When the gossip port is already taken, +/// construction FAILS rather than coming up with a reliable plane and no gossip. +#[cfg(feature = "tokio")] +#[tokio::test] +async fn taken_gossip_port_fails_construction() { + use agnostic::tokio::TokioRuntime; + + use super::TlsTransport; + use crate::{FirstAddrResolver, SocketAddrResolver, transport::Transport}; + + let squatter = std::net::UdpSocket::bind("127.0.0.1:0").expect("squat a UDP port"); + let taken = squatter.local_addr().expect("the squatted address"); + + let err = as Transport>::new( + TlsTransportOptions::new() + .with_local_id(SmolStr::new("squatted")) + .with_advertise_addr(MaybeResolved::Resolved(taken)) + .with_tls_options(test_tls_options()), + &SocketAddrResolver, + &FirstAddrResolver, + ) + .await + .err() + .expect("a node cannot come up without its gossip plane"); + assert!( + matches!(err, crate::SerfError::Io(_)), + "a taken gossip port is an I/O failure, got {err:?}" + ); + + // Freeing the port makes the very same construction succeed — the failure was + // the squatter, not the address. + drop(squatter); + let transport = + as Transport>::new( + TlsTransportOptions::new() + .with_local_id(SmolStr::new("unsquatted")) + .with_advertise_addr(MaybeResolved::Resolved(taken)) + .with_tls_options(test_tls_options()), + &SocketAddrResolver, + &FirstAddrResolver, + ) + .await + .expect("the released gossip port lets the transport bind"); + assert_eq!(*transport.advertise_address(), taken); +} + +/// A resolver that resolves the advertise address to NO candidate fails +/// construction rather than booting a node with no reachable contact. +#[cfg(feature = "tokio")] +#[tokio::test] +async fn advertise_resolution_failure_fails_construction() { + use agnostic::tokio::TokioRuntime; + + use super::TlsTransport; + use crate::{FirstAddrResolver, Resolver, transport::Transport}; + + /// Resolves nothing — a bootstrap outage the advertise picker must refuse. + struct EmptyResolver; + impl Resolver for EmptyResolver { + type Address = SocketAddr; + type Error = std::io::Error; + fn resolve( + &self, + _addr: &SocketAddr, + ) -> impl core::future::Future>> + Send + '_ { + // The candidate set is empty whatever the input, so the future borrows nothing. + let out = Vec::new(); + async move { Ok(out) } + } + } + + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let err = as Transport>::new( + TlsTransportOptions::new() + .with_local_id(SmolStr::new("unresolvable")) + .with_advertise_addr(MaybeResolved::Unresolved(bind)) + .with_tls_options(test_tls_options()), + &EmptyResolver, + &FirstAddrResolver, + ) + .await + .err() + .expect("an advertise address that resolves to nothing cannot boot a node"); + assert!( + matches!(err, crate::SerfError::Resolve(_)), + "an empty candidate set is a resolution failure, got {err:?}" + ); +} diff --git a/serf-reactor/src/transport/mod.rs b/serf-reactor/src/transport/mod.rs index 29fd310a..003e2bf1 100644 --- a/serf-reactor/src/transport/mod.rs +++ b/serf-reactor/src/transport/mod.rs @@ -198,3 +198,6 @@ pub(crate) fn reject_cross_cipher_keyring( } Ok(()) } + +#[cfg(all(test, any(feature = "tcp", feature = "quic")))] +mod tests; diff --git a/serf-reactor/src/transport/tests.rs b/serf-reactor/src/transport/tests.rs new file mode 100644 index 00000000..217a6ba5 --- /dev/null +++ b/serf-reactor/src/transport/tests.rs @@ -0,0 +1,150 @@ +//! Unit tests for the shared pre-flight gates every backend's `Transport::new` +//! runs: the advertise-address admission check (which rejects a contact peers +//! could not dial or the wire could not carry) and the construction-time +//! cross-cipher keyring refusal. + +use std::net::SocketAddr; + +use super::validate_advertise_addr; + +/// A routable loopback / unicast contact with a concrete port is admitted, in +/// both address families. +#[test] +fn unicast_advertise_addr_is_admitted() { + for addr in [ + "127.0.0.1:7946", + "192.168.1.10:7946", + "8.8.8.8:1", + "[::1]:7946", + "[2001:db8::1]:7946", + ] { + let addr: SocketAddr = addr.parse().expect("advertise addr"); + assert!( + validate_advertise_addr(&addr).is_ok(), + "{addr} is a routable unicast contact and must be admitted" + ); + } +} + +/// The wildcard-bind address is not a contact: its `local_addr()` readback keeps +/// the unspecified IP, so peers would learn a member they cannot dial. +#[test] +fn unspecified_advertise_addr_is_rejected() { + for addr in ["0.0.0.0:7946", "[::]:7946"] { + let addr: SocketAddr = addr.parse().expect("advertise addr"); + let err = validate_advertise_addr(&addr).expect_err("an unspecified IP is not a contact"); + assert!( + matches!(err, crate::SerfError::InvalidAdvertiseAddr(_)), + "{addr} must be refused as an invalid advertise address, got {err:?}" + ); + } +} + +/// A multicast group address and the IPv4 broadcast address name a group, not a +/// single peer's unicast contact. +#[test] +fn group_advertise_addr_is_rejected() { + for addr in ["224.0.0.1:7946", "[ff02::1]:7946", "255.255.255.255:7946"] { + let addr: SocketAddr = addr.parse().expect("advertise addr"); + let err = validate_advertise_addr(&addr).expect_err("a group address is not a unicast contact"); + assert!( + matches!(err, crate::SerfError::InvalidAdvertiseAddr(_)), + "{addr} must be refused as an invalid advertise address, got {err:?}" + ); + } +} + +/// A zero port is undialable — the bound socket's readback must have resolved an +/// ephemeral `:0` to a concrete port before this gate runs. +#[test] +fn zero_port_advertise_addr_is_rejected() { + let addr: SocketAddr = "127.0.0.1:0".parse().expect("advertise addr"); + let err = validate_advertise_addr(&addr).expect_err("a zero port is undialable"); + assert!( + matches!(err, crate::SerfError::InvalidAdvertiseAddr(_)), + "a zero port must be refused, got {err:?}" + ); +} + +/// The compact `[16B IP][2B port]` wire layout carries neither the IPv6 scope id +/// nor the flow label, so a scoped/flow-labelled address is not representable as +/// a peer-decodable contact and is refused rather than silently truncated. +#[test] +fn scoped_or_flow_labelled_ipv6_advertise_addr_is_rejected() { + use std::net::{Ipv6Addr, SocketAddrV6}; + + let scoped = SocketAddr::V6(SocketAddrV6::new( + Ipv6Addr::new(0xfe80, 0, 0, 0, 0, 0, 0, 1), + 7946, + 0, + 3, + )); + let err = validate_advertise_addr(&scoped).expect_err("a scoped IPv6 addr is not wire-carryable"); + assert!( + matches!(err, crate::SerfError::InvalidAdvertiseAddr(_)), + "a nonzero scope_id must be refused, got {err:?}" + ); + + let flow_labelled = SocketAddr::V6(SocketAddrV6::new( + Ipv6Addr::new(0x2001, 0xdb8, 0, 0, 0, 0, 0, 1), + 7946, + 7, + 0, + )); + let err = + validate_advertise_addr(&flow_labelled).expect_err("a flow label is not wire-carryable"); + assert!( + matches!(err, crate::SerfError::InvalidAdvertiseAddr(_)), + "a nonzero flowinfo must be refused, got {err:?}" + ); +} + +/// The rejection carries the offending address and a non-empty reason, so an +/// operator can tell WHICH address was refused and why. +#[test] +fn rejection_reports_the_offending_address() { + let addr: SocketAddr = "0.0.0.0:7946".parse().expect("advertise addr"); + match validate_advertise_addr(&addr) { + Err(crate::SerfError::InvalidAdvertiseAddr(e)) => { + assert_eq!(e.addr(), addr, "the error names the refused address"); + assert!(!e.reason().is_empty(), "the error carries a reason"); + } + other => panic!("expected InvalidAdvertiseAddr, got {other:?}"), + } +} + +/// A seed keyring whose keys collide across ciphers — the same raw bytes under +/// two cipher variants — is refused at construction: the coordinator's rotation +/// ops match on bytes alone, so such a ring would let a later `use`/`remove` +/// promote or drop the wrong cipher's key. A same-cipher multi-key ring, and a +/// cross-cipher ring with DISTINCT bytes, both stay admissible. +#[cfg(all(feature = "aes-gcm", feature = "chacha20-poly1305"))] +#[test] +fn cross_cipher_twin_keyring_is_rejected_at_construction() { + use memberlist_proto::{EncryptionOptions, Keyring, SecretKey}; + + use super::reject_cross_cipher_keyring; + + let aes = |b: u8| SecretKey::Aes256([b; 32]); + let chacha = |b: u8| SecretKey::ChaCha20Poly1305([b; 32]); + + let mut twinned = Keyring::new(aes(1)); + twinned.insert_secondary(chacha(1)); + let err = reject_cross_cipher_keyring(&EncryptionOptions::new().with_keyring(twinned)) + .expect_err("a cross-cipher byte twin makes every later key op ambiguous"); + assert!( + matches!(err, crate::SerfError::Io(ref e) if e.kind() == std::io::ErrorKind::InvalidInput), + "the twin refusal is an InvalidInput, got {err:?}" + ); + + let clean = Keyring::with_secondaries(aes(1), [aes(2), chacha(3)]); + assert!( + reject_cross_cipher_keyring(&EncryptionOptions::new().with_keyring(clean)).is_ok(), + "distinct key bytes across ciphers are unambiguous and stay admissible" + ); + + assert!( + reject_cross_cipher_keyring(&EncryptionOptions::new()).is_ok(), + "a node with no keyring configured has nothing to refuse" + ); +} diff --git a/serf-reactor/tests/cluster/mod.rs b/serf-reactor/tests/cluster/mod.rs index 9461e6c9..6642f9ea 100644 --- a/serf-reactor/tests/cluster/mod.rs +++ b/serf-reactor/tests/cluster/mod.rs @@ -1,21 +1,29 @@ //! Reusable multi-node fault-injection fixture for the reactor's real-node serf //! driver, shared by the runtime-generic scenario bodies in the sibling test -//! binaries. +//! binaries (TCP, TLS, QUIC). //! -//! A [`Cluster`] spins up N ephemeral loopback nodes through the ergonomic -//! `Serf::tcp` constructor with fast SWIM failure-detection timing (the transport -//! probe / gossip / suspicion overrides), so an abruptly-killed peer is detected -//! as Failed in well under a second. Each node runs a detached collector that -//! drains its event stream into a shared log; the log is a separate `Arc`, so it -//! survives an abrupt [`kill_abrupt`](Cluster::kill_abrupt) (which drops the -//! node's last handle) and later assertions read an ordered per-node member-event -//! history. +//! A [`Cluster`] spins up N ephemeral loopback nodes with fast SWIM +//! failure-detection timing (the transport probe / gossip / suspicion +//! overrides), so an abruptly-killed peer is detected as Failed in well under a +//! second. Each node runs a detached collector that drains its event stream into +//! a shared log; the log is a separate `Arc`, so it survives an abrupt +//! [`kill_abrupt`](Cluster::kill_abrupt) (which drops the node's last handle) and +//! later assertions read an ordered per-node member-event history. +//! +//! The fixture is TRANSPORT-AGNOSTIC: each test binary supplies a [`Backend`] +//! that maps the fixture's id / bind / [`ClusterTiming`] onto its own transport +//! options block and ergonomic `Serf::{tcp,tls,quic}` constructor, so the same +//! scenario bodies run over every reliable plane. //! //! Mirrors the legacy Go-parity cluster helpers — `wait_until_num_nodes` / //! `test_events` in `legacy/serf-core/src/serf/base/tests.rs` — adapted to the //! reactor's `Send`/`agnostic` model. -use core::time::Duration; +// The fixture is a shared harness: each test binary exercises the subset of its +// surface its own scenarios need. +#![allow(dead_code)] + +use core::{future::Future, time::Duration}; use std::{ net::SocketAddr, sync::{Arc, Mutex}, @@ -28,15 +36,34 @@ use serf_proto::{ members::MemberStatus, options::Options as SerfOptions, }; -use serf_reactor::{ - FirstAddrResolver, MaybeResolved, RuntimeOptions, Serf, SocketAddrResolver, TcpTransportOptions, - VoidDelegate, -}; +use serf_reactor::{MaybeResolved, Serf, SocketAddrResolver}; use smol_str::SmolStr; -/// A reactor TCP node handle over the agnostic runtime `R`. +/// A reactor node handle over the agnostic runtime `R`. Every backend binds the +/// same id and membership-address types; only the reliable record layer differs. pub type Node = Serf; +/// The per-transport half of the fixture: how to build one node at `bind` with +/// the fixture's [`ClusterTiming`]. +/// +/// Implemented once per test binary (TCP / TLS / QUIC) over its own transport +/// options block, so the scenario bodies below stay transport-agnostic. A +/// backend MUST apply every timing knob it can carry — the fault-injection +/// scenarios depend on the fast failure detection and the reap / reconnect +/// windows the fixture configures. +pub trait Backend: Send + Sync + 'static +where + R: Runtime, +{ + /// Build (and start) a node with id `id` bound to `bind` — an ephemeral + /// `127.0.0.1:0` for a fresh node, or a concrete address for a restart. + fn build( + id: &str, + bind: SocketAddr, + timing: &ClusterTiming, + ) -> impl Future>> + Send; +} + /// Wall-clock ceiling for every fixture poll loop, so a convergence or detection /// regression surfaces as a bounded timeout rather than a hang. const POLL_TIMEOUT: Duration = Duration::from_secs(20); @@ -115,6 +142,17 @@ impl ClusterTiming { self } + /// Override the direct-ping timeout — how long an unanswered probe waits + /// before escalating. A transport whose first probe to a peer must also + /// establish a session (QUIC's pooled connection) needs a wider timeout than + /// a connectionless datagram round-trip, or a LIVE peer is falsely suspected + /// while its session is still being set up. + #[must_use] + pub fn with_probe_timeout(mut self, v: Duration) -> Self { + self.probe_timeout = v; + self + } + /// Override the reconnect re-dial cadence — how often a survivor attempts to /// re-establish contact with a Failed member. Raise it beyond the test window /// to park the re-dial (and the push/pull merge it runs) out of the scenario. @@ -157,31 +195,32 @@ impl ClusterTiming { self } - /// The transport options for one node: the fast SWIM overrides plus this node's - /// id and advertise address. - fn transport_opts( - &self, - id: &str, - advertise: SocketAddr, - ) -> TcpTransportOptions { - let mut opts = TcpTransportOptions::::new() - .with_local_id(SmolStr::new(id)) - .with_advertise_addr(MaybeResolved::Resolved(advertise)) - .with_probe_interval(self.probe_interval) - .with_probe_timeout(self.probe_timeout) - .with_gossip_interval(self.gossip_interval) - .with_suspicion_mult(self.suspicion_mult); - if let Some(v) = self.dead_node_reclaim { - opts = opts.with_dead_node_reclaim_time(v); - } - if let Some(v) = self.push_pull_interval { - opts = opts.with_push_pull_interval(v); - } - opts + /// The SWIM failure-detection cadence a [`Backend`] applies to its transport + /// options block. + pub fn probe_interval(&self) -> Duration { + self.probe_interval + } + + /// The SWIM direct-ping timeout a [`Backend`] applies to its transport options + /// block. + pub fn probe_timeout(&self) -> Duration { + self.probe_timeout + } + + /// The gossip flush cadence a [`Backend`] applies to its transport options + /// block. + pub fn gossip_interval(&self) -> Duration { + self.gossip_interval + } + + /// The SWIM suspicion multiplier a [`Backend`] applies to its transport options + /// block. + pub fn suspicion_mult(&self) -> u32 { + self.suspicion_mult } /// The serf `Options` for every node: the fast reap / reconnect timing. - fn serf_opts(&self) -> SerfOptions { + pub fn serf_opts(&self) -> SerfOptions { SerfOptions::new() .with_reap_interval(self.reap_interval) .with_reconnect_interval(self.reconnect_interval) @@ -212,18 +251,22 @@ where log: Arc>>, } -/// A live multi-node loopback cluster with per-node member-event logs. -pub struct Cluster +/// A live multi-node loopback cluster with per-node member-event logs, driven +/// over the reliable plane the [`Backend`] `B` builds. +pub struct Cluster where R: Runtime, + B: Backend, { timing: ClusterTiming, slots: Vec>, + backend: core::marker::PhantomData B>, } -impl Cluster +impl Cluster where R: Runtime, + B: Backend, { /// Spawn `ids.len()` ephemeral loopback nodes with `timing`, attach a per-node /// event collector, join every non-seed node to the first (a star), and wait @@ -231,9 +274,9 @@ where pub async fn spawn(ids: &[&str], timing: ClusterTiming) -> Self { let mut slots = Vec::with_capacity(ids.len()); for id in ids { - let serf = build_node::(id, loopback_ephemeral(), &timing) + let serf = B::build(id, loopback_ephemeral(), &timing) .await - .expect("spawn serf tcp node"); + .expect("spawn serf node"); let addr = serf.advertise_address(); let log = Arc::new(Mutex::new(Vec::new())); // Attach the collector before the handle moves into the slot, so no member @@ -246,7 +289,11 @@ where log, }); } - let cluster = Self { timing, slots }; + let cluster = Self { + timing, + slots, + backend: core::marker::PhantomData, + }; let seed = cluster.slots[0].addr; for i in 1..cluster.slots.len() { cluster @@ -342,7 +389,7 @@ where const REBIND_RETRIES: usize = 25; let mut attempt = 0usize; let serf = loop { - match build_node::(id.as_str(), addr, &self.timing).await { + match B::build(id.as_str(), addr, &self.timing).await { Ok(serf) => break serf, // Ignoring Err: a transient rebind race (the freed port not yet reusable) // is retried; only the final attempt's error is fatal. @@ -377,13 +424,9 @@ where const DISTINCT_PORT_RETRIES: usize = 25; let mut attempt = 0usize; let serf = loop { - let serf = build_node::( - id.as_str(), - "127.0.0.1:0".parse().expect("loopback addr"), - &self.timing, - ) - .await - .expect("an ephemeral rebind cannot collide"); + let serf = B::build(id.as_str(), loopback_ephemeral(), &self.timing) + .await + .expect("an ephemeral rebind cannot collide"); if serf.advertise_address() != old_addr { break serf; } @@ -589,36 +632,10 @@ where } /// An ephemeral loopback bind address (`127.0.0.1:0`). -fn loopback_ephemeral() -> SocketAddr { +pub fn loopback_ephemeral() -> SocketAddr { "127.0.0.1:0".parse().expect("loopback addr") } -/// Spawn a fixture node at `bind` (an ephemeral `:0` for a fresh node, or a -/// concrete addr for a restart) with `timing`. -async fn build_node( - id: &str, - bind: SocketAddr, - timing: &ClusterTiming, -) -> serf_reactor::Result> -where - R: Runtime, -{ - Serf::::tcp( - timing.transport_opts(id, bind), - &SocketAddrResolver, - &FirstAddrResolver, - VoidDelegate::::new(), - RuntimeOptions::new(), - timing.serf_opts(), - None, - None, - None, - #[cfg(encryption)] - std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), - ) - .await -} - /// Attach a detached collector that drains `serf`'s event stream into `log`, /// recording every member event (kind + named ids). The collector holds only the /// stream, so a later kill (last-handle drop) still tears the node down while the diff --git a/serf-reactor/tests/quic.rs b/serf-reactor/tests/quic.rs index 8cc61ddb..d0b2b229 100644 --- a/serf-reactor/tests/quic.rs +++ b/serf-reactor/tests/quic.rs @@ -31,7 +31,7 @@ use rustls::{ use rustls_pki_types::{CertificateDer, PrivateKeyDer}; use serf_proto::{ event::{Event, MemberEventKind}, - members::SerfState, + members::{MemberStatus, SerfState}, options::Options as SerfOptions, }; #[cfg(encryption)] @@ -42,6 +42,10 @@ use serf_reactor::{ }; use smol_str::SmolStr; +/// The reusable multi-node fault-injection fixture, shared with the TCP and TLS +/// suites. +mod cluster; + /// A reactor QUIC node handle over the agnostic runtime `R`. type Node = Serf; @@ -127,9 +131,9 @@ fn test_client() -> quinn_proto::ClientConfig { } /// A fresh QUIC config bundle with a 20s idle timeout (well past a localhost -/// handshake) and datagram-mode unreliable transport. A fresh bundle is built per +/// handshake) and the given unreliable-transport mode. A fresh bundle is built per /// node so each owns its own cert and quinn endpoint config. -fn test_quic_options() -> QuicOptions { +fn quic_options_with(unreliable: UnreliableTransport) -> QuicOptions { let mut transport = quinn_proto::TransportConfig::default(); transport.max_idle_timeout(Some( quinn_proto::IdleTimeout::try_from(Duration::from_secs(20)).expect("idle timeout"), @@ -140,10 +144,79 @@ fn test_quic_options() -> QuicOptions { test_client(), transport, "localhost", - UnreliableTransport::Datagram, + unreliable, ) } +/// The default test bundle: datagram-mode unreliable transport. +fn test_quic_options() -> QuicOptions { + quic_options_with(UnreliableTransport::Datagram) +} + +/// An ephemeral loopback bind (`127.0.0.1:0`). +fn ephemeral_bind() -> SocketAddr { + "127.0.0.1:0".parse().expect("loopback addr") +} + +/// SWIM timing for the QUIC fault-injection cluster. +/// +/// QUIC's first probe to a peer must also establish the pooled connection, so the +/// direct-ping timeout is widened relative to the shared `fast()` profile: a LIVE +/// peer whose session is still handshaking must not be falsely suspected, while an +/// abrupt kill is still detected in well under a second. +fn quic_timing() -> cluster::ClusterTiming { + cluster::ClusterTiming::fast() + .with_probe_interval(Duration::from_millis(200)) + .with_probe_timeout(Duration::from_millis(150)) +} + +/// The fixture's QUIC backend: the fast-SWIM timing mapped onto a +/// [`QuicTransportOptions`] block carrying a fresh per-node quinn bundle. +struct Quic; + +impl cluster::Backend for Quic +where + R: Runtime, +{ + async fn build( + id: &str, + bind: SocketAddr, + timing: &cluster::ClusterTiming, + ) -> serf_reactor::Result> { + let mut opts = QuicTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(bind)) + .with_quic_config(test_quic_options()) + .with_probe_interval(timing.probe_interval()) + .with_probe_timeout(timing.probe_timeout()) + .with_gossip_interval(timing.gossip_interval()) + .with_suspicion_mult(timing.suspicion_mult()); + if let Some(v) = timing.dead_node_reclaim { + opts = opts.with_dead_node_reclaim_time(v); + } + if let Some(v) = timing.push_pull_interval { + opts = opts.with_push_pull_interval(v); + } + Serf::::quic( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + timing.serf_opts(), + None, + None, + None, + #[cfg(encryption)] + std::sync::Arc::new(VoidKeyringDelegate), + ) + .await + } +} + +/// The QUIC fault-injection cluster. +type QuicCluster = cluster::Cluster; + /// Build and spawn a reactor QUIC node on an ephemeral loopback port through the /// ergonomic `Serf::quic` constructor. async fn spawn_node(id: &str) -> Node @@ -945,145 +1018,1973 @@ where let _ = std::fs::remove_file(&path); } -// The tokio cells: the runtime-generic scenarios driven on tokio's multi-thread -// runtime. Gated on the `tokio` feature so the `--test quic -- smol` build (which -// enables only `smol`) can drop the `agnostic/tokio` code path. -#[cfg(feature = "tokio")] -mod tokio_cells { - use agnostic::tokio::TokioRuntime; +/// Over QUIC: node A joins node B, then B is abruptly killed. A must observe +/// Join → Failed → Reap about B — a Failed (not a Leave), proving the kill +/// discards the graceful-leave datagram, followed by the reaper removing the +/// failed member under the shortened reconnect timeout. +async fn serf_events_failed() +where + R: Runtime, +{ + let mut cluster = QuicCluster::::spawn(&["q-failed-a", "q-failed-b"], quic_timing()).await; + let subject = cluster.id(1); + + cluster.kill_abrupt(1).await; + cluster.await_num_members(0, 1).await; + + cluster + .assert_member_events( + 0, + subject.as_str(), + &[ + MemberEventKind::Join, + MemberEventKind::Failed, + MemberEventKind::Reap, + ], + ) + .await; + + cluster.shutdown_all().await; +} - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn two_node_quic_join_converges() { - super::two_node_quic_join_converges::().await; - } +/// Over QUIC: a graceful leave reaches the peer as a Leave (never a Failed) and +/// lands the leaver in the observer's Left tombstone view. +async fn serf_events_leave() +where + R: Runtime, +{ + let mut cluster = QuicCluster::::spawn( + &["q-leave-a", "q-leave-b"], + quic_timing().with_tombstone_timeout(Duration::from_secs(30)), + ) + .await; + let subject = cluster.id(1); - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn snapshot_leave_gate_controls_rejoin() { - super::snapshot_leave_gate_controls_rejoin::().await; - } + cluster.leave_graceful(1).await; - #[cfg(all(unix, encryption))] - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn file_backed_rotation_gates_the_response_on_persistence() { - super::file_backed_rotation_gates_the_response_on_persistence::().await; - } + cluster + .assert_member_events( + 0, + subject.as_str(), + &[MemberEventKind::Join, MemberEventKind::Leave], + ) + .await; + cluster.await_left_tombstone(0, subject.as_str()).await; - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn user_event_delivered() { - super::user_event_delivered::().await; - } + cluster.shutdown_all().await; +} - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn datagram_mode_gossip_rides_quic_datagrams() { - super::datagram_mode_gossip_rides_quic_datagrams::().await; - } +/// Over QUIC: a failed node that returns at the SAME address reconnects rather +/// than being reaped — the survivor's reconnect re-dial re-establishes the QUIC +/// connection and the member revives (Join → Failed → Join). +async fn serf_reconnect() +where + R: Runtime, +{ + let mut cluster = QuicCluster::::spawn( + &["q-recon-a", "q-recon-b"], + quic_timing().with_reconnect_timeout(Duration::from_secs(30)), + ) + .await; + let subject = cluster.id(1); + + cluster.kill_abrupt(1).await; + cluster + .await_member_event(0, subject.as_str(), MemberEventKind::Failed) + .await; + + cluster.restart(1).await; + cluster.await_num_members(0, 2).await; + + cluster + .assert_member_events( + 0, + subject.as_str(), + &[ + MemberEventKind::Join, + MemberEventKind::Failed, + MemberEventKind::Join, + ], + ) + .await; + + cluster.shutdown_all().await; +} - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn query_round_trip() { - super::query_round_trip::().await; - } +/// Over QUIC: an operator force-leaving a FAILED member transitions it to Left on +/// every surviving node rather than leaving it to linger Failed until the reap. +async fn serf_force_leave_failed() +where + R: Runtime, +{ + let mut cluster = QuicCluster::::spawn( + &["q-fl-a", "q-fl-b", "q-fl-c"], + quic_timing() + .with_tombstone_timeout(Duration::from_secs(120)) + .with_reconnect_timeout(Duration::from_secs(120)), + ) + .await; + let subject = cluster.id(2); + + cluster.kill_abrupt(2).await; + cluster + .await_member_event(0, subject.as_str(), MemberEventKind::Failed) + .await; + cluster + .await_member_event(1, subject.as_str(), MemberEventKind::Failed) + .await; + + cluster + .node(0) + .force_leave(subject.clone(), false) + .await + .expect("force_leave dispatches for a failed member"); + + cluster.await_left_tombstone(0, subject.as_str()).await; + cluster.await_left_tombstone(1, subject.as_str()).await; + cluster + .assert_member_events( + 0, + subject.as_str(), + &[ + MemberEventKind::Join, + MemberEventKind::Failed, + MemberEventKind::Leave, + ], + ) + .await; + + cluster.shutdown_all().await; +} - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn leave_emits_left_cluster() { - super::leave_emits_left_cluster::().await; - } +/// Over QUIC: removing a failed member WITH prune erases it outright on every +/// survivor — the membership count drops without waiting out the Left tombstone. +async fn serf_remove_failed_node_prune_erases() +where + R: Runtime, +{ + let mut cluster = QuicCluster::::spawn( + &["q-prune-a", "q-prune-b", "q-prune-c"], + quic_timing().with_reconnect_timeout(Duration::from_secs(120)), + ) + .await; + let subject = cluster.id(2); + + cluster.kill_abrupt(2).await; + cluster + .await_member_event(0, subject.as_str(), MemberEventKind::Failed) + .await; + cluster + .await_member_event(1, subject.as_str(), MemberEventKind::Failed) + .await; + + cluster + .node(0) + .remove_failed_node_prune(subject.clone()) + .await + .expect("remove_failed_node_prune dispatches"); - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn leave_with_racing_shutdown_reaches_peer_as_leave() { - super::leave_with_racing_shutdown_reaches_peer_as_leave::().await; - } + cluster.await_num_members(0, 2).await; + cluster.await_num_members(1, 2).await; - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn quic_shutdown_releases_bound_address_for_rebind() { - super::quic_shutdown_releases_bound_address_for_rebind::().await; - } + cluster.shutdown_all().await; +} - #[cfg(encryption)] - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn two_node_quic_gossip_convergence_encrypted() { - super::two_node_quic_gossip_convergence_encrypted::().await; - } +/// Over QUIC: a tag change propagates — the observer records an Update member +/// event for the setter and its member view carries the new value. +async fn serf_set_tags_propagates() +where + R: Runtime, +{ + let mut cluster = QuicCluster::::spawn(&["q-tags-a", "q-tags-b"], quic_timing()).await; + let b_id = cluster.id(1); + + let mut tags = serf_proto::Tags::new(); + tags.0.insert(SmolStr::new("role"), SmolStr::new("worker")); + cluster + .node(1) + .set_tags(tags) + .await + .expect("B re-tags itself"); + cluster + .await_member_event(0, b_id.as_str(), MemberEventKind::Update) + .await; + + let seen = cluster + .node(0) + .members() + .iter() + .find(|m| m.node().id_ref().as_str() == b_id.as_str()) + .map(|m| m.tags().0.get("role").cloned()) + .expect("A tracks B"); + assert_eq!( + seen.as_deref(), + Some("worker"), + "A's view of B carries the propagated role tag" + ); - #[cfg(encryption)] - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn mismatched_keyring_gossip_does_not_cross() { - super::mismatched_keyring_gossip_does_not_cross::().await; - } + cluster.shutdown_all().await; } -// The smol cells: the identical scenarios instantiated over `SmolRuntime` and -// driven by smol's `block_on`. The reactor QUIC poll task runs on smol's global -// executor, so the same scenario bodies verify the driver under a second runtime. -// `cargo test --test quic -- smol` selects exactly these. -#[cfg(feature = "smol")] -mod smol_cells { - use agnostic::{RuntimeLite, smol::SmolRuntime}; +/// Over QUIC: after a peer leaves gracefully, the observer reaps it while the +/// leaver holds its own self `Left` tombstone (never a self Reap) beside the +/// still-Alive peer. +async fn serf_join_leave() +where + R: Runtime, +{ + let mut cluster = QuicCluster::::spawn(&["q-jl-a", "q-jl-b"], quic_timing()).await; + let peer = cluster.id(0); + let leaver = cluster.id(1); - #[test] - fn snapshot_leave_gate_controls_rejoin_smol() { - SmolRuntime::block_on(super::snapshot_leave_gate_controls_rejoin::()); - } + cluster.leave_in_place(1).await; + cluster.await_num_members(0, 1).await; - #[cfg(all(unix, encryption))] - #[test] - fn file_backed_rotation_gates_the_response_on_persistence_smol() { - SmolRuntime::block_on( - super::file_backed_rotation_gates_the_response_on_persistence::(), + assert_eq!( + cluster.member_event_kinds(1, leaver.as_str()), + vec![MemberEventKind::Join, MemberEventKind::Leave], + "the leaver holds its self tombstone: Join then Leave, never a self Reap" + ); + cluster + .await_member_status(1, leaver.as_str(), MemberStatus::Left) + .await; + cluster + .await_member_status(1, peer.as_str(), MemberStatus::Alive) + .await; + + cluster.shutdown_all().await; +} + +/// `join_many` over two seeds — one reachable, one an unroutable blackhole port — +/// returns only the reached seed's address once both exchanges terminate. The +/// blackhole seed drives the QUIC driver's dial-failure path: a seed that never +/// establishes must retire its exchange, not hang the join. +async fn join_many_returns_only_reached_seeds() +where + R: Runtime, +{ + let b = spawn_node::("q-jm-b").await; + let a = spawn_node::("q-jm-a").await; + let b_addr = b.advertise_address(); + let blackhole: SocketAddr = "127.0.0.1:7219".parse().expect("loopback addr"); + + let reached = a + .join_many( + &SocketAddrResolver, + [ + MaybeResolved::Resolved(b_addr), + MaybeResolved::Resolved(blackhole), + ] + .into_iter(), + false, + ) + .await + .expect("join_many reaches the reachable seed"); + + assert_eq!(reached.len(), 1, "only the reachable seed is contacted"); + assert_eq!( + reached[0], b_addr, + "the reached set carries node B's address" + ); + + a.shutdown().await.expect("q-jm-a shuts down"); + b.shutdown().await.expect("q-jm-b shuts down"); +} + +/// Over QUIC: removing a name that is not a member reports success as a no-op and +/// leaves the membership view unchanged. +async fn remove_failed_node_absent_is_a_noop() +where + R: Runtime, +{ + let a = spawn_node::("q-absent-a").await; + a.remove_failed_node(SmolStr::new("no-such-node")) + .await + .expect("removing an absent member is an accepted no-op"); + assert_eq!(a.num_members(), 1, "the membership view is unchanged"); + a.shutdown().await.expect("q-absent-a shuts down"); +} + +/// Every mutating operation is REFUSED with `NotRunning` once the node has left +/// the cluster — `leave()` stops the periodic schedulers, so a post-leave mutation +/// would be issued by a node no longer participating. The read-only coordinate +/// probe is the deliberate exception: post-leave introspection stays valid. +/// +/// `respond` is included by capturing a live query token BEFORE the leave, which +/// is the only way its post-leave arm can be reached at all. +async fn post_leave_operations_report_not_running() +where + R: Runtime, +{ + let a = spawn_node::("q-nr-a").await; + let b = spawn_node::("q-nr-b").await; + let a_addr = a.advertise_address(); + + let mut b_events = b.events(); + b.join(&SocketAddrResolver, MaybeResolved::Resolved(a_addr), false) + .await + .expect("join reaches node A"); + converge(&a, &b).await; + + // Capture a live query token on B, so the post-leave `respond` below has + // something real to answer. + a.query("probe", Bytes::from_static(b"q"), a.default_query_param()) + .await + .expect("query issued"); + let token = R::timeout(Duration::from_secs(30), async { + loop { + match b_events.next().await { + Some(Event::Query(qe)) if qe.name() == "probe" => break Some(qe), + Some(_) => {} + None => break None, + } + } + }) + .await + .expect("B surfaces the query within the timeout") + .expect("B's event stream stays open"); + + b.leave().await.expect("B leaves the cluster"); + + let not_running = |e: &serf_reactor::SerfError| matches!(e, serf_reactor::SerfError::NotRunning); + + let err = b + .join(&SocketAddrResolver, MaybeResolved::Resolved(a_addr), false) + .await + .expect_err("a left node cannot rejoin in place"); + assert!(not_running(&err), "join after leave: got {err:?}"); + + for (what, res) in [ + ( + "user_event", + b.user_event("nope", Bytes::from_static(b"x"), false).await, + ), + ( + "query", + b.query("nope", Bytes::from_static(b"x"), b.default_query_param()) + .await + .map(|_| ()), + ), + ("respond", b.respond(token, Bytes::from_static(b"x")).await), + ("set_tags", b.set_tags(serf_proto::Tags::new()).await), + ( + "force_leave", + b.force_leave(SmolStr::new("q-nr-a"), false).await, + ), + ] { + let err = res.expect_err("a left node refuses to mutate"); + assert!( + not_running(&err), + "{what} after leave must report NotRunning, got {err:?}" ); } - #[test] - fn two_node_quic_join_converges_smol() { - SmolRuntime::block_on(super::two_node_quic_join_converges::()); - } + // The read-only coordinate probe still answers after the leave. + #[cfg(feature = "coordinates")] + b.cached_coordinate(SmolStr::new("q-nr-a")) + .await + .expect("a read-only coordinate probe stays answerable after leave"); - #[test] - fn user_event_delivered_smol() { - SmolRuntime::block_on(super::user_event_delivered::()); - } + a.shutdown().await.expect("q-nr-a shuts down"); + b.shutdown().await.expect("q-nr-b shuts down"); +} - #[test] - fn datagram_mode_gossip_rides_quic_datagrams_smol() { - SmolRuntime::block_on(super::datagram_mode_gossip_rides_quic_datagrams::< - SmolRuntime, - >()); +/// The key-management operations refuse to run on a node that has left — the same +/// `NotRunning` gate the membership mutations take, so a departed node can never +/// originate a cluster-wide rotation. +#[cfg(encryption)] +async fn post_leave_key_operations_report_not_running() +where + R: Runtime, +{ + let node = spawn_encrypted_node::( + "q-nrkey", + EncryptionOptions::new().with_keyring(Keyring::new(test_secret_key(0x71))), + ) + .await; + node.leave().await.expect("the node leaves the cluster"); + + let k = test_secret_key(0x72); + for (what, res) in [ + ("install_key", node.install_key(k).await.map(|_| ())), + ("use_key", node.use_key(k).await.map(|_| ())), + ("remove_key", node.remove_key(k).await.map(|_| ())), + ("list_keys", node.list_keys().await.map(|_| ())), + ] { + let err = res.expect_err("a left node refuses a key rotation"); + assert!( + matches!(err, serf_reactor::SerfError::NotRunning), + "{what} after leave must report NotRunning, got {err:?}" + ); } - #[test] - fn query_round_trip_smol() { - SmolRuntime::block_on(super::query_round_trip::()); - } + node.shutdown().await.expect("q-nrkey shuts down"); +} - #[test] - fn leave_emits_left_cluster_smol() { - SmolRuntime::block_on(super::leave_emits_left_cluster::()); - } +/// The full key-rotation lifecycle over QUIC: `install_key` adds a secondary on +/// every node, `use_key` promotes it to primary, `list_keys` reports the resulting +/// ring from BOTH nodes, and `remove_key` drops the retired key. Each step is +/// collected as a cluster-wide `KeyResponse`, so the assertions pin that the +/// rotation reached the peer — not merely the originator's own ring. +#[cfg(encryption)] +async fn key_rotation_lifecycle() +where + R: Runtime, +{ + let k1 = test_secret_key(0x81); + let k2 = test_secret_key(0x82); + let enc = || EncryptionOptions::new().with_keyring(Keyring::new(k1)); - #[test] - fn leave_with_racing_shutdown_reaches_peer_as_leave_smol() { - SmolRuntime::block_on(super::leave_with_racing_shutdown_reaches_peer_as_leave::< - SmolRuntime, - >()); - } + let b = spawn_encrypted_node::("q-rot-b", enc()).await; + let a = spawn_encrypted_node::("q-rot-a", enc()).await; + let b_addr = b.advertise_address(); - #[test] - fn quic_shutdown_releases_bound_address_for_rebind_smol() { - SmolRuntime::block_on(super::quic_shutdown_releases_bound_address_for_rebind::< - SmolRuntime, - >()); - } + let mut a_events = a.events(); + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B over the encrypted QUIC cluster"); + converge(&a, &b).await; - #[cfg(encryption)] - #[test] - fn two_node_quic_gossip_convergence_encrypted_smol() { - SmolRuntime::block_on(super::two_node_quic_gossip_convergence_encrypted::< - SmolRuntime, - >()); + /// Collect the next cluster-wide key response off A's stream. + async fn next_key_response( + events: &mut serf_reactor::EventStream, + ) -> serf_proto::event::KeyResponse + where + R: Runtime, + { + R::timeout(Duration::from_secs(30), async { + loop { + match events.next().await { + Some(Event::KeyResponse(kr)) => break kr, + Some(_) => {} + None => panic!("the event stream ended before the key response"), + } + } + }) + .await + .expect("the key response is collected within the query window") } - #[cfg(encryption)] - #[test] - fn mismatched_keyring_gossip_does_not_cross_smol() { - SmolRuntime::block_on(super::mismatched_keyring_gossip_does_not_cross::()); + a.install_key(k2).await.expect("install_key dispatched"); + let kr = next_key_response::(&mut a_events).await; + assert_eq!(kr.num_err, 0, "install_key must succeed on every node"); + assert!( + kr.num_resp >= 2, + "install_key must be answered by BOTH nodes (num_resp={})", + kr.num_resp + ); + + a.use_key(k2).await.expect("use_key dispatched"); + let kr = next_key_response::(&mut a_events).await; + assert_eq!(kr.num_err, 0, "use_key must succeed on every node"); + assert!(kr.num_resp >= 2, "use_key must be answered by BOTH nodes"); + + a.list_keys().await.expect("list_keys dispatched"); + let kr = next_key_response::(&mut a_events).await; + assert_eq!(kr.num_err, 0, "list_keys must succeed on every node"); + assert!( + kr.num_resp >= 2, + "list_keys must be answered by BOTH nodes — the rotation reached the peer" + ); + + // The promoted key is now the primary everywhere, so retiring the ORIGINAL key + // must be accepted (removing a live primary is refused). + a.remove_key(k1).await.expect("remove_key dispatched"); + let kr = next_key_response::(&mut a_events).await; + assert_eq!( + kr.num_err, 0, + "remove_key must succeed on every node once the key is no longer primary" + ); + assert!( + kr.num_resp >= 2, + "remove_key must be answered by BOTH nodes" + ); + + // The cluster still gossips under the rotated key: a user event still crosses. + let mut a_gossip = a.events(); + b.user_event("post-rotation", Bytes::from_static(b"ok"), false) + .await + .expect("user event dispatched"); + let got = R::timeout(GOSSIP_DELIVERY_WINDOW, async { + loop { + match a_gossip.next().await { + Some(Event::User(u)) if u.name.as_str() == "post-rotation" => { + break Some(u.payload.clone()); + } + Some(_) => {} + None => break None, + } + } + }) + .await + .expect("the rotated cluster still carries gossip"); + assert_eq!( + got, + Some(Bytes::from_static(b"ok")), + "gossip still crosses under the rotated primary key" + ); + + a.shutdown().await.expect("q-rot-a shuts down"); + b.shutdown().await.expect("q-rot-b shuts down"); +} + +/// The `Udp` unreliable-transport opt-out routes ALL gossip over the shared plain +/// UDP socket instead of the QUIC datagram plane: two nodes still join and gossip +/// a user event, and the sender's `datagrams_sent` stays at ZERO — the exact +/// discriminator the datagram-mode test asserts the opposite of. +async fn udp_mode_gossip_bypasses_the_datagram_plane() +where + R: Runtime, +{ + async fn spawn_udp_node(id: &str) -> Node + where + R: Runtime, + { + Serf::::quic( + QuicTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(ephemeral_bind())) + .with_quic_config(quic_options_with(UnreliableTransport::Udp)), + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + None, + None, + None, + #[cfg(encryption)] + std::sync::Arc::new(VoidKeyringDelegate), + ) + .await + .expect("spawn udp-mode serf quic node") + } + + let b = spawn_udp_node::("udp-b").await; + let a = spawn_udp_node::("udp-a").await; + let b_addr = b.advertise_address(); + + let mut a_events = a.events(); + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B over the QUIC reliable plane"); + converge(&a, &b).await; + + b.user_event("greet", Bytes::from_static(b"hello"), false) + .await + .expect("user event dispatched"); + + let got = R::timeout(Duration::from_secs(30), async { + loop { + match a_events.next().await { + Some(Event::User(u)) if u.name.as_str() == "greet" => break Some(u.payload.clone()), + Some(_) => {} + None => break None, + } + } + }) + .await + .expect("A observes B's user event over plain-UDP gossip"); + assert_eq!( + got, + Some(Bytes::from_static(b"hello")), + "the event crosses over the plain-UDP gossip plane" + ); + + // The discriminator: in `Udp` mode NO gossip payload may ride a QUIC datagram. + // The counter advances only on a `DatagramSendStatus::Queued`, so a driver that + // ignored the configured mode would leave it non-zero here. + assert_eq!( + b.datagrams_sent(), + 0, + "the Udp opt-out must route every gossip payload over the plain socket" + ); + assert_eq!(a.datagrams_sent(), 0); + + a.shutdown().await.expect("udp-a shuts down"); + b.shutdown().await.expect("udp-b shuts down"); +} + +/// The QUIC transport's construction gate: each field `QuicTransport::new` +/// requires is refused when absent, so a half-built options block can never bind +/// a socket. +async fn construction_requires_id_advertise_and_quic_config() +where + R: Runtime, +{ + async fn build( + opts: QuicTransportOptions, + ) -> serf_reactor::Result> + where + R: Runtime, + { + Serf::::quic( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + None, + None, + None, + #[cfg(encryption)] + std::sync::Arc::new(VoidKeyringDelegate), + ) + .await + } + + let cases: [(&str, QuicTransportOptions); 3] = [ + ( + "local_id", + QuicTransportOptions::new() + .with_advertise_addr(MaybeResolved::Resolved(ephemeral_bind())) + .with_quic_config(test_quic_options()), + ), + ( + "advertise_addr", + QuicTransportOptions::new() + .with_local_id(SmolStr::new("no-addr")) + .with_quic_config(test_quic_options()), + ), + ( + "quic_config", + QuicTransportOptions::new() + .with_local_id(SmolStr::new("no-cfg")) + .with_advertise_addr(MaybeResolved::Resolved(ephemeral_bind())), + ), + ]; + for (missing, opts) in cases { + let err = build::(opts) + .await + .err() + .unwrap_or_else(|| panic!("a node missing {missing} cannot be built")); + assert!( + matches!(err, serf_reactor::SerfError::Io(ref e) if e.kind() == std::io::ErrorKind::InvalidInput), + "a missing {missing} is an InvalidInput, got {err:?}" + ); + } +} + +/// A wildcard advertise address is refused AFTER the bind: the readback keeps the +/// unspecified IP, which peers could not route back to, so construction fails and +/// the bound socket is released rather than the node joining as an undialable +/// member. The released port is proven free by an immediate successful rebind of +/// the very port the failed attempt had claimed. +async fn wildcard_advertise_is_refused_and_releases_the_bind() +where + R: Runtime, +{ + let probe = spawn_node::("q-wild-probe").await; + let port = probe.advertise_address().port(); + probe.shutdown().await.expect("probe shuts down"); + + let build = |id: &'static str, addr: SocketAddr| async move { + Serf::::quic( + QuicTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(addr)) + .with_quic_config(test_quic_options()), + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + None, + None, + None, + #[cfg(encryption)] + std::sync::Arc::new(VoidKeyringDelegate), + ) + .await + }; + + let wildcard: SocketAddr = format!("0.0.0.0:{port}").parse().expect("wildcard addr"); + let err = build("q-wild", wildcard) + .await + .err() + .expect("a wildcard advertise address is not a routable contact"); + assert!( + matches!(err, serf_reactor::SerfError::InvalidAdvertiseAddr(_)), + "a wildcard bind must be refused as an invalid advertise address, got {err:?}" + ); + + // The refused construction released the bound socket: the same port rebinds. + let after = build( + "q-wild-after", + format!("127.0.0.1:{port}").parse().expect("loopback addr"), + ) + .await + .expect("the refused construction released the port it had bound"); + assert_eq!(after.advertise_address().port(), port); + after.shutdown().await.expect("q-wild-after shuts down"); +} + +/// An UNRESOLVED advertise address is resolved at construction through the +/// caller's resolvers, and the node comes up on the resolved contact. +async fn unresolved_advertise_addr_is_resolved_at_construction() +where + R: Runtime, +{ + let node = Serf::::quic( + QuicTransportOptions::::new() + .with_local_id(SmolStr::new("q-resolve-me")) + .with_advertise_addr(MaybeResolved::Unresolved(ephemeral_bind())) + .with_quic_config(test_quic_options()), + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + None, + None, + None, + #[cfg(encryption)] + std::sync::Arc::new(VoidKeyringDelegate), + ) + .await + .expect("the unresolved advertise address resolves through the supplied resolver"); + + let bound = node.advertise_address(); + assert!(bound.ip().is_loopback(), "the resolved contact is loopback"); + assert_ne!( + bound.port(), + 0, + "the ephemeral bind resolved to a concrete port" + ); + node.shutdown().await.expect("q-resolve-me shuts down"); +} + +/// The constructor-supplied merge delegate is the predicate the machine consults +/// on the QUIC join push/pull: with a recording accept-all delegate installed on +/// B, A's join drives at least one `notify_merge` on B carrying A's node state. +async fn merge_delegate_is_consulted_on_join() +where + R: Runtime, +{ + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct RecordingMerge { + hits: Arc, + saw_peer: Arc, + } + impl serf_reactor::MergeDelegate for RecordingMerge { + fn notify_merge( + &self, + peers: memberlist_proto::MaybeOwned< + '_, + [memberlist_proto::typed::NodeState], + >, + ) -> bool { + self.hits.fetch_add(1, Ordering::Relaxed); + if peers.iter().any(|p| p.id_ref().as_str() == "q-merge-a") { + self.saw_peer.fetch_add(1, Ordering::Relaxed); + } + true + } + } + + let hits = Arc::new(AtomicUsize::new(0)); + let saw_peer = Arc::new(AtomicUsize::new(0)); + + let b = Serf::::quic( + QuicTransportOptions::::new() + .with_local_id(SmolStr::new("q-merge-b")) + .with_advertise_addr(MaybeResolved::Resolved(ephemeral_bind())) + .with_quic_config(test_quic_options()), + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + None, + Some(Box::new(RecordingMerge { + hits: hits.clone(), + saw_peer: saw_peer.clone(), + })), + None, + #[cfg(encryption)] + std::sync::Arc::new(VoidKeyringDelegate), + ) + .await + .expect("spawn q-merge-b"); + let a = spawn_node::("q-merge-a").await; + let b_addr = b.advertise_address(); + + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + converge(&a, &b).await; + + assert!( + hits.load(Ordering::Relaxed) > 0, + "the machine must consult the constructor-supplied merge delegate on the join push/pull" + ); + assert!( + saw_peer.load(Ordering::Relaxed) > 0, + "the consulted peer set must carry the joining node's state" + ); + + a.shutdown().await.expect("q-merge-a shuts down"); + b.shutdown().await.expect("q-merge-b shuts down"); +} + +/// After a two-node QUIC join, the snapshot read-forwarders reflect the joined +/// cluster: `members` returns both nodes, `local_member` / `local_id` return this +/// node, `state` is `Alive`, `advertise_node` composes id + advertise, and the +/// operator aggregate reports the converged, healthy view. +async fn snapshot_forwarders_reflect_joined_cluster() +where + R: Runtime, +{ + let b = spawn_node::("q-snap-b").await; + let a = spawn_node::("q-snap-a").await; + let b_addr = b.advertise_address(); + let a_id = SmolStr::new("q-snap-a"); + let b_id = SmolStr::new("q-snap-b"); + + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + converge(&a, &b).await; + + let members = a.members(); + assert_eq!(members.len(), 2, "members() returns the 2-member cluster"); + let ids: Vec<&SmolStr> = members.iter().map(|m| m.node().id_ref()).collect(); + assert!(ids.contains(&&a_id), "members() includes local node A"); + assert!(ids.contains(&&b_id), "members() includes peer node B"); + + assert_eq!(a.local_member().node().id_ref(), &a_id); + assert_eq!(a.local_id(), a_id); + assert_eq!(a.state(), SerfState::Alive, "state() is Alive after join"); + assert_eq!(a.advertise_node().id_ref(), &a_id); + assert_eq!(a.advertise_node().addr_ref(), &a.advertise_address()); + + let stats = a.stats(); + assert_eq!(stats.members(), 2); + assert_eq!(stats.failed(), 0); + assert_eq!(stats.left(), 0); + assert_eq!(stats.health_score(), 0, "a healthy node scores 0"); + assert!(!a.encryption_enabled(), "no keyring is configured"); + + a.shutdown().await.expect("q-snap-a shuts down"); + b.shutdown().await.expect("q-snap-b shuts down"); +} + +/// After a two-node QUIC join, probe round-trips feed the Vivaldi coordinate +/// client: the local coordinate surfaces through the published snapshot, and the +/// peer's coordinate surfaces through the driver round-trip (`cached_coordinate`). +/// Both must go `Some` within the probe cadence — the discriminator that the QUIC +/// driver actually forwards coordinates rather than merely compiling the feature. +#[cfg(feature = "coordinates")] +async fn coordinates_surface_on_the_handle() +where + R: Runtime, +{ + let mut cluster = QuicCluster::::spawn(&["q-coord-a", "q-coord-b"], quic_timing()).await; + let b_id = cluster.id(1); + + let deadline = std::time::Instant::now() + Duration::from_secs(30); + loop { + let local = cluster.node(0).coordinate(); + let cached = cluster + .node(0) + .cached_coordinate(b_id.clone()) + .await + .expect("cached_coordinate round-trips through the driver"); + if local.is_some() && cached.is_some() { + break; + } + assert!( + std::time::Instant::now() < deadline, + "coordinates must surface on the handle: local={local:?} cached={cached:?}" + ); + R::sleep(Duration::from_millis(50)).await; + } + + cluster.shutdown_all().await; +} + +/// Leave is a SHARED in-flight operation: two concurrent `leave()` calls JOIN one +/// leave — the machine's `leave()` is invoked once and its single `LeftCluster` +/// resolves BOTH callers `Ok`. A THIRD leave issued after the chain completed is +/// an accepted no-op, not an error. +async fn concurrent_leaves_share_one_in_flight_leave() +where + R: Runtime, +{ + let b = spawn_node::("q-dl-b").await; + let a = spawn_node::("q-dl-a").await; + let b_addr = b.advertise_address(); + + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + converge(&a, &b).await; + + let (first, second) = future::join(a.leave(), a.leave()).await; + first.expect("the initiating leave resolves Ok"); + second.expect("the leave that JOINED the in-flight one resolves Ok too"); + + // The shared leave still drove the endpoint to Left (poll to absorb the + // snapshot-refresh race after the leave chain completes). + R::timeout(Duration::from_secs(5), async { + loop { + if a.state() == SerfState::Left { + break; + } + R::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("the shared leave drives the endpoint to Left"); + + a.leave() + .await + .expect("leaving an already-left node is an accepted no-op"); + + a.shutdown().await.expect("q-dl-a shuts down"); + b.shutdown().await.expect("q-dl-b shuts down"); +} + +/// An `ignore_old` join records each seed's exchange as a one-shot replay-suppress +/// target: the join still merges membership, and the joiner does NOT replay the +/// seed's pre-join user event. A plain join is the control — it DOES surface the +/// buffered event — so the suppression is proven, not merely asserted as absence. +async fn ignore_old_join_suppresses_the_replay() +where + R: Runtime, +{ + let seed = spawn_node::("q-io-seed").await; + let seed_addr = seed.advertise_address(); + + seed + .user_event("old-news", Bytes::from_static(b"stale"), false) + .await + .expect("the seed buffers a pre-join user event"); + + let plain = spawn_node::("q-io-plain").await; + let mut plain_events = plain.events(); + plain + .join( + &SocketAddrResolver, + MaybeResolved::Resolved(seed_addr), + false, + ) + .await + .expect("the plain join reaches the seed"); + let replayed = R::timeout(Duration::from_secs(30), async { + loop { + match plain_events.next().await { + Some(Event::User(u)) if u.name.as_str() == "old-news" => break true, + Some(_) => {} + None => break false, + } + } + }) + .await + .expect("a plain join replays the seed's buffered user event"); + assert!( + replayed, + "the control join must surface the buffered event, or the suppression below is vacuous" + ); + + let quiet = spawn_node::("q-io-quiet").await; + let mut quiet_events = quiet.events(); + let reached = quiet + .join( + &SocketAddrResolver, + MaybeResolved::Resolved(seed_addr), + true, + ) + .await + .expect("the ignore_old join reaches the seed"); + assert_eq!(reached, seed_addr, "the ignore_old join still merges"); + R::timeout(Duration::from_secs(30), async { + loop { + if quiet.num_members() >= 2 { + break; + } + R::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("the ignore_old join still converges the membership"); + + let saw = R::timeout(Duration::from_secs(2), async { + loop { + match quiet_events.next().await { + Some(Event::User(u)) if u.name.as_str() == "old-news" => break true, + Some(_) => {} + None => break false, + } + } + }) + .await + .unwrap_or(false); + assert!( + !saw, + "an ignore_old join must not replay the seed's pre-join user events" + ); + + quiet.shutdown().await.expect("q-io-quiet shuts down"); + plain.shutdown().await.expect("q-io-plain shuts down"); + seed.shutdown().await.expect("q-io-seed shuts down"); +} + +/// A datagram the gossip plane cannot parse is DROPPED and the node keeps serving. +/// On QUIC the shared UDP socket also carries quinn's own packets, so the ingress +/// must shrug off junk on BOTH demux paths without poisoning the pump. +async fn malformed_gossip_datagram_is_ignored() +where + R: Runtime, +{ + let a = spawn_node::("q-junk-a").await; + let a_addr = a.advertise_address(); + + let raw = std::net::UdpSocket::bind("127.0.0.1:0").expect("bind a raw sender"); + for payload in [ + &b"\xff\xff\xff\xff\xff\xff\xff\xff"[..], + &b""[..], + &[0x7fu8; 400][..], + ] { + raw + .send_to(payload, a_addr) + .expect("the garbage datagram is sent"); + } + + let b = spawn_node::("q-junk-b").await; + b.join(&SocketAddrResolver, MaybeResolved::Resolved(a_addr), false) + .await + .expect("the node still serves joins after the malformed datagrams"); + converge(&a, &b).await; + + a.shutdown().await.expect("q-junk-a shuts down"); + b.shutdown().await.expect("q-junk-b shuts down"); +} + +/// A subscriber that stops draining its `EventStream` must NOT stall the QUIC +/// driver: the fan-out sheds the events it cannot deliver and COUNTS them, and the +/// node keeps serving. +async fn slow_subscriber_sheds_events_and_counts_them() +where + R: Runtime, +{ + let node = Serf::::quic( + QuicTransportOptions::::new() + .with_local_id(SmolStr::new("q-shed-a")) + .with_advertise_addr(MaybeResolved::Resolved(ephemeral_bind())) + .with_quic_config(test_quic_options()), + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new().with_event_queue_cap(1), + SerfOptions::new(), + None, + None, + None, + #[cfg(encryption)] + std::sync::Arc::new(VoidKeyringDelegate), + ) + .await + .expect("spawn q-shed-a"); + + // Subscribe and NEVER poll the stream: the fan-out queue fills at once. + let _stalled = node.events(); + + for i in 0..64u32 { + node + .user_event("flood", Bytes::from(i.to_be_bytes().to_vec()), false) + .await + .expect("the driver keeps accepting commands while the subscriber stalls"); + } + + R::timeout(Duration::from_secs(20), async { + loop { + if node.events_dropped() > 0 { + break; + } + R::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("the fan-out sheds events a stalled subscriber cannot take, and counts them"); + + assert_eq!(node.num_members(), 1); + node.shutdown().await.expect("q-shed-a shuts down"); +} + +/// A delegate hook that PARKS must not wedge the QUIC pump: the observation channel +/// backs up, the pump retains what application data it can (a bounded overflow), +/// and once THAT is full it sheds the excess and COUNTS the loss — while the node +/// keeps accepting commands throughout. +async fn stalling_delegate_sheds_observations_and_counts_them() +where + R: Runtime, +{ + /// Enough payload events to fill the two-slot channel, then the pump's bounded + /// retry overflow (1024 entries), and still leave a surplus that must be shed. + const FLOOD: u32 = 1200; + + /// A delegate whose user-event hook parks until `gate`'s sender is dropped. It + /// does NOT override the test-only message-dropper hook, so the composite's + /// default (drop nothing) applies. + struct StallingDelegate { + gate: flume::Receiver<()>, + } + + impl serf_reactor::MemberDelegate for StallingDelegate { + type Id = SmolStr; + type Address = SocketAddr; + } + impl serf_reactor::QueryDelegate for StallingDelegate { + type Id = SmolStr; + type Address = SocketAddr; + } + impl serf_reactor::UserEventDelegate for StallingDelegate { + fn notify_user_event( + &self, + _event: &serf_proto::typed::UserEventMessage, + ) -> impl core::future::Future + Send + '_ { + let gate = self.gate.clone(); + async move { + // Ignoring Err: a disconnected gate means the test released the hook. + let _ = gate.recv_async().await; + } + } + } + impl serf_reactor::Delegate for StallingDelegate { + type Id = SmolStr; + type Address = SocketAddr; + } + + let (release, gate) = flume::bounded::<()>(0); + + let node = Serf::::quic( + QuicTransportOptions::::new() + .with_local_id(SmolStr::new("q-stall-a")) + .with_advertise_addr(MaybeResolved::Resolved(ephemeral_bind())) + .with_quic_config(test_quic_options()), + &SocketAddrResolver, + &FirstAddrResolver, + StallingDelegate { gate }, + RuntimeOptions::new().with_observation_channel(serf_reactor::Channel::Bounded(2)), + SerfOptions::new(), + None, + None, + None, + #[cfg(encryption)] + std::sync::Arc::new(VoidKeyringDelegate), + ) + .await + .expect("spawn q-stall-a"); + + let payload = Bytes::from(vec![0x5au8; 32]); + for _ in 0..FLOOD { + node + .user_event("flood", payload.clone(), false) + .await + .expect("the pump keeps accepting commands while the delegate parks"); + } + + R::timeout(Duration::from_secs(20), async { + loop { + if node.observation_dropped() > 0 { + break; + } + R::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("the pump sheds observations a parked delegate cannot take, and counts them"); + + assert_eq!(node.num_members(), 1); + + drop(release); + node.shutdown().await.expect("q-stall-a shuts down"); +} + +/// A shutdown racing an IN-FLIGHT await-result join resolves that join +/// `Err(Shutdown)` — never leaves it parked forever. The seed address is a bound +/// UDP socket that speaks no QUIC, so the handshake never completes and the +/// exchange is still pending inside the driver when the teardown reaps the waiter. +async fn shutdown_racing_an_inflight_join_resolves_it() +where + R: Runtime, +{ + // A UDP socket that binds the port but never answers a QUIC Initial. + let silent = std::net::UdpSocket::bind("127.0.0.1:0").expect("bind the silent seed"); + let seed_addr = silent.local_addr().expect("the silent seed's address"); + + let node = spawn_node::("q-race-join").await; + + let (join, shutdown) = future::join( + node.join( + &SocketAddrResolver, + MaybeResolved::Resolved(seed_addr), + false, + ), + async { + // Let the Initial go out before the teardown begins, so the waiter is + // genuinely in flight rather than never dispatched. + R::sleep(Duration::from_millis(200)).await; + node.shutdown().await + }, + ) + .await; + + shutdown.expect("the node shuts down"); + let err = join.expect_err("a join racing a shutdown cannot succeed against a silent seed"); + assert!( + matches!( + err, + serf_reactor::SerfError::Shutdown | serf_reactor::SerfError::JoinAllFailed(_) + ), + "the in-flight join must be resolved by the teardown, not stranded (got {err:?})" + ); +} + +/// A second `shutdown()` — issued once the driver has already exited and closed +/// its command queue — still resolves `Ok`, and only AFTER the bind address is +/// actually free: the late caller parks on the teardown-completion latch instead +/// of returning into a still-bound port. +async fn second_shutdown_awaits_teardown_completion() +where + R: Runtime, +{ + let node = spawn_node::("q-twice-a").await; + let addr = node.advertise_address(); + + node.shutdown().await.expect("the first shutdown resolves"); + node + .shutdown() + .await + .expect("a second shutdown after teardown still resolves Ok"); + + let err = node + .user_event("post", Bytes::from_static(b"x"), false) + .await + .expect_err("a shut-down node accepts no commands"); + assert!( + matches!(err, serf_reactor::SerfError::Shutdown), + "a post-shutdown command reports Shutdown, got {err:?}" + ); + + // The latch fired only once the bind address was free: rebinding it succeeds. + let reborn = Serf::::quic( + QuicTransportOptions::::new() + .with_local_id(SmolStr::new("q-twice-b")) + .with_advertise_addr(MaybeResolved::Resolved(addr)) + .with_quic_config(test_quic_options()), + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + None, + None, + None, + #[cfg(encryption)] + std::sync::Arc::new(VoidKeyringDelegate), + ) + .await + .expect("the freed address rebinds after the awaited teardown"); + assert_eq!(reborn.advertise_address(), addr); + reborn.shutdown().await.expect("q-twice-b shuts down"); +} + +/// The fire-and-forget `dispatch_join` reports how many seeds it DISPATCHED +/// without waiting for any of them, and the membership it started still converges +/// through the driver's own exchange completion. A dispatch on a LEFT node is +/// refused rather than silently swallowed. +async fn dispatch_join_reports_the_dispatched_seed_count() +where + R: Runtime, +{ + let b = spawn_node::("q-dj-b").await; + let a = spawn_node::("q-dj-a").await; + let b_addr = b.advertise_address(); + let blackhole: SocketAddr = "127.0.0.1:7219".parse().expect("loopback addr"); + + let dispatched = a + .dispatch_join( + &SocketAddrResolver, + &[ + MaybeResolved::Resolved(b_addr), + MaybeResolved::Resolved(blackhole), + ], + ) + .await + .expect("the join dispatches"); + assert_eq!( + dispatched, 2, + "dispatch_join counts every seed it started an exchange against, reachable or not" + ); + + // The reachable seed still merges — the fire-and-forget join is real, not a no-op. + converge(&a, &b).await; + + // A dispatch on a left node is refused. + a.leave().await.expect("A leaves the cluster"); + let err = a + .dispatch_join(&SocketAddrResolver, &[MaybeResolved::Resolved(b_addr)]) + .await + .expect_err("a left node cannot dispatch a join"); + assert!( + matches!(err, serf_reactor::SerfError::NotRunning), + "a post-leave dispatch_join reports NotRunning, got {err:?}" + ); + + a.shutdown().await.expect("q-dj-a shuts down"); + b.shutdown().await.expect("q-dj-b shuts down"); +} + +/// Every SWIM override the QUIC transport options carry is threaded into the +/// coordinator the driver builds. Two nodes configured with the FULL override set +/// — including the reclaim window and the suspicion ceiling that no other scenario +/// sets — still join, converge, and detect an abrupt kill, so no override is +/// dropped or mis-wired on the way through `Transport::run`. +async fn full_swim_override_set_is_threaded_into_the_coordinator() +where + R: Runtime, +{ + async fn spawn_tuned(id: &str) -> Node + where + R: Runtime, + { + Serf::::quic( + QuicTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(ephemeral_bind())) + .with_quic_config(test_quic_options()) + .with_probe_interval(Duration::from_millis(200)) + .with_probe_timeout(Duration::from_millis(150)) + .with_gossip_interval(Duration::from_millis(20)) + .with_suspicion_mult(3) + .with_suspicion_max_timeout_mult(4) + .with_dead_node_reclaim_time(Duration::from_millis(1)) + .with_push_pull_interval(Duration::from_millis(500)), + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + None, + None, + None, + #[cfg(encryption)] + std::sync::Arc::new(VoidKeyringDelegate), + ) + .await + .expect("spawn a fully-tuned serf quic node") + } + + let b = spawn_tuned::("q-tuned-b").await; + let a = spawn_tuned::("q-tuned-a").await; + let b_addr = b.advertise_address(); + let b_id = SmolStr::new("q-tuned-b"); + + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("the fully-tuned nodes still join"); + converge(&a, &b).await; + + // The tuned failure detection still fires: an abrupt kill is detected. + b.shutdown().await.expect("q-tuned-b shuts down abruptly"); + R::timeout(Duration::from_secs(30), async { + loop { + let failed = a + .members() + .iter() + .any(|m| m.node().id_ref() == &b_id && m.status() != MemberStatus::Alive); + if failed { + break; + } + R::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("the tuned SWIM overrides still detect the killed peer"); + + a.shutdown().await.expect("q-tuned-a shuts down"); +} + +/// A resolver that resolves the advertise address to NO candidate fails +/// construction rather than booting a QUIC node with no reachable contact. +async fn advertise_resolution_failure_fails_construction() +where + R: Runtime, +{ + /// Resolves nothing — a bootstrap outage the advertise picker must refuse. + struct EmptyResolver; + impl serf_reactor::Resolver for EmptyResolver { + type Address = SocketAddr; + type Error = std::io::Error; + fn resolve( + &self, + _addr: &SocketAddr, + ) -> impl core::future::Future>> + Send + '_ { + // The candidate set is empty whatever the input, so the future borrows nothing. + let out = Vec::new(); + async move { Ok(out) } + } + } + + let err = Serf::::quic( + QuicTransportOptions::::new() + .with_local_id(SmolStr::new("q-unresolvable")) + .with_advertise_addr(MaybeResolved::Unresolved(ephemeral_bind())) + .with_quic_config(test_quic_options()), + &EmptyResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + None, + None, + None, + #[cfg(encryption)] + std::sync::Arc::new(VoidKeyringDelegate), + ) + .await + .err() + .expect("an advertise address that resolves to nothing cannot boot a node"); + assert!( + matches!(err, serf_reactor::SerfError::Resolve(_)), + "an empty candidate set is a resolution failure, got {err:?}" + ); +} + +/// A key-management request on a node with NO keyring is REFUSED, not silently +/// applied: the responder answers `result = false` with an explanatory message and +/// leaves the wire untouched, so the originator's collected response carries the +/// error rather than a false success. +#[cfg(encryption)] +async fn key_op_without_a_keyring_is_refused() +where + R: Runtime, +{ + // Two PLAINTEXT nodes — neither carries a gossip keyring. + let b = spawn_node::("q-nokey-b").await; + let a = spawn_node::("q-nokey-a").await; + let b_addr = b.advertise_address(); + + let mut a_events = a.events(); + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + converge(&a, &b).await; + + a.install_key(test_secret_key(0x78)) + .await + .expect("the request itself dispatches"); + + let kr = R::timeout(Duration::from_secs(30), async { + loop { + match a_events.next().await { + Some(Event::KeyResponse(kr)) => break kr, + Some(_) => {} + None => panic!("the event stream ended before the key response"), + } + } + }) + .await + .expect("A collects the key response within the query window"); + + assert!( + kr.num_err > 0, + "a node with no keyring must REFUSE the key op, not report success (num_err={}, num_resp={})", + kr.num_err, + kr.num_resp + ); + assert!( + !kr.messages.is_empty(), + "the refusal carries an explanatory message" + ); + assert!( + !a.encryption_enabled() && !b.encryption_enabled(), + "the refused op left both nodes plaintext" + ); + + a.shutdown().await.expect("q-nokey-a shuts down"); + b.shutdown().await.expect("q-nokey-b shuts down"); +} + +// The tokio cells: the runtime-generic scenarios driven on tokio's multi-thread +// runtime. Gated on the `tokio` feature so the `--test quic -- smol` build (which +// enables only `smol`) can drop the `agnostic/tokio` code path. +#[cfg(feature = "tokio")] +mod tokio_cells { + use agnostic::tokio::TokioRuntime; + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn two_node_quic_join_converges() { + super::two_node_quic_join_converges::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn concurrent_leaves_share_one_in_flight_leave() { + super::concurrent_leaves_share_one_in_flight_leave::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn ignore_old_join_suppresses_the_replay() { + super::ignore_old_join_suppresses_the_replay::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn malformed_gossip_datagram_is_ignored() { + super::malformed_gossip_datagram_is_ignored::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn slow_subscriber_sheds_events_and_counts_them() { + super::slow_subscriber_sheds_events_and_counts_them::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn stalling_delegate_sheds_observations_and_counts_them() { + super::stalling_delegate_sheds_observations_and_counts_them::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn shutdown_racing_an_inflight_join_resolves_it() { + super::shutdown_racing_an_inflight_join_resolves_it::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn second_shutdown_awaits_teardown_completion() { + super::second_shutdown_awaits_teardown_completion::().await; + } + + #[cfg(encryption)] + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn key_op_without_a_keyring_is_refused() { + super::key_op_without_a_keyring_is_refused::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn dispatch_join_reports_the_dispatched_seed_count() { + super::dispatch_join_reports_the_dispatched_seed_count::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn full_swim_override_set_is_threaded_into_the_coordinator() { + super::full_swim_override_set_is_threaded_into_the_coordinator::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn advertise_resolution_failure_fails_construction() { + super::advertise_resolution_failure_fails_construction::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_events_failed() { + super::serf_events_failed::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_events_leave() { + super::serf_events_leave::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_reconnect() { + super::serf_reconnect::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_force_leave_failed() { + super::serf_force_leave_failed::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_remove_failed_node_prune_erases() { + super::serf_remove_failed_node_prune_erases::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_set_tags_propagates() { + super::serf_set_tags_propagates::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_join_leave() { + super::serf_join_leave::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn join_many_returns_only_reached_seeds() { + super::join_many_returns_only_reached_seeds::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn remove_failed_node_absent_is_a_noop() { + super::remove_failed_node_absent_is_a_noop::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn post_leave_operations_report_not_running() { + super::post_leave_operations_report_not_running::().await; + } + + #[cfg(encryption)] + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn post_leave_key_operations_report_not_running() { + super::post_leave_key_operations_report_not_running::().await; + } + + #[cfg(encryption)] + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn key_rotation_lifecycle() { + super::key_rotation_lifecycle::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn udp_mode_gossip_bypasses_the_datagram_plane() { + super::udp_mode_gossip_bypasses_the_datagram_plane::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn construction_requires_id_advertise_and_quic_config() { + super::construction_requires_id_advertise_and_quic_config::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn wildcard_advertise_is_refused_and_releases_the_bind() { + super::wildcard_advertise_is_refused_and_releases_the_bind::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn unresolved_advertise_addr_is_resolved_at_construction() { + super::unresolved_advertise_addr_is_resolved_at_construction::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn merge_delegate_is_consulted_on_join() { + super::merge_delegate_is_consulted_on_join::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn snapshot_forwarders_reflect_joined_cluster() { + super::snapshot_forwarders_reflect_joined_cluster::().await; + } + + #[cfg(feature = "coordinates")] + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn coordinates_surface_on_the_handle() { + super::coordinates_surface_on_the_handle::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn snapshot_leave_gate_controls_rejoin() { + super::snapshot_leave_gate_controls_rejoin::().await; + } + + #[cfg(all(unix, encryption))] + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn file_backed_rotation_gates_the_response_on_persistence() { + super::file_backed_rotation_gates_the_response_on_persistence::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn user_event_delivered() { + super::user_event_delivered::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn datagram_mode_gossip_rides_quic_datagrams() { + super::datagram_mode_gossip_rides_quic_datagrams::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn query_round_trip() { + super::query_round_trip::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn leave_emits_left_cluster() { + super::leave_emits_left_cluster::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn leave_with_racing_shutdown_reaches_peer_as_leave() { + super::leave_with_racing_shutdown_reaches_peer_as_leave::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn quic_shutdown_releases_bound_address_for_rebind() { + super::quic_shutdown_releases_bound_address_for_rebind::().await; + } + + #[cfg(encryption)] + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn two_node_quic_gossip_convergence_encrypted() { + super::two_node_quic_gossip_convergence_encrypted::().await; + } + + #[cfg(encryption)] + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn mismatched_keyring_gossip_does_not_cross() { + super::mismatched_keyring_gossip_does_not_cross::().await; + } +} + +// The smol cells: the identical scenarios instantiated over `SmolRuntime` and +// driven by smol's `block_on`. The reactor QUIC poll task runs on smol's global +// executor, so the same scenario bodies verify the driver under a second runtime. +// `cargo test --test quic -- smol` selects exactly these. +#[cfg(feature = "smol")] +mod smol_cells { + use agnostic::{RuntimeLite, smol::SmolRuntime}; + + #[test] + fn snapshot_leave_gate_controls_rejoin_smol() { + SmolRuntime::block_on(super::snapshot_leave_gate_controls_rejoin::()); + } + + #[cfg(all(unix, encryption))] + #[test] + fn file_backed_rotation_gates_the_response_on_persistence_smol() { + SmolRuntime::block_on( + super::file_backed_rotation_gates_the_response_on_persistence::(), + ); + } + + #[test] + fn two_node_quic_join_converges_smol() { + SmolRuntime::block_on(super::two_node_quic_join_converges::()); + } + + #[test] + fn user_event_delivered_smol() { + SmolRuntime::block_on(super::user_event_delivered::()); + } + + #[test] + fn datagram_mode_gossip_rides_quic_datagrams_smol() { + SmolRuntime::block_on(super::datagram_mode_gossip_rides_quic_datagrams::< + SmolRuntime, + >()); + } + + #[test] + fn query_round_trip_smol() { + SmolRuntime::block_on(super::query_round_trip::()); + } + + #[test] + fn leave_emits_left_cluster_smol() { + SmolRuntime::block_on(super::leave_emits_left_cluster::()); + } + + #[test] + fn leave_with_racing_shutdown_reaches_peer_as_leave_smol() { + SmolRuntime::block_on(super::leave_with_racing_shutdown_reaches_peer_as_leave::< + SmolRuntime, + >()); + } + + #[test] + fn quic_shutdown_releases_bound_address_for_rebind_smol() { + SmolRuntime::block_on(super::quic_shutdown_releases_bound_address_for_rebind::< + SmolRuntime, + >()); + } + + #[cfg(encryption)] + #[test] + fn two_node_quic_gossip_convergence_encrypted_smol() { + SmolRuntime::block_on(super::two_node_quic_gossip_convergence_encrypted::< + SmolRuntime, + >()); + } + + #[cfg(encryption)] + #[test] + fn mismatched_keyring_gossip_does_not_cross_smol() { + SmolRuntime::block_on(super::mismatched_keyring_gossip_does_not_cross::()); + } + + #[test] + fn serf_events_failed_smol() { + SmolRuntime::block_on(super::serf_events_failed::()); + } + + #[test] + fn serf_events_leave_smol() { + SmolRuntime::block_on(super::serf_events_leave::()); + } + + #[test] + fn serf_reconnect_smol() { + SmolRuntime::block_on(super::serf_reconnect::()); + } + + #[test] + fn serf_force_leave_failed_smol() { + SmolRuntime::block_on(super::serf_force_leave_failed::()); + } + + #[test] + fn serf_remove_failed_node_prune_erases_smol() { + SmolRuntime::block_on(super::serf_remove_failed_node_prune_erases::()); + } + + #[test] + fn serf_set_tags_propagates_smol() { + SmolRuntime::block_on(super::serf_set_tags_propagates::()); + } + + #[test] + fn serf_join_leave_smol() { + SmolRuntime::block_on(super::serf_join_leave::()); + } + + #[test] + fn join_many_returns_only_reached_seeds_smol() { + SmolRuntime::block_on(super::join_many_returns_only_reached_seeds::()); + } + + #[test] + fn remove_failed_node_absent_is_a_noop_smol() { + SmolRuntime::block_on(super::remove_failed_node_absent_is_a_noop::()); + } + + #[test] + fn post_leave_operations_report_not_running_smol() { + SmolRuntime::block_on(super::post_leave_operations_report_not_running::()); + } + + #[cfg(encryption)] + #[test] + fn post_leave_key_operations_report_not_running_smol() { + SmolRuntime::block_on(super::post_leave_key_operations_report_not_running::< + SmolRuntime, + >()); + } + + #[cfg(encryption)] + #[test] + fn key_rotation_lifecycle_smol() { + SmolRuntime::block_on(super::key_rotation_lifecycle::()); + } + + #[test] + fn udp_mode_gossip_bypasses_the_datagram_plane_smol() { + SmolRuntime::block_on(super::udp_mode_gossip_bypasses_the_datagram_plane::< + SmolRuntime, + >()); + } + + #[test] + fn construction_requires_id_advertise_and_quic_config_smol() { + SmolRuntime::block_on(super::construction_requires_id_advertise_and_quic_config::< + SmolRuntime, + >()); + } + + #[test] + fn wildcard_advertise_is_refused_and_releases_the_bind_smol() { + SmolRuntime::block_on( + super::wildcard_advertise_is_refused_and_releases_the_bind::(), + ); + } + + #[test] + fn unresolved_advertise_addr_is_resolved_at_construction_smol() { + SmolRuntime::block_on( + super::unresolved_advertise_addr_is_resolved_at_construction::(), + ); + } + + #[test] + fn merge_delegate_is_consulted_on_join_smol() { + SmolRuntime::block_on(super::merge_delegate_is_consulted_on_join::()); + } + + #[test] + fn snapshot_forwarders_reflect_joined_cluster_smol() { + SmolRuntime::block_on(super::snapshot_forwarders_reflect_joined_cluster::< + SmolRuntime, + >()); + } + + #[cfg(feature = "coordinates")] + #[test] + fn coordinates_surface_on_the_handle_smol() { + SmolRuntime::block_on(super::coordinates_surface_on_the_handle::()); + } + + #[test] + fn concurrent_leaves_share_one_in_flight_leave_smol() { + SmolRuntime::block_on(super::concurrent_leaves_share_one_in_flight_leave::< + SmolRuntime, + >()); + } + + #[test] + fn ignore_old_join_suppresses_the_replay_smol() { + SmolRuntime::block_on(super::ignore_old_join_suppresses_the_replay::()); + } + + #[test] + fn malformed_gossip_datagram_is_ignored_smol() { + SmolRuntime::block_on(super::malformed_gossip_datagram_is_ignored::()); + } + + #[test] + fn slow_subscriber_sheds_events_and_counts_them_smol() { + SmolRuntime::block_on(super::slow_subscriber_sheds_events_and_counts_them::< + SmolRuntime, + >()); + } + + #[test] + fn stalling_delegate_sheds_observations_and_counts_them_smol() { + SmolRuntime::block_on( + super::stalling_delegate_sheds_observations_and_counts_them::(), + ); + } + + #[test] + fn shutdown_racing_an_inflight_join_resolves_it_smol() { + SmolRuntime::block_on(super::shutdown_racing_an_inflight_join_resolves_it::< + SmolRuntime, + >()); + } + + #[test] + fn second_shutdown_awaits_teardown_completion_smol() { + SmolRuntime::block_on(super::second_shutdown_awaits_teardown_completion::< + SmolRuntime, + >()); + } + + #[cfg(encryption)] + #[test] + fn key_op_without_a_keyring_is_refused_smol() { + SmolRuntime::block_on(super::key_op_without_a_keyring_is_refused::()); + } + + #[test] + fn dispatch_join_reports_the_dispatched_seed_count_smol() { + SmolRuntime::block_on(super::dispatch_join_reports_the_dispatched_seed_count::< + SmolRuntime, + >()); + } + + #[test] + fn full_swim_override_set_is_threaded_into_the_coordinator_smol() { + SmolRuntime::block_on( + super::full_swim_override_set_is_threaded_into_the_coordinator::(), + ); + } + + #[test] + fn advertise_resolution_failure_fails_construction_smol() { + SmolRuntime::block_on(super::advertise_resolution_failure_fails_construction::< + SmolRuntime, + >()); } } diff --git a/serf-reactor/tests/tcp.rs b/serf-reactor/tests/tcp.rs index 801e1a70..2ed49d62 100644 --- a/serf-reactor/tests/tcp.rs +++ b/serf-reactor/tests/tcp.rs @@ -31,8 +31,8 @@ use serf_proto::{ #[cfg(encryption)] use serf_reactor::{EncryptionOptions, Keyring, SecretKey, VoidKeyringDelegate}; use serf_reactor::{ - FirstAddrResolver, MaybeResolved, RuntimeOptions, Serf, SocketAddrResolver, TcpTransportOptions, - VoidDelegate, + FirstAddrResolver, MaybeResolved, RuntimeOptions, Serf, SerfError, SocketAddrResolver, + TcpTransportOptions, VoidDelegate, }; use smol_str::SmolStr; @@ -43,6 +43,52 @@ mod cluster; /// A reactor TCP node handle over the agnostic runtime `R`. type Node = Serf; +/// The fixture's plain-TCP backend: the fast-SWIM timing mapped onto a +/// [`TcpTransportOptions`] block. +struct Tcp; + +impl cluster::Backend for Tcp +where + R: Runtime, +{ + async fn build( + id: &str, + bind: SocketAddr, + timing: &cluster::ClusterTiming, + ) -> serf_reactor::Result> { + let mut opts = TcpTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(bind)) + .with_probe_interval(timing.probe_interval()) + .with_probe_timeout(timing.probe_timeout()) + .with_gossip_interval(timing.gossip_interval()) + .with_suspicion_mult(timing.suspicion_mult()); + if let Some(v) = timing.dead_node_reclaim { + opts = opts.with_dead_node_reclaim_time(v); + } + if let Some(v) = timing.push_pull_interval { + opts = opts.with_push_pull_interval(v); + } + Serf::::tcp( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + timing.serf_opts(), + None, + None, + None, + #[cfg(encryption)] + std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), + ) + .await + } +} + +/// The plain-TCP fault-injection cluster. +type TcpCluster = cluster::Cluster; + /// A [`ReconnectDelegate`](serf_proto::ReconnectDelegate) that forces an immediate /// reap (zero timeout) for one target member id and passes every other member /// through the configured base timeout unchanged. @@ -503,7 +549,7 @@ async fn serf_events_failed() where R: Runtime, { - let mut cluster = cluster::Cluster::::spawn( + let mut cluster = TcpCluster::::spawn( &["events-failed-a", "events-failed-b"], cluster::ClusterTiming::fast(), ) @@ -542,7 +588,7 @@ where // Raise A's tombstone timeout past the test window so the reaper holds B's // Left tombstone rather than appending a trailing Reap to the observed // sequence (the fast profile otherwise reaps a left member sub-second). - let mut cluster = cluster::Cluster::::spawn( + let mut cluster = TcpCluster::::spawn( &["events-leave-a", "events-leave-b"], cluster::ClusterTiming::fast().with_tombstone_timeout(Duration::from_secs(30)), ) @@ -584,7 +630,7 @@ where R: Runtime, { // Hold B's Left tombstone past the test window, as in `serf_events_leave`. - let mut cluster = cluster::Cluster::::spawn( + let mut cluster = TcpCluster::::spawn( &["leave-race-a", "leave-race-b"], cluster::ClusterTiming::fast().with_tombstone_timeout(Duration::from_secs(30)), ) @@ -620,7 +666,7 @@ where R: Runtime, { let mut cluster = - cluster::Cluster::::spawn(&["coord-a", "coord-b"], cluster::ClusterTiming::fast()).await; + TcpCluster::::spawn(&["coord-a", "coord-b"], cluster::ClusterTiming::fast()).await; let b_id = cluster.id(1); // Probe RTTs accumulate at the fast profile's 100ms cadence; both surfaces @@ -961,7 +1007,7 @@ async fn serf_reconnect() where R: Runtime, { - let mut cluster = cluster::Cluster::::spawn( + let mut cluster = TcpCluster::::spawn( &["reconnect-a", "reconnect-b"], cluster::ClusterTiming::fast().with_reconnect_timeout(Duration::from_secs(30)), ) @@ -1005,7 +1051,7 @@ where // whole assertion window, and a long reconnect timeout keeps the FAILED // member from being reaped out of the views before the intent lands (the // reference tests run with the default day-scale reconnect timeout). - let mut cluster = cluster::Cluster::::spawn( + let mut cluster = TcpCluster::::spawn( &["fleave-a", "fleave-b", "fleave-c"], cluster::ClusterTiming::fast() .with_tombstone_timeout(Duration::from_secs(120)) @@ -1064,7 +1110,7 @@ where // exclusivity the causal fence below relies on (anti-entropy also // witnesses remote clocks and would replay the Left state, masking the // fresh intent). - let mut cluster = cluster::Cluster::::spawn( + let mut cluster = TcpCluster::::spawn( &["fleft-a", "fleft-b", "fleft-c"], cluster::ClusterTiming::fast() .with_tombstone_timeout(Duration::from_secs(120)) @@ -1179,7 +1225,7 @@ where { // A long tombstone keeps the removed member observable as Left for the // whole assertion window. - let mut cluster = cluster::Cluster::::spawn( + let mut cluster = TcpCluster::::spawn( &["remove-a", "remove-b", "remove-c"], cluster::ClusterTiming::fast() .with_tombstone_timeout(Duration::from_secs(120)) @@ -1222,7 +1268,7 @@ where { // Hold the failed member (no reap, no reconnect eviction) so the prune — // not the reaper — is what erases it. - let mut cluster = cluster::Cluster::::spawn( + let mut cluster = TcpCluster::::spawn( &["prune-a", "prune-b", "prune-c"], cluster::ClusterTiming::fast().with_reconnect_timeout(Duration::from_secs(120)), ) @@ -1275,7 +1321,7 @@ where // The reclaim window is what allows a SAME-name member to revive at a NEW // address at all — without it a different-address Alive is a name conflict, // exactly as in the reference implementation's dead-node reclaim. - let mut cluster = cluster::Cluster::::spawn( + let mut cluster = TcpCluster::::spawn( &["sameip-a", "sameip-b"], cluster::ClusterTiming::fast() .with_reconnect_timeout(Duration::from_secs(30)) @@ -1439,7 +1485,7 @@ where R: Runtime, { let mut cluster = - cluster::Cluster::::spawn(&["tags-a", "tags-b"], cluster::ClusterTiming::fast()).await; + TcpCluster::::spawn(&["tags-a", "tags-b"], cluster::ClusterTiming::fast()).await; let a_id = cluster.id(0); let b_id = cluster.id(1); @@ -1510,7 +1556,7 @@ async fn serf_update_after_rejoin() where R: Runtime, { - let mut cluster = cluster::Cluster::::spawn( + let mut cluster = TcpCluster::::spawn( &["upd-a", "upd-b"], // The explicit rejoin below is the single revival path: the survivor's // own reconnect re-dial is parked out of the window so its push/pull @@ -1583,12 +1629,8 @@ where /// Poll `observer`'s member view until `subject`'s `version` tag equals /// `want` — the propagation fence for a retag. -async fn await_version_tag( - cluster: &cluster::Cluster, - observer: usize, - subject: &str, - want: &str, -) where +async fn await_version_tag(cluster: &TcpCluster, observer: usize, subject: &str, want: &str) +where R: Runtime, { R::timeout(Duration::from_secs(20), async { @@ -1649,7 +1691,7 @@ async fn serf_update_after_restart_with_changed_tags() where R: Runtime, { - let mut cluster = cluster::Cluster::::spawn( + let mut cluster = TcpCluster::::spawn( &["updm-a", "updm-b"], cluster::ClusterTiming::fast() .with_probe_interval(Duration::from_secs(600)) @@ -1873,8 +1915,7 @@ async fn serf_join_leave() where R: Runtime, { - let mut cluster = - cluster::Cluster::::spawn(&["jl-a", "jl-b"], cluster::ClusterTiming::fast()).await; + let mut cluster = TcpCluster::::spawn(&["jl-a", "jl-b"], cluster::ClusterTiming::fast()).await; let peer = cluster.id(0); let leaver = cluster.id(1); @@ -1914,7 +1955,7 @@ async fn serf_join_leave_join() where R: Runtime, { - let mut cluster = cluster::Cluster::::spawn( + let mut cluster = TcpCluster::::spawn( &["jlj-a", "jlj-b"], cluster::ClusterTiming::fast().with_tombstone_timeout(Duration::from_secs(30)), ) @@ -1952,7 +1993,7 @@ async fn serf_leave_rejoin_different_role() where R: Runtime, { - let mut cluster = cluster::Cluster::::spawn( + let mut cluster = TcpCluster::::spawn( &["lrr-a", "lrr-b"], cluster::ClusterTiming::fast().with_tombstone_timeout(Duration::from_secs(30)), ) @@ -2580,160 +2621,1611 @@ where b.shutdown().await.expect("mis-b shuts down"); } -// The tokio cells: the runtime-generic scenarios driven on tokio's multi-thread -// runtime. Gated on the `tokio` feature so the `--test tcp -- smol` build (which -// enables only `smol`) can drop the `agnostic/tokio` code path. -#[cfg(feature = "tokio")] -mod tokio_cells { - use agnostic::tokio::TokioRuntime; +/// Every mutating operation is REFUSED with `NotRunning` once the node has left +/// the cluster — `leave()` stops the periodic schedulers, so a post-leave mutation +/// would be issued by a node no longer participating. The read-only coordinate +/// probe is the deliberate exception: post-leave introspection stays valid. +/// +/// `respond` is included by capturing a live query token BEFORE the leave, which +/// is the only way its post-leave arm can be reached at all. +async fn post_leave_operations_report_not_running() +where + R: Runtime, +{ + let a = spawn_node::("nr-a").await; + let b = spawn_node::("nr-b").await; + let a_addr = a.advertise_address(); - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn two_node_join_converges() { - super::two_node_join_converges::().await; - } + let mut b_events = b.events(); + b.join(&SocketAddrResolver, MaybeResolved::Resolved(a_addr), false) + .await + .expect("join reaches node A"); + converge(&a, &b).await; - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn user_event_delivered() { - super::user_event_delivered::().await; - } + // Capture a live query token on B, so the post-leave `respond` below has + // something real to answer. + a.query("probe", Bytes::from_static(b"q"), a.default_query_param()) + .await + .expect("query issued"); + let token = R::timeout(Duration::from_secs(20), async { + loop { + match b_events.next().await { + Some(Event::Query(qe)) if qe.name() == "probe" => break Some(qe), + Some(_) => {} + None => break None, + } + } + }) + .await + .expect("B surfaces the query within the timeout") + .expect("B's event stream stays open"); - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn query_round_trip() { - super::query_round_trip::().await; - } + b.leave().await.expect("B leaves the cluster"); - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn leave_emits_left_cluster() { - super::leave_emits_left_cluster::().await; - } + let not_running = |e: &serf_reactor::SerfError| matches!(e, serf_reactor::SerfError::NotRunning); - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn reconnect_delegate_reaps_left_member() { - super::reconnect_delegate_reaps_left_member::().await; + let err = b + .join(&SocketAddrResolver, MaybeResolved::Resolved(a_addr), false) + .await + .expect_err("a left node cannot rejoin in place"); + assert!(not_running(&err), "join after leave: got {err:?}"); + + for (what, res) in [ + ( + "user_event", + b.user_event("nope", Bytes::from_static(b"x"), false).await, + ), + ( + "query", + b.query("nope", Bytes::from_static(b"x"), b.default_query_param()) + .await + .map(|_| ()), + ), + ("respond", b.respond(token, Bytes::from_static(b"x")).await), + ("set_tags", b.set_tags(serf_proto::Tags::new()).await), + ( + "force_leave", + b.force_leave(SmolStr::new("nr-a"), false).await, + ), + ] { + let err = res.expect_err("a left node refuses to mutate"); + assert!( + not_running(&err), + "{what} after leave must report NotRunning, got {err:?}" + ); } - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn snapshot_forwarders_reflect_joined_cluster() { - super::snapshot_forwarders_reflect_joined_cluster::().await; - } + // The read-only coordinate probe still answers after the leave. + #[cfg(feature = "coordinates")] + b.cached_coordinate(SmolStr::new("nr-a")) + .await + .expect("a read-only coordinate probe stays answerable after leave"); - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn join_many_returns_only_reached_seeds() { - super::join_many_returns_only_reached_seeds::().await; - } + a.shutdown().await.expect("nr-a shuts down"); + b.shutdown().await.expect("nr-b shuts down"); +} - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn remove_failed_node_alias_succeeds() { - super::remove_failed_node_alias_succeeds::().await; +/// The key-management operations refuse to run on a node that has left — the same +/// `NotRunning` gate the membership mutations take, so a departed node can never +/// originate a cluster-wide rotation. +#[cfg(encryption)] +async fn post_leave_key_operations_report_not_running() +where + R: Runtime, +{ + let node = spawn_encrypted_node::( + "nrkey", + EncryptionOptions::new().with_keyring(Keyring::new(test_secret_key(0x61))), + ) + .await; + node.leave().await.expect("the node leaves the cluster"); + + let k = test_secret_key(0x62); + for (what, res) in [ + ("install_key", node.install_key(k).await.map(|_| ())), + ("use_key", node.use_key(k).await.map(|_| ())), + ("remove_key", node.remove_key(k).await.map(|_| ())), + ("list_keys", node.list_keys().await.map(|_| ())), + ] { + let err = res.expect_err("a left node refuses a key rotation"); + assert!( + matches!(err, serf_reactor::SerfError::NotRunning), + "{what} after leave must report NotRunning, got {err:?}" + ); } - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn serf_events_failed() { - super::serf_events_failed::().await; - } + node.shutdown().await.expect("nrkey shuts down"); +} - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn serf_events_leave() { - super::serf_events_leave::().await; - } +/// Leave is a SHARED in-flight operation: two concurrent `leave()` calls JOIN one +/// leave — the machine's `leave()` is invoked once and its single `LeftCluster` +/// resolves BOTH callers `Ok`. (Re-invoking it would be a terminal no-op emitting +/// no second `LeftCluster`, so the second caller would hang to its timeout.) A +/// THIRD leave issued after the chain completed is an accepted no-op, not an +/// error. +async fn concurrent_leaves_share_one_in_flight_leave() +where + R: Runtime, +{ + let b = spawn_node::("dl-b").await; + let a = spawn_node::("dl-a").await; + let b_addr = b.advertise_address(); - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn serf_events_leave_with_racing_shutdown() { - super::serf_events_leave_with_racing_shutdown::().await; - } + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + converge(&a, &b).await; - #[cfg(feature = "coordinates")] - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn coordinates_surface_on_the_handle() { - super::coordinates_surface_on_the_handle::().await; - } + let (first, second) = future::join(a.leave(), a.leave()).await; + first.expect("the initiating leave resolves Ok"); + second.expect("the leave that JOINED the in-flight one resolves Ok too"); - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn snapshot_restart_rejoins_the_cluster() { - super::snapshot_restart_rejoins_the_cluster::().await; - } + // The shared leave still drove the endpoint to Left (poll to absorb the + // snapshot-refresh race after the leave chain completes). + R::timeout(Duration::from_secs(5), async { + loop { + if a.state() == SerfState::Left { + break; + } + R::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("the shared leave drives the endpoint to Left"); - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn snapshot_leave_gate_controls_rejoin() { - super::snapshot_leave_gate_controls_rejoin::().await; - } + // A leave on an already-left node is a terminal no-op, replied to immediately. + a.leave() + .await + .expect("leaving an already-left node is an accepted no-op"); - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn merge_delegate_is_consulted_on_join() { - super::merge_delegate_is_consulted_on_join::().await; - } + a.shutdown().await.expect("dl-a shuts down"); + b.shutdown().await.expect("dl-b shuts down"); +} - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn leave_with_zero_timeout_racing_shutdown_times_out() { - super::leave_with_zero_timeout_racing_shutdown_times_out::().await; - } +/// An `ignore_old` join records each seed's exchange as a one-shot replay-suppress +/// target, and the driver clears the recording when the exchange terminates: the +/// join still merges membership, and the joiner does NOT replay the seed's +/// pre-join user event onto its fresh event stream. A plain (non-ignoring) join is +/// the control — it DOES surface the buffered event — so the suppression is +/// proven, not merely asserted as an absence. +async fn ignore_old_join_suppresses_the_replay() +where + R: Runtime, +{ + let seed = spawn_node::("io-seed").await; + let seed_addr = seed.advertise_address(); - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn serf_reconnect() { - super::serf_reconnect::().await; - } + // The seed buffers a user event BEFORE anyone joins. + seed + .user_event("old-news", Bytes::from_static(b"stale"), false) + .await + .expect("the seed buffers a pre-join user event"); - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn serf_force_leave_failed() { - super::serf_force_leave_failed::().await; - } + // The control: a plain join replays the buffered event to the joiner. + let plain = spawn_node::("io-plain").await; + let mut plain_events = plain.events(); + plain + .join( + &SocketAddrResolver, + MaybeResolved::Resolved(seed_addr), + false, + ) + .await + .expect("the plain join reaches the seed"); + let replayed = R::timeout(Duration::from_secs(20), async { + loop { + match plain_events.next().await { + Some(Event::User(u)) if u.name.as_str() == "old-news" => break true, + Some(_) => {} + None => break false, + } + } + }) + .await + .expect("a plain join replays the seed's buffered user event"); + assert!( + replayed, + "the control join must surface the buffered event, or the suppression below is vacuous" + ); - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn serf_force_leave_left_is_idempotent() { - super::serf_force_leave_left_is_idempotent::().await; - } + // The subject: an `ignore_old` join merges membership but suppresses the replay. + let quiet = spawn_node::("io-quiet").await; + let mut quiet_events = quiet.events(); + let reached = quiet + .join( + &SocketAddrResolver, + MaybeResolved::Resolved(seed_addr), + true, + ) + .await + .expect("the ignore_old join reaches the seed"); + assert_eq!(reached, seed_addr, "the ignore_old join still merges"); + R::timeout(Duration::from_secs(20), async { + loop { + if quiet.num_members() >= 2 { + break; + } + R::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("the ignore_old join still converges the membership"); - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn serf_remove_failed_node_propagates() { - super::serf_remove_failed_node_propagates::().await; - } + // No replay reaches the ignoring joiner across a window the control proved is + // ample. + let saw = R::timeout(Duration::from_secs(2), async { + loop { + match quiet_events.next().await { + Some(Event::User(u)) if u.name.as_str() == "old-news" => break true, + Some(_) => {} + None => break false, + } + } + }) + .await + .unwrap_or(false); + assert!( + !saw, + "an ignore_old join must not replay the seed's pre-join user events" + ); - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn serf_remove_failed_node_prune_erases() { - super::serf_remove_failed_node_prune_erases::().await; - } + quiet.shutdown().await.expect("io-quiet shuts down"); + plain.shutdown().await.expect("io-plain shuts down"); + seed.shutdown().await.expect("io-seed shuts down"); +} - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn remove_failed_node_absent_is_a_noop() { - super::remove_failed_node_absent_is_a_noop::().await; - } +/// A datagram the gossip plane cannot parse is DROPPED and the node keeps serving: +/// the ingress decode failure must not poison the pump. Garbage is injected from a +/// raw UDP socket (no serf peer involved), then the node is proven still live by a +/// fresh peer joining and converging afterwards. +async fn malformed_gossip_datagram_is_ignored() +where + R: Runtime, +{ + let a = spawn_node::("junk-a").await; + let a_addr = a.advertise_address(); - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn serf_reconnect_same_ip() { - super::serf_reconnect_same_ip::().await; + // Raw garbage straight at the gossip socket: not a label frame, not a message. + let raw = std::net::UdpSocket::bind("127.0.0.1:0").expect("bind a raw sender"); + for payload in [ + &b"\xff\xff\xff\xff\xff\xff\xff\xff"[..], + &b""[..], + &[0x7fu8; 400][..], + ] { + raw + .send_to(payload, a_addr) + .expect("the garbage datagram is sent"); } - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn serf_join_cancel() { - super::serf_join_cancel::().await; - } + // The node survived the junk: a fresh peer still joins and converges. + let b = spawn_node::("junk-b").await; + b.join(&SocketAddrResolver, MaybeResolved::Resolved(a_addr), false) + .await + .expect("the node still serves joins after the malformed datagrams"); + converge(&a, &b).await; - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn serf_set_tags_propagates() { - super::serf_set_tags_propagates::().await; - } + a.shutdown().await.expect("junk-a shuts down"); + b.shutdown().await.expect("junk-b shuts down"); +} - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn serf_update_after_rejoin() { - super::serf_update_after_rejoin::().await; - } +/// An encrypted node DROPS a datagram it cannot authenticate — the AEAD open +/// fails, the pump moves on, and the node keeps serving. The garbage is +/// indistinguishable from a forged datagram, so this is the gossip plane's +/// unauthenticated-input gate. +#[cfg(encryption)] +async fn unauthenticatable_gossip_datagram_is_ignored() +where + R: Runtime, +{ + let key = EncryptionOptions::new().with_keyring(Keyring::new(test_secret_key(0x51))); + let a = spawn_encrypted_node::("forge-a", key.clone()).await; + let a_addr = a.advertise_address(); - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn serf_update_after_restart_with_changed_tags() { - super::serf_update_after_restart_with_changed_tags::().await; + let raw = std::net::UdpSocket::bind("127.0.0.1:0").expect("bind a raw sender"); + for payload in [&[0x00u8; 64][..], &[0xabu8; 300][..]] { + raw + .send_to(payload, a_addr) + .expect("the forged datagram is sent"); } - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn serf_query_filter() { - super::serf_query_filter::().await; - } + // The node survived the forgeries: a keyring-sharing peer still joins. + let b = spawn_encrypted_node::("forge-b", key).await; + b.join(&SocketAddrResolver, MaybeResolved::Resolved(a_addr), false) + .await + .expect("the node still serves joins after the unauthenticatable datagrams"); + converge(&a, &b).await; - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] - async fn serf_join_leave() { - super::serf_join_leave::().await; - } + a.shutdown().await.expect("forge-a shuts down"); + b.shutdown().await.expect("forge-b shuts down"); +} - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] +/// A node that has LEFT admits no further inbound reliable exchange: the accept +/// still happens (the listener is bound until shutdown), but the machine refuses +/// the connection and the driver drops the stream rather than bridging an exchange +/// it will never feed — so the raw peer sees an immediate EOF. +async fn left_node_refuses_new_inbound_exchanges() +where + R: Runtime, +{ + use std::io::Read; + + let a = spawn_node::("closed-a").await; + let a_addr = a.advertise_address(); + a.leave().await.expect("A leaves the cluster"); + + // A raw reliable dial after the leave: the connection is accepted and then + // dropped, so the read reaches EOF without a byte of protocol. + let eof = R::spawn_blocking(move || { + let mut sock = std::net::TcpStream::connect(a_addr).expect("the listener is still bound"); + sock + .set_read_timeout(Some(Duration::from_secs(10))) + .expect("set read timeout"); + // A push/pull request the machine would answer if it were still running. + let mut buf = [0u8; 64]; + sock.read(&mut buf) + }) + .await + .expect("the blocking dial completes"); + + match eof { + Ok(0) => {} + other => panic!("a left node must drop the accepted stream (EOF), got {other:?}"), + } + + a.shutdown().await.expect("closed-a shuts down"); +} + +/// A peer that RESETS mid-exchange must FAIL the join, not complete it: a reset is +/// a transport error, and a one-way frame maps a clean EOF to a SUCCESSFUL +/// completion — so routing the reset down the benign-EOF path would report the +/// reliable exchange as having succeeded against a peer that never answered. +/// +/// The evil peer accepts the connection and drops it WITHOUT reading the joiner's +/// already-sent request, so the unread bytes in its receive queue make the close a +/// RST rather than a FIN. +async fn peer_reset_mid_exchange_fails_the_join() +where + R: Runtime, +{ + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind the evil peer"); + let peer_addr = listener.local_addr().expect("the evil peer's address"); + + let evil = std::thread::spawn(move || { + if let Ok((stream, _)) = listener.accept() { + // Let the joiner's push/pull request land in the receive queue unread — the + // close then resets the connection instead of half-closing it. + std::thread::sleep(Duration::from_millis(250)); + drop(stream); + } + }); + + let a = spawn_node::("rst-a").await; + let outcome = a + .join( + &SocketAddrResolver, + MaybeResolved::Resolved(peer_addr), + false, + ) + .await; + assert!( + outcome.is_err(), + "a peer that never answers must fail the join, never complete it (got {outcome:?})" + ); + assert_eq!(a.num_members(), 1, "nothing was merged from the evil peer"); + + // The driver survived the reset: a real peer still joins. + let b = spawn_node::("rst-b").await; + let a_addr = a.advertise_address(); + b.join(&SocketAddrResolver, MaybeResolved::Resolved(a_addr), false) + .await + .expect("the node still serves joins after the reset exchange"); + converge(&a, &b).await; + + a.shutdown().await.expect("rst-a shuts down"); + b.shutdown().await.expect("rst-b shuts down"); + evil.join().expect("the evil peer thread exits"); +} + +/// A subscriber that stops draining its `EventStream` must NOT stall the driver: +/// the fan-out sheds the events it cannot deliver and COUNTS them, and the node +/// keeps serving. The shed count is the observable — a driver that instead blocked +/// on the full queue would never reach it. +async fn slow_subscriber_sheds_events_and_counts_them() +where + R: Runtime, +{ + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let node = Serf::::tcp( + TcpTransportOptions::::new() + .with_local_id(SmolStr::new("shed-a")) + .with_advertise_addr(MaybeResolved::Resolved(bind)), + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + // A one-slot event queue: the second undelivered event already overflows. + RuntimeOptions::new().with_event_queue_cap(1), + SerfOptions::new(), + None, + None, + None, + #[cfg(encryption)] + std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), + ) + .await + .expect("spawn shed-a"); + + // Subscribe and NEVER poll the stream: the fan-out queue fills at once. + let _stalled = node.events(); + + for i in 0..64u32 { + node + .user_event("flood", Bytes::from(i.to_be_bytes().to_vec()), false) + .await + .expect("the driver keeps accepting commands while the subscriber stalls"); + } + + R::timeout(Duration::from_secs(20), async { + loop { + if node.events_dropped() > 0 { + break; + } + R::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("the fan-out sheds events a stalled subscriber cannot take, and counts them"); + + // The node is still live and answering. + assert_eq!(node.num_members(), 1); + node.shutdown().await.expect("shed-a shuts down"); +} + +/// A delegate hook that PARKS must not wedge the driver: the observation channel +/// backs up, the pump retains what application data it can (a bounded overflow), +/// and once THAT is full it sheds the excess and COUNTS the loss — while the node +/// keeps accepting commands throughout. The shed count is the observable; a pump +/// that blocked on the delegate would never reach it. +async fn stalling_delegate_sheds_observations_and_counts_them() +where + R: Runtime, +{ + /// Enough payload events to fill the two-slot channel, then the pump's bounded + /// retry overflow (1024 entries), and still leave a surplus that must be shed. + const FLOOD: u32 = 1200; + + /// A delegate whose user-event hook parks until `release` is dropped. It does + /// NOT override the test-only message-dropper hook, so the composite's default + /// (drop nothing) applies. + struct StallingDelegate { + gate: flume::Receiver<()>, + } + + impl serf_reactor::MemberDelegate for StallingDelegate { + type Id = SmolStr; + type Address = SocketAddr; + } + impl serf_reactor::QueryDelegate for StallingDelegate { + type Id = SmolStr; + type Address = SocketAddr; + } + impl serf_reactor::UserEventDelegate for StallingDelegate { + fn notify_user_event( + &self, + _event: &serf_proto::typed::UserEventMessage, + ) -> impl core::future::Future + Send + '_ { + let gate = self.gate.clone(); + async move { + // Ignoring Err: a disconnected gate means the test released the hook. + let _ = gate.recv_async().await; + } + } + } + impl serf_reactor::Delegate for StallingDelegate { + type Id = SmolStr; + type Address = SocketAddr; + } + + // Hold the sender: while it lives, every `notify_user_event` parks. + let (release, gate) = flume::bounded::<()>(0); + + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let node = Serf::::tcp( + TcpTransportOptions::::new() + .with_local_id(SmolStr::new("stall-a")) + .with_advertise_addr(MaybeResolved::Resolved(bind)), + &SocketAddrResolver, + &FirstAddrResolver, + StallingDelegate { gate }, + // A two-slot observation channel: the parked hook fills it immediately, so the + // pump must shed rather than block. + RuntimeOptions::new().with_observation_channel(serf_reactor::Channel::Bounded(2)), + SerfOptions::new(), + None, + None, + None, + #[cfg(encryption)] + std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), + ) + .await + .expect("spawn stall-a"); + + // Flood payload events past the pump's bounded retry overflow while the hook parks. + let payload = Bytes::from(vec![0x5au8; 32]); + for _ in 0..FLOOD { + node + .user_event("flood", payload.clone(), false) + .await + .expect("the pump keeps accepting commands while the delegate parks"); + } + + R::timeout(Duration::from_secs(20), async { + loop { + if node.observation_dropped() > 0 { + break; + } + R::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("the pump sheds observations a parked delegate cannot take, and counts them"); + + // The node is still live and answering commands. + assert_eq!(node.num_members(), 1); + + // Release the hook so the observation task can drain, then shut down. + drop(release); + node.shutdown().await.expect("stall-a shuts down"); +} + +/// An UNBOUNDED observation channel never sheds: with no cap there is no byte +/// backstop and no overflow, so a node that surfaces many payload events reports a +/// zero observation-drop count while still delivering them. +async fn unbounded_observation_channel_never_sheds() +where + R: Runtime, +{ + let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); + let node = Serf::::tcp( + TcpTransportOptions::::new() + .with_local_id(SmolStr::new("unb-a")) + .with_advertise_addr(MaybeResolved::Resolved(bind)), + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new().with_observation_channel(serf_reactor::Channel::Unbounded), + SerfOptions::new(), + None, + None, + None, + #[cfg(encryption)] + std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), + ) + .await + .expect("spawn unb-a"); + + let mut events = node.events(); + let payload = Bytes::from(vec![0x11u8; 480]); + for _ in 0..128u32 { + node + .user_event("flood", payload.clone(), false) + .await + .expect("user event dispatched"); + } + + // Every event still arrives, and nothing was shed at the observation channel. + let mut seen = 0usize; + R::timeout(Duration::from_secs(20), async { + loop { + match events.next().await { + Some(Event::User(u)) if u.name.as_str() == "flood" => { + seen += 1; + if seen == 128 { + break; + } + } + Some(_) => {} + None => break, + } + } + }) + .await + .expect("an unbounded observation channel delivers every event"); + assert_eq!(seen, 128, "every flooded event reached the subscriber"); + assert_eq!( + node.observation_dropped(), + 0, + "an unbounded observation channel has no backstop to shed against" + ); + + node.shutdown().await.expect("unb-a shuts down"); +} + +/// A shutdown racing an IN-FLIGHT await-result join resolves that join +/// `Err(Shutdown)` — never leaves it parked forever. The seed accepts the TCP +/// connection but never answers, so the exchange is still pending inside the +/// driver when the teardown reaps the waiter. +async fn shutdown_racing_an_inflight_join_resolves_it() +where + R: Runtime, +{ + // A seed that accepts and then goes silent: the join's push/pull never completes. + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind the silent seed"); + let seed_addr = listener.local_addr().expect("the silent seed's address"); + let (stop_tx, stop_rx) = std::sync::mpsc::channel::<()>(); + let silent = std::thread::spawn(move || { + let held = listener.accept(); + // Hold the accepted connection open (no reply) until the test releases us. + // Ignoring Err: a disconnected sender means the test finished. + let _ = stop_rx.recv(); + drop(held); + }); + + let node = spawn_node::("race-join").await; + + // The join parks on the silent seed; the shutdown races it. + let (join, shutdown) = future::join( + node.join( + &SocketAddrResolver, + MaybeResolved::Resolved(seed_addr), + false, + ), + async { + // Let the dial connect and the request go out before the teardown begins, so + // the waiter is genuinely in flight rather than never dispatched. + R::sleep(Duration::from_millis(200)).await; + node.shutdown().await + }, + ) + .await; + + shutdown.expect("the node shuts down"); + let err = join.expect_err("a join racing a shutdown cannot succeed against a silent seed"); + assert!( + matches!( + err, + serf_reactor::SerfError::Shutdown | serf_reactor::SerfError::JoinAllFailed(_) + ), + "the in-flight join must be resolved by the teardown, not stranded (got {err:?})" + ); + + // Ignoring Err: the silent-seed thread may already have exited. + let _ = stop_tx.send(()); + silent.join().expect("the silent seed thread exits"); +} + +/// A second `shutdown()` — issued once the driver has already exited and closed +/// its command queue — still resolves `Ok`, and only AFTER the bind address is +/// actually free: the late caller parks on the teardown-completion latch instead +/// of returning into a still-bound port. The freed port is proven rebindable +/// immediately after. +async fn second_shutdown_awaits_teardown_completion() +where + R: Runtime, +{ + let node = spawn_node::("twice-a").await; + let addr = node.advertise_address(); + + node.shutdown().await.expect("the first shutdown resolves"); + node + .shutdown() + .await + .expect("a second shutdown after teardown still resolves Ok"); + + // Every command path fails fast once the queue is closed, rather than hanging. + let err = node + .user_event("post", Bytes::from_static(b"x"), false) + .await + .expect_err("a shut-down node accepts no commands"); + assert!( + matches!(err, serf_reactor::SerfError::Shutdown), + "a post-shutdown command reports Shutdown, got {err:?}" + ); + + // The latch fired only once the bind address was free: rebinding it succeeds. + let reborn = spawn_node_with_snapshot::( + "twice-b", + addr, + serf_reactor::SnapshotOptions::new(snapshot_path::("twice")), + false, + ) + .await + .expect("the freed address rebinds after the awaited teardown"); + assert_eq!(reborn.advertise_address(), addr); + reborn.shutdown().await.expect("twice-b shuts down"); +} + +/// A key-management request on a node with NO keyring is REFUSED, not silently +/// applied: the responder answers `result = false` with an explanatory message and +/// leaves the wire untouched, so the originator's collected response carries the +/// error rather than a false success. +#[cfg(encryption)] +async fn key_op_without_a_keyring_is_refused() +where + R: Runtime, +{ + // Two PLAINTEXT nodes — neither carries a keyring. + let b = spawn_node::("nokey-b").await; + let a = spawn_node::("nokey-a").await; + let b_addr = b.advertise_address(); + + let mut a_events = a.events(); + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + converge(&a, &b).await; + + a.install_key(test_secret_key(0x77)) + .await + .expect("the request itself dispatches"); + + let kr = R::timeout(Duration::from_secs(20), async { + loop { + match a_events.next().await { + Some(Event::KeyResponse(kr)) => break kr, + Some(_) => {} + None => panic!("the event stream ended before the key response"), + } + } + }) + .await + .expect("A collects the key response within the query window"); + + assert!( + kr.num_err > 0, + "a node with no keyring must REFUSE the key op, not report success (num_err={}, num_resp={})", + kr.num_err, + kr.num_resp + ); + assert!( + !kr.messages.is_empty(), + "the refusal carries an explanatory message" + ); + assert!( + !a.encryption_enabled() && !b.encryption_enabled(), + "the refused op left both nodes plaintext" + ); + + a.shutdown().await.expect("nokey-a shuts down"); + b.shutdown().await.expect("nokey-b shuts down"); +} + +/// Past its threshold the snapshot file is COMPACTED to the live state rather than +/// growing without bound: after a churn of membership events the file stays small +/// and still replays the live cluster — a restarted node recovers its peer from the +/// compacted file. +async fn snapshot_compaction_rewrites_the_live_state() +where + R: Runtime, +{ + let path = snapshot_path::("compact"); + let a = spawn_node::("cmp-a").await; + let a_addr = a.advertise_address(); + + // A tiny threshold: any membership churn crosses it and forces a rewrite. + let b = spawn_node_with_snapshot::( + "cmp-b", + ephemeral_bind(), + serf_reactor::SnapshotOptions::new(&path).with_compact_threshold(1), + false, + ) + .await + .expect("spawn snapshot-backed serf tcp node"); + + b.join(&SocketAddrResolver, MaybeResolved::Resolved(a_addr), false) + .await + .expect("join reaches node A"); + converge(&a, &b).await; + + // Churn: repeated tag updates append member records the compaction must fold. + for i in 0..8u32 { + let mut tags = serf_proto::Tags::new(); + tags + .0 + .insert(SmolStr::new("gen"), SmolStr::new(i.to_string())); + a.set_tags(tags).await.expect("A re-tags itself"); + R::sleep(Duration::from_millis(50)).await; + } + R::sleep(Duration::from_millis(300)).await; + + // The compacted file holds the LIVE alive-set, not the whole append history: a + // fold of two members plus their clock floors stays far below the churn's + // un-compacted footprint. + let len = std::fs::metadata(&path).expect("the snapshot exists").len(); + assert!( + len < 4096, + "the compaction must rewrite the file to the live state, but it grew to {len} bytes" + ); + + b.shutdown().await.expect("cmp-b shuts down"); + + // The compacted file still replays: a restarted node recovers its peer from it. + let b2 = spawn_node_with_snapshot::( + "cmp-b", + ephemeral_bind(), + serf_reactor::SnapshotOptions::new(&path).with_compact_threshold(1), + false, + ) + .await + .expect("spawn snapshot-backed serf tcp node"); + converge(&a, &b2).await; + assert_eq!( + b2.num_members(), + 2, + "the compacted snapshot still recovers the live membership" + ); + + a.shutdown().await.expect("cmp-a shuts down"); + b2.shutdown().await.expect("cmp-b2 shuts down"); + // Ignoring Err: best-effort test-file cleanup. + let _ = std::fs::remove_file(&path); +} + +/// Build a plain-TCP node from `opts` through the ergonomic constructor. +async fn build_tcp(opts: TcpTransportOptions) -> Result, SerfError> +where + R: Runtime, +{ + Serf::::tcp( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + None, + None, + None, + #[cfg(encryption)] + std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), + ) + .await +} + +/// The TCP transport's construction gate: each field `TcpTransport::new` requires +/// is refused when absent, so a half-built options block can never bind a socket. +async fn construction_requires_id_and_advertise_addr() +where + R: Runtime, +{ + let err = build_tcp::( + TcpTransportOptions::new().with_advertise_addr(MaybeResolved::Resolved(ephemeral_bind())), + ) + .await + .err() + .expect("a node with no id cannot be built"); + assert!( + matches!(err, SerfError::Io(ref e) if e.kind() == std::io::ErrorKind::InvalidInput), + "a missing local_id is an InvalidInput, got {err:?}" + ); + + let err = build_tcp::(TcpTransportOptions::new().with_local_id(SmolStr::new("no-addr"))) + .await + .err() + .expect("a node with no advertise address cannot be built"); + assert!( + matches!(err, SerfError::Io(ref e) if e.kind() == std::io::ErrorKind::InvalidInput), + "a missing advertise_addr is an InvalidInput, got {err:?}" + ); +} + +/// A wildcard advertise address is refused AFTER the bind: the readback keeps the +/// unspecified IP, which peers could not route back to, so construction fails and +/// BOTH bound sockets are released rather than the node joining as an undialable +/// member. The released port is proven free by an immediate successful rebind of +/// the very port the failed attempt had claimed. +async fn wildcard_advertise_is_refused_and_releases_the_bind() +where + R: Runtime, +{ + // Claim a concrete port through a successful node, then free it, so the wildcard + // attempt below binds a KNOWN port we can prove was released. + let probe = spawn_node::("wild-probe").await; + let port = probe.advertise_address().port(); + probe.shutdown().await.expect("probe shuts down"); + + let wildcard: SocketAddr = format!("0.0.0.0:{port}").parse().expect("wildcard addr"); + let err = build_tcp::( + TcpTransportOptions::new() + .with_local_id(SmolStr::new("wild")) + .with_advertise_addr(MaybeResolved::Resolved(wildcard)), + ) + .await + .err() + .expect("a wildcard advertise address is not a routable contact"); + assert!( + matches!(err, SerfError::InvalidAdvertiseAddr(_)), + "a wildcard bind must be refused as an invalid advertise address, got {err:?}" + ); + + // The refused construction released BOTH bound sockets: the same port rebinds. + let after = build_tcp::( + TcpTransportOptions::new() + .with_local_id(SmolStr::new("wild-after")) + .with_advertise_addr(MaybeResolved::Resolved( + format!("127.0.0.1:{port}").parse().expect("loopback addr"), + )), + ) + .await + .expect("the refused construction released the ports it had bound"); + assert_eq!(after.advertise_address().port(), port); + after.shutdown().await.expect("wild-after shuts down"); +} + +/// An UNRESOLVED advertise address is resolved at construction through the +/// caller's resolvers, and the node comes up on the resolved contact. +async fn unresolved_advertise_addr_is_resolved_at_construction() +where + R: Runtime, +{ + let node = build_tcp::( + TcpTransportOptions::new() + .with_local_id(SmolStr::new("resolve-me")) + .with_advertise_addr(MaybeResolved::Unresolved(ephemeral_bind())), + ) + .await + .expect("the unresolved advertise address resolves through the supplied resolver"); + + let bound = node.advertise_address(); + assert!(bound.ip().is_loopback(), "the resolved contact is loopback"); + assert_ne!( + bound.port(), + 0, + "the ephemeral bind resolved to a concrete port" + ); + node.shutdown().await.expect("resolve-me shuts down"); +} + +/// A resolver that FAILS, and one that resolves to NO candidate, both fail +/// construction rather than booting an addressless node: an advertise outage must +/// be loud, never a node that gossips a contact nobody can dial. +async fn advertise_resolution_failure_fails_construction() +where + R: Runtime, +{ + /// Resolves nothing — a bootstrap outage the advertise picker must refuse. + struct EmptyResolver; + impl serf_reactor::Resolver for EmptyResolver { + type Address = SocketAddr; + type Error = std::io::Error; + fn resolve( + &self, + _addr: &SocketAddr, + ) -> impl core::future::Future>> + Send + '_ { + // The candidate set is empty whatever the input, so the future borrows nothing. + let out = Vec::new(); + async move { Ok(out) } + } + } + + /// Fails outright — the DNS-down shape. + struct FailingResolver; + impl serf_reactor::Resolver for FailingResolver { + type Address = SocketAddr; + type Error = std::io::Error; + fn resolve( + &self, + _addr: &SocketAddr, + ) -> impl core::future::Future>> + Send + '_ { + // The failure is unconditional, so the future borrows nothing. + let err = std::io::Error::other("resolver is down"); + async move { Err(err) } + } + } + + async fn build_with(resolver: &RES) -> Result, SerfError> + where + R: Runtime, + RES: serf_reactor::Resolver
, + { + Serf::::tcp( + TcpTransportOptions::new() + .with_local_id(SmolStr::new("unresolvable")) + .with_advertise_addr(MaybeResolved::Unresolved(ephemeral_bind())), + resolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + None, + None, + None, + #[cfg(encryption)] + std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), + ) + .await + } + + let err = build_with::(&EmptyResolver) + .await + .err() + .expect("an advertise address that resolves to nothing cannot boot a node"); + assert!( + matches!(err, SerfError::Resolve(_)), + "an empty candidate set is a resolution failure, got {err:?}" + ); + + let err = build_with::(&FailingResolver) + .await + .err() + .expect("a failing resolver cannot boot a node"); + assert!( + matches!(err, SerfError::Resolve(_)), + "a resolver error is a resolution failure, got {err:?}" + ); +} + +/// A node needs BOTH planes: the reliable TCP listener and the gossip UDP socket +/// on the same port. When the gossip port is already taken, construction FAILS +/// rather than coming up with a reliable plane and no gossip — a node that could +/// merge membership but never probe, gossip, or be detected as failed. +async fn taken_gossip_port_fails_construction() +where + R: Runtime, +{ + // Take a concrete UDP port, leaving the same TCP port free. + let squatter = std::net::UdpSocket::bind("127.0.0.1:0").expect("squat a UDP port"); + let taken = squatter.local_addr().expect("the squatted address"); + + let err = build_tcp::( + TcpTransportOptions::new() + .with_local_id(SmolStr::new("squatted")) + .with_advertise_addr(MaybeResolved::Resolved(taken)), + ) + .await + .err() + .expect("a node cannot come up without its gossip plane"); + assert!( + matches!(err, SerfError::Io(_)), + "a taken gossip port is an I/O failure, got {err:?}" + ); + + // Freeing the port makes the very same construction succeed — the failure was + // the squatter, not the address. + drop(squatter); + let node = build_tcp::( + TcpTransportOptions::new() + .with_local_id(SmolStr::new("unsquatted")) + .with_advertise_addr(MaybeResolved::Resolved(taken)), + ) + .await + .expect("the released gossip port lets the node bind"); + assert_eq!(node.advertise_address(), taken); + node.shutdown().await.expect("unsquatted shuts down"); +} + +/// The handle's operator readouts are all live and consistent on a healthy joined +/// node: nothing has been shed at any of the four drop counters, the awareness +/// score is healthy, and the QUIC datagram counter stays at zero on a stream +/// transport (it counts only datagram-plane gossip, which plain TCP never sends). +async fn handle_readouts_are_quiet_on_a_healthy_node() +where + R: Runtime, +{ + let b = spawn_node::("obs-b").await; + let a = spawn_node::("obs-a").await; + let b_addr = b.advertise_address(); + + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + converge(&a, &b).await; + + // A subscriber that keeps up, a delegate that never parks, and no coalescing + // pressure: every shed counter must be zero. + assert_eq!(a.events_dropped(), 0, "no subscriber was starved"); + assert_eq!(a.observation_dropped(), 0, "no delegate stalled the pump"); + assert_eq!( + a.coalesced_user_events_dropped(), + 0, + "no user event was shed by the coalescer" + ); + assert_eq!( + a.coalesced_member_events_dropped(), + 0, + "no member change was shed by the coalescer" + ); + assert_eq!( + a.datagrams_sent(), + 0, + "a stream transport never rides the QUIC datagram plane" + ); + assert_eq!(a.health_score(), 0, "a healthy node scores 0"); + assert_eq!( + a.health_score(), + a.stats().health_score(), + "the standalone readout and the aggregate agree" + ); + + a.shutdown().await.expect("obs-a shuts down"); + b.shutdown().await.expect("obs-b shuts down"); +} + +/// The seed-resolution failure paths are LOUD on every join entry point: a +/// resolver that fails surfaces the error from `join_many` and `dispatch_join` +/// rather than reporting a healthy zero-contact join, and a join issued after +/// shutdown reports `Shutdown` rather than parking on a reply that can never come. +async fn join_entry_points_surface_their_failures() +where + R: Runtime, +{ + /// Fails outright — the DNS-down shape. + struct FailingResolver; + impl serf_reactor::Resolver for FailingResolver { + type Address = SocketAddr; + type Error = std::io::Error; + fn resolve( + &self, + _addr: &SocketAddr, + ) -> impl core::future::Future>> + Send + '_ { + // The failure is unconditional, so the future borrows nothing. + let err = std::io::Error::other("resolver is down"); + async move { Err(err) } + } + } + + let node = spawn_node::("seed-fail").await; + // An UNRESOLVED seed is the one the resolver is actually consulted for (an + // already-resolved seed passes straight through). + let unresolved = || { + MaybeResolved::Unresolved( + "127.0.0.1:7219" + .parse::() + .expect("loopback addr"), + ) + }; + + // `join_many` propagates the resolver failure with an empty reached set. + let (reached, err) = node + .join_many(&FailingResolver, [unresolved()].into_iter(), false) + .await + .expect_err("an unresolvable seed set cannot report a healthy join"); + assert!(reached.is_empty(), "no seed was reached"); + assert!( + matches!(err, SerfError::Resolve(_)), + "the resolver failure reaches the caller, got {err:?}" + ); + + // `dispatch_join` propagates it too — a fire-and-forget join must not swallow it. + let err = node + .dispatch_join(&FailingResolver, &[unresolved()]) + .await + .expect_err("a fire-and-forget join still surfaces the resolver failure"); + assert!( + matches!(err, SerfError::Resolve(_)), + "the resolver failure reaches the caller, got {err:?}" + ); + + // A dispatch_join on a LEFT node is refused, not silently dispatched. + node.leave().await.expect("the node leaves the cluster"); + let err = node + .dispatch_join( + &SocketAddrResolver, + &[MaybeResolved::Resolved( + "127.0.0.1:7219".parse().expect("loopback addr"), + )], + ) + .await + .expect_err("a left node cannot dispatch a join"); + assert!( + matches!(err, SerfError::NotRunning), + "a post-leave dispatch_join reports NotRunning, got {err:?}" + ); + + // After shutdown the command never even reaches a driver: the send fails fast. + node.shutdown().await.expect("seed-fail shuts down"); + let err = node + .join( + &SocketAddrResolver, + MaybeResolved::Resolved("127.0.0.1:7219".parse().expect("loopback addr")), + false, + ) + .await + .expect_err("a shut-down node cannot join"); + assert!( + matches!(err, SerfError::Shutdown), + "a post-shutdown join reports Shutdown, got {err:?}" + ); +} + +/// A burst of concurrent operations racing a shutdown must all RESOLVE — with a +/// success, a `NotRunning`, or a `Shutdown` — and never strand a caller on a reply +/// that can never come. Whichever commands the teardown finds still queued are +/// failed by it rather than dropped, and the node still frees its bind address. +async fn concurrent_commands_racing_shutdown_all_resolve() +where + R: Runtime, +{ + let node = spawn_node::("storm-a").await; + let addr = node.advertise_address(); + + // 64 command-issuing tasks against one shutdown: whichever land in the queue as + // the teardown closes it must be failed by the teardown, not stranded. + let mut tasks = Vec::new(); + for i in 0..64u32 { + let node = node.clone(); + tasks.push(R::spawn(async move { + let mut outcomes = Vec::new(); + for _ in 0..8 { + outcomes.push( + node + .user_event("storm", Bytes::from(i.to_be_bytes().to_vec()), false) + .await, + ); + outcomes.push(node.set_tags(serf_proto::Tags::new()).await); + } + outcomes + })); + } + + let shutdown = node.shutdown().await; + shutdown.expect("the node shuts down under the command storm"); + + for t in tasks { + for outcome in t.await.expect("no command task panics or is stranded") { + if let Err(e) = outcome { + assert!( + matches!( + e, + SerfError::Shutdown | SerfError::NotRunning | SerfError::CommandSend + ), + "a command racing the shutdown must resolve with a terminal reason, got {e:?}" + ); + } + } + } + + // The teardown still completed: the bind address is free. + let reborn = build_tcp::( + TcpTransportOptions::new() + .with_local_id(SmolStr::new("storm-b")) + .with_advertise_addr(MaybeResolved::Resolved(addr)), + ) + .await + .expect("the storm did not wedge the teardown; the address is free"); + reborn.shutdown().await.expect("storm-b shuts down"); +} + +/// Every SWIM override the transport options carry is threaded into the +/// coordinator the driver builds. Two nodes configured with the FULL override set +/// — including the reclaim window and the suspicion ceiling that no other scenario +/// sets — still join, converge, and detect an abrupt kill, so no override is +/// dropped or mis-wired on the way through `Transport::run`. +async fn full_swim_override_set_is_threaded_into_the_coordinator() +where + R: Runtime, +{ + async fn spawn_tuned(id: &str) -> Node + where + R: Runtime, + { + build_tcp::( + TcpTransportOptions::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(ephemeral_bind())) + .with_probe_interval(Duration::from_millis(100)) + .with_probe_timeout(Duration::from_millis(50)) + .with_gossip_interval(Duration::from_millis(20)) + .with_suspicion_mult(3) + .with_suspicion_max_timeout_mult(4) + .with_dead_node_reclaim_time(Duration::from_millis(1)) + .with_push_pull_interval(Duration::from_millis(500)) + .with_stream( + serf_reactor::StreamTransportOptions::new() + .with_dial_timeout(Duration::from_secs(5)) + .with_close_timeout(Duration::from_secs(5)), + ), + ) + .await + .expect("spawn a fully-tuned serf tcp node") + } + + let b = spawn_tuned::("tuned-b").await; + let a = spawn_tuned::("tuned-a").await; + let b_addr = b.advertise_address(); + let b_id = SmolStr::new("tuned-b"); + + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("the fully-tuned nodes still join"); + converge(&a, &b).await; + + // The tuned failure detection still fires: an abrupt kill is detected Failed. + b.shutdown().await.expect("tuned-b shuts down abruptly"); + R::timeout(Duration::from_secs(20), async { + loop { + let failed = a + .members() + .iter() + .any(|m| m.node().id_ref() == &b_id && m.status() != MemberStatus::Alive); + if failed { + break; + } + R::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("the tuned SWIM overrides still detect the killed peer"); + + a.shutdown().await.expect("tuned-a shuts down"); +} + +// The tokio cells: the runtime-generic scenarios driven on tokio's multi-thread +// runtime. Gated on the `tokio` feature so the `--test tcp -- smol` build (which +// enables only `smol`) can drop the `agnostic/tokio` code path. +#[cfg(feature = "tokio")] +mod tokio_cells { + use agnostic::tokio::TokioRuntime; + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn two_node_join_converges() { + super::two_node_join_converges::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn construction_requires_id_and_advertise_addr() { + super::construction_requires_id_and_advertise_addr::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn wildcard_advertise_is_refused_and_releases_the_bind() { + super::wildcard_advertise_is_refused_and_releases_the_bind::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn unresolved_advertise_addr_is_resolved_at_construction() { + super::unresolved_advertise_addr_is_resolved_at_construction::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn advertise_resolution_failure_fails_construction() { + super::advertise_resolution_failure_fails_construction::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn taken_gossip_port_fails_construction() { + super::taken_gossip_port_fails_construction::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn full_swim_override_set_is_threaded_into_the_coordinator() { + super::full_swim_override_set_is_threaded_into_the_coordinator::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn handle_readouts_are_quiet_on_a_healthy_node() { + super::handle_readouts_are_quiet_on_a_healthy_node::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn join_entry_points_surface_their_failures() { + super::join_entry_points_surface_their_failures::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn concurrent_commands_racing_shutdown_all_resolve() { + super::concurrent_commands_racing_shutdown_all_resolve::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn post_leave_operations_report_not_running() { + super::post_leave_operations_report_not_running::().await; + } + + #[cfg(encryption)] + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn post_leave_key_operations_report_not_running() { + super::post_leave_key_operations_report_not_running::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn concurrent_leaves_share_one_in_flight_leave() { + super::concurrent_leaves_share_one_in_flight_leave::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn ignore_old_join_suppresses_the_replay() { + super::ignore_old_join_suppresses_the_replay::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn malformed_gossip_datagram_is_ignored() { + super::malformed_gossip_datagram_is_ignored::().await; + } + + #[cfg(encryption)] + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn unauthenticatable_gossip_datagram_is_ignored() { + super::unauthenticatable_gossip_datagram_is_ignored::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn left_node_refuses_new_inbound_exchanges() { + super::left_node_refuses_new_inbound_exchanges::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn peer_reset_mid_exchange_fails_the_join() { + super::peer_reset_mid_exchange_fails_the_join::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn slow_subscriber_sheds_events_and_counts_them() { + super::slow_subscriber_sheds_events_and_counts_them::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn stalling_delegate_sheds_observations_and_counts_them() { + super::stalling_delegate_sheds_observations_and_counts_them::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn unbounded_observation_channel_never_sheds() { + super::unbounded_observation_channel_never_sheds::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn shutdown_racing_an_inflight_join_resolves_it() { + super::shutdown_racing_an_inflight_join_resolves_it::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn second_shutdown_awaits_teardown_completion() { + super::second_shutdown_awaits_teardown_completion::().await; + } + + #[cfg(encryption)] + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn key_op_without_a_keyring_is_refused() { + super::key_op_without_a_keyring_is_refused::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn snapshot_compaction_rewrites_the_live_state() { + super::snapshot_compaction_rewrites_the_live_state::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn user_event_delivered() { + super::user_event_delivered::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn query_round_trip() { + super::query_round_trip::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn leave_emits_left_cluster() { + super::leave_emits_left_cluster::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn reconnect_delegate_reaps_left_member() { + super::reconnect_delegate_reaps_left_member::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn snapshot_forwarders_reflect_joined_cluster() { + super::snapshot_forwarders_reflect_joined_cluster::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn join_many_returns_only_reached_seeds() { + super::join_many_returns_only_reached_seeds::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn remove_failed_node_alias_succeeds() { + super::remove_failed_node_alias_succeeds::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_events_failed() { + super::serf_events_failed::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_events_leave() { + super::serf_events_leave::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_events_leave_with_racing_shutdown() { + super::serf_events_leave_with_racing_shutdown::().await; + } + + #[cfg(feature = "coordinates")] + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn coordinates_surface_on_the_handle() { + super::coordinates_surface_on_the_handle::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn snapshot_restart_rejoins_the_cluster() { + super::snapshot_restart_rejoins_the_cluster::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn snapshot_leave_gate_controls_rejoin() { + super::snapshot_leave_gate_controls_rejoin::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn merge_delegate_is_consulted_on_join() { + super::merge_delegate_is_consulted_on_join::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn leave_with_zero_timeout_racing_shutdown_times_out() { + super::leave_with_zero_timeout_racing_shutdown_times_out::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_reconnect() { + super::serf_reconnect::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_force_leave_failed() { + super::serf_force_leave_failed::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_force_leave_left_is_idempotent() { + super::serf_force_leave_left_is_idempotent::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_remove_failed_node_propagates() { + super::serf_remove_failed_node_propagates::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_remove_failed_node_prune_erases() { + super::serf_remove_failed_node_prune_erases::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn remove_failed_node_absent_is_a_noop() { + super::remove_failed_node_absent_is_a_noop::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_reconnect_same_ip() { + super::serf_reconnect_same_ip::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_join_cancel() { + super::serf_join_cancel::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_set_tags_propagates() { + super::serf_set_tags_propagates::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_update_after_rejoin() { + super::serf_update_after_rejoin::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_update_after_restart_with_changed_tags() { + super::serf_update_after_restart_with_changed_tags::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_query_filter() { + super::serf_query_filter::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_join_leave() { + super::serf_join_leave::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn serf_join_leave_join() { super::serf_join_leave_join::().await; } @@ -2982,4 +4474,159 @@ mod smol_cells { super::mismatched_keyring_nodes_do_not_exchange_membership::(), ); } + + #[test] + fn post_leave_operations_report_not_running_smol() { + SmolRuntime::block_on(super::post_leave_operations_report_not_running::()); + } + + #[cfg(encryption)] + #[test] + fn post_leave_key_operations_report_not_running_smol() { + SmolRuntime::block_on(super::post_leave_key_operations_report_not_running::< + SmolRuntime, + >()); + } + + #[test] + fn concurrent_leaves_share_one_in_flight_leave_smol() { + SmolRuntime::block_on(super::concurrent_leaves_share_one_in_flight_leave::< + SmolRuntime, + >()); + } + + #[test] + fn ignore_old_join_suppresses_the_replay_smol() { + SmolRuntime::block_on(super::ignore_old_join_suppresses_the_replay::()); + } + + #[test] + fn malformed_gossip_datagram_is_ignored_smol() { + SmolRuntime::block_on(super::malformed_gossip_datagram_is_ignored::()); + } + + #[cfg(encryption)] + #[test] + fn unauthenticatable_gossip_datagram_is_ignored_smol() { + SmolRuntime::block_on(super::unauthenticatable_gossip_datagram_is_ignored::< + SmolRuntime, + >()); + } + + #[test] + fn left_node_refuses_new_inbound_exchanges_smol() { + SmolRuntime::block_on(super::left_node_refuses_new_inbound_exchanges::()); + } + + #[test] + fn peer_reset_mid_exchange_fails_the_join_smol() { + SmolRuntime::block_on(super::peer_reset_mid_exchange_fails_the_join::()); + } + + #[test] + fn slow_subscriber_sheds_events_and_counts_them_smol() { + SmolRuntime::block_on(super::slow_subscriber_sheds_events_and_counts_them::< + SmolRuntime, + >()); + } + + #[test] + fn stalling_delegate_sheds_observations_and_counts_them_smol() { + SmolRuntime::block_on( + super::stalling_delegate_sheds_observations_and_counts_them::(), + ); + } + + #[test] + fn unbounded_observation_channel_never_sheds_smol() { + SmolRuntime::block_on(super::unbounded_observation_channel_never_sheds::< + SmolRuntime, + >()); + } + + #[test] + fn shutdown_racing_an_inflight_join_resolves_it_smol() { + SmolRuntime::block_on(super::shutdown_racing_an_inflight_join_resolves_it::< + SmolRuntime, + >()); + } + + #[test] + fn second_shutdown_awaits_teardown_completion_smol() { + SmolRuntime::block_on(super::second_shutdown_awaits_teardown_completion::< + SmolRuntime, + >()); + } + + #[cfg(encryption)] + #[test] + fn key_op_without_a_keyring_is_refused_smol() { + SmolRuntime::block_on(super::key_op_without_a_keyring_is_refused::()); + } + + #[test] + fn snapshot_compaction_rewrites_the_live_state_smol() { + SmolRuntime::block_on(super::snapshot_compaction_rewrites_the_live_state::< + SmolRuntime, + >()); + } + + #[test] + fn construction_requires_id_and_advertise_addr_smol() { + SmolRuntime::block_on(super::construction_requires_id_and_advertise_addr::< + SmolRuntime, + >()); + } + + #[test] + fn wildcard_advertise_is_refused_and_releases_the_bind_smol() { + SmolRuntime::block_on( + super::wildcard_advertise_is_refused_and_releases_the_bind::(), + ); + } + + #[test] + fn unresolved_advertise_addr_is_resolved_at_construction_smol() { + SmolRuntime::block_on( + super::unresolved_advertise_addr_is_resolved_at_construction::(), + ); + } + + #[test] + fn advertise_resolution_failure_fails_construction_smol() { + SmolRuntime::block_on(super::advertise_resolution_failure_fails_construction::< + SmolRuntime, + >()); + } + + #[test] + fn taken_gossip_port_fails_construction_smol() { + SmolRuntime::block_on(super::taken_gossip_port_fails_construction::()); + } + + #[test] + fn full_swim_override_set_is_threaded_into_the_coordinator_smol() { + SmolRuntime::block_on( + super::full_swim_override_set_is_threaded_into_the_coordinator::(), + ); + } + + #[test] + fn handle_readouts_are_quiet_on_a_healthy_node_smol() { + SmolRuntime::block_on(super::handle_readouts_are_quiet_on_a_healthy_node::< + SmolRuntime, + >()); + } + + #[test] + fn join_entry_points_surface_their_failures_smol() { + SmolRuntime::block_on(super::join_entry_points_surface_their_failures::()); + } + + #[test] + fn concurrent_commands_racing_shutdown_all_resolve_smol() { + SmolRuntime::block_on(super::concurrent_commands_racing_shutdown_all_resolve::< + SmolRuntime, + >()); + } } diff --git a/serf-reactor/tests/tls.rs b/serf-reactor/tests/tls.rs index 38c53a2b..2f412806 100644 --- a/serf-reactor/tests/tls.rs +++ b/serf-reactor/tests/tls.rs @@ -1,13 +1,15 @@ -//! Real-node TLS serf tests: two loopback nodes exercising the reactor stream -//! driver with the rustls record layer end-to-end. Each test spins up ephemeral +//! Real-node TLS serf tests: loopback nodes exercising the reactor stream driver +//! with the rustls record layer end-to-end. Each test spins up ephemeral //! `127.0.0.1:0` nodes via the ergonomic [`Serf::tls`] constructor and drives the //! full pump — TLS handshake-on-dial, join push/pull, coordinator merge, gossip, -//! user events, queries, and graceful leave/shutdown. +//! user events, queries, membership fault injection, and graceful leave/shutdown. //! //! The scenario bodies are runtime-generic `async fn ` helpers, so the //! SAME scenario runs as a `#[tokio::test]` cell over `TokioRuntime` and as a //! `_smol` cell driven by `SmolRuntime::block_on` — mirroring memberlist-reactor's -//! runtime-parameterized suite. +//! runtime-parameterized suite. The multi-node fault-injection scenarios drive the +//! shared `cluster` fixture through a TLS [`cluster::Backend`], so the same bodies +//! that pin the TCP membership lifecycle pin it over the TLS record layer. //! //! Mirrors `tests/tcp.rs` (TLS rides the same stream driver as plain TCP, differing //! only in the record layer) and serf-compio's / memberlist-reactor's TLS harness: @@ -25,18 +27,27 @@ use agnostic::Runtime; use bytes::Bytes; use futures_util::{StreamExt, future}; use rustls::{ + RootCertStore, client::danger::{HandshakeSignatureValid, ServerCertVerified}, crypto::CryptoProvider, pki_types::CertificateDer, version::TLS13, }; -use serf_proto::{event::Event, members::SerfState, options::Options as SerfOptions}; +use serf_proto::{ + event::{Event, MemberEventKind}, + members::{MemberStatus, SerfState}, + options::Options as SerfOptions, +}; use serf_reactor::{ FirstAddrResolver, MaybeResolved, RuntimeOptions, Serf, SocketAddrResolver, TlsOptions, TlsTransportOptions, VoidDelegate, }; use smol_str::SmolStr; +/// The reusable multi-node fault-injection fixture, shared with the TCP and QUIC +/// suites. +mod cluster; + /// A reactor TLS node handle over the agnostic runtime `R`. type Node = Serf; @@ -86,14 +97,23 @@ fn crypto_provider() -> Arc { .unwrap_or_else(|| Arc::new(rustls::crypto::ring::default_provider())) } -/// Build a self-signed localhost-SAN `ServerConfig` + accept-any `ClientConfig` -/// bundle. A fresh bundle is built per node so each owns its own cert. -fn test_tls_options() -> TlsOptions { +/// A fresh self-signed localhost-SAN cert + key, and the DER the client side needs +/// to pin it as a trust anchor. +fn self_signed() -> ( + Vec>, + rustls::pki_types::PrivateKeyDer<'static>, +) { let ck = rcgen::generate_simple_self_signed(vec!["localhost".into()]) .expect("rcgen generate_simple_self_signed"); let chain = vec![CertificateDer::from(ck.cert.der().to_vec())]; let key = rustls::pki_types::PrivateKeyDer::Pkcs8(ck.signing_key.serialize_der().into()); + (chain, key) +} +/// Build a self-signed localhost-SAN `ServerConfig` + accept-any `ClientConfig` +/// bundle. A fresh bundle is built per node so each owns its own cert. +fn test_tls_options() -> TlsOptions { + let (chain, key) = self_signed(); let provider = crypto_provider(); let server_cfg = rustls::ServerConfig::builder_with_provider(provider.clone()) @@ -113,6 +133,86 @@ fn test_tls_options() -> TlsOptions { TlsOptions::new(server_cfg, client_cfg) } +/// A bundle whose CLIENT side trusts exactly one root: its OWN self-signed cert. +/// A node built from it can serve peers, but can only complete an outbound +/// handshake against a peer presenting that same cert — the trust-anchor boundary +/// the accept-any bundle above deliberately waives. +fn self_trusting_tls_options() -> TlsOptions { + let (chain, key) = self_signed(); + let provider = crypto_provider(); + + let server_cfg = rustls::ServerConfig::builder_with_provider(provider.clone()) + .with_protocol_versions(&[&TLS13]) + .expect("TLS 1.3 supported") + .with_no_client_auth() + .with_single_cert(chain.clone(), key) + .expect("valid self-signed cert"); + + let mut roots = RootCertStore::empty(); + roots + .add(chain[0].clone()) + .expect("its own cert is a valid root"); + let client_cfg = rustls::ClientConfig::builder_with_provider(provider) + .with_protocol_versions(&[&TLS13]) + .expect("TLS 1.3 supported") + .with_root_certificates(roots) + .with_no_client_auth(); + + TlsOptions::new(server_cfg, client_cfg) +} + +/// An ephemeral loopback bind (`127.0.0.1:0`). +fn ephemeral_bind() -> SocketAddr { + "127.0.0.1:0".parse().expect("loopback addr") +} + +/// The fixture's TLS backend: the fast-SWIM timing mapped onto a +/// [`TlsTransportOptions`] block carrying a fresh per-node cert bundle. +struct Tls; + +impl cluster::Backend for Tls +where + R: Runtime, +{ + async fn build( + id: &str, + bind: SocketAddr, + timing: &cluster::ClusterTiming, + ) -> serf_reactor::Result> { + let mut opts = TlsTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(bind)) + .with_tls_options(test_tls_options()) + .with_probe_interval(timing.probe_interval()) + .with_probe_timeout(timing.probe_timeout()) + .with_gossip_interval(timing.gossip_interval()) + .with_suspicion_mult(timing.suspicion_mult()); + if let Some(v) = timing.dead_node_reclaim { + opts = opts.with_dead_node_reclaim_time(v); + } + if let Some(v) = timing.push_pull_interval { + opts = opts.with_push_pull_interval(v); + } + Serf::::tls( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + timing.serf_opts(), + None, + None, + None, + #[cfg(encryption)] + std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), + ) + .await + } +} + +/// The TLS fault-injection cluster. +type TlsCluster = cluster::Cluster; + /// Build and spawn a reactor TLS node on an ephemeral loopback port through the /// ergonomic `Serf::tls` constructor. The default SNI provider (`Some("localhost")`) /// matches the self-signed cert SAN. @@ -120,10 +220,9 @@ async fn spawn_node(id: &str) -> Node where R: Runtime, { - let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); let opts = TlsTransportOptions::::new() .with_local_id(SmolStr::new(id)) - .with_advertise_addr(MaybeResolved::Resolved(bind)) + .with_advertise_addr(MaybeResolved::Resolved(ephemeral_bind())) .with_tls_options(test_tls_options()); Serf::::tls( opts, @@ -353,11 +452,10 @@ async fn spawn_node_with_snapshot( where R: Runtime, { - let bind: SocketAddr = "127.0.0.1:0".parse().expect("loopback addr"); Serf::::tls( TlsTransportOptions::::new() .with_local_id(SmolStr::new(id)) - .with_advertise_addr(MaybeResolved::Resolved(bind)) + .with_advertise_addr(MaybeResolved::Resolved(ephemeral_bind())) .with_tls_options(test_tls_options()), &SocketAddrResolver, &FirstAddrResolver, @@ -428,6 +526,638 @@ where let _ = std::fs::remove_file(&path); } +/// Over TLS: node A joins node B, then B is abruptly killed. A must observe +/// Join → Failed → Reap about B — a Failed (not a Leave), proving the kill +/// discards the graceful-leave datagram, followed by the reaper removing the +/// failed member under the shortened reconnect timeout. The TLS reliable plane +/// carries the join; the SWIM failure detection that follows rides the plain-UDP +/// gossip plane the TLS transport binds alongside it. +async fn serf_events_failed() +where + R: Runtime, +{ + let mut cluster = TlsCluster::::spawn( + &["tls-failed-a", "tls-failed-b"], + cluster::ClusterTiming::fast(), + ) + .await; + let subject = cluster.id(1); + + cluster.kill_abrupt(1).await; + cluster.await_num_members(0, 1).await; + + cluster + .assert_member_events( + 0, + subject.as_str(), + &[ + MemberEventKind::Join, + MemberEventKind::Failed, + MemberEventKind::Reap, + ], + ) + .await; + + cluster.shutdown_all().await; +} + +/// Over TLS: a graceful leave reaches the peer as a Leave (never a Failed) and +/// lands the leaver in the observer's Left tombstone view. +async fn serf_events_leave() +where + R: Runtime, +{ + let mut cluster = TlsCluster::::spawn( + &["tls-leave-a", "tls-leave-b"], + cluster::ClusterTiming::fast().with_tombstone_timeout(Duration::from_secs(30)), + ) + .await; + let subject = cluster.id(1); + + cluster.leave_graceful(1).await; + + cluster + .assert_member_events( + 0, + subject.as_str(), + &[MemberEventKind::Join, MemberEventKind::Leave], + ) + .await; + cluster.await_left_tombstone(0, subject.as_str()).await; + + cluster.shutdown_all().await; +} + +/// Over TLS: a failed node that returns at the SAME address reconnects rather +/// than being reaped — the survivor's reconnect re-dial re-establishes the TLS +/// session and the member revives (Join → Failed → Join). +async fn serf_reconnect() +where + R: Runtime, +{ + let mut cluster = TlsCluster::::spawn( + &["tls-recon-a", "tls-recon-b"], + cluster::ClusterTiming::fast().with_reconnect_timeout(Duration::from_secs(30)), + ) + .await; + let subject = cluster.id(1); + + cluster.kill_abrupt(1).await; + cluster + .await_member_event(0, subject.as_str(), MemberEventKind::Failed) + .await; + + cluster.restart(1).await; + cluster.await_num_members(0, 2).await; + + cluster + .assert_member_events( + 0, + subject.as_str(), + &[ + MemberEventKind::Join, + MemberEventKind::Failed, + MemberEventKind::Join, + ], + ) + .await; + + cluster.shutdown_all().await; +} + +/// Over TLS: an operator force-leaving a FAILED member transitions it to Left on +/// every surviving node rather than leaving it to linger Failed until the reap. +async fn serf_force_leave_failed() +where + R: Runtime, +{ + let mut cluster = TlsCluster::::spawn( + &["tls-fl-a", "tls-fl-b", "tls-fl-c"], + cluster::ClusterTiming::fast() + .with_tombstone_timeout(Duration::from_secs(120)) + .with_reconnect_timeout(Duration::from_secs(120)), + ) + .await; + let subject = cluster.id(2); + + cluster.kill_abrupt(2).await; + cluster + .await_member_event(0, subject.as_str(), MemberEventKind::Failed) + .await; + cluster + .await_member_event(1, subject.as_str(), MemberEventKind::Failed) + .await; + + cluster + .node(0) + .force_leave(subject.clone(), false) + .await + .expect("force_leave dispatches for a failed member"); + + cluster.await_left_tombstone(0, subject.as_str()).await; + cluster.await_left_tombstone(1, subject.as_str()).await; + cluster + .assert_member_events( + 0, + subject.as_str(), + &[ + MemberEventKind::Join, + MemberEventKind::Failed, + MemberEventKind::Leave, + ], + ) + .await; + + cluster.shutdown_all().await; +} + +/// Over TLS: a tag change propagates in both directions — the observer records an +/// Update member event for the setter and its member view carries the new value. +async fn serf_set_tags_propagates() +where + R: Runtime, +{ + let mut cluster = TlsCluster::::spawn( + &["tls-tags-a", "tls-tags-b"], + cluster::ClusterTiming::fast(), + ) + .await; + let b_id = cluster.id(1); + + let mut tags = serf_proto::Tags::new(); + tags.0.insert(SmolStr::new("role"), SmolStr::new("worker")); + cluster + .node(1) + .set_tags(tags) + .await + .expect("B re-tags itself"); + cluster + .await_member_event(0, b_id.as_str(), MemberEventKind::Update) + .await; + + let seen = cluster + .node(0) + .members() + .iter() + .find(|m| m.node().id_ref().as_str() == b_id.as_str()) + .map(|m| m.tags().0.get("role").cloned()) + .expect("A tracks B"); + assert_eq!( + seen.as_deref(), + Some("worker"), + "A's view of B carries the propagated role tag" + ); + + cluster.shutdown_all().await; +} + +/// Over TLS: after a peer leaves gracefully, the departure settles on BOTH sides +/// under the default tombstone timeout — the observer reaps it while the leaver +/// holds its own self `Left` tombstone (never a self Reap) beside the still-Alive +/// peer. +async fn serf_join_leave() +where + R: Runtime, +{ + let mut cluster = + TlsCluster::::spawn(&["tls-jl-a", "tls-jl-b"], cluster::ClusterTiming::fast()).await; + let peer = cluster.id(0); + let leaver = cluster.id(1); + + cluster.leave_in_place(1).await; + cluster.await_num_members(0, 1).await; + + assert_eq!( + cluster.member_event_kinds(1, leaver.as_str()), + vec![MemberEventKind::Join, MemberEventKind::Leave], + "the leaver holds its self tombstone: Join then Leave, never a self Reap" + ); + cluster + .await_member_status(1, leaver.as_str(), MemberStatus::Left) + .await; + cluster + .await_member_status(1, peer.as_str(), MemberStatus::Alive) + .await; + + cluster.shutdown_all().await; +} + +/// The constructor-supplied merge delegate is the predicate the machine consults +/// on the TLS join push/pull: with a recording accept-all delegate installed on B, +/// A's join drives at least one `notify_merge` on B carrying A's node state. +async fn merge_delegate_is_consulted_on_join() +where + R: Runtime, +{ + use std::sync::atomic::{AtomicUsize, Ordering}; + + struct RecordingMerge { + hits: Arc, + saw_peer: Arc, + } + impl serf_reactor::MergeDelegate for RecordingMerge { + fn notify_merge( + &self, + peers: memberlist_proto::MaybeOwned< + '_, + [memberlist_proto::typed::NodeState], + >, + ) -> bool { + self.hits.fetch_add(1, Ordering::Relaxed); + if peers.iter().any(|p| p.id_ref().as_str() == "tls-merge-a") { + self.saw_peer.fetch_add(1, Ordering::Relaxed); + } + true + } + } + + let hits = Arc::new(AtomicUsize::new(0)); + let saw_peer = Arc::new(AtomicUsize::new(0)); + + let b = Serf::::tls( + TlsTransportOptions::::new() + .with_local_id(SmolStr::new("tls-merge-b")) + .with_advertise_addr(MaybeResolved::Resolved(ephemeral_bind())) + .with_tls_options(test_tls_options()), + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + None, + Some(Box::new(RecordingMerge { + hits: hits.clone(), + saw_peer: saw_peer.clone(), + })), + None, + #[cfg(encryption)] + std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), + ) + .await + .expect("spawn tls-merge-b"); + let a = spawn_node::("tls-merge-a").await; + let b_addr = b.advertise_address(); + + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("join reaches node B"); + converge(&a, &b).await; + + assert!( + hits.load(Ordering::Relaxed) > 0, + "the machine must consult the constructor-supplied merge delegate on the join push/pull" + ); + assert!( + saw_peer.load(Ordering::Relaxed) > 0, + "the consulted peer set must carry the joining node's state" + ); + + a.shutdown().await.expect("tls-merge-a shuts down"); + b.shutdown().await.expect("tls-merge-b shuts down"); +} + +/// An SNI provider that refuses a peer (`None`) aborts the outbound dial BEFORE +/// the handshake, so the await-result join fails and no membership merges. The +/// server side is untouched — B never sees a completed exchange — which is the +/// discriminator against a provider that merely supplied the wrong name (that +/// would fail INSIDE the handshake instead). +async fn sni_provider_refusal_aborts_the_dial() +where + R: Runtime, +{ + let b = spawn_node::("sni-b").await; + let b_addr = b.advertise_address(); + + let a = Serf::::tls( + TlsTransportOptions::::new() + .with_local_id(SmolStr::new("sni-a")) + .with_advertise_addr(MaybeResolved::Resolved(ephemeral_bind())) + .with_tls_options(test_tls_options()) + .with_sni_provider(Box::new(|_| None)), + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + None, + None, + None, + #[cfg(encryption)] + std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), + ) + .await + .expect("a refusing SNI provider still builds a node"); + + let outcome = a + .join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await; + assert!( + outcome.is_err(), + "an SNI refusal must fail the join, not silently merge (got {outcome:?})" + ); + assert_eq!( + a.num_members(), + 1, + "no peer is admitted when the dial never handshakes" + ); + + a.shutdown().await.expect("sni-a shuts down"); + b.shutdown().await.expect("sni-b shuts down"); +} + +/// The TLS trust anchor IS the reliable-plane boundary for the DIALER: a node whose +/// client config trusts only its OWN self-signed cert cannot complete a handshake +/// against a peer presenting a different one, so its join fails and neither side +/// merges membership. The paired accept-any nodes (every other scenario here) prove +/// the same join SUCCEEDS once the verifier accepts the peer's cert, so this +/// failure is real verification, not a broken bundle. +async fn untrusted_peer_cert_fails_the_join() +where + R: Runtime, +{ + // B presents its own fresh self-signed cert; A trusts only A's cert. + let b = spawn_node::("trust-b").await; + let b_addr = b.advertise_address(); + + let a = Serf::::tls( + TlsTransportOptions::::new() + .with_local_id(SmolStr::new("trust-a")) + .with_advertise_addr(MaybeResolved::Resolved(ephemeral_bind())) + .with_tls_options(self_trusting_tls_options()), + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + None, + None, + None, + #[cfg(encryption)] + std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), + ) + .await + .expect("spawn trust-a"); + + let outcome = a + .join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await; + assert!( + outcome.is_err(), + "a peer outside the trust anchor must fail the handshake, not merge (got {outcome:?})" + ); + assert_eq!(a.num_members(), 1, "the dialer admitted nothing"); + assert_eq!( + b.num_members(), + 1, + "the seed never completed an exchange either" + ); + + a.shutdown().await.expect("trust-a shuts down"); + b.shutdown().await.expect("trust-b shuts down"); +} + +/// The TLS transport's construction gate: each field `TlsTransport::new` requires +/// is refused when absent, so a half-built options block can never bind a socket. +async fn construction_requires_id_advertise_and_tls_options() +where + R: Runtime, +{ + async fn build(opts: TlsTransportOptions) -> serf_reactor::Result> + where + R: Runtime, + { + Serf::::tls( + opts, + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + None, + None, + None, + #[cfg(encryption)] + std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), + ) + .await + } + + // No local id. + let err = build::( + TlsTransportOptions::new() + .with_advertise_addr(MaybeResolved::Resolved(ephemeral_bind())) + .with_tls_options(test_tls_options()), + ) + .await + .err() + .expect("a node with no id cannot be built"); + assert!( + matches!(err, serf_reactor::SerfError::Io(ref e) if e.kind() == std::io::ErrorKind::InvalidInput), + "a missing local_id is an InvalidInput, got {err:?}" + ); + + // No advertise address. + let err = build::( + TlsTransportOptions::new() + .with_local_id(SmolStr::new("no-addr")) + .with_tls_options(test_tls_options()), + ) + .await + .err() + .expect("a node with no advertise address cannot be built"); + assert!( + matches!(err, serf_reactor::SerfError::Io(ref e) if e.kind() == std::io::ErrorKind::InvalidInput), + "a missing advertise_addr is an InvalidInput, got {err:?}" + ); + + // No TLS bundle: the record layer has no cert/key to run. + let err = build::( + TlsTransportOptions::new() + .with_local_id(SmolStr::new("no-tls")) + .with_advertise_addr(MaybeResolved::Resolved(ephemeral_bind())), + ) + .await + .err() + .expect("a TLS node with no cert bundle cannot be built"); + assert!( + matches!(err, serf_reactor::SerfError::Io(ref e) if e.kind() == std::io::ErrorKind::InvalidInput), + "a missing tls_options is an InvalidInput, got {err:?}" + ); +} + +/// A wildcard advertise address is refused AFTER the bind: the readback keeps the +/// unspecified IP, which peers could not route back to, so construction fails and +/// the bound sockets are released rather than the node joining as an undialable +/// member. The released port is proven free by an immediate successful rebind of +/// the very port the failed attempt had claimed. +async fn wildcard_advertise_is_refused_and_releases_the_bind() +where + R: Runtime, +{ + // Claim a concrete port through a successful node, then free it, so the wildcard + // attempt below binds a KNOWN port we can prove was released. + let probe = spawn_node::("wild-probe").await; + let port = probe.advertise_address().port(); + probe.shutdown().await.expect("probe shuts down"); + + let wildcard: SocketAddr = format!("0.0.0.0:{port}").parse().expect("wildcard addr"); + let err = Serf::::tls( + TlsTransportOptions::::new() + .with_local_id(SmolStr::new("wild")) + .with_advertise_addr(MaybeResolved::Resolved(wildcard)) + .with_tls_options(test_tls_options()), + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + None, + None, + None, + #[cfg(encryption)] + std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), + ) + .await + .err() + .expect("a wildcard advertise address is not a routable contact"); + assert!( + matches!(err, serf_reactor::SerfError::InvalidAdvertiseAddr(_)), + "a wildcard bind must be refused as an invalid advertise address, got {err:?}" + ); + + // The refused construction released BOTH bound sockets: the same port rebinds. + let after = Serf::::tls( + TlsTransportOptions::::new() + .with_local_id(SmolStr::new("wild-after")) + .with_advertise_addr(MaybeResolved::Resolved( + format!("127.0.0.1:{port}").parse().expect("loopback addr"), + )) + .with_tls_options(test_tls_options()), + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + None, + None, + None, + #[cfg(encryption)] + std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), + ) + .await + .expect("the refused construction released the port it had bound"); + assert_eq!(after.advertise_address().port(), port); + after.shutdown().await.expect("wild-after shuts down"); +} + +/// Every SWIM override the TLS transport options carry is threaded into the +/// coordinator the driver builds. Two nodes configured with the FULL override set +/// — including the reclaim window and the suspicion ceiling that no other scenario +/// sets — still handshake, join, and converge, so no override is dropped or +/// mis-wired on the way through `Transport::run`. +async fn full_swim_override_set_is_threaded_into_the_coordinator() +where + R: Runtime, +{ + async fn spawn_tuned(id: &str) -> Node + where + R: Runtime, + { + Serf::::tls( + TlsTransportOptions::::new() + .with_local_id(SmolStr::new(id)) + .with_advertise_addr(MaybeResolved::Resolved(ephemeral_bind())) + .with_tls_options(test_tls_options()) + .with_probe_interval(Duration::from_millis(100)) + .with_probe_timeout(Duration::from_millis(50)) + .with_gossip_interval(Duration::from_millis(20)) + .with_suspicion_mult(3) + .with_suspicion_max_timeout_mult(4) + .with_dead_node_reclaim_time(Duration::from_millis(1)) + .with_push_pull_interval(Duration::from_millis(500)) + .with_stream( + serf_reactor::StreamTransportOptions::new().with_dial_timeout(Duration::from_secs(5)), + ), + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + None, + None, + None, + #[cfg(encryption)] + std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), + ) + .await + .expect("spawn a fully-tuned serf tls node") + } + + let b = spawn_tuned::("tls-tuned-b").await; + let a = spawn_tuned::("tls-tuned-a").await; + let b_addr = b.advertise_address(); + let b_id = SmolStr::new("tls-tuned-b"); + + a.join(&SocketAddrResolver, MaybeResolved::Resolved(b_addr), false) + .await + .expect("the fully-tuned nodes still handshake and join"); + converge(&a, &b).await; + + // The tuned failure detection still fires: an abrupt kill is detected. + b.shutdown().await.expect("tls-tuned-b shuts down abruptly"); + R::timeout(Duration::from_secs(20), async { + loop { + let failed = a + .members() + .iter() + .any(|m| m.node().id_ref() == &b_id && m.status() != MemberStatus::Alive); + if failed { + break; + } + R::sleep(Duration::from_millis(20)).await; + } + }) + .await + .expect("the tuned SWIM overrides still detect the killed peer"); + + a.shutdown().await.expect("tls-tuned-a shuts down"); +} + +/// An UNRESOLVED advertise address is resolved at construction through the +/// caller's resolvers, and the node comes up on the resolved contact. +async fn unresolved_advertise_addr_is_resolved_at_construction() +where + R: Runtime, +{ + let node = Serf::::tls( + TlsTransportOptions::::new() + .with_local_id(SmolStr::new("resolve-me")) + .with_advertise_addr(MaybeResolved::Unresolved(ephemeral_bind())) + .with_tls_options(test_tls_options()), + &SocketAddrResolver, + &FirstAddrResolver, + VoidDelegate::::new(), + RuntimeOptions::new(), + SerfOptions::new(), + None, + None, + None, + #[cfg(encryption)] + std::sync::Arc::new(serf_reactor::VoidKeyringDelegate), + ) + .await + .expect("the unresolved advertise address resolves through the supplied resolver"); + + let bound = node.advertise_address(); + assert!(bound.ip().is_loopback(), "the resolved contact is loopback"); + assert_ne!( + bound.port(), + 0, + "the ephemeral bind resolved to a concrete port" + ); + node.shutdown().await.expect("resolve-me shuts down"); +} + // The tokio cells: the runtime-generic scenarios driven on tokio's multi-thread // runtime. Gated on the `tokio` feature so the `--test tls -- smol` build (which // enables only `smol`) can drop the `agnostic/tokio` code path. @@ -459,6 +1189,71 @@ mod tokio_cells { async fn leave_emits_left_cluster() { super::leave_emits_left_cluster::().await; } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_events_failed() { + super::serf_events_failed::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_events_leave() { + super::serf_events_leave::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_reconnect() { + super::serf_reconnect::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_force_leave_failed() { + super::serf_force_leave_failed::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_set_tags_propagates() { + super::serf_set_tags_propagates::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn serf_join_leave() { + super::serf_join_leave::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn merge_delegate_is_consulted_on_join() { + super::merge_delegate_is_consulted_on_join::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn sni_provider_refusal_aborts_the_dial() { + super::sni_provider_refusal_aborts_the_dial::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn untrusted_peer_cert_fails_the_join() { + super::untrusted_peer_cert_fails_the_join::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn construction_requires_id_advertise_and_tls_options() { + super::construction_requires_id_advertise_and_tls_options::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn wildcard_advertise_is_refused_and_releases_the_bind() { + super::wildcard_advertise_is_refused_and_releases_the_bind::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn unresolved_advertise_addr_is_resolved_at_construction() { + super::unresolved_advertise_addr_is_resolved_at_construction::().await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn full_swim_override_set_is_threaded_into_the_coordinator() { + super::full_swim_override_set_is_threaded_into_the_coordinator::().await; + } } // The smol cells: the identical scenarios instantiated over `SmolRuntime` and @@ -492,4 +1287,77 @@ mod smol_cells { fn leave_emits_left_cluster_smol() { SmolRuntime::block_on(super::leave_emits_left_cluster::()); } + + #[test] + fn serf_events_failed_smol() { + SmolRuntime::block_on(super::serf_events_failed::()); + } + + #[test] + fn serf_events_leave_smol() { + SmolRuntime::block_on(super::serf_events_leave::()); + } + + #[test] + fn serf_reconnect_smol() { + SmolRuntime::block_on(super::serf_reconnect::()); + } + + #[test] + fn serf_force_leave_failed_smol() { + SmolRuntime::block_on(super::serf_force_leave_failed::()); + } + + #[test] + fn serf_set_tags_propagates_smol() { + SmolRuntime::block_on(super::serf_set_tags_propagates::()); + } + + #[test] + fn serf_join_leave_smol() { + SmolRuntime::block_on(super::serf_join_leave::()); + } + + #[test] + fn merge_delegate_is_consulted_on_join_smol() { + SmolRuntime::block_on(super::merge_delegate_is_consulted_on_join::()); + } + + #[test] + fn sni_provider_refusal_aborts_the_dial_smol() { + SmolRuntime::block_on(super::sni_provider_refusal_aborts_the_dial::()); + } + + #[test] + fn untrusted_peer_cert_fails_the_join_smol() { + SmolRuntime::block_on(super::untrusted_peer_cert_fails_the_join::()); + } + + #[test] + fn construction_requires_id_advertise_and_tls_options_smol() { + SmolRuntime::block_on(super::construction_requires_id_advertise_and_tls_options::< + SmolRuntime, + >()); + } + + #[test] + fn wildcard_advertise_is_refused_and_releases_the_bind_smol() { + SmolRuntime::block_on( + super::wildcard_advertise_is_refused_and_releases_the_bind::(), + ); + } + + #[test] + fn unresolved_advertise_addr_is_resolved_at_construction_smol() { + SmolRuntime::block_on( + super::unresolved_advertise_addr_is_resolved_at_construction::(), + ); + } + + #[test] + fn full_swim_override_set_is_threaded_into_the_coordinator_smol() { + SmolRuntime::block_on( + super::full_swim_override_set_is_threaded_into_the_coordinator::(), + ); + } } diff --git a/serf-smoltcp/src/config/tests.rs b/serf-smoltcp/src/config/tests.rs index 95e3ca0f..224b9f3b 100644 --- a/serf-smoltcp/src/config/tests.rs +++ b/serf-smoltcp/src/config/tests.rs @@ -1,5 +1,39 @@ use super::*; +/// The close timeout is overridable, and a CIDR policy installed through the +/// builder is the one the engine will admit peers against. +#[test] +fn close_timeout_and_cidr_policy_are_installable() { + let c = Options::new().with_close_timeout(Duration::from_secs(42)); + assert_eq!(c.close_timeout, Duration::from_secs(42)); + + #[cfg(feature = "cidr")] + { + use core::net::{IpAddr, Ipv4Addr}; + + let mut policy = serf_embedded::CidrPolicy::block_all(); + policy.add( + "10.0.0.0/8" + .parse::() + .expect("a well-formed CIDR parses"), + ); + + let c = Options::new().with_cidr_policy(policy); + let installed = c.cidr_policy.expect("the policy is installed"); + assert!( + installed.is_allowed(&IpAddr::V4(Ipv4Addr::new(10, 1, 2, 3))), + "an address inside the allow-list is admitted" + ); + assert!( + installed.is_blocked(&IpAddr::V4(Ipv4Addr::new(192, 168, 1, 1))), + "an address outside the allow-list is refused" + ); + + // The default posture installs no policy at all (every address admitted). + assert!(Options::new().cidr_policy.is_none()); + } +} + #[test] fn defaults_are_sane_and_overridable() { let c = Options::new(); diff --git a/serf-smoltcp/src/error/tests.rs b/serf-smoltcp/src/error/tests.rs index 7a691fda..76e2fe38 100644 --- a/serf-smoltcp/src/error/tests.rs +++ b/serf-smoltcp/src/error/tests.rs @@ -201,3 +201,129 @@ fn source_chains_only_for_wrapping_variants() { .is_none() ); } + +/// A resolver error with a recognisable rendering, so the boxed `Resolve` arms can +/// be checked for actually carrying their cause into the message and source chain. +#[derive(Debug)] +struct ResolverFault; + +impl core::fmt::Display for ResolverFault { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str("the resolver gave up") + } +} + +impl core::error::Error for ResolverFault {} + +/// The advertise-resolution arms name the step that failed and carry the resolver's +/// own error into the message, so a caller need not guess which lookup broke. +#[test] +fn advertise_resolution_failures_render_their_cause() { + assert_eq!( + format!("{}", InitError::Resolve(Box::new(ResolverFault))), + "advertise address resolution failed: the resolver gave up" + ); + assert_eq!( + format!("{}", InitError::NoAddresses), + "advertise address resolution returned no addresses" + ); +} + +/// The boxed resolver error chains, so a caller that knows its concrete resolver can +/// downcast to it. +#[cfg(feature = "std")] +#[test] +fn a_resolver_failure_chains_as_the_source() { + use std::error::Error as _; + + let err = InitError::Resolve(Box::new(ResolverFault)); + let source = err.source().expect("the boxed resolver error chains"); + assert_eq!(format!("{source}"), "the resolver gave up"); + assert!( + source.downcast_ref::().is_some(), + "the concrete resolver error survives boxing" + ); + assert!(InitError::NoAddresses.source().is_none()); +} + +/// `from_embedded` maps BOTH halves of the engine's construction error: its serf +/// options half passes through as the driver's typed cause, and its memberlist half +/// is remapped through `from_memberlist`. +#[test] +fn from_embedded_maps_both_engine_halves() { + let invalid = crate::SerfOptions::new() + .with_max_user_event_size(crate::SerfOptions::DEFAULT_USER_EVENT_SIZE_LIMIT + 1) + .validate() + .expect_err("an over-ceiling max_user_event_size is invalid"); + assert!(matches!( + InitError::from_embedded(serf_embedded::InitError::InvalidSerfOptions(invalid)), + InitError::InvalidSerfOptions(_) + )); + + assert!(matches!( + InitError::from_embedded(serf_embedded::InitError::Memberlist( + serf_embedded::MemberlistInitError::ZeroPort + )), + InitError::ZeroPort + )); +} + +/// The join failure modes each name the step that failed, and the wrapping ones +/// chain their cause. +#[test] +fn join_error_renders_and_chains_each_cause() { + assert_eq!( + format!("{}", JoinError::Resolve(Box::new(ResolverFault))), + "seed address resolution failed: the resolver gave up" + ); + assert_eq!( + format!("{}", JoinError::NoAddresses), + "no wire address resolved for any seed" + ); + + // The control arm names the rejection and carries the engine's own message. + let rejected = serf_embedded::SerfError::BadLeaveState(crate::SerfState::Leaving); + let rendered = format!("{}", JoinError::Control(rejected)); + assert!(rendered.starts_with("join was rejected: "), "{rendered}"); + assert!( + rendered.contains(&format!( + "{}", + serf_embedded::SerfError::BadLeaveState(crate::SerfState::Leaving) + )), + "{rendered}" + ); +} + +/// An engine rejection converts into the join error's control arm rather than being +/// flattened into a message. +#[test] +fn join_error_converts_from_an_engine_rejection() { + let err: JoinError = serf_embedded::SerfError::LeaveClockExhausted.into(); + assert!(err.is_control()); + assert!(!err.is_resolve()); + assert!(!err.is_no_addresses()); + assert!(matches!( + err, + JoinError::Control(serf_embedded::SerfError::LeaveClockExhausted) + )); +} + +#[cfg(feature = "std")] +#[test] +fn join_error_source_chains_only_for_wrapping_variants() { + use std::error::Error as _; + + let resolve = JoinError::Resolve(Box::new(ResolverFault)); + assert_eq!( + format!("{}", resolve.source().expect("the resolver error chains")), + "the resolver gave up" + ); + + let control = JoinError::Control(serf_embedded::SerfError::LeaveClockExhausted); + assert!(control.source().is_some()); + + assert!( + JoinError::NoAddresses.source().is_none(), + "a discovery failure carries no inner cause" + ); +} diff --git a/serf-smoltcp/src/serf/tests.rs b/serf-smoltcp/src/serf/tests.rs index f2d23652..1981a154 100644 --- a/serf-smoltcp/src/serf/tests.rs +++ b/serf-smoltcp/src/serf/tests.rs @@ -16,9 +16,9 @@ use smoltcp::{ }; use crate::{ - DEFAULT_EVENT_BUFFER_CAP, EndpointOptions, HardwareAddress, InterfaceOptions, IpAddress, IpCidr, - Options, Serf, SerfOptions, SerfState, SocketAddrResolver, TransformOptions, - stream_io::SmoltcpStream, + DEFAULT_EVENT_BUFFER_CAP, EndpointOptions, EthernetAddress, HardwareAddress, InterfaceOptions, + IpAddress, IpCidr, Ipv4Address, Options, Route, Serf, SerfOptions, SerfState, SocketAddrResolver, + TransformOptions, stream_io::SmoltcpStream, }; /// A `Medium::Ip` device that never delivers a frame — enough to construct a node @@ -217,15 +217,11 @@ impl GossipIo for FloodGossip { fn send(&mut self, _bytes: &[u8], _dest: SocketAddr) {} } -/// Encode a well-formed gossip datagram carrying a single `Alive` for a distinct -/// node id/address, using the default (unlabelled, unencrypted) codec so the engine +/// Encode a well-formed gossip datagram carrying a single `Alive` for `id` at +/// `advertise`, using the default (unlabelled, unencrypted) codec so the engine /// decodes it exactly as any inbound gossip frame. -fn alive_frame(i: usize) -> Vec { - let ip = Ipv4Addr::new(10, 1, (i / 250) as u8, (i % 250 + 1) as u8); - let node = Node::new( - SmolStr::from(std::format!("flood-{i}")), - SocketAddr::new(IpAddr::V4(ip), 7946), - ); +fn alive_frame_at(id: &str, advertise: SocketAddr) -> Vec { + let node = Node::new(SmolStr::from(id), advertise); let alive = Alive::new(1, node) .with_meta(Meta::empty()) .with_protocol_version(ProtocolVersion::V1) @@ -236,6 +232,15 @@ fn alive_frame(i: usize) -> Vec { .to_vec() } +/// A distinct `Alive` per index, each at its own address. +fn alive_frame(i: usize) -> Vec { + let ip = Ipv4Addr::new(10, 1, (i / 250) as u8, (i % 250 + 1) as u8); + alive_frame_at( + &std::format!("flood-{i}"), + SocketAddr::new(IpAddr::V4(ip), 7946), + ) +} + /// A single pump that produces more passive observations than the event buffer cap /// makes the ENGINE shed the excess into its own `events_dropped` counter, while the /// cap-sized remainder still fits the driver's empty queue (no driver-side drop). @@ -350,3 +355,842 @@ fn advertise_not_local_is_rejected() { }; assert!(matches!(err, crate::InitError::AdvertiseAddrNotLocal(_))); } + +// ── link-layer and sizing validation ────────────────────────────────────────── + +/// An `Medium::Ethernet` device, for the checks that only apply to an L2 medium. +struct EthernetDevice; + +impl Device for EthernetDevice { + type RxToken<'a> = NRx; + type TxToken<'a> = NTx; + + fn receive(&mut self, _t: SmolInstant) -> Option<(NRx, NTx)> { + None + } + + fn transmit(&mut self, _t: SmolInstant) -> Option { + Some(NTx) + } + + fn capabilities(&self) -> DeviceCapabilities { + let mut caps = DeviceCapabilities::default(); + caps.medium = Medium::Ethernet; + caps.max_transmission_unit = 1500; + caps.checksum = ChecksumCapabilities::ignored(); + caps + } +} + +const LOCAL: SocketAddr = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 1)), 7946); + +/// Construct over an arbitrary [`Options`] / [`InterfaceOptions`] pair and device, +/// returning only the error — `Serf` is not `Debug` (it holds a smoltcp +/// `Interface`), so the success case cannot be unwrapped for a message. +fn init_error(cfg: Options, iface: InterfaceOptions, device: &mut D) -> crate::InitError +where + D: Device, +{ + let Err(err) = Serf::::try_new( + cfg, + iface, + TransformOptions::default(), + EndpointOptions::new(SmolStr::new("a"), LOCAL), + SerfOptions::new(), + &SocketAddrResolver, + device, + now(), + ) else { + panic!("the configuration under test must be rejected"); + }; + err +} + +/// smoltcp's `Interface::new` asserts the hardware address's medium equals the +/// device's; the driver derives it first and rejects the mismatch as a typed error +/// rather than reaching that assert. +#[test] +fn a_medium_mismatch_is_rejected() { + let mut dev = NullDevice; // Medium::Ip + let iface = InterfaceOptions::new(HardwareAddress::Ethernet(EthernetAddress([ + 0x02, 0, 0, 0, 0, 1, + ]))) + .with_ip_addr(IpCidr::new(IpAddress::v4(10, 0, 0, 1), 24)) + .with_random_seed(1); + + let err = init_error(Options::new(), iface, &mut dev); + let crate::InitError::MediumMismatch(m) = err else { + panic!("an Ethernet hardware address on an IP device must be rejected: {err:?}"); + }; + assert_eq!(m.expected, Medium::Ethernet); + assert_eq!(m.actual, Medium::Ip); +} + +/// smoltcp stores the configured hardware address WITHOUT re-checking it, so a +/// multicast MAC would silently install an invalid L2 identity. The driver rejects +/// it instead. +#[test] +fn a_non_unicast_hardware_address_is_rejected() { + let mut dev = EthernetDevice; + // The low bit of the first octet marks a multicast MAC. + let mac = EthernetAddress([0x01, 0, 0, 0, 0, 1]); + let iface = InterfaceOptions::new(HardwareAddress::Ethernet(mac)) + .with_ip_addr(IpCidr::new(IpAddress::v4(10, 0, 0, 1), 24)) + .with_random_seed(1); + + let err = init_error(Options::new(), iface, &mut dev); + assert!( + matches!(err, crate::InitError::NonUnicastHardwareAddress(_)), + "{err:?}" + ); +} + +/// An interface with no address accepts no packets at all — a silently deaf node. +#[test] +fn an_interface_without_an_address_is_rejected() { + let mut dev = NullDevice; + let iface = InterfaceOptions::new(HardwareAddress::Ip).with_random_seed(1); + assert!(matches!( + init_error(Options::new(), iface, &mut dev), + crate::InitError::MissingIpAddress + )); +} + +/// smoltcp's `check_ip_addrs` `panic!`s on an address that is neither unicast nor +/// unspecified; the driver mirrors the condition and returns a typed error. +#[test] +fn a_non_unicast_interface_address_is_rejected() { + let mut dev = NullDevice; + let iface = InterfaceOptions::new(HardwareAddress::Ip) + // A multicast address is neither unicast nor unspecified. + .with_ip_addr(IpCidr::new(IpAddress::v4(224, 0, 0, 1), 24)) + .with_random_seed(1); + assert!(matches!( + init_error(Options::new(), iface, &mut dev), + crate::InitError::NonUnicastIpAddress(_) + )); +} + +/// A route whose gateway is not unicast would crash the running node at its first +/// off-link egress (smoltcp's neighbor lookup asserts the address is unicast), so +/// it is refused at construction. +#[test] +fn a_non_unicast_route_gateway_is_rejected() { + let mut dev = NullDevice; + let iface = InterfaceOptions::new(HardwareAddress::Ip) + .with_ip_addr(IpCidr::new(IpAddress::v4(10, 0, 0, 1), 24)) + .with_route(Route::new_ipv4_gateway(Ipv4Address::new(224, 0, 0, 1))) + .with_random_seed(1); + assert!(matches!( + init_error(Options::new(), iface, &mut dev), + crate::InitError::NonUnicastRouteGateway(_) + )); +} + +/// A route to an IPv4 prefix via an IPv6 gateway can never resolve a next hop, so +/// it is refused rather than silently failing at egress. +#[test] +fn a_route_family_mismatch_is_rejected() { + let mut dev = NullDevice; + let crossed = Route { + cidr: IpCidr::new(IpAddress::v4(10, 0, 0, 0), 24), + via_router: IpAddress::v6(0xfe80, 0, 0, 0, 0, 0, 0, 1), + preferred_until: None, + expires_at: None, + }; + let iface = InterfaceOptions::new(HardwareAddress::Ip) + .with_ip_addr(IpCidr::new(IpAddress::v4(10, 0, 0, 1), 24)) + .with_route(crossed) + .with_random_seed(1); + assert!(matches!( + init_error(Options::new(), iface, &mut dev), + crate::InitError::RouteFamilyMismatch(_) + )); +} + +/// More addresses (or routes) than smoltcp's fixed interface tables can hold is a +/// typed error, not a silently-truncated interface that would drop the traffic +/// bound for the addresses that did not fit. +#[test] +fn over_capacity_interface_tables_are_rejected() { + let mut dev = NullDevice; + + // smoltcp's address table is a fixed `heapless::Vec`; overflow it comfortably. + let mut iface = InterfaceOptions::new(HardwareAddress::Ip).with_random_seed(1); + for i in 1..=32u8 { + iface = iface.with_ip_addr(IpCidr::new(IpAddress::v4(10, 0, 0, i), 24)); + } + assert!(matches!( + init_error(Options::new(), iface, &mut dev), + crate::InitError::TooManyIpAddresses + )); + + // Likewise the route table. + let mut iface = InterfaceOptions::new(HardwareAddress::Ip) + .with_ip_addr(IpCidr::new(IpAddress::v4(10, 0, 0, 1), 24)) + .with_random_seed(1); + for i in 1..=32u8 { + iface = iface.with_route(Route::new_ipv4_gateway(Ipv4Address::new(10, 0, 0, i))); + } + assert!(matches!( + init_error(Options::new(), iface, &mut dev), + crate::InitError::TooManyRoutes + )); +} + +/// The socket-sizing screens: each one guards a configuration smoltcp would either +/// panic on or silently accept as a permanently-dead plane. +#[test] +fn unusable_socket_sizing_is_rejected() { + let iface = || { + InterfaceOptions::new(HardwareAddress::Ip) + .with_ip_addr(IpCidr::new(IpAddress::v4(10, 0, 0, 1), 24)) + .with_random_seed(1) + }; + + // A listener plus one dial/accept socket is the functional minimum: with one + // socket the listener holds it and the node could never dial a seed. + let mut dev = NullDevice; + let mut cfg = Options::new(); + cfg.tcp_pool_size = 1; + assert!(matches!( + init_error(cfg, iface(), &mut dev), + crate::InitError::TcpPoolTooSmall + )); + + // smoltcp's `RingBuffer::new` accepts a zero-length ring and builds a socket that + // can never receive — a silently-dead reliable plane. + let mut cfg = Options::new(); + cfg.tcp_socket_rx_bytes = 0; + assert!(matches!( + init_error(cfg, iface(), &mut dev), + crate::InitError::ZeroTcpSocketBuffer + )); + + // smoltcp `panic!`s on a receive buffer past 1 GiB. + let mut cfg = Options::new(); + cfg.tcp_socket_rx_bytes = (1 << 30) + 1; + assert!(matches!( + init_error(cfg, iface(), &mut dev), + crate::InitError::TcpRxBufferTooLarge + )); + + // Zero packet-metadata slots is a gossip ring that can never enqueue a datagram. + let mut cfg = Options::new(); + cfg.udp_rx_packets = 0; + assert!(matches!( + init_error(cfg, iface(), &mut dev), + crate::InitError::ZeroUdpPackets + )); +} + +/// The close timeout bounds the graceful reliable-close drain; a zero one would +/// force-abort every close immediately, truncating an in-flight push/pull response. +#[test] +fn a_zero_close_timeout_is_rejected() { + let mut dev = NullDevice; + let iface = InterfaceOptions::new(HardwareAddress::Ip) + .with_ip_addr(IpCidr::new(IpAddress::v4(10, 0, 0, 1), 24)) + .with_random_seed(1); + assert!(matches!( + init_error( + Options::new().with_close_timeout(core::time::Duration::ZERO), + iface, + &mut dev + ), + crate::InitError::ZeroCloseTimeout + )); +} + +// ── the handle's read surface and command forwards ──────────────────────────── + +/// Pump `node` once over `gossip` at `at`, giving the engine the real +/// reliable-plane view. Also turns the smoltcp stack so a dialing socket advances. +fn pump_at( + node: &mut Serf, + gossip: &mut FloodGossip, + at: Instant, +) { + { + let sockets = RefCell::new(&mut node.sockets); + let mut stream = SmoltcpStream::new(&mut node.iface, &sockets); + node.engine.pump(at, gossip, &mut stream); + } + let mut dev = NullDevice; + node.iface.poll( + crate::addr::to_smoltcp_instant(at), + &mut dev, + &mut node.sockets, + ); +} + +/// Pump `node` once at the fixed base instant. +fn pump_over(node: &mut Serf, gossip: &mut FloodGossip) { + pump_at(node, gossip, now()); +} + +/// The base instant advanced by `secs`. +fn advanced(secs: u64) -> Instant { + Instant::from_origin(core::time::Duration::from_secs(86_400 + secs)) +} + +/// A gossip seam that delivers nothing — for pumps that only need to turn the +/// engine's own schedulers. +fn silent_gossip() -> FloodGossip { + FloodGossip { + frames: Vec::new(), + idx: 0, + src: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 2)), 7946), + } +} + +/// The reliable-plane and clock accessors report the engine's live state: a fresh +/// node has armed its listener out of the pool with nothing in flight, and each +/// command advances exactly the clock it owns. +#[test] +fn the_read_surface_reports_the_live_engine_view() { + let mut dev = NullDevice; + let mut node = try_build(LOCAL, &mut dev).expect("a valid configuration constructs"); + node.start(now()); + + // The listener is armed from the pool; nothing else is in flight yet. + assert!(node.listener_present(), "construction arms the listener"); + assert_eq!( + node.pool_free_count(), + Options::new().tcp_pool_size - 1, + "the listener holds one pooled socket; the rest are free to dial" + ); + assert_eq!(node.accepted_inbound_count(), 0); + assert_eq!(node.closing_count(), 0); + assert_eq!(node.half_closed_count(), 0); + assert_eq!(node.pending_dial_count(), 0); + assert_eq!(node.pending_join_count(), 0); + assert_eq!(node.events_dropped(), 0); + + // Each command advances only its own Lamport clock. + let (m0, e0, q0) = (node.member_time(), node.event_time(), node.query_time()); + node + .user_event("evt", bytes::Bytes::from_static(b"p"), false, now()) + .expect("a user event is accepted while running"); + assert!( + node.event_time() > e0, + "a user event advances the event clock" + ); + assert_eq!(node.query_time(), q0, "and leaves the query clock alone"); + + node + .query( + "q", + bytes::Bytes::from_static(b"p"), + Default::default(), + now(), + ) + .expect("a query is accepted while running"); + assert!(node.query_time() > q0, "a query advances the query clock"); + + // Replacing the local tags re-advertises the node's metadata and refreshes the + // local member in the store. It is a metadata refresh, NOT a membership event, + // so serf's member clock — which orders join/leave intents — must not move. + let tags: crate::Tags = [("role", "leader")].into_iter().collect(); + node.set_tags(tags, now()).expect("set_tags while running"); + let local = node + .members() + .into_iter() + .find(|m| m.node().id_ref() == "a") + .expect("the local node is a member of its own view"); + assert_eq!( + local.tags().0.get("role").map(smol_str::SmolStr::as_str), + Some("leader"), + "the refreshed local member must carry the new tags" + ); + assert_eq!( + node.member_time(), + m0, + "a tag refresh is not a membership intent and must not advance the member clock" + ); + assert_eq!(node.members().len(), node.num_members()); +} + +/// A queued seed is visible as pending work until it is dispatched, and cancelling +/// the join reaps it: the handle must not leak a seed the caller gave up on. +#[test] +fn a_cancelled_join_reaps_its_queued_seed() { + use crate::MaybeResolved; + + let mut dev = NullDevice; + let mut node = try_build(LOCAL, &mut dev).expect("a valid configuration constructs"); + node.start(now()); + + let seed = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 2)), 7946); + let handle = node + .join( + &SocketAddrResolver, + &[MaybeResolved::Resolved(seed)], + false, + now(), + ) + .expect("a join from a running node is accepted"); + assert_eq!( + node.pending_join_count(), + 1, + "the seed is queued for the next tick's dispatch" + ); + assert!( + node.poll_join(handle).is_none(), + "the join is still in flight" + ); + + node.cancel_join(handle); + assert_eq!( + node.pending_join_count(), + 0, + "cancelling must drop the still-queued seed" + ); + assert!( + node.poll_join(handle).is_none(), + "a cancelled join never resolves a caller reply" + ); + + // The cancel reaped the entry outright (no exchange had started), so a repeat + // cancel of the now-unknown handle is a no-op rather than a panic. + node.cancel_join(handle); + assert_eq!(node.pending_join_count(), 0); + assert!(node.poll_join(handle).is_none()); +} + +/// A non-empty seed set that resolves to no wire address is a discovery FAILURE, not +/// a successful no-op join — otherwise a caller would believe it had joined a +/// cluster it never contacted. +#[test] +fn a_seed_set_resolving_to_nothing_is_a_join_failure() { + use crate::MaybeResolved; + + /// A resolver that finds no address for any name. + struct EmptyResolver; + + impl crate::Resolver for EmptyResolver { + type Address = SocketAddr; + type Error = core::convert::Infallible; + + fn resolve(&self, _addr: &Self::Address) -> Result { + Ok(serf_embedded::ResolvedAddrs::new()) + } + } + + let mut dev = NullDevice; + let mut node = try_build(LOCAL, &mut dev).expect("a valid configuration constructs"); + node.start(now()); + + let seed = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 2)), 7946); + let Err(err) = node.join( + &EmptyResolver, + &[MaybeResolved::Unresolved(seed)], + false, + now(), + ) else { + panic!("a seed set that resolves to nothing must not report a successful join"); + }; + assert!(err.is_no_addresses(), "{err}"); + assert_eq!(node.pending_join_count(), 0, "no seed was queued"); +} + +/// An operator-driven `force_leave` takes a known peer out of the local view rather +/// than being a silent no-op, and a per-member reconnect policy can be installed and +/// cleared on a running node. +#[test] +fn force_leave_removes_a_known_peer_from_the_local_view() { + use core::time::Duration; + use serf_proto::members::Member; + + /// A reconnect policy pinning every member to a fixed timeout. + struct FixedReconnect(Duration); + + impl serf_embedded::ReconnectDelegate for FixedReconnect { + fn reconnect_timeout(&self, _m: &Member, _t: Duration) -> Duration { + self.0 + } + } + + let mut dev = NullDevice; + let mut node = try_build(LOCAL, &mut dev).expect("a valid configuration constructs"); + node.start(now()); + + node.set_reconnect_delegate(Some(std::boxed::Box::new(FixedReconnect( + Duration::from_secs(30), + )))); + + // Learn one peer over the gossip plane. + let mut gossip = FloodGossip { + frames: std::vec![alive_frame(0)], + idx: 0, + src: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 2)), 7946), + }; + pump_over(&mut node, &mut gossip); + let peer = SmolStr::from("flood-0"); + assert!( + node.members().iter().any(|m| m.node().id_ref() == &peer), + "the gossiped alive must be admitted as a member" + ); + + node + .force_leave(peer.clone(), false, now()) + .expect("force_leave of a known member is accepted"); + pump_over(&mut node, &mut silent_gossip()); + + let seen = node + .members() + .into_iter() + .find(|m| m.node().id_ref() == &peer); + match seen { + None => {} + Some(m) => assert_ne!( + m.status(), + crate::MemberStatus::Alive, + "a force-left peer must not remain Alive in the local view" + ), + } + + // Clearing the override is equally accepted on a running node. + node.set_reconnect_delegate(None); +} + +// ── the engine's dial-failure, pool, and join-abandonment paths ─────────────── + +/// Build a node over an arbitrary [`Options`], advertising [`LOCAL`]. +fn try_build_with( + cfg: Options, + device: &mut NullDevice, +) -> Result, crate::InitError> { + Serf::try_new( + cfg, + ip_iface(1), + TransformOptions::default(), + EndpointOptions::new(SmolStr::new("a"), LOCAL), + SerfOptions::new(), + &SocketAddrResolver, + device, + now(), + ) +} + +/// A policy admitting only `allow`, blocking everything else. +#[cfg(feature = "cidr")] +fn only(allow: &str) -> serf_embedded::CidrPolicy { + let mut policy = serf_embedded::CidrPolicy::block_all(); + policy.add(allow.parse().expect("a well-formed CIDR parses")); + policy +} + +/// A seed the CIDR policy blocks is a dial FAILURE, not a benign no-op: the socket +/// is reclaimed to the pool, the exchange terminalizes, and the await-result join +/// resolves as reaching NO seed — never as a silent success. +#[cfg(feature = "cidr")] +#[test] +fn a_cidr_blocked_seed_fails_the_join_and_reclaims_its_socket() { + use crate::MaybeResolved; + + let mut dev = NullDevice; + // Only the local node's own address is admitted; the seed below is blocked. + let mut node = try_build_with( + Options::new().with_cidr_policy(only("10.0.0.1/32")), + &mut dev, + ) + .expect("a valid configuration constructs"); + node.start(now()); + let free_before = node.pool_free_count(); + + let seed = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 2)), 7946); + // `ignore_old` records an ignore token per started exchange; the terminal must + // clear it, or the machine's ignore set leaks. + let handle = node + .join( + &SocketAddrResolver, + &[MaybeResolved::Resolved(seed)], + true, + now(), + ) + .expect("a join from a running node is accepted"); + + // The pump dispatches the seed, dials it, and the CIDR screen fails the dial. + // `poll_join` delivers exactly once, so the outcome is captured as it resolves. + let mut outcome = None; + for _ in 0..8 { + pump_over(&mut node, &mut silent_gossip()); + if let Some(o) = node.poll_join(handle) { + outcome = Some(o); + break; + } + } + + let Some(Err(failed)) = outcome else { + panic!("a blocked seed must resolve the join as reaching none, got {outcome:?}"); + }; + assert_eq!(failed.requested(), 1); + assert_eq!(failed.contacted(), 0); + + assert_eq!( + node.pool_free_count(), + free_before, + "the failed dial must return its socket to the pool" + ); + assert_eq!( + node.pending_dial_count(), + 0, + "no dial may stay parked after the failure" + ); + assert_eq!( + node.pending_join_count(), + 0, + "the join was fully dispatched" + ); +} + +/// With the pool exhausted, a dial intent is PARKED rather than lost, and picked up +/// once a slot frees — otherwise a join with more seeds than pooled sockets would +/// silently never dispatch the surplus. +/// +/// The seeds are routable but unanswerable (the null device delivers nothing), so +/// the first dial HOLDS its socket in the TCP handshake until the machine's +/// exchange deadline force-aborts it and returns the slot. +#[test] +fn an_exhausted_pool_parks_a_dial_until_a_slot_frees() { + use crate::MaybeResolved; + + let mut dev = NullDevice; + // Two sockets: one becomes the listener, leaving exactly ONE to dial with — so a + // two-seed join must park its second dial. + let mut cfg = Options::new(); + cfg.tcp_pool_size = 2; + let mut node = try_build_with(cfg, &mut dev).expect("a valid configuration constructs"); + node.start(now()); + assert_eq!( + node.pool_free_count(), + 1, + "the listener holds the other slot" + ); + + let seeds = [ + MaybeResolved::Resolved(SocketAddr::new( + IpAddr::V4(Ipv4Addr::new(10, 0, 0, 2)), + 7946, + )), + MaybeResolved::Resolved(SocketAddr::new( + IpAddr::V4(Ipv4Addr::new(10, 0, 0, 3)), + 7946, + )), + ]; + let handle = node + .join(&SocketAddrResolver, &seeds, false, now()) + .expect("a join from a running node is accepted"); + + // The first dial consumes the only free slot and stays in the handshake; the + // second has nowhere to go and MUST be parked rather than dropped. + pump_over(&mut node, &mut silent_gossip()); + assert_eq!( + node.pool_free_count(), + 0, + "the sole free slot backs the first dial" + ); + assert_eq!( + node.pending_dial_count(), + 1, + "the surplus dial must be parked, not lost" + ); + + // A pump while the pool is still empty leaves the parked dial parked. + pump_over(&mut node, &mut silent_gossip()); + assert_eq!( + node.pending_dial_count(), + 1, + "a parked dial stays parked while no slot is free" + ); + + // Past the machine's exchange deadline both dials terminalize; the freed slot is + // handed to the parked dial, and the join resolves having reached no seed. + let mut outcome = None; + for step in 1..=40u64 { + let at = advanced(step * 5); + pump_at(&mut node, &mut silent_gossip(), at); + if let Some(o) = node.poll_join(handle) { + outcome = Some(o); + break; + } + } + + let Some(Err(failed)) = outcome else { + panic!("both unanswerable seeds must resolve the join as reaching none, got {outcome:?}"); + }; + assert_eq!( + failed.requested(), + 2, + "BOTH seeds must have been dispatched, including the parked one" + ); + assert_eq!(failed.contacted(), 0); + assert_eq!( + node.pending_dial_count(), + 0, + "no dial intent may be left parked once the exchanges terminalize" + ); +} + +/// The CIDR policy composes with the built-in routable filter at membership +/// admission (logical AND), so a peer whose datagram arrives from an ADMITTED source +/// but whose self-advertised address is outside the policy is not admitted — its bad +/// address is never stored and never re-gossiped. +#[cfg(feature = "cidr")] +#[test] +fn the_cidr_policy_composes_with_the_routable_filter_at_admission() { + let mut dev = NullDevice; + // The gossip source (10.0.0.2) is admitted at the transport boundary; only + // addresses inside 10.0.0.0/24 pass membership admission. + let mut node = try_build_with( + Options::new().with_cidr_policy(only("10.0.0.0/24")), + &mut dev, + ) + .expect("a valid configuration constructs"); + node.start(now()); + + let admitted = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 5)), 7946); + let outside = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 1, 0, 1)), 7946); + let mut gossip = FloodGossip { + frames: std::vec![ + alive_frame_at("inside", admitted), + alive_frame_at("outside", outside), + ], + idx: 0, + src: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 2)), 7946), + }; + pump_over(&mut node, &mut gossip); + + let ids: std::vec::Vec = node + .members() + .iter() + .map(|m| m.node().id_ref().clone()) + .collect(); + assert!( + ids.iter().any(|id| id == "inside"), + "a peer advertising an admitted address must join: {ids:?}" + ); + assert!( + !ids.iter().any(|id| id == "outside"), + "a peer advertising an address outside the policy must NOT be admitted: {ids:?}" + ); +} + +/// Cancelling a join whose push/pull already dispatched forgets the caller reply +/// (so `poll_join` never yields) while the exchange runs to its own terminal, and a +/// repeat cancel of a reaped handle is a no-op rather than a panic. +#[test] +fn cancelling_a_dispatched_join_forgets_its_reply() { + use crate::MaybeResolved; + + let mut dev = NullDevice; + let mut node = try_build(LOCAL, &mut dev).expect("a valid configuration constructs"); + node.start(now()); + + let seed = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 2)), 7946); + let handle = node + .join( + &SocketAddrResolver, + &[MaybeResolved::Resolved(seed)], + false, + now(), + ) + .expect("a join from a running node is accepted"); + + // One pump dispatches the seed, so the exchange has STARTED when the cancel lands. + pump_over(&mut node, &mut silent_gossip()); + assert_eq!(node.pending_join_count(), 1); + + node.cancel_join(handle); + assert!( + node.poll_join(handle).is_none(), + "a cancelled join must never hand back a caller reply" + ); + + // Cancelling again — a handle the fold may already have reaped — is a no-op. + node.cancel_join(handle); + assert!(node.poll_join(handle).is_none()); +} + +/// Leaving hands every in-flight await-result join its reply once, from the set it +/// had actually reached: the pump initiates no further push/pull after a leave, so a +/// join that reached nothing must resolve as FAILED rather than hang forever. +#[test] +fn leaving_resolves_every_in_flight_join() { + use crate::MaybeResolved; + + let mut dev = NullDevice; + let mut node = try_build(LOCAL, &mut dev).expect("a valid configuration constructs"); + node.start(now()); + + let seed = SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 2)), 7946); + let handle = node + .join( + &SocketAddrResolver, + &[MaybeResolved::Resolved(seed)], + false, + now(), + ) + .expect("a join from a running node is accepted"); + assert!( + node.poll_join(handle).is_none(), + "the join is still in flight before the leave" + ); + + node.leave(now()).expect("leave from a running node"); + + let Some(Err(failed)) = node.poll_join(handle) else { + panic!("a leave must resolve the in-flight join rather than strand it"); + }; + assert_eq!(failed.contacted(), 0, "the join had reached no seed"); + assert_eq!( + node.pending_join_count(), + 0, + "the leave dropped the queued seed" + ); +} + +/// An app that never drains `poll_event` cannot grow the driver's buffer without +/// bound: past the cap the OLDEST buffered events are shed and counted, and the +/// public counter reports the loss rather than hiding it. +#[test] +fn the_app_event_buffer_sheds_the_oldest_when_the_app_never_polls() { + let mut dev = NullDevice; + let mut node = try_build(LOCAL, &mut dev).expect("a valid configuration constructs"); + node.start(now()); + + // Two floods, each drained into the driver's queue without the app polling: the + // first fills it to the cap, the second must shed to stay bounded. + let mut next_id = 0usize; + for _ in 0..2 { + let frames: std::vec::Vec> = (0..DEFAULT_EVENT_BUFFER_CAP) + .map(|_| { + let f = alive_frame(next_id); + next_id += 1; + f + }) + .collect(); + let mut gossip = FloodGossip { + frames, + idx: 0, + src: SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, 2)), 7946), + }; + pump_over(&mut node, &mut gossip); + node.drain_engine_events(now()); + } + + assert!( + node.app_events.len() <= DEFAULT_EVENT_BUFFER_CAP, + "the driver's buffer must stay bounded at the cap, got {}", + node.app_events.len() + ); + assert!( + node.app_events_dropped > 0, + "the second drain must shed the oldest buffered events" + ); + assert!( + node.events_dropped() >= node.app_events_dropped, + "the public counter must include the driver-side shedding" + ); +} diff --git a/serf-smoltcp/src/stream_io/tests.rs b/serf-smoltcp/src/stream_io/tests.rs index 3d924fa0..ed44a0f1 100644 --- a/serf-smoltcp/src/stream_io/tests.rs +++ b/serf-smoltcp/src/stream_io/tests.rs @@ -227,6 +227,102 @@ fn reset_is_not_reported_as_eof() { ); } +/// The view owns NO free-list: the engine's `ReliablePlane::pool` is the single +/// authority, so the view must report an empty pool rather than a phantom one an +/// engine could draw a socket from twice. `give` is the matching no-op — handing a +/// socket back to the view must never make it appear allocatable here. +#[test] +fn the_view_never_claims_a_socket_pool() { + let (mut a, _b) = established(); + let cell = RefCell::new(&mut a.1); + let mut view = SmoltcpStream::new(&mut a.0, &cell); + + assert!( + view.take_free().is_none(), + "the view must not hand out sockets; the engine's pool is the sole authority" + ); + assert_eq!(view.free_count(), 0); + + // Returning a live handle to the view must not register it as free. + view.give(a.3); + assert!(view.take_free().is_none()); + assert_eq!( + view.free_count(), + 0, + "a socket given back must never surface as allocatable on the view" + ); +} + +/// The send/recv capability predicates track the real socket state: an established +/// socket is send-capable, recv-capable, open, and "established" for the engine's +/// promotion check; an aborted one is none of those. +#[test] +fn the_capability_predicates_track_the_socket_state() { + let (mut a, mut b) = established(); + + { + let cell = RefCell::new(&mut a.1); + let view = SmoltcpStream::new(&mut a.0, &cell); + assert!(view.may_send(a.3), "an established socket can send"); + assert!(view.may_recv(a.3), "an established socket can receive"); + assert!(view.is_open(a.3)); + assert!( + view.is_established(a.3), + "the engine promotes on send-capability" + ); + assert_eq!( + view.send_queue(a.3), + 0, + "nothing is queued on a fresh established socket" + ); + } + + // B aborts: the RST drives A's socket to Closed, so every capability drops. + b.1.get_mut::(b.3).abort(); + settle(&mut a, &mut b, 100); + assert_eq!(a.1.get::(a.3).state(), tcp::State::Closed); + + let cell = RefCell::new(&mut a.1); + let view = SmoltcpStream::new(&mut a.0, &cell); + assert!(!view.may_send(a.3), "a reset socket cannot send"); + assert!(!view.may_recv(a.3), "a reset socket cannot receive"); + assert!(!view.is_open(a.3)); + assert!( + !view.is_established(a.3), + "a reset socket must not be promoted as established" + ); +} + +/// `abort` through the view tears the connection down immediately: the local socket +/// reaches `Closed` without a FIN handshake, and the peer sees the reset. +#[test] +fn abort_through_the_view_resets_the_connection() { + let (mut a, mut b) = established(); + + { + let cell = RefCell::new(&mut a.1); + let mut view = SmoltcpStream::new(&mut a.0, &cell); + view.abort(a.3); + } + assert_eq!( + a.1.get::(a.3).state(), + tcp::State::Closed, + "abort must close the local socket outright, with no FIN handshake" + ); + + // The RST reaches B, which likewise drops to Closed rather than half-closing. + settle(&mut a, &mut b, 100); + assert_eq!( + b.1.get::(b.3).state(), + tcp::State::Closed, + "the peer must observe the reset" + ); + assert!( + !recv_finished(&mut b), + "a reset is a failure on the peer, never a graceful EOF" + ); +} + /// A RST that arrives mid-stream (the peer aborts after sending some bytes, the /// classic abrupt teardown) is likewise not a clean EOF: the buffered bytes are /// dropped by the reset and `recv_finished` stays `false`. diff --git a/serf-smoltcp/tests/inbound_admission.rs b/serf-smoltcp/tests/inbound_admission.rs new file mode 100644 index 00000000..76183589 --- /dev/null +++ b/serf-smoltcp/tests/inbound_admission.rs @@ -0,0 +1,197 @@ +//! Inbound reliable-plane ADMISSION end-to-end: which TCP connections a node lets +//! into its serf machine, and what happens to the ones it refuses. +//! +//! A node's listener accepts a socket long before serf sees it, so refusal has to +//! happen at the driver's accept gate. Two refusals are pinned here over a real TCP +//! handshake across the paired device: +//! +//! - a peer the node's CIDR policy excludes is aborted at the transport boundary and +//! never registered as an exchange, and +//! - a peer that dials a node which has already LEFT is refused by the machine. +//! +//! In both cases the refused socket must be returned to the pool and a fresh listener +//! re-armed — otherwise every refusal would shrink the finite socket pool by one and +//! a node under a hostile or stale peer would eventually have no reliable plane at +//! all. The joiner, for its part, must see a FAILED join rather than hang. + +mod harness; + +use core::net::{IpAddr, Ipv4Addr, SocketAddr}; + +use serf_smoltcp::{ + EndpointOptions, MaybeResolved, Options, Serf, SerfOptions, SocketAddrResolver, TransformOptions, +}; +use smol_str::SmolStr; + +fn addr(ip: u8, port: u16) -> SocketAddr { + SocketAddr::new(IpAddr::V4(Ipv4Addr::new(10, 0, 0, ip)), port) +} + +/// Build a node at `10.0.0.{ip}` over `device`, with an optional CIDR policy. +fn node( + ip: u8, + cfg: Options, + device: &mut harness::PairedDevice, + now: serf_smoltcp::Instant, +) -> Serf { + let id = if ip == 1 { "a" } else { "b" }; + Serf::new( + cfg, + harness::ip_iface(IpAddr::V4(Ipv4Addr::new(10, 0, 0, ip))), + TransformOptions::default(), + EndpointOptions::new(SmolStr::new(id), addr(ip, 7946)), + SerfOptions::new(), + &SocketAddrResolver, + device, + now, + ) +} + +/// A policy admitting only `allow`, blocking every other address. +#[cfg(feature = "cidr")] +fn only(allow: &str) -> serf_smoltcp::CidrPolicy { + let mut policy = serf_smoltcp::CidrPolicy::block_all(); + policy.add(allow.parse().expect("a well-formed CIDR parses")); + policy +} + +/// A's CIDR policy excludes B. B dials A over real TCP; A's accept gate aborts the +/// connected socket WITHOUT registering an exchange, returns it to the pool, and +/// re-arms its listener — so the refusal costs A nothing and B's join fails rather +/// than hanging, and A never admits B as a member. +#[cfg(feature = "cidr")] +#[test] +fn a_cidr_blocked_peer_is_refused_at_the_accept_gate() { + const BUDGET: u32 = 2000; + + let (mut da, mut db) = harness::link(1500); + let mut clk = harness::Clock::new(); + let now = clk.now(); + + // A admits only itself; B (10.0.0.2) is outside the policy. + let mut a = node( + 1, + Options::new().with_cidr_policy(only("10.0.0.1/32")), + &mut da, + now, + ); + let mut b = node(2, Options::new(), &mut db, now); + a.start(now); + b.start(now); + + let free_before = a.pool_free_count(); + assert!(a.listener_present(), "A starts with an armed listener"); + + let jid = b + .join( + &SocketAddrResolver, + &[MaybeResolved::Resolved(addr(1, 7946))], + false, + now, + ) + .expect("join from a running node"); + + // Drive both until B's join resolves. `poll_join` delivers exactly once, so the + // outcome is captured as it lands. + let mut outcome = None; + for _ in 0..BUDGET { + let _ = a.poll(clk.now(), &mut da); + let _ = b.poll(clk.now(), &mut db); + while a.poll_event().is_some() {} + while b.poll_event().is_some() {} + if let Some(o) = b.poll_join(jid) { + outcome = Some(o); + break; + } + clk.advance_ms(10); + } + + let Some(Err(failed)) = outcome else { + panic!("a join into a policy that excludes the joiner must FAIL, got {outcome:?}"); + }; + assert_eq!(failed.contacted(), 0, "the blocked join reached no seed"); + + assert_eq!( + a.num_members(), + 1, + "A must never admit the excluded peer as a member" + ); + assert!( + a.listener_present(), + "A must re-arm its listener after refusing the connection" + ); + assert_eq!( + a.pool_free_count(), + free_before, + "a refused connection must return its socket to the pool, not shrink it" + ); + assert_eq!( + a.accepted_inbound_count(), + 0, + "a refused connection is never counted as an accepted exchange" + ); +} + +/// A has LEFT the cluster when B dials it. The machine refuses the inbound exchange, +/// and the driver must still return the connected socket to the pool and re-arm the +/// listener — a rejection must not shrink the finite pool one slot at a time. +#[test] +fn a_peer_dialing_a_departed_node_is_refused_without_leaking_its_socket() { + const BUDGET: u32 = 2000; + + let (mut da, mut db) = harness::link(1500); + let mut clk = harness::Clock::new(); + let now = clk.now(); + + let mut a = node(1, Options::new(), &mut da, now); + let mut b = node(2, Options::new(), &mut db, now); + a.start(now); + b.start(now); + + let free_before = a.pool_free_count(); + + // A leaves before B ever contacts it, so the inbound exchange arrives at a node + // that is no longer an admissible member of the cluster. + a.leave(clk.now()).expect("leave from a running node"); + for _ in 0..50 { + let _ = a.poll(clk.now(), &mut da); + while a.poll_event().is_some() {} + clk.advance_ms(10); + } + + let jid = b + .join( + &SocketAddrResolver, + &[MaybeResolved::Resolved(addr(1, 7946))], + false, + now, + ) + .expect("join from a running node"); + + let mut outcome = None; + for _ in 0..BUDGET { + let _ = a.poll(clk.now(), &mut da); + let _ = b.poll(clk.now(), &mut db); + while a.poll_event().is_some() {} + while b.poll_event().is_some() {} + if let Some(o) = b.poll_join(jid) { + outcome = Some(o); + break; + } + clk.advance_ms(10); + } + + assert!( + outcome.is_some(), + "a join into a departed node must resolve, never hang" + ); + assert!( + a.listener_present(), + "A must re-arm its listener after refusing the exchange" + ); + assert_eq!( + a.pool_free_count(), + free_before, + "a refused exchange must return its socket to the pool, not shrink it" + ); +}