From cff32468040bcc4844f44066cb30da5cf05bf857 Mon Sep 17 00:00:00 2001 From: Roman Khafizianov Date: Mon, 6 Jul 2026 19:08:52 +0200 Subject: [PATCH 1/5] feat(pubsub): stateless space-scoped pub/sub over any-sync Add an ephemeral, fire-and-forget, at-most-once publish/subscribe channel scoped to a space, carried over a dedicated PubSub DRPC stream that is fully isolated from the sync engine. commonspace/pubsub: - Wire protocol (pubsubproto): PubSubStream bidi RPC with Subscribe/ Unsubscribe/Publish/Status frames. - Flat + NATS-style wildcard topics (* one segment, > tail) matched by a per-space sublist-shaped trie; reserved acc/.../ self-owned namespace. - ACL-gated subscribe and publish; per-message account-key signatures (length-prefixed, domain-separated) verified by receivers; payloads encrypted with the space ReadKey via a pluggable Crypto. - Node relay with the one-hop rule (client-originated msgs forwarded once to other responsible nodes, relayed msgs never re-forwarded); LAN peers symmetric; echo and duplicate-path suppression via a bounded msgId ring; per-peer publish token bucket. - Serving-side interest keyed by streamId (trie refcounts subscribing streams) so a reconnecting peer's fresh stream keeps its interest and a mid-subscribe close cannot orphan state. - Reconnect watcher (periodic + on-close resync) and peer TTL so a pure subscriber survives stream drops and idle pool GC. - CloseSpace teardown and EvictMember (active drop on ACL removal, strips tags so delivery stops even on an open stream). - Receive path runs cheap filters (interest, membership, ownership, timestamp staleness) before the Ed25519 verify and records dedup only after verify, shedding forged-signature floods and closing a replay window. net/streampool: - NewStreamPool standalone constructor with WithStreamCloseHook (carries streamId) and WithMetric options; CtxStreamId accessor; RemoveTagsById; cross-tag stream dedup in Broadcast. docs/stateless-pubsub: research and design docs. --- Makefile | 1 + commonspace/pubsub/dedup.go | 48 + commonspace/pubsub/dedup_test.go | 41 + commonspace/pubsub/deps.go | 141 ++ commonspace/pubsub/pubsubproto/errors.go | 20 + .../pubsub/pubsubproto/protos/pubsub.proto | 84 + commonspace/pubsub/pubsubproto/pubsub.pb.go | 623 +++++++ .../pubsub/pubsubproto/pubsub_drpc.pb.go | 149 ++ .../pubsub/pubsubproto/pubsub_vtproto.pb.go | 1501 +++++++++++++++++ commonspace/pubsub/ratelimit.go | 55 + commonspace/pubsub/reconnect_test.go | 249 +++ commonspace/pubsub/rpchandler.go | 21 + commonspace/pubsub/service.go | 940 +++++++++++ commonspace/pubsub/service_test.go | 493 ++++++ commonspace/pubsub/sign.go | 69 + commonspace/pubsub/sign_test.go | 82 + commonspace/pubsub/topic.go | 112 ++ commonspace/pubsub/topic_test.go | 72 + commonspace/pubsub/trie.go | 176 ++ commonspace/pubsub/trie_test.go | 118 ++ docs/stateless-pubsub/DESIGN.md | 635 +++++++ docs/stateless-pubsub/RESEARCH.md | 250 +++ net/streampool/context.go | 9 + net/streampool/streampool.go | 93 +- 24 files changed, 5980 insertions(+), 2 deletions(-) create mode 100644 commonspace/pubsub/dedup.go create mode 100644 commonspace/pubsub/dedup_test.go create mode 100644 commonspace/pubsub/deps.go create mode 100644 commonspace/pubsub/pubsubproto/errors.go create mode 100644 commonspace/pubsub/pubsubproto/protos/pubsub.proto create mode 100644 commonspace/pubsub/pubsubproto/pubsub.pb.go create mode 100644 commonspace/pubsub/pubsubproto/pubsub_drpc.pb.go create mode 100644 commonspace/pubsub/pubsubproto/pubsub_vtproto.pb.go create mode 100644 commonspace/pubsub/ratelimit.go create mode 100644 commonspace/pubsub/reconnect_test.go create mode 100644 commonspace/pubsub/rpchandler.go create mode 100644 commonspace/pubsub/service.go create mode 100644 commonspace/pubsub/service_test.go create mode 100644 commonspace/pubsub/sign.go create mode 100644 commonspace/pubsub/sign_test.go create mode 100644 commonspace/pubsub/topic.go create mode 100644 commonspace/pubsub/topic_test.go create mode 100644 commonspace/pubsub/trie.go create mode 100644 commonspace/pubsub/trie_test.go create mode 100644 docs/stateless-pubsub/DESIGN.md create mode 100644 docs/stateless-pubsub/RESEARCH.md diff --git a/Makefile b/Makefile index 5fdf6cfa3..346d37f2d 100644 --- a/Makefile +++ b/Makefile @@ -53,6 +53,7 @@ proto: $(call generate_drpc,,net/secureservice/handshake/handshakeproto/protos) $(call generate_drpc,,net/rpc/limiter/limiterproto/protos) + $(call generate_drpc,,commonspace/pubsub/pubsubproto/protos) $(call generate_drpc,$(PKGMAP),coordinator/coordinatorproto/protos) $(call generate_drpc,,consensus/consensusproto/protos) $(call generate_drpc,,identityrepo/identityrepoproto/protos) diff --git a/commonspace/pubsub/dedup.go b/commonspace/pubsub/dedup.go new file mode 100644 index 000000000..42b582816 --- /dev/null +++ b/commonspace/pubsub/dedup.go @@ -0,0 +1,48 @@ +package pubsub + +import "sync" + +const msgIdLen = 16 + +// msgIdDedup is a fixed-size duplicate-suppression cache keyed by message id. +// It is a FIFO ring: adding a new id evicts the oldest one, keeping memory flat. +type msgIdDedup struct { + mu sync.Mutex + set map[[msgIdLen]byte]struct{} + ring [][msgIdLen]byte + pos int + full bool +} + +func newMsgIdDedup(size int) *msgIdDedup { + return &msgIdDedup{ + set: make(map[[msgIdLen]byte]struct{}, size), + ring: make([][msgIdLen]byte, size), + } +} + +// seen reports whether id was already recorded; if not, it records it, +// evicting the oldest entry when the cache is full. +func (d *msgIdDedup) seen(id []byte) bool { + if len(id) != msgIdLen { + return false + } + var key [msgIdLen]byte + copy(key[:], id) + d.mu.Lock() + defer d.mu.Unlock() + if _, ok := d.set[key]; ok { + return true + } + if d.full { + delete(d.set, d.ring[d.pos]) + } + d.ring[d.pos] = key + d.set[key] = struct{}{} + d.pos++ + if d.pos == len(d.ring) { + d.pos = 0 + d.full = true + } + return false +} diff --git a/commonspace/pubsub/dedup_test.go b/commonspace/pubsub/dedup_test.go new file mode 100644 index 000000000..b0f02835f --- /dev/null +++ b/commonspace/pubsub/dedup_test.go @@ -0,0 +1,41 @@ +package pubsub + +import ( + "encoding/binary" + "testing" + + "github.com/stretchr/testify/require" +) + +func testMsgId(n int) []byte { + id := make([]byte, msgIdLen) + binary.LittleEndian.PutUint64(id, uint64(n)) + return id +} + +func TestDedupSeen(t *testing.T) { + d := newMsgIdDedup(4) + require.False(t, d.seen(testMsgId(1))) + require.True(t, d.seen(testMsgId(1))) + require.False(t, d.seen(testMsgId(2))) + require.True(t, d.seen(testMsgId(2))) +} + +func TestDedupEviction(t *testing.T) { + d := newMsgIdDedup(4) + for i := 1; i <= 4; i++ { + require.False(t, d.seen(testMsgId(i))) + } + // adding a fifth evicts the oldest (1) + require.False(t, d.seen(testMsgId(5))) + require.False(t, d.seen(testMsgId(1)), "oldest id should have been evicted") + // 1 was re-recorded above, evicting 2 + require.True(t, d.seen(testMsgId(5))) + require.False(t, d.seen(testMsgId(2))) +} + +func TestDedupInvalidLength(t *testing.T) { + d := newMsgIdDedup(4) + require.False(t, d.seen([]byte("short"))) + require.False(t, d.seen([]byte("short")), "invalid ids are never recorded") +} diff --git a/commonspace/pubsub/deps.go b/commonspace/pubsub/deps.go new file mode 100644 index 000000000..c85375f76 --- /dev/null +++ b/commonspace/pubsub/deps.go @@ -0,0 +1,141 @@ +package pubsub + +import ( + "context" + "time" + + "github.com/anyproto/any-sync/commonspace/pubsub/pubsubproto" + "github.com/anyproto/any-sync/metric" + "github.com/anyproto/any-sync/net/peer" + "github.com/anyproto/any-sync/net/streampool" + "github.com/anyproto/any-sync/util/crypto" +) + +// Handler receives a decrypted, signature-verified message on a subscribed topic. +// Handlers run on a bounded dispatch queue and must not block. +type Handler func(spaceId, topic string, identity crypto.PubKey, payload []byte) + +// MembershipChecker gates subscribe and publish on space membership. +// Implementations resolve the space's ACL state; any non-nil error rejects. +type MembershipChecker interface { + CheckMember(ctx context.Context, spaceId string, identity crypto.PubKey) error +} + +// Crypto encrypts and decrypts payloads with the space read key. +// A nil Crypto means plaintext payloads (keyless spaces). +type Crypto interface { + // Encrypt returns the key id used and the ciphertext. + Encrypt(spaceId string, payload []byte) (keyId string, encrypted []byte, err error) + // Decrypt resolves keyId to a historical read key and decrypts. + Decrypt(spaceId, keyId string, encrypted []byte) ([]byte, error) +} + +// PeerProvider resolves the peers a client sends publishes and interest to for a +// space: the responsible sync node plus any directly connected LAN peers. +type PeerProvider interface { + SpacePeers(ctx context.Context, spaceId string) ([]peer.Peer, error) +} + +// Relay is implemented only on responsible nodes; nil on clients. +type Relay interface { + // IsResponsible reports whether this node is responsible for the space. + IsResponsible(spaceId string) bool + // IsResponsibleNode reports whether peerId is a responsible node for the space + // (used to authorize inbound relayed publishes). + IsResponsibleNode(spaceId, peerId string) bool + // OtherResponsiblePeers returns the other responsible nodes, excluding self. + OtherResponsiblePeers(ctx context.Context, spaceId string) ([]peer.Peer, error) +} + +// StatusHandler observes Status frames received from serving peers (rejections). +type StatusHandler func(peerId string, status *pubsubproto.Status) + +// Deps carries the pluggable pieces wired by the node or client host. +type Deps struct { + Membership MembershipChecker + Crypto Crypto + Peers PeerProvider // client side; may be nil on nodes + Relay Relay // node side; nil on clients + OnStatus StatusHandler + // Metric, if set, registers the private pool's prometheus metrics so a node + // relaying at scale is observable. Optional. + Metric metric.Metric + Config Config +} + +// Config bounds the engine per DESIGN.md §9; zero values take defaults. +type Config struct { + MaxPayloadSize int + MaxPatternsPerStream int + MaxPatternsPerSpace int + PublishRps float64 + PublishBurst int + WriteQueueSize int + DispatchQueueSize int + DedupSize int + // MaxTimestampSkew bounds how stale (or future) a received message's timestamp + // may be before it is dropped, raising the replay bar even after dedup eviction. + MaxTimestampSkew time.Duration + // ResyncInterval is how often a client re-pushes its interest to space peers, + // keeping server-side interest alive across reconnects. + ResyncInterval time.Duration + // PeerTTL keeps a pubsub stream's peer from being reaped by idle pool GC. + PeerTTL time.Duration + // DialQueueWorkers/DialQueueSize size the pool's outbound dial pool. + DialQueueWorkers int + DialQueueSize int +} + +func (c Config) withDefaults() Config { + if c.MaxPayloadSize <= 0 { + c.MaxPayloadSize = 64 * 1024 + } + if c.MaxPatternsPerStream <= 0 { + c.MaxPatternsPerStream = 1000 + } + if c.MaxPatternsPerSpace <= 0 { + c.MaxPatternsPerSpace = 100 + } + if c.PublishRps <= 0 { + c.PublishRps = 30 + } + if c.PublishBurst <= 0 { + c.PublishBurst = 60 + } + if c.WriteQueueSize <= 0 { + c.WriteQueueSize = 100 + } + if c.DispatchQueueSize <= 0 { + c.DispatchQueueSize = 100 + } + if c.DedupSize <= 0 { + c.DedupSize = 4096 + } + if c.MaxTimestampSkew <= 0 { + c.MaxTimestampSkew = 5 * time.Minute + } + if c.ResyncInterval <= 0 { + c.ResyncInterval = 20 * time.Second + } + if c.PeerTTL <= 0 { + c.PeerTTL = time.Hour + } + if c.DialQueueWorkers <= 0 { + c.DialQueueWorkers = 4 + } + if c.DialQueueSize <= 0 { + c.DialQueueSize = 100 + } + return c +} + +// streamPoolConfig maps the pubsub config onto the streampool's own config. Only +// the dial knobs are consumed by the pool; per-stream queue size is passed +// explicitly to AddStream/ReadStream via WriteQueueSize. +func (c Config) streamPoolConfig() streampool.StreamConfig { + return streampool.StreamConfig{ + SendQueueSize: c.WriteQueueSize, + DialQueueWorkers: c.DialQueueWorkers, + DialQueueSize: c.DialQueueSize, + } +} diff --git a/commonspace/pubsub/pubsubproto/errors.go b/commonspace/pubsub/pubsubproto/errors.go new file mode 100644 index 000000000..3390ff7a0 --- /dev/null +++ b/commonspace/pubsub/pubsubproto/errors.go @@ -0,0 +1,20 @@ +package pubsubproto + +import ( + "errors" + + "github.com/anyproto/any-sync/net/rpc/rpcerr" +) + +var ( + errGroup = rpcerr.ErrGroup(ErrCodes_ErrorOffset) + + ErrUnexpected = errGroup.Register(errors.New("unexpected error"), uint64(ErrCodes_Unexpected)) + ErrNotAMember = errGroup.Register(errors.New("identity is not a space member"), uint64(ErrCodes_NotAMember)) + ErrNotResponsible = errGroup.Register(errors.New("peer is not responsible for space"), uint64(ErrCodes_NotResponsible)) + ErrRateLimited = errGroup.Register(errors.New("publish rate limit exceeded"), uint64(ErrCodes_RateLimited)) + ErrTooManyTopics = errGroup.Register(errors.New("too many topic patterns"), uint64(ErrCodes_TooManyTopics)) + ErrInvalidMessage = errGroup.Register(errors.New("invalid message"), uint64(ErrCodes_InvalidMessage)) + ErrTopicNotOwned = errGroup.Register(errors.New("topic is owned by another account"), uint64(ErrCodes_TopicNotOwned)) + ErrInvalidTopic = errGroup.Register(errors.New("invalid topic or pattern"), uint64(ErrCodes_InvalidTopic)) +) diff --git a/commonspace/pubsub/pubsubproto/protos/pubsub.proto b/commonspace/pubsub/pubsubproto/protos/pubsub.proto new file mode 100644 index 000000000..ecc49672c --- /dev/null +++ b/commonspace/pubsub/pubsubproto/protos/pubsub.proto @@ -0,0 +1,84 @@ +syntax = "proto3"; +package pubsub; + +option go_package = "commonspace/pubsub/pubsubproto"; + +service PubSub { + // PubSubStream is a long-lived bidirectional stream multiplexing all spaces and topics between two peers + rpc PubSubStream(stream PubSubMessage) returns (stream PubSubMessage); +} + +enum ErrCodes { + Unexpected = 0; + // NotAMember - identity has no permissions in the space's ACL + NotAMember = 1; + // NotResponsible - the serving node is not responsible for the space + NotResponsible = 2; + // RateLimited - the per-peer publish rate limit was exceeded + RateLimited = 3; + // TooManyTopics - the per-stream or per-space pattern cap was exceeded + TooManyTopics = 4; + // InvalidMessage - malformed frame, oversized payload, identity mismatch or bad signature + InvalidMessage = 5; + // TopicNotOwned - publish into the acc/ namespace by an identity other than the topic owner + TopicNotOwned = 6; + // InvalidTopic - malformed topic or pattern: bad wildcard placement, reserved chars, non-canonical form + InvalidTopic = 7; + ErrorOffset = 800; +} + +// PubSubMessage is the single frame type carried by PubSubStream +message PubSubMessage { + oneof content { + Subscribe subscribe = 1; + Unsubscribe unsubscribe = 2; + Publish publish = 3; + Status status = 4; + } +} + +// Subscribe adds topic patterns to the stream's interest set for the space. +// Patterns may contain wildcards: '*' matches exactly one segment, '>' matches one-or-more trailing segments (tail-only) +message Subscribe { + string spaceId = 1; + repeated string topics = 2; +} + +// Unsubscribe removes patterns from the stream's interest set, matched verbatim (not expanded); +// empty topics means remove all patterns of the space +message Unsubscribe { + string spaceId = 1; + repeated string topics = 2; +} + +// Publish carries one ephemeral fire-and-forget message to a fully-qualified topic +message Publish { + string spaceId = 1; + // topic is a fully-qualified '/'-separated topic; wildcards are not allowed + string topic = 2; + // msgId is 16 random bytes generated by the publisher, used for duplicate suppression + bytes msgId = 3; + // keyId is the space ReadKey id used to encrypt the payload; empty means plaintext (keyless spaces) + string keyId = 4; + // payload is ciphertext, or plaintext iff keyId is empty + bytes payload = 5; + // identity is the sender's marshalled account public key + bytes identity = 6; + // signature is the account-key signature over "anysync:pubsub:v1" | spaceId | topic | msgId | keyId | le64(timestampMilli) | payload + bytes signature = 7; + // timestampMilli is the sender's wall clock, informational + int64 timestampMilli = 8; + // relayed is set by a node when forwarding node-to-node; a relayed message is never forwarded again; excluded from the signature + bool relayed = 9; +} + +// Status is sent by the serving peer on a rejected subscribe or publish; success is silent +message Status { + string spaceId = 1; + // topics echoes the offending patterns (or the topic of a rejected publish) + repeated string topics = 2; + ErrCodes code = 3; + // msgId echoes a rejected publish's id so the caller can correlate the + // rejection to a specific Publish; empty for subscribe rejections + bytes msgId = 4; +} diff --git a/commonspace/pubsub/pubsubproto/pubsub.pb.go b/commonspace/pubsub/pubsubproto/pubsub.pb.go new file mode 100644 index 000000000..ac1d45856 --- /dev/null +++ b/commonspace/pubsub/pubsubproto/pubsub.pb.go @@ -0,0 +1,623 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v7.35.1 +// source: commonspace/pubsub/pubsubproto/protos/pubsub.proto + +package pubsubproto + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ErrCodes int32 + +const ( + ErrCodes_Unexpected ErrCodes = 0 + // NotAMember - identity has no permissions in the space's ACL + ErrCodes_NotAMember ErrCodes = 1 + // NotResponsible - the serving node is not responsible for the space + ErrCodes_NotResponsible ErrCodes = 2 + // RateLimited - the per-peer publish rate limit was exceeded + ErrCodes_RateLimited ErrCodes = 3 + // TooManyTopics - the per-stream or per-space pattern cap was exceeded + ErrCodes_TooManyTopics ErrCodes = 4 + // InvalidMessage - malformed frame, oversized payload, identity mismatch or bad signature + ErrCodes_InvalidMessage ErrCodes = 5 + // TopicNotOwned - publish into the acc/ namespace by an identity other than the topic owner + ErrCodes_TopicNotOwned ErrCodes = 6 + // InvalidTopic - malformed topic or pattern: bad wildcard placement, reserved chars, non-canonical form + ErrCodes_InvalidTopic ErrCodes = 7 + ErrCodes_ErrorOffset ErrCodes = 800 +) + +// Enum value maps for ErrCodes. +var ( + ErrCodes_name = map[int32]string{ + 0: "Unexpected", + 1: "NotAMember", + 2: "NotResponsible", + 3: "RateLimited", + 4: "TooManyTopics", + 5: "InvalidMessage", + 6: "TopicNotOwned", + 7: "InvalidTopic", + 800: "ErrorOffset", + } + ErrCodes_value = map[string]int32{ + "Unexpected": 0, + "NotAMember": 1, + "NotResponsible": 2, + "RateLimited": 3, + "TooManyTopics": 4, + "InvalidMessage": 5, + "TopicNotOwned": 6, + "InvalidTopic": 7, + "ErrorOffset": 800, + } +) + +func (x ErrCodes) Enum() *ErrCodes { + p := new(ErrCodes) + *p = x + return p +} + +func (x ErrCodes) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ErrCodes) Descriptor() protoreflect.EnumDescriptor { + return file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_enumTypes[0].Descriptor() +} + +func (ErrCodes) Type() protoreflect.EnumType { + return &file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_enumTypes[0] +} + +func (x ErrCodes) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ErrCodes.Descriptor instead. +func (ErrCodes) EnumDescriptor() ([]byte, []int) { + return file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDescGZIP(), []int{0} +} + +// PubSubMessage is the single frame type carried by PubSubStream +type PubSubMessage struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Content: + // + // *PubSubMessage_Subscribe + // *PubSubMessage_Unsubscribe + // *PubSubMessage_Publish + // *PubSubMessage_Status + Content isPubSubMessage_Content `protobuf_oneof:"content"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PubSubMessage) Reset() { + *x = PubSubMessage{} + mi := &file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PubSubMessage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PubSubMessage) ProtoMessage() {} + +func (x *PubSubMessage) ProtoReflect() protoreflect.Message { + mi := &file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PubSubMessage.ProtoReflect.Descriptor instead. +func (*PubSubMessage) Descriptor() ([]byte, []int) { + return file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDescGZIP(), []int{0} +} + +func (x *PubSubMessage) GetContent() isPubSubMessage_Content { + if x != nil { + return x.Content + } + return nil +} + +func (x *PubSubMessage) GetSubscribe() *Subscribe { + if x != nil { + if x, ok := x.Content.(*PubSubMessage_Subscribe); ok { + return x.Subscribe + } + } + return nil +} + +func (x *PubSubMessage) GetUnsubscribe() *Unsubscribe { + if x != nil { + if x, ok := x.Content.(*PubSubMessage_Unsubscribe); ok { + return x.Unsubscribe + } + } + return nil +} + +func (x *PubSubMessage) GetPublish() *Publish { + if x != nil { + if x, ok := x.Content.(*PubSubMessage_Publish); ok { + return x.Publish + } + } + return nil +} + +func (x *PubSubMessage) GetStatus() *Status { + if x != nil { + if x, ok := x.Content.(*PubSubMessage_Status); ok { + return x.Status + } + } + return nil +} + +type isPubSubMessage_Content interface { + isPubSubMessage_Content() +} + +type PubSubMessage_Subscribe struct { + Subscribe *Subscribe `protobuf:"bytes,1,opt,name=subscribe,proto3,oneof"` +} + +type PubSubMessage_Unsubscribe struct { + Unsubscribe *Unsubscribe `protobuf:"bytes,2,opt,name=unsubscribe,proto3,oneof"` +} + +type PubSubMessage_Publish struct { + Publish *Publish `protobuf:"bytes,3,opt,name=publish,proto3,oneof"` +} + +type PubSubMessage_Status struct { + Status *Status `protobuf:"bytes,4,opt,name=status,proto3,oneof"` +} + +func (*PubSubMessage_Subscribe) isPubSubMessage_Content() {} + +func (*PubSubMessage_Unsubscribe) isPubSubMessage_Content() {} + +func (*PubSubMessage_Publish) isPubSubMessage_Content() {} + +func (*PubSubMessage_Status) isPubSubMessage_Content() {} + +// Subscribe adds topic patterns to the stream's interest set for the space. +// Patterns may contain wildcards: '*' matches exactly one segment, '>' matches one-or-more trailing segments (tail-only) +type Subscribe struct { + state protoimpl.MessageState `protogen:"open.v1"` + SpaceId string `protobuf:"bytes,1,opt,name=spaceId,proto3" json:"spaceId,omitempty"` + Topics []string `protobuf:"bytes,2,rep,name=topics,proto3" json:"topics,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Subscribe) Reset() { + *x = Subscribe{} + mi := &file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Subscribe) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Subscribe) ProtoMessage() {} + +func (x *Subscribe) ProtoReflect() protoreflect.Message { + mi := &file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Subscribe.ProtoReflect.Descriptor instead. +func (*Subscribe) Descriptor() ([]byte, []int) { + return file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDescGZIP(), []int{1} +} + +func (x *Subscribe) GetSpaceId() string { + if x != nil { + return x.SpaceId + } + return "" +} + +func (x *Subscribe) GetTopics() []string { + if x != nil { + return x.Topics + } + return nil +} + +// Unsubscribe removes patterns from the stream's interest set, matched verbatim (not expanded); +// empty topics means remove all patterns of the space +type Unsubscribe struct { + state protoimpl.MessageState `protogen:"open.v1"` + SpaceId string `protobuf:"bytes,1,opt,name=spaceId,proto3" json:"spaceId,omitempty"` + Topics []string `protobuf:"bytes,2,rep,name=topics,proto3" json:"topics,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Unsubscribe) Reset() { + *x = Unsubscribe{} + mi := &file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Unsubscribe) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Unsubscribe) ProtoMessage() {} + +func (x *Unsubscribe) ProtoReflect() protoreflect.Message { + mi := &file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Unsubscribe.ProtoReflect.Descriptor instead. +func (*Unsubscribe) Descriptor() ([]byte, []int) { + return file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDescGZIP(), []int{2} +} + +func (x *Unsubscribe) GetSpaceId() string { + if x != nil { + return x.SpaceId + } + return "" +} + +func (x *Unsubscribe) GetTopics() []string { + if x != nil { + return x.Topics + } + return nil +} + +// Publish carries one ephemeral fire-and-forget message to a fully-qualified topic +type Publish struct { + state protoimpl.MessageState `protogen:"open.v1"` + SpaceId string `protobuf:"bytes,1,opt,name=spaceId,proto3" json:"spaceId,omitempty"` + // topic is a fully-qualified '/'-separated topic; wildcards are not allowed + Topic string `protobuf:"bytes,2,opt,name=topic,proto3" json:"topic,omitempty"` + // msgId is 16 random bytes generated by the publisher, used for duplicate suppression + MsgId []byte `protobuf:"bytes,3,opt,name=msgId,proto3" json:"msgId,omitempty"` + // keyId is the space ReadKey id used to encrypt the payload; empty means plaintext (keyless spaces) + KeyId string `protobuf:"bytes,4,opt,name=keyId,proto3" json:"keyId,omitempty"` + // payload is ciphertext, or plaintext iff keyId is empty + Payload []byte `protobuf:"bytes,5,opt,name=payload,proto3" json:"payload,omitempty"` + // identity is the sender's marshalled account public key + Identity []byte `protobuf:"bytes,6,opt,name=identity,proto3" json:"identity,omitempty"` + // signature is the account-key signature over "anysync:pubsub:v1" | spaceId | topic | msgId | keyId | le64(timestampMilli) | payload + Signature []byte `protobuf:"bytes,7,opt,name=signature,proto3" json:"signature,omitempty"` + // timestampMilli is the sender's wall clock, informational + TimestampMilli int64 `protobuf:"varint,8,opt,name=timestampMilli,proto3" json:"timestampMilli,omitempty"` + // relayed is set by a node when forwarding node-to-node; a relayed message is never forwarded again; excluded from the signature + Relayed bool `protobuf:"varint,9,opt,name=relayed,proto3" json:"relayed,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Publish) Reset() { + *x = Publish{} + mi := &file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Publish) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Publish) ProtoMessage() {} + +func (x *Publish) ProtoReflect() protoreflect.Message { + mi := &file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Publish.ProtoReflect.Descriptor instead. +func (*Publish) Descriptor() ([]byte, []int) { + return file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDescGZIP(), []int{3} +} + +func (x *Publish) GetSpaceId() string { + if x != nil { + return x.SpaceId + } + return "" +} + +func (x *Publish) GetTopic() string { + if x != nil { + return x.Topic + } + return "" +} + +func (x *Publish) GetMsgId() []byte { + if x != nil { + return x.MsgId + } + return nil +} + +func (x *Publish) GetKeyId() string { + if x != nil { + return x.KeyId + } + return "" +} + +func (x *Publish) GetPayload() []byte { + if x != nil { + return x.Payload + } + return nil +} + +func (x *Publish) GetIdentity() []byte { + if x != nil { + return x.Identity + } + return nil +} + +func (x *Publish) GetSignature() []byte { + if x != nil { + return x.Signature + } + return nil +} + +func (x *Publish) GetTimestampMilli() int64 { + if x != nil { + return x.TimestampMilli + } + return 0 +} + +func (x *Publish) GetRelayed() bool { + if x != nil { + return x.Relayed + } + return false +} + +// Status is sent by the serving peer on a rejected subscribe or publish; success is silent +type Status struct { + state protoimpl.MessageState `protogen:"open.v1"` + SpaceId string `protobuf:"bytes,1,opt,name=spaceId,proto3" json:"spaceId,omitempty"` + // topics echoes the offending patterns (or the topic of a rejected publish) + Topics []string `protobuf:"bytes,2,rep,name=topics,proto3" json:"topics,omitempty"` + Code ErrCodes `protobuf:"varint,3,opt,name=code,proto3,enum=pubsub.ErrCodes" json:"code,omitempty"` + // msgId echoes a rejected publish's id so the caller can correlate the + // rejection to a specific Publish; empty for subscribe rejections + MsgId []byte `protobuf:"bytes,4,opt,name=msgId,proto3" json:"msgId,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Status) Reset() { + *x = Status{} + mi := &file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Status) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Status) ProtoMessage() {} + +func (x *Status) ProtoReflect() protoreflect.Message { + mi := &file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Status.ProtoReflect.Descriptor instead. +func (*Status) Descriptor() ([]byte, []int) { + return file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDescGZIP(), []int{4} +} + +func (x *Status) GetSpaceId() string { + if x != nil { + return x.SpaceId + } + return "" +} + +func (x *Status) GetTopics() []string { + if x != nil { + return x.Topics + } + return nil +} + +func (x *Status) GetCode() ErrCodes { + if x != nil { + return x.Code + } + return ErrCodes_Unexpected +} + +func (x *Status) GetMsgId() []byte { + if x != nil { + return x.MsgId + } + return nil +} + +var File_commonspace_pubsub_pubsubproto_protos_pubsub_proto protoreflect.FileDescriptor + +const file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDesc = "" + + "\n" + + "2commonspace/pubsub/pubsubproto/protos/pubsub.proto\x12\x06pubsub\"\xdd\x01\n" + + "\rPubSubMessage\x121\n" + + "\tsubscribe\x18\x01 \x01(\v2\x11.pubsub.SubscribeH\x00R\tsubscribe\x127\n" + + "\vunsubscribe\x18\x02 \x01(\v2\x13.pubsub.UnsubscribeH\x00R\vunsubscribe\x12+\n" + + "\apublish\x18\x03 \x01(\v2\x0f.pubsub.PublishH\x00R\apublish\x12(\n" + + "\x06status\x18\x04 \x01(\v2\x0e.pubsub.StatusH\x00R\x06statusB\t\n" + + "\acontent\"=\n" + + "\tSubscribe\x12\x18\n" + + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x12\x16\n" + + "\x06topics\x18\x02 \x03(\tR\x06topics\"?\n" + + "\vUnsubscribe\x12\x18\n" + + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x12\x16\n" + + "\x06topics\x18\x02 \x03(\tR\x06topics\"\xfb\x01\n" + + "\aPublish\x12\x18\n" + + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x12\x14\n" + + "\x05topic\x18\x02 \x01(\tR\x05topic\x12\x14\n" + + "\x05msgId\x18\x03 \x01(\fR\x05msgId\x12\x14\n" + + "\x05keyId\x18\x04 \x01(\tR\x05keyId\x12\x18\n" + + "\apayload\x18\x05 \x01(\fR\apayload\x12\x1a\n" + + "\bidentity\x18\x06 \x01(\fR\bidentity\x12\x1c\n" + + "\tsignature\x18\a \x01(\fR\tsignature\x12&\n" + + "\x0etimestampMilli\x18\b \x01(\x03R\x0etimestampMilli\x12\x18\n" + + "\arelayed\x18\t \x01(\bR\arelayed\"v\n" + + "\x06Status\x12\x18\n" + + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x12\x16\n" + + "\x06topics\x18\x02 \x03(\tR\x06topics\x12$\n" + + "\x04code\x18\x03 \x01(\x0e2\x10.pubsub.ErrCodesR\x04code\x12\x14\n" + + "\x05msgId\x18\x04 \x01(\fR\x05msgId*\xad\x01\n" + + "\bErrCodes\x12\x0e\n" + + "\n" + + "Unexpected\x10\x00\x12\x0e\n" + + "\n" + + "NotAMember\x10\x01\x12\x12\n" + + "\x0eNotResponsible\x10\x02\x12\x0f\n" + + "\vRateLimited\x10\x03\x12\x11\n" + + "\rTooManyTopics\x10\x04\x12\x12\n" + + "\x0eInvalidMessage\x10\x05\x12\x11\n" + + "\rTopicNotOwned\x10\x06\x12\x10\n" + + "\fInvalidTopic\x10\a\x12\x10\n" + + "\vErrorOffset\x10\xa0\x062J\n" + + "\x06PubSub\x12@\n" + + "\fPubSubStream\x12\x15.pubsub.PubSubMessage\x1a\x15.pubsub.PubSubMessage(\x010\x01B Z\x1ecommonspace/pubsub/pubsubprotob\x06proto3" + +var ( + file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDescOnce sync.Once + file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDescData []byte +) + +func file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDescGZIP() []byte { + file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDescOnce.Do(func() { + file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDesc), len(file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDesc))) + }) + return file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDescData +} + +var file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_goTypes = []any{ + (ErrCodes)(0), // 0: pubsub.ErrCodes + (*PubSubMessage)(nil), // 1: pubsub.PubSubMessage + (*Subscribe)(nil), // 2: pubsub.Subscribe + (*Unsubscribe)(nil), // 3: pubsub.Unsubscribe + (*Publish)(nil), // 4: pubsub.Publish + (*Status)(nil), // 5: pubsub.Status +} +var file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_depIdxs = []int32{ + 2, // 0: pubsub.PubSubMessage.subscribe:type_name -> pubsub.Subscribe + 3, // 1: pubsub.PubSubMessage.unsubscribe:type_name -> pubsub.Unsubscribe + 4, // 2: pubsub.PubSubMessage.publish:type_name -> pubsub.Publish + 5, // 3: pubsub.PubSubMessage.status:type_name -> pubsub.Status + 0, // 4: pubsub.Status.code:type_name -> pubsub.ErrCodes + 1, // 5: pubsub.PubSub.PubSubStream:input_type -> pubsub.PubSubMessage + 1, // 6: pubsub.PubSub.PubSubStream:output_type -> pubsub.PubSubMessage + 6, // [6:7] is the sub-list for method output_type + 5, // [5:6] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name +} + +func init() { file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_init() } +func file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_init() { + if File_commonspace_pubsub_pubsubproto_protos_pubsub_proto != nil { + return + } + file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_msgTypes[0].OneofWrappers = []any{ + (*PubSubMessage_Subscribe)(nil), + (*PubSubMessage_Unsubscribe)(nil), + (*PubSubMessage_Publish)(nil), + (*PubSubMessage_Status)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDesc), len(file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDesc)), + NumEnums: 1, + NumMessages: 5, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_goTypes, + DependencyIndexes: file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_depIdxs, + EnumInfos: file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_enumTypes, + MessageInfos: file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_msgTypes, + }.Build() + File_commonspace_pubsub_pubsubproto_protos_pubsub_proto = out.File + file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_goTypes = nil + file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_depIdxs = nil +} diff --git a/commonspace/pubsub/pubsubproto/pubsub_drpc.pb.go b/commonspace/pubsub/pubsubproto/pubsub_drpc.pb.go new file mode 100644 index 000000000..923f1aa20 --- /dev/null +++ b/commonspace/pubsub/pubsubproto/pubsub_drpc.pb.go @@ -0,0 +1,149 @@ +// Code generated by protoc-gen-go-drpc. DO NOT EDIT. +// protoc-gen-go-drpc version: v1.0.0 +// source: commonspace/pubsub/pubsubproto/protos/pubsub.proto + +package pubsubproto + +import ( + context "context" + errors "errors" + drpc1 "github.com/planetscale/vtprotobuf/codec/drpc" + drpc "storj.io/drpc" + drpcerr "storj.io/drpc/drpcerr" +) + +type drpcEncoding_File_commonspace_pubsub_pubsubproto_protos_pubsub_proto struct{} + +func (drpcEncoding_File_commonspace_pubsub_pubsubproto_protos_pubsub_proto) Marshal(msg drpc.Message) ([]byte, error) { + return drpc1.Marshal(msg) +} + +func (drpcEncoding_File_commonspace_pubsub_pubsubproto_protos_pubsub_proto) Unmarshal(buf []byte, msg drpc.Message) error { + return drpc1.Unmarshal(buf, msg) +} + +func (drpcEncoding_File_commonspace_pubsub_pubsubproto_protos_pubsub_proto) JSONMarshal(msg drpc.Message) ([]byte, error) { + return drpc1.JSONMarshal(msg) +} + +func (drpcEncoding_File_commonspace_pubsub_pubsubproto_protos_pubsub_proto) JSONUnmarshal(buf []byte, msg drpc.Message) error { + return drpc1.JSONUnmarshal(buf, msg) +} + +type DRPCPubSubClient interface { + DRPCConn() drpc.Conn + + PubSubStream(ctx context.Context) (DRPCPubSub_PubSubStreamClient, error) +} + +type drpcPubSubClient struct { + cc drpc.Conn +} + +func NewDRPCPubSubClient(cc drpc.Conn) DRPCPubSubClient { + return &drpcPubSubClient{cc} +} + +func (c *drpcPubSubClient) DRPCConn() drpc.Conn { return c.cc } + +func (c *drpcPubSubClient) PubSubStream(ctx context.Context) (DRPCPubSub_PubSubStreamClient, error) { + stream, err := c.cc.NewStream(ctx, "/pubsub.PubSub/PubSubStream", drpcEncoding_File_commonspace_pubsub_pubsubproto_protos_pubsub_proto{}) + if err != nil { + return nil, err + } + x := &drpcPubSub_PubSubStreamClient{stream} + return x, nil +} + +type DRPCPubSub_PubSubStreamClient interface { + drpc.Stream + Send(*PubSubMessage) error + Recv() (*PubSubMessage, error) +} + +type drpcPubSub_PubSubStreamClient struct { + drpc.Stream +} + +func (x *drpcPubSub_PubSubStreamClient) GetStream() drpc.Stream { + return x.Stream +} + +func (x *drpcPubSub_PubSubStreamClient) Send(m *PubSubMessage) error { + return x.MsgSend(m, drpcEncoding_File_commonspace_pubsub_pubsubproto_protos_pubsub_proto{}) +} + +func (x *drpcPubSub_PubSubStreamClient) Recv() (*PubSubMessage, error) { + m := new(PubSubMessage) + if err := x.MsgRecv(m, drpcEncoding_File_commonspace_pubsub_pubsubproto_protos_pubsub_proto{}); err != nil { + return nil, err + } + return m, nil +} + +func (x *drpcPubSub_PubSubStreamClient) RecvMsg(m *PubSubMessage) error { + return x.MsgRecv(m, drpcEncoding_File_commonspace_pubsub_pubsubproto_protos_pubsub_proto{}) +} + +type DRPCPubSubServer interface { + PubSubStream(DRPCPubSub_PubSubStreamStream) error +} + +type DRPCPubSubUnimplementedServer struct{} + +func (s *DRPCPubSubUnimplementedServer) PubSubStream(DRPCPubSub_PubSubStreamStream) error { + return drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) +} + +type DRPCPubSubDescription struct{} + +func (DRPCPubSubDescription) NumMethods() int { return 1 } + +func (DRPCPubSubDescription) Method(n int) (string, drpc.Encoding, drpc.Receiver, interface{}, bool) { + switch n { + case 0: + return "/pubsub.PubSub/PubSubStream", drpcEncoding_File_commonspace_pubsub_pubsubproto_protos_pubsub_proto{}, + func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) { + return nil, srv.(DRPCPubSubServer). + PubSubStream( + &drpcPubSub_PubSubStreamStream{in1.(drpc.Stream)}, + ) + }, DRPCPubSubServer.PubSubStream, true + default: + return "", nil, nil, nil, false + } +} + +func DRPCRegisterPubSub(mux drpc.Mux, impl DRPCPubSubServer) error { + return mux.Register(impl, DRPCPubSubDescription{}) +} + +type DRPCPubSub_PubSubStreamStream interface { + drpc.Stream + Send(*PubSubMessage) error + Recv() (*PubSubMessage, error) +} + +type drpcPubSub_PubSubStreamStream struct { + drpc.Stream +} + +func (x *drpcPubSub_PubSubStreamStream) GetStream() drpc.Stream { + return x.Stream +} + +func (x *drpcPubSub_PubSubStreamStream) Send(m *PubSubMessage) error { + return x.MsgSend(m, drpcEncoding_File_commonspace_pubsub_pubsubproto_protos_pubsub_proto{}) +} + +func (x *drpcPubSub_PubSubStreamStream) Recv() (*PubSubMessage, error) { + m := new(PubSubMessage) + if err := x.MsgRecv(m, drpcEncoding_File_commonspace_pubsub_pubsubproto_protos_pubsub_proto{}); err != nil { + return nil, err + } + return m, nil +} + +func (x *drpcPubSub_PubSubStreamStream) RecvMsg(m *PubSubMessage) error { + return x.MsgRecv(m, drpcEncoding_File_commonspace_pubsub_pubsubproto_protos_pubsub_proto{}) +} diff --git a/commonspace/pubsub/pubsubproto/pubsub_vtproto.pb.go b/commonspace/pubsub/pubsubproto/pubsub_vtproto.pb.go new file mode 100644 index 000000000..362a0e66a --- /dev/null +++ b/commonspace/pubsub/pubsubproto/pubsub_vtproto.pb.go @@ -0,0 +1,1501 @@ +// Code generated by protoc-gen-go-vtproto. DO NOT EDIT. +// protoc-gen-go-vtproto version: v0.6.0 +// source: commonspace/pubsub/pubsubproto/protos/pubsub.proto + +package pubsubproto + +import ( + fmt "fmt" + protohelpers "github.com/planetscale/vtprotobuf/protohelpers" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + io "io" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +func (m *PubSubMessage) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PubSubMessage) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *PubSubMessage) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if vtmsg, ok := m.Content.(interface { + MarshalToSizedBufferVT([]byte) (int, error) + }); ok { + size, err := vtmsg.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + } + return len(dAtA) - i, nil +} + +func (m *PubSubMessage_Subscribe) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *PubSubMessage_Subscribe) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + i := len(dAtA) + if m.Subscribe != nil { + size, err := m.Subscribe.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} +func (m *PubSubMessage_Unsubscribe) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *PubSubMessage_Unsubscribe) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + i := len(dAtA) + if m.Unsubscribe != nil { + size, err := m.Unsubscribe.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x12 + } + return len(dAtA) - i, nil +} +func (m *PubSubMessage_Publish) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *PubSubMessage_Publish) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + i := len(dAtA) + if m.Publish != nil { + size, err := m.Publish.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x1a + } + return len(dAtA) - i, nil +} +func (m *PubSubMessage_Status) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *PubSubMessage_Status) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + i := len(dAtA) + if m.Status != nil { + size, err := m.Status.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x22 + } + return len(dAtA) - i, nil +} +func (m *Subscribe) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Subscribe) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *Subscribe) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Topics) > 0 { + for iNdEx := len(m.Topics) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Topics[iNdEx]) + copy(dAtA[i:], m.Topics[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Topics[iNdEx]))) + i-- + dAtA[i] = 0x12 + } + } + if len(m.SpaceId) > 0 { + i -= len(m.SpaceId) + copy(dAtA[i:], m.SpaceId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *Unsubscribe) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Unsubscribe) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *Unsubscribe) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Topics) > 0 { + for iNdEx := len(m.Topics) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Topics[iNdEx]) + copy(dAtA[i:], m.Topics[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Topics[iNdEx]))) + i-- + dAtA[i] = 0x12 + } + } + if len(m.SpaceId) > 0 { + i -= len(m.SpaceId) + copy(dAtA[i:], m.SpaceId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *Publish) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Publish) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *Publish) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.Relayed { + i-- + if m.Relayed { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x48 + } + if m.TimestampMilli != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.TimestampMilli)) + i-- + dAtA[i] = 0x40 + } + if len(m.Signature) > 0 { + i -= len(m.Signature) + copy(dAtA[i:], m.Signature) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Signature))) + i-- + dAtA[i] = 0x3a + } + if len(m.Identity) > 0 { + i -= len(m.Identity) + copy(dAtA[i:], m.Identity) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Identity))) + i-- + dAtA[i] = 0x32 + } + if len(m.Payload) > 0 { + i -= len(m.Payload) + copy(dAtA[i:], m.Payload) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Payload))) + i-- + dAtA[i] = 0x2a + } + if len(m.KeyId) > 0 { + i -= len(m.KeyId) + copy(dAtA[i:], m.KeyId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.KeyId))) + i-- + dAtA[i] = 0x22 + } + if len(m.MsgId) > 0 { + i -= len(m.MsgId) + copy(dAtA[i:], m.MsgId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.MsgId))) + i-- + dAtA[i] = 0x1a + } + if len(m.Topic) > 0 { + i -= len(m.Topic) + copy(dAtA[i:], m.Topic) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Topic))) + i-- + dAtA[i] = 0x12 + } + if len(m.SpaceId) > 0 { + i -= len(m.SpaceId) + copy(dAtA[i:], m.SpaceId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *Status) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Status) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *Status) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.MsgId) > 0 { + i -= len(m.MsgId) + copy(dAtA[i:], m.MsgId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.MsgId))) + i-- + dAtA[i] = 0x22 + } + if m.Code != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Code)) + i-- + dAtA[i] = 0x18 + } + if len(m.Topics) > 0 { + for iNdEx := len(m.Topics) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Topics[iNdEx]) + copy(dAtA[i:], m.Topics[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Topics[iNdEx]))) + i-- + dAtA[i] = 0x12 + } + } + if len(m.SpaceId) > 0 { + i -= len(m.SpaceId) + copy(dAtA[i:], m.SpaceId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *PubSubMessage) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if vtmsg, ok := m.Content.(interface{ SizeVT() int }); ok { + n += vtmsg.SizeVT() + } + n += len(m.unknownFields) + return n +} + +func (m *PubSubMessage_Subscribe) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Subscribe != nil { + l = m.Subscribe.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + return n +} +func (m *PubSubMessage_Unsubscribe) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Unsubscribe != nil { + l = m.Unsubscribe.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + return n +} +func (m *PubSubMessage_Publish) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Publish != nil { + l = m.Publish.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + return n +} +func (m *PubSubMessage_Status) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Status != nil { + l = m.Status.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + return n +} +func (m *Subscribe) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.SpaceId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if len(m.Topics) > 0 { + for _, s := range m.Topics { + l = len(s) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *Unsubscribe) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.SpaceId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if len(m.Topics) > 0 { + for _, s := range m.Topics { + l = len(s) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *Publish) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.SpaceId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Topic) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.MsgId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.KeyId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Payload) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Identity) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Signature) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.TimestampMilli != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.TimestampMilli)) + } + if m.Relayed { + n += 2 + } + n += len(m.unknownFields) + return n +} + +func (m *Status) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.SpaceId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if len(m.Topics) > 0 { + for _, s := range m.Topics { + l = len(s) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + if m.Code != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Code)) + } + l = len(m.MsgId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + +func (m *PubSubMessage) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PubSubMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PubSubMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Subscribe", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if oneof, ok := m.Content.(*PubSubMessage_Subscribe); ok { + if err := oneof.Subscribe.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + } else { + v := &Subscribe{} + if err := v.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Content = &PubSubMessage_Subscribe{Subscribe: v} + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Unsubscribe", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if oneof, ok := m.Content.(*PubSubMessage_Unsubscribe); ok { + if err := oneof.Unsubscribe.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + } else { + v := &Unsubscribe{} + if err := v.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Content = &PubSubMessage_Unsubscribe{Unsubscribe: v} + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Publish", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if oneof, ok := m.Content.(*PubSubMessage_Publish); ok { + if err := oneof.Publish.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + } else { + v := &Publish{} + if err := v.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Content = &PubSubMessage_Publish{Publish: v} + } + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Status", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if oneof, ok := m.Content.(*PubSubMessage_Status); ok { + if err := oneof.Status.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + } else { + v := &Status{} + if err := v.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Content = &PubSubMessage_Status{Status: v} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Subscribe) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Subscribe: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Subscribe: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Topics", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Topics = append(m.Topics, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Unsubscribe) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Unsubscribe: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Unsubscribe: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Topics", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Topics = append(m.Topics, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Publish) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Publish: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Publish: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Topic", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Topic = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field MsgId", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.MsgId = append(m.MsgId[:0], dAtA[iNdEx:postIndex]...) + if m.MsgId == nil { + m.MsgId = []byte{} + } + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field KeyId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.KeyId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Payload", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Payload = append(m.Payload[:0], dAtA[iNdEx:postIndex]...) + if m.Payload == nil { + m.Payload = []byte{} + } + iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Identity", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Identity = append(m.Identity[:0], dAtA[iNdEx:postIndex]...) + if m.Identity == nil { + m.Identity = []byte{} + } + iNdEx = postIndex + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Signature", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Signature = append(m.Signature[:0], dAtA[iNdEx:postIndex]...) + if m.Signature == nil { + m.Signature = []byte{} + } + iNdEx = postIndex + case 8: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field TimestampMilli", wireType) + } + m.TimestampMilli = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.TimestampMilli |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 9: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Relayed", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Relayed = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Status) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Status: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Status: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Topics", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Topics = append(m.Topics, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Code", wireType) + } + m.Code = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Code |= ErrCodes(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field MsgId", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.MsgId = append(m.MsgId[:0], dAtA[iNdEx:postIndex]...) + if m.MsgId == nil { + m.MsgId = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} diff --git a/commonspace/pubsub/ratelimit.go b/commonspace/pubsub/ratelimit.go new file mode 100644 index 000000000..c83d42d63 --- /dev/null +++ b/commonspace/pubsub/ratelimit.go @@ -0,0 +1,55 @@ +package pubsub + +import ( + "sync" + "time" + + "golang.org/x/time/rate" +) + +const rateLimiterIdleTimeout = time.Minute + +// peerRateLimiter is a per-peer publish token bucket with lazy garbage collection +// of idle entries, so memory stays O(recently active peers). +type peerRateLimiter struct { + mu sync.Mutex + peers map[string]*peerRate + rps rate.Limit + burst int + lastGC time.Time +} + +type peerRate struct { + *rate.Limiter + lastUsage time.Time +} + +func newPeerRateLimiter(rps float64, burst int) *peerRateLimiter { + return &peerRateLimiter{ + peers: make(map[string]*peerRate), + rps: rate.Limit(rps), + burst: burst, + lastGC: time.Now(), + } +} + +func (r *peerRateLimiter) allow(peerId string) bool { + now := time.Now() + r.mu.Lock() + defer r.mu.Unlock() + if now.Sub(r.lastGC) > rateLimiterIdleTimeout { + for id, pr := range r.peers { + if now.Sub(pr.lastUsage) > rateLimiterIdleTimeout { + delete(r.peers, id) + } + } + r.lastGC = now + } + pr, ok := r.peers[peerId] + if !ok { + pr = &peerRate{Limiter: rate.NewLimiter(r.rps, r.burst)} + r.peers[peerId] = pr + } + pr.lastUsage = now + return pr.Allow() +} diff --git a/commonspace/pubsub/reconnect_test.go b/commonspace/pubsub/reconnect_test.go new file mode 100644 index 000000000..d2f351bb6 --- /dev/null +++ b/commonspace/pubsub/reconnect_test.go @@ -0,0 +1,249 @@ +package pubsub + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/anyproto/any-sync/commonspace/pubsub/pubsubproto" +) + +// rawStream opens a direct PubSubStream from client fx to server target, bypassing +// the client engine's pool so a test can drive two independent streams from the +// same peerId (as a reconnect produces server-side). +func rawStream(t *testing.T, client, target *engineFx) pubsubproto.DRPCPubSub_PubSubStreamClient { + p := connect(t, client, target) + conn, err := p.AcquireDrpcConn(testCtx) + require.NoError(t, err) + stream, err := pubsubproto.NewDRPCPubSubClient(conn).PubSubStream(p.Context()) + require.NoError(t, err) + return stream +} + +func remoteMatchCount(fx *engineFx, spaceId, topic string) int { + fx.svc.remoteMu.Lock() + defer fx.svc.remoteMu.Unlock() + si := fx.svc.remote[spaceId] + if si == nil { + return 0 + } + return len(si.trie.Match(topic, nil)) +} + +func waitMatchCount(t *testing.T, fx *engineFx, spaceId, topic string, want int) { + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if remoteMatchCount(fx, spaceId, topic) == want { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("match count for %s never reached %d (got %d)", topic, want, remoteMatchCount(fx, spaceId, topic)) +} + +// TestReconnectKeepsInterest is the regression test for the streamId-keyed interest +// fix: two streams from the SAME peer each hold interest, and closing one must not +// wipe the other's — a stale reconnecting stream must not take the fresh one down. +func TestReconnectKeepsInterest(t *testing.T) { + membership := &fakeMembership{} + relay := &fakeRelay{} + node := newEngineFx(t, "node", membership, relay) + defer node.finish() + client := newEngineFx(t, "client", membership, nil) + defer client.finish() + membership.allow(client.identity()) + + subMsg := &pubsubproto.PubSubMessage{Content: &pubsubproto.PubSubMessage_Subscribe{ + Subscribe: &pubsubproto.Subscribe{SpaceId: testSpace, Topics: []string{"chat/>"}}, + }} + + // stream 1 subscribes + s1 := rawStream(t, client, node) + require.NoError(t, s1.Send(subMsg)) + waitMatchCount(t, node, testSpace, "chat/x", 1) + + // stream 2 (same peerId) subscribes to the same pattern — the node must track + // it independently, so the trie refcount is now 2 even though Match still + // returns the single pattern + s2 := rawStream(t, client, node) + require.NoError(t, s2.Send(subMsg)) + // give the node time to process s2's subscribe + time.Sleep(200 * time.Millisecond) + require.Equal(t, 1, remoteMatchCount(node, testSpace, "chat/x")) + + // close stream 1 — the stale stream must NOT take down stream 2's interest + require.NoError(t, s1.Close()) + // interest must remain after s1's close is processed + time.Sleep(300 * time.Millisecond) + require.Equal(t, 1, remoteMatchCount(node, testSpace, "chat/x"), + "closing the stale stream wiped the fresh stream's interest") + + // closing stream 2 finally clears it + require.NoError(t, s2.Close()) + waitMatchCount(t, node, testSpace, "chat/x", 0) + + // and the per-stream bookkeeping is fully drained + node.svc.remoteMu.Lock() + remainingStreams := len(node.svc.streams) + remainingSpaces := len(node.svc.remote) + node.svc.remoteMu.Unlock() + require.Equal(t, 0, remainingStreams, "stream records leaked") + require.Equal(t, 0, remainingSpaces, "space records leaked") +} + +// TestStreamCloseDrainsInterest verifies a single stream's interest and all +// bookkeeping is removed when it closes without an explicit unsubscribe. +func TestStreamCloseDrainsInterest(t *testing.T) { + membership := &fakeMembership{} + relay := &fakeRelay{} + node := newEngineFx(t, "node", membership, relay) + defer node.finish() + client := newEngineFx(t, "client", membership, nil) + defer client.finish() + membership.allow(client.identity()) + + s := rawStream(t, client, node) + require.NoError(t, s.Send(&pubsubproto.PubSubMessage{Content: &pubsubproto.PubSubMessage_Subscribe{ + Subscribe: &pubsubproto.Subscribe{SpaceId: testSpace, Topics: []string{"a/*", "b/>", "c"}}, + }})) + waitMatchCount(t, node, testSpace, "a/x", 1) + + require.NoError(t, s.Close()) + waitMatchCount(t, node, testSpace, "a/x", 0) + node.svc.remoteMu.Lock() + defer node.svc.remoteMu.Unlock() + require.Empty(t, node.svc.streams) + require.Empty(t, node.svc.remote) +} + +// TestInvalidSpaceIdRejected ensures a spaceId containing '/' (which would break +// tag parsing) is rejected at subscribe with InvalidTopic. +func TestInvalidSpaceIdRejected(t *testing.T) { + membership := &fakeMembership{} + node := newEngineFx(t, "node", membership, &fakeRelay{}) + defer node.finish() + client := newEngineFx(t, "client", membership, nil) + defer client.finish() + membership.allow(client.identity()) + + s := rawStream(t, client, node) + require.NoError(t, s.Send(&pubsubproto.PubSubMessage{Content: &pubsubproto.PubSubMessage_Subscribe{ + Subscribe: &pubsubproto.Subscribe{SpaceId: "bad/space", Topics: []string{"chat/>"}}, + }})) + // the node replies with a Status frame we can read directly off the raw stream + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + msg, err := s.Recv() + require.NoError(t, err) + if st := msg.GetStatus(); st != nil { + require.Equal(t, pubsubproto.ErrCodes_InvalidTopic, st.Code) + return + } + } + t.Fatal("no InvalidTopic status received") +} + +// TestEvictMemberStopsDelivery verifies §6.4 active eviction: after EvictMember, +// a removed member's stream stops receiving even though its stream stays open, and +// another member subscribed to the same pattern keeps receiving. +func TestEvictMemberStopsDelivery(t *testing.T) { + n := newNetFx(t) + defer n.finish() + + _, err := n.clientA.svc.Subscribe(testSpace, "chat/>", n.clientA.handler()) + require.NoError(t, err) + _, err = n.clientC.svc.Subscribe(testSpace, "chat/>", n.clientC.handler()) + require.NoError(t, err) + waitMatchCount(t, n.nodeB, testSpace, "chat/x", 1) + // wait until both members' interest is registered on the node + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + n.nodeB.svc.remoteMu.Lock() + nstreams := len(n.nodeB.svc.streams) + n.nodeB.svc.remoteMu.Unlock() + if nstreams == 2 { + break + } + time.Sleep(10 * time.Millisecond) + } + + // evict clientA at the node + n.nodeB.svc.EvictMember(testSpace, n.clientA.identity()) + + // clientC (still a member) publishes; clientC receives, clientA does not + require.NoError(t, n.clientC.svc.Publish(testCtx, testSpace, "chat/x", []byte("members-only"))) + require.Equal(t, "members-only", waitReceived(t, n.clientC).payload) + expectSilence(t, n.clientA, 400*time.Millisecond) +} + +// TestCloseSpaceDropsInterest verifies CloseSpace tears down both local and +// serving-side interest so a global service retains nothing for a closed space. +func TestCloseSpaceDropsInterest(t *testing.T) { + n := newNetFx(t) + defer n.finish() + + _, err := n.clientA.svc.Subscribe(testSpace, "chat/>", n.clientA.handler()) + require.NoError(t, err) + waitMatchCount(t, n.nodeB, testSpace, "chat/x", 1) + + // node unloads the space + n.nodeB.svc.CloseSpace(testSpace) + waitMatchCount(t, n.nodeB, testSpace, "chat/x", 0) + + // client closes the space: local interest is gone + n.clientA.svc.CloseSpace(testSpace) + n.clientA.svc.localMu.Lock() + _, hasTrie := n.clientA.svc.localTrie[testSpace] + n.clientA.svc.localMu.Unlock() + require.False(t, hasTrie, "local interest retained after CloseSpace") +} + +// TestResyncRestoresDeliveryAfterDrop verifies the reconnect watcher: after a +// subscriber's stream drops and a fresh peer replaces it, the periodic resync +// re-pushes interest and delivery resumes without the app re-subscribing. +func TestResyncRestoresDeliveryAfterDrop(t *testing.T) { + membership := &fakeMembership{} + relay := &fakeRelay{} + node := newEngineFx(t, "node", membership, relay) + defer node.finish() + sub := newEngineFx(t, "sub", membership, nil) + defer sub.finish() + pub := newEngineFx(t, "pub", membership, nil) + defer pub.finish() + membership.allow(sub.identity(), pub.identity()) + + sub.peers.add(connect(t, sub, node)) + pub.peers.add(connect(t, pub, node)) + + _, err := sub.svc.Subscribe(testSpace, "chat/>", sub.handler()) + require.NoError(t, err) + waitInterest(t, node, "chat/x") + + require.NoError(t, pub.svc.Publish(testCtx, testSpace, "chat/x", []byte("before"))) + require.Equal(t, "before", waitReceived(t, sub).payload) + + // drop the subscriber's connection and replace it with a fresh peer, as a real + // reconnect would; the node sees the stale stream close and (eventually) a new one + for _, p := range sub.ownedPeers { + _ = p.Close() + } + sub.peers.replace(connect(t, sub, node)) + + // the periodic resync re-pushes interest over the new peer; publish is + // fire-and-forget so retry until a message lands (delivery resumes once resync + // has reopened the stream and re-registered interest on the node) + deadline := time.Now().Add(3 * time.Second) + for { + require.NoError(t, pub.svc.Publish(testCtx, testSpace, "chat/x", []byte("after"))) + select { + case r := <-sub.received: + require.Equal(t, "after", r.payload) + return + case <-time.After(200 * time.Millisecond): + } + if time.Now().After(deadline) { + t.Fatal("delivery did not resume after reconnect") + } + } +} diff --git a/commonspace/pubsub/rpchandler.go b/commonspace/pubsub/rpchandler.go new file mode 100644 index 000000000..c80771741 --- /dev/null +++ b/commonspace/pubsub/rpchandler.go @@ -0,0 +1,21 @@ +package pubsub + +import ( + "storj.io/drpc" + + "github.com/anyproto/any-sync/commonspace/pubsub/pubsubproto" +) + +// rpcHandler adapts Service to the generated DRPC server interface. +type rpcHandler struct { + s Service +} + +func (r rpcHandler) PubSubStream(stream pubsubproto.DRPCPubSub_PubSubStreamStream) error { + return r.s.HandleStream(stream) +} + +// RegisterRpc registers the pubsub DRPC service on the given mux. +func RegisterRpc(mux drpc.Mux, s Service) error { + return pubsubproto.DRPCRegisterPubSub(mux, rpcHandler{s: s}) +} diff --git a/commonspace/pubsub/service.go b/commonspace/pubsub/service.go new file mode 100644 index 000000000..6a1489274 --- /dev/null +++ b/commonspace/pubsub/service.go @@ -0,0 +1,940 @@ +package pubsub + +import ( + "context" + "crypto/rand" + "errors" + "sync" + "time" + + "github.com/cheggaaa/mb/v3" + "go.uber.org/zap" + "storj.io/drpc" + + "github.com/anyproto/any-sync/accountservice" + "github.com/anyproto/any-sync/app" + "github.com/anyproto/any-sync/app/logger" + "github.com/anyproto/any-sync/commonspace/object/accountdata" + "github.com/anyproto/any-sync/commonspace/pubsub/pubsubproto" + "github.com/anyproto/any-sync/net/peer" + "github.com/anyproto/any-sync/net/streampool" + "github.com/anyproto/any-sync/util/crypto" +) + +const CName = "common.commonspace.pubsub" + +var log = logger.NewNamed(CName) + +var ( + ErrClosed = errors.New("pubsub service closed") +) + +// Service is the shared pubsub engine used by nodes (relay role, Deps.Relay set) +// and clients (Deps.Peers set) alike. One long-lived PubSubStream per peer pair +// multiplexes all spaces and topics; interest and routing state are in-memory only +// and die with the streams. +type Service interface { + app.ComponentRunnable + // Publish encrypts, signs and fire-and-forgets payload to the topic within the space. + Publish(ctx context.Context, spaceId, topic string, payload []byte) error + // Subscribe registers a local handler for a pattern and pushes the interest to + // the space's peers. The returned func unregisters and unsubscribes. + Subscribe(spaceId, pattern string, h Handler) (unsubscribe func(), err error) + // SyncInterest (re)sends all local interest for the space to its current peers; + // call after (re)connecting to a space's peers. + SyncInterest(ctx context.Context, spaceId string) error + // CloseSpace drops all local and serving-side interest for the space and + // withdraws the local interest from its peers. Hosts call it on space + // unload/close so a global service doesn't retain closed-space state. + CloseSpace(spaceId string) + // EvictMember drops all serving-side interest of an identity in a space, + // enforcing DESIGN §6.4 (active drop on ACL removal). Nodes wire it to their + // ACL-update hook. No-op on clients (they hold no serving interest). + EvictMember(spaceId string, identity crypto.PubKey) + // HandleStream serves an inbound PubSubStream; blocks for the stream lifetime. + HandleStream(stream drpc.Stream) error +} + +func New(deps Deps) Service { + return &service{deps: deps} +} + +type service struct { + deps Deps + cfg Config + account *accountdata.AccountKeys + pool streampool.StreamPool + + // remote interest: what peers subscribed on us (serving side). + // Interest is keyed by streamId (not peerId): two streams from the same peer + // are independent, so a reconnecting peer's fresh stream keeps its interest + // when the stale stream closes, and the trie refcount counts subscribing + // streams. streams[streamId] is the authoritative record cleaned on close. + remoteMu sync.Mutex + remote map[string]*spaceInterest // spaceId -> match trie (refcount = subscribing streams) + streams map[uint32]*streamInterest // streamId -> what that stream subscribed + + // local interest: what our handlers subscribed to (client side) + localMu sync.Mutex + localTrie map[string]*patternTrie // spaceId -> trie of local patterns + localSubs map[string]map[string][]localSub // spaceId -> pattern -> handlers + nextSubId uint64 + localTopic map[string]int // spaceId -> distinct local patterns (cap bookkeeping) + + dedup *msgIdDedup + rate *peerRateLimiter + dispatch *mb.MB[dispatchItem] + resyncNow chan struct{} + loops sync.WaitGroup + + ctx context.Context + ctxCancel context.CancelFunc +} + +type spaceInterest struct { + trie *patternTrie +} + +// streamInterest records the patterns one inbound stream subscribed, per space, +// so a stream close can withdraw exactly its own contribution regardless of what +// tags the pool recorded. +type streamInterest struct { + peerId string + account string // subscriber account id, for member eviction + bySpace map[string]map[string]struct{} // spaceId -> patterns + total int // total patterns across spaces on this stream +} + +type localSub struct { + id uint64 + h Handler +} + +type dispatchItem struct { + patterns []string + spaceId string + topic string + identity crypto.PubKey + payload []byte +} + +func (s *service) Init(a *app.App) (err error) { + s.cfg = s.deps.Config.withDefaults() + s.account = a.MustComponent(accountservice.CName).(accountservice.Service).Account() + poolOpts := []streampool.Option{streampool.WithStreamCloseHook(s.onStreamClose)} + if s.deps.Metric != nil { + poolOpts = append(poolOpts, streampool.WithMetric(s.deps.Metric, "pubsub")) + } + s.pool = streampool.NewStreamPool(s, s.cfg.streamPoolConfig(), poolOpts...) + s.remote = make(map[string]*spaceInterest) + s.streams = make(map[uint32]*streamInterest) + s.localTrie = make(map[string]*patternTrie) + s.localSubs = make(map[string]map[string][]localSub) + s.localTopic = make(map[string]int) + s.dedup = newMsgIdDedup(s.cfg.DedupSize) + s.rate = newPeerRateLimiter(s.cfg.PublishRps, s.cfg.PublishBurst) + s.dispatch = mb.New[dispatchItem](s.cfg.DispatchQueueSize) + s.resyncNow = make(chan struct{}, 1) + s.ctx, s.ctxCancel = context.WithCancel(context.Background()) + return nil +} + +func (s *service) Name() string { return CName } + +func (s *service) Run(ctx context.Context) error { + if err := s.pool.Run(ctx); err != nil { + return err + } + s.loops.Add(1) + go func() { + defer s.loops.Done() + s.dispatchLoop() + }() + if s.deps.Peers != nil { + s.loops.Add(1) + go func() { + defer s.loops.Done() + s.resyncLoop() + }() + } + return nil +} + +// resyncLoop keeps server-side interest alive across reconnects. streampool opens +// streams lazily and never re-sends interest on a fresh stream, so without this a +// subscriber goes silent after its stream drops (a routine event: idle pool GC +// reaps quiet streams). The loop re-pushes all local interest periodically, and +// immediately when a client-side stream closes. Re-sends are idempotent +// (handleSubscribe dedups per stream) and bounded by the local subscription set. +func (s *service) resyncLoop() { + ticker := time.NewTicker(s.cfg.ResyncInterval) + defer ticker.Stop() + for { + select { + case <-s.ctx.Done(): + return + case <-ticker.C: + case <-s.resyncNow: + } + s.localMu.Lock() + spaces := make([]string, 0, len(s.localSubs)) + for spaceId := range s.localSubs { + spaces = append(spaces, spaceId) + } + s.localMu.Unlock() + for _, spaceId := range spaces { + if err := s.SyncInterest(s.ctx, spaceId); err != nil { + log.Debug("resync interest failed", zap.String("spaceId", spaceId), zap.Error(err)) + } + } + } +} + +// triggerResync asks the resync loop to re-push interest now (non-blocking). +func (s *service) triggerResync() { + select { + case s.resyncNow <- struct{}{}: + default: + } +} + +func (s *service) Close(ctx context.Context) error { + s.ctxCancel() + _ = s.dispatch.Close() + s.loops.Wait() + return s.pool.Close(ctx) +} + +// +// public API (client side) +// + +func (s *service) Publish(ctx context.Context, spaceId, topic string, payload []byte) error { + if err := ValidateTopic(topic); err != nil { + return err + } + if len(payload) > s.cfg.MaxPayloadSize { + return pubsubproto.ErrInvalidMessage + } + if owner := TopicOwner(topic); owner != "" && owner != s.account.SignKey.GetPublic().Account() { + return pubsubproto.ErrTopicNotOwned + } + p := &pubsubproto.Publish{ + SpaceId: spaceId, + Topic: topic, + MsgId: make([]byte, msgIdLen), + TimestampMilli: time.Now().UnixMilli(), + } + if _, err := rand.Read(p.MsgId); err != nil { + return err + } + if s.deps.Crypto != nil { + keyId, enc, err := s.deps.Crypto.Encrypt(spaceId, payload) + if err != nil { + return err + } + p.KeyId, p.Payload = keyId, enc + } else { + p.Payload = payload + } + if err := signPublish(s.account.SignKey, p); err != nil { + return err + } + // record our own msgId so network echoes are suppressed, + // and deliver to local handlers synchronously with the plaintext + s.dedup.seen(p.MsgId) + s.enqueueLocal(spaceId, topic, s.account.SignKey.GetPublic(), payload) + + if s.deps.Peers == nil { + return nil + } + msg := wrapPublish(p) + return s.pool.Send(ctx, msg, func(ctx context.Context) ([]peer.Peer, error) { + return s.deps.Peers.SpacePeers(ctx, spaceId) + }) +} + +func (s *service) Subscribe(spaceId, pattern string, h Handler) (func(), error) { + if err := ValidatePattern(pattern); err != nil { + return nil, err + } + s.localMu.Lock() + if s.localTopic[spaceId] >= s.cfg.MaxPatternsPerSpace { + s.localMu.Unlock() + return nil, pubsubproto.ErrTooManyTopics + } + trie, ok := s.localTrie[spaceId] + if !ok { + trie = newPatternTrie() + s.localTrie[spaceId] = trie + s.localSubs[spaceId] = make(map[string][]localSub) + } + s.nextSubId++ + id := s.nextSubId + firstForPattern := len(s.localSubs[spaceId][pattern]) == 0 + s.localSubs[spaceId][pattern] = append(s.localSubs[spaceId][pattern], localSub{id: id, h: h}) + if firstForPattern { + trie.Add(pattern) + s.localTopic[spaceId]++ + } + s.localMu.Unlock() + + if firstForPattern { + s.sendInterest(spaceId, []string{pattern}, true) + } + return func() { s.unsubscribe(spaceId, pattern, id) }, nil +} + +func (s *service) unsubscribe(spaceId, pattern string, id uint64) { + s.localMu.Lock() + subs := s.localSubs[spaceId][pattern] + for i, sub := range subs { + if sub.id == id { + subs = append(subs[:i], subs[i+1:]...) + break + } + } + lastForPattern := len(subs) == 0 + if lastForPattern { + delete(s.localSubs[spaceId], pattern) + if trie := s.localTrie[spaceId]; trie != nil { + trie.Remove(pattern) + s.localTopic[spaceId]-- + if trie.Len() == 0 { + delete(s.localTrie, spaceId) + delete(s.localSubs, spaceId) + delete(s.localTopic, spaceId) + } + } + } else { + s.localSubs[spaceId][pattern] = subs + } + s.localMu.Unlock() + + if lastForPattern { + s.sendInterest(spaceId, []string{pattern}, false) + } +} + +func (s *service) SyncInterest(ctx context.Context, spaceId string) error { + s.localMu.Lock() + var patterns []string + for pattern := range s.localSubs[spaceId] { + patterns = append(patterns, pattern) + } + s.localMu.Unlock() + if len(patterns) == 0 || s.deps.Peers == nil { + return nil + } + msg := &pubsubproto.PubSubMessage{Content: &pubsubproto.PubSubMessage_Subscribe{ + Subscribe: &pubsubproto.Subscribe{SpaceId: spaceId, Topics: patterns}, + }} + return s.pool.Send(ctx, msg, func(ctx context.Context) ([]peer.Peer, error) { + return s.deps.Peers.SpacePeers(ctx, spaceId) + }) +} + +func (s *service) CloseSpace(spaceId string) { + // client side: drop local subscriptions and withdraw interest from peers + s.localMu.Lock() + var patterns []string + for pattern := range s.localSubs[spaceId] { + patterns = append(patterns, pattern) + } + delete(s.localTrie, spaceId) + delete(s.localSubs, spaceId) + delete(s.localTopic, spaceId) + s.localMu.Unlock() + if len(patterns) > 0 { + s.sendInterest(spaceId, patterns, false) + } + + // serving side: drop the space trie and every stream's interest in it, + // stripping the routing tags so no lingering tag delivers after close + s.remoteMu.Lock() + delete(s.remote, spaceId) + for streamId, strm := range s.streams { + spacePatterns := strm.bySpace[spaceId] + if len(spacePatterns) == 0 { + continue + } + tags := make([]string, 0, len(spacePatterns)) + for pattern := range spacePatterns { + tags = append(tags, interestTag(spaceId, pattern)) + strm.total-- + } + delete(strm.bySpace, spaceId) + _ = s.pool.RemoveTagsById(streamId, tags...) + if strm.total == 0 { + delete(s.streams, streamId) + } + } + s.remoteMu.Unlock() +} + +func (s *service) EvictMember(spaceId string, identity crypto.PubKey) { + account := identity.Account() + s.remoteMu.Lock() + si := s.remote[spaceId] + for streamId, strm := range s.streams { + if strm.account != account { + continue + } + spacePatterns := strm.bySpace[spaceId] + if len(spacePatterns) == 0 { + continue + } + tags := make([]string, 0, len(spacePatterns)) + for pattern := range spacePatterns { + tags = append(tags, interestTag(spaceId, pattern)) + strm.total-- + if si != nil { + si.trie.Remove(pattern) + } + } + delete(strm.bySpace, spaceId) + _ = s.pool.RemoveTagsById(streamId, tags...) + if strm.total == 0 { + delete(s.streams, streamId) + } + } + if si != nil { + s.pruneSpace(spaceId, si) + } + s.remoteMu.Unlock() +} + +func (s *service) sendInterest(spaceId string, patterns []string, subscribe bool) { + if s.deps.Peers == nil { + return + } + var msg *pubsubproto.PubSubMessage + if subscribe { + msg = &pubsubproto.PubSubMessage{Content: &pubsubproto.PubSubMessage_Subscribe{ + Subscribe: &pubsubproto.Subscribe{SpaceId: spaceId, Topics: patterns}, + }} + } else { + msg = &pubsubproto.PubSubMessage{Content: &pubsubproto.PubSubMessage_Unsubscribe{ + Unsubscribe: &pubsubproto.Unsubscribe{SpaceId: spaceId, Topics: patterns}, + }} + } + if err := s.pool.Send(s.ctx, msg, func(ctx context.Context) ([]peer.Peer, error) { + return s.deps.Peers.SpacePeers(ctx, spaceId) + }); err != nil { + log.Info("send interest failed", zap.String("spaceId", spaceId), zap.Error(err)) + } +} + +// +// streamhandler.StreamHandler (the engine is its own pool handler) +// + +func (s *service) OpenStream(ctx context.Context, p peer.Peer) (stream drpc.Stream, tags []string, queueSize int, err error) { + // hold the peer so idle pool GC (default ~1m TTL) does not silently reap a + // quiet pubsub stream out from under a subscriber + p.SetTTL(s.cfg.PeerTTL) + conn, err := p.AcquireDrpcConn(ctx) + if err != nil { + return nil, nil, 0, err + } + objectStream, err := pubsubproto.NewDRPCPubSubClient(conn).PubSubStream(ctx) + if err != nil { + return nil, nil, 0, err + } + return objectStream, nil, s.cfg.WriteQueueSize, nil +} + +func (s *service) NewReadMessage() drpc.Message { + return &pubsubproto.PubSubMessage{} +} + +func (s *service) HandleMessage(ctx context.Context, peerId string, msg drpc.Message) error { + m, ok := msg.(*pubsubproto.PubSubMessage) + if !ok { + return pubsubproto.ErrUnexpected + } + switch { + case m.GetSubscribe() != nil: + s.handleSubscribe(ctx, peerId, m.GetSubscribe()) + case m.GetUnsubscribe() != nil: + s.handleUnsubscribe(ctx, peerId, m.GetUnsubscribe()) + case m.GetPublish() != nil: + s.handlePublish(ctx, peerId, m.GetPublish()) + case m.GetStatus() != nil: + st := m.GetStatus() + log.Debug("pubsub status from peer", + zap.String("peerId", peerId), zap.String("spaceId", st.SpaceId), + zap.String("code", st.Code.String()), zap.Strings("topics", st.Topics)) + if s.deps.OnStatus != nil { + s.deps.OnStatus(peerId, st) + } + } + // frame-level rejections answer with Status; never break the stream + return nil +} + +// HandleStream serves an inbound PubSubStream (DRPC server entry); blocks. +func (s *service) HandleStream(stream drpc.Stream) error { + return s.pool.ReadStream(stream, s.cfg.WriteQueueSize) +} + +// +// serving-side frame handling +// + +func (s *service) handleSubscribe(ctx context.Context, peerId string, sub *pubsubproto.Subscribe) { + streamId, ok := streampool.CtxStreamId(ctx) + if !ok { + s.sendStatus(ctx, peerId, sub.SpaceId, sub.Topics, pubsubproto.ErrCodes_InvalidMessage) + return + } + identity, err := peer.CtxPubKey(ctx) + if err != nil { + s.sendStatus(ctx, peerId, sub.SpaceId, sub.Topics, pubsubproto.ErrCodes_InvalidMessage) + return + } + if err = validateSpaceId(sub.SpaceId); err != nil { + s.sendStatus(ctx, peerId, sub.SpaceId, sub.Topics, pubsubproto.ErrCodes_InvalidTopic) + return + } + if s.deps.Relay != nil && !s.deps.Relay.IsResponsible(sub.SpaceId) { + s.sendStatus(ctx, peerId, sub.SpaceId, sub.Topics, pubsubproto.ErrCodes_NotResponsible) + return + } + for _, pattern := range sub.Topics { + if err = ValidatePattern(pattern); err != nil { + s.sendStatus(ctx, peerId, sub.SpaceId, sub.Topics, pubsubproto.ErrCodes_InvalidTopic) + return + } + } + if s.deps.Membership != nil { + if err = s.deps.Membership.CheckMember(ctx, sub.SpaceId, identity); err != nil { + s.sendStatus(ctx, peerId, sub.SpaceId, sub.Topics, pubsubproto.ErrCodes_NotAMember) + return + } + } + + // Record interest and register the routing tags under a single remoteMu hold. + // remoteMu -> pool.mu is a consistent lock order (removeStream releases pool.mu + // before onStreamClose takes remoteMu), so holding across AddTagsCtx is safe and + // makes interest+tag atomic w.r.t. a concurrent close: if the stream was already + // removed, AddTagsCtx fails and we roll the interest back, so nothing leaks. + s.remoteMu.Lock() + si := s.remote[sub.SpaceId] + if si == nil { + si = &spaceInterest{trie: newPatternTrie()} + s.remote[sub.SpaceId] = si + } + strm := s.streams[streamId] + if strm == nil { + strm = &streamInterest{peerId: peerId, account: identity.Account(), bySpace: make(map[string]map[string]struct{})} + s.streams[streamId] = strm + } + spacePatterns := strm.bySpace[sub.SpaceId] + if spacePatterns == nil { + spacePatterns = make(map[string]struct{}) + strm.bySpace[sub.SpaceId] = spacePatterns + } + var accepted []string + var capExceeded bool + for _, pattern := range sub.Topics { + if _, exists := spacePatterns[pattern]; exists { + continue + } + if len(spacePatterns) >= s.cfg.MaxPatternsPerSpace || strm.total >= s.cfg.MaxPatternsPerStream { + capExceeded = true + break + } + spacePatterns[pattern] = struct{}{} + strm.total++ + si.trie.Add(pattern) + accepted = append(accepted, pattern) + } + if len(accepted) > 0 { + tags := make([]string, len(accepted)) + for i, pattern := range accepted { + tags[i] = interestTag(sub.SpaceId, pattern) + } + if err = s.pool.AddTagsCtx(ctx, tags...); err != nil { + // the stream was removed between accept and tagging: undo the interest + // so onStreamClose (which found nothing) leaves no orphan behind + for _, pattern := range accepted { + s.removeStreamPattern(strm, si, sub.SpaceId, pattern) + } + s.pruneStream(streamId, strm) + s.pruneSpace(sub.SpaceId, si) + } + } + s.remoteMu.Unlock() + + if capExceeded { + s.sendStatus(ctx, peerId, sub.SpaceId, sub.Topics, pubsubproto.ErrCodes_TooManyTopics) + } +} + +func (s *service) handleUnsubscribe(ctx context.Context, peerId string, unsub *pubsubproto.Unsubscribe) { + streamId, ok := streampool.CtxStreamId(ctx) + if !ok { + return + } + s.remoteMu.Lock() + strm := s.streams[streamId] + si := s.remote[unsub.SpaceId] + if strm == nil || si == nil { + s.remoteMu.Unlock() + return + } + patterns := unsub.Topics + if len(patterns) == 0 { // empty means all patterns of the space + for pattern := range strm.bySpace[unsub.SpaceId] { + patterns = append(patterns, pattern) + } + } + var removed []string + for _, pattern := range patterns { + if s.removeStreamPattern(strm, si, unsub.SpaceId, pattern) { + removed = append(removed, pattern) + } + } + s.pruneStream(streamId, strm) + s.pruneSpace(unsub.SpaceId, si) + s.remoteMu.Unlock() + + if len(removed) > 0 { + tags := make([]string, len(removed)) + for i, pattern := range removed { + tags[i] = interestTag(unsub.SpaceId, pattern) + } + if err := s.pool.RemoveTagsCtx(ctx, tags...); err != nil { + log.Warn("remove tags failed", zap.Error(err)) + } + } +} + +func (s *service) handlePublish(ctx context.Context, peerId string, p *pubsubproto.Publish) { + if len(p.MsgId) != msgIdLen || len(p.Payload) > s.cfg.MaxPayloadSize { + s.sendPubStatus(ctx, peerId, p, pubsubproto.ErrCodes_InvalidMessage) + return + } + if ValidateTopic(p.Topic) != nil { + s.sendPubStatus(ctx, peerId, p, pubsubproto.ErrCodes_InvalidTopic) + return + } + if s.deps.Relay != nil { + s.relayPublish(ctx, peerId, p) + return + } + // client role: deliver locally only, never forward (relay rule 1) + s.receivePublish(ctx, p) +} + +// relayPublish is the node ingress path: authorize, fan out to subscribed streams, +// and forward client-originated messages once to the other responsible nodes. +func (s *service) relayPublish(ctx context.Context, peerId string, p *pubsubproto.Publish) { + if !s.deps.Relay.IsResponsible(p.SpaceId) { + s.sendPubStatus(ctx, peerId, p, pubsubproto.ErrCodes_NotResponsible) + return + } + if p.Relayed { + // only responsible peer nodes may relay; relayed messages are never re-forwarded + if !s.deps.Relay.IsResponsibleNode(p.SpaceId, peerId) { + return + } + s.fanout(ctx, p) + return + } + // client-originated: bind attribution to the handshake-proven identity. + // Reject an empty identity explicitly: CtxIdentity returns (nil,nil) for an + // unverified inbound, and bytesEqual(nil,nil) would otherwise pass the bind. + ctxIdentity, err := peer.CtxIdentity(ctx) + if err != nil || len(ctxIdentity) == 0 || len(p.Identity) == 0 || !bytesEqual(ctxIdentity, p.Identity) { + s.sendPubStatus(ctx, peerId, p, pubsubproto.ErrCodes_InvalidMessage) + return + } + if s.deps.Membership != nil { + identity, err := peer.CtxPubKey(ctx) + if err != nil || s.deps.Membership.CheckMember(ctx, p.SpaceId, identity) != nil { + s.sendPubStatus(ctx, peerId, p, pubsubproto.ErrCodes_NotAMember) + return + } + } + if owner := TopicOwner(p.Topic); owner != "" { + identity, err := peer.CtxPubKey(ctx) + if err != nil || identity.Account() != owner { + s.sendPubStatus(ctx, peerId, p, pubsubproto.ErrCodes_TopicNotOwned) + return + } + } + if !s.rate.allow(peerId) { + s.sendPubStatus(ctx, peerId, p, pubsubproto.ErrCodes_RateLimited) + return + } + s.fanout(ctx, p) + // forward exactly once to the other responsible nodes; + // byte-slice fields are shared with the original but never mutated + relayed := &pubsubproto.Publish{ + SpaceId: p.SpaceId, + Topic: p.Topic, + MsgId: p.MsgId, + KeyId: p.KeyId, + Payload: p.Payload, + Identity: p.Identity, + Signature: p.Signature, + TimestampMilli: p.TimestampMilli, + Relayed: true, + } + if err := s.pool.Send(ctx, wrapPublish(relayed), func(ctx context.Context) ([]peer.Peer, error) { + return s.deps.Relay.OtherResponsiblePeers(ctx, p.SpaceId) + }); err != nil { + log.Info("forward to responsible nodes failed", zap.Error(err)) + } +} + +// fanout writes the message to every stream whose interest matches the topic. +func (s *service) fanout(ctx context.Context, p *pubsubproto.Publish) { + s.remoteMu.Lock() + si := s.remote[p.SpaceId] + var patterns []string + if si != nil { + patterns = si.trie.Match(p.Topic, nil) + } + s.remoteMu.Unlock() + if len(patterns) == 0 { + return + } + tags := make([]string, len(patterns)) + for i, pattern := range patterns { + tags[i] = interestTag(p.SpaceId, pattern) + } + // Broadcast dedups streams subscribed to several matching patterns + if err := s.pool.Broadcast(ctx, wrapPublish(p), tags...); err != nil { + log.Info("fanout failed", zap.Error(err)) + } +} + +// receivePublish is the client receive path. Cheap filters (local interest, +// membership, ownership, timestamp) run before the expensive Ed25519 verify so a +// relay/LAN-peer flood of forged messages is shed cheaply; the dedup ring is +// recorded only after verify so junk msgIds can't evict legitimate ones. +func (s *service) receivePublish(ctx context.Context, p *pubsubproto.Publish) { + s.localMu.Lock() + var patterns []string + if trie := s.localTrie[p.SpaceId]; trie != nil { + patterns = trie.Match(p.Topic, nil) + } + s.localMu.Unlock() + if len(patterns) == 0 { + return + } + // cheap: unmarshal the claimed identity (not yet trusted) to run filters + identity, err := identityOf(p) + if err != nil { + log.Debug("dropping publish with bad identity", zap.String("topic", p.Topic), zap.Error(err)) + return + } + if s.deps.Membership != nil { + if err = s.deps.Membership.CheckMember(ctx, p.SpaceId, identity); err != nil { + log.Debug("dropping publish from non-member", zap.String("topic", p.Topic)) + return + } + } + // end-to-end acc/ ownership: the signature covers the topic, so not even a + // malicious relay can inject into someone else's self-owned topic + if owner := TopicOwner(p.Topic); owner != "" && identity.Account() != owner { + log.Debug("dropping publish into unowned topic", zap.String("topic", p.Topic)) + return + } + if s.isStale(p.TimestampMilli) { + log.Debug("dropping stale publish", zap.String("topic", p.Topic)) + return + } + // expensive: verify only after the cheap filters pass + if err = verifySignature(identity, p); err != nil { + log.Debug("dropping publish with bad signature", zap.String("topic", p.Topic), zap.Error(err)) + return + } + // record dedup only for messages we would deliver, so forged floods can't + // flush the ring and re-open a replay window + if s.dedup.seen(p.MsgId) { + return + } + payload := p.Payload + if p.KeyId != "" { + if s.deps.Crypto == nil { + return + } + if payload, err = s.deps.Crypto.Decrypt(p.SpaceId, p.KeyId, p.Payload); err != nil { + log.Debug("dropping undecryptable publish", zap.String("topic", p.Topic), zap.Error(err)) + return + } + } + s.enqueueLocalMatched(patterns, p.SpaceId, p.Topic, identity, payload) +} + +// isStale reports whether a signed timestamp is outside the accepted skew window, +// raising the replay bar even after dedup eviction. Zero is treated as absent +// (never stale) to stay compatible with senders that omit it. +func (s *service) isStale(timestampMilli int64) bool { + if timestampMilli == 0 { + return false + } + skew := s.cfg.MaxTimestampSkew.Milliseconds() + now := time.Now().UnixMilli() + delta := now - timestampMilli + return delta > skew || delta < -skew +} + +// +// local dispatch +// + +func (s *service) enqueueLocal(spaceId, topic string, identity crypto.PubKey, payload []byte) { + s.localMu.Lock() + var patterns []string + if trie := s.localTrie[spaceId]; trie != nil { + patterns = trie.Match(topic, nil) + } + s.localMu.Unlock() + if len(patterns) == 0 { + return + } + s.enqueueLocalMatched(patterns, spaceId, topic, identity, payload) +} + +func (s *service) enqueueLocalMatched(patterns []string, spaceId, topic string, identity crypto.PubKey, payload []byte) { + err := s.dispatch.TryAdd(dispatchItem{ + patterns: patterns, + spaceId: spaceId, + topic: topic, + identity: identity, + payload: payload, + }) + if err != nil && !errors.Is(err, mb.ErrClosed) { + log.Debug("dispatch queue overflow, message dropped", zap.String("topic", topic)) + } +} + +func (s *service) dispatchLoop() { + for { + item, err := s.dispatch.WaitOne(s.ctx) + if err != nil { + return + } + s.localMu.Lock() + var handlers []Handler + for _, pattern := range item.patterns { + for _, sub := range s.localSubs[item.spaceId][pattern] { + handlers = append(handlers, sub.h) + } + } + s.localMu.Unlock() + for _, h := range handlers { + h(item.spaceId, item.topic, item.identity, item.payload) + } + } +} + +// +// housekeeping +// + +// onStreamClose withdraws exactly the closed stream's interest, keyed by streamId. +// It uses the engine's own per-stream record (streams[streamId]) rather than the +// pool's tag snapshot, so a stream that closed before its tags were registered is +// still cleaned, and a sibling stream of the same peer is never touched. +func (s *service) onStreamClose(streamId uint32, _ string, _ []string) { + // A client-side outbound stream dropping means our pushed interest is gone on + // the peer; re-push it promptly rather than waiting for the periodic tick. + if s.deps.Peers != nil { + s.triggerResync() + } + s.remoteMu.Lock() + defer s.remoteMu.Unlock() + strm := s.streams[streamId] + if strm == nil { + return + } + for spaceId, patterns := range strm.bySpace { + si := s.remote[spaceId] + if si == nil { + continue + } + for pattern := range patterns { + si.trie.Remove(pattern) + } + s.pruneSpace(spaceId, si) + } + delete(s.streams, streamId) +} + +// removeStreamPattern withdraws one pattern of one space from a stream's record and +// the space trie. Returns true if the pattern was present. Caller holds remoteMu. +func (s *service) removeStreamPattern(strm *streamInterest, si *spaceInterest, spaceId, pattern string) bool { + patterns := strm.bySpace[spaceId] + if _, ok := patterns[pattern]; !ok { + return false + } + delete(patterns, pattern) + strm.total-- + if len(patterns) == 0 { + delete(strm.bySpace, spaceId) + } + si.trie.Remove(pattern) + return true +} + +// pruneStream drops an empty stream record. Caller holds remoteMu. +func (s *service) pruneStream(streamId uint32, strm *streamInterest) { + if strm.total == 0 { + delete(s.streams, streamId) + } +} + +// pruneSpace drops an empty space trie. Caller holds remoteMu. +func (s *service) pruneSpace(spaceId string, si *spaceInterest) { + if si.trie.Len() == 0 { + delete(s.remote, spaceId) + } +} + +func (s *service) sendStatus(ctx context.Context, peerId, spaceId string, topics []string, code pubsubproto.ErrCodes) { + s.sendStatusMsg(ctx, peerId, &pubsubproto.Status{SpaceId: spaceId, Topics: topics, Code: code}) +} + +// sendPubStatus reports a rejected publish, echoing its msgId so the caller can +// correlate the rejection to the originating Publish. +func (s *service) sendPubStatus(ctx context.Context, peerId string, p *pubsubproto.Publish, code pubsubproto.ErrCodes) { + s.sendStatusMsg(ctx, peerId, &pubsubproto.Status{ + SpaceId: p.SpaceId, + Topics: []string{p.Topic}, + Code: code, + MsgId: p.MsgId, + }) +} + +func (s *service) sendStatusMsg(ctx context.Context, peerId string, st *pubsubproto.Status) { + msg := &pubsubproto.PubSubMessage{Content: &pubsubproto.PubSubMessage_Status{Status: st}} + if err := s.pool.SendById(ctx, msg, peerId); err != nil { + log.Debug("send status failed", zap.String("peerId", peerId), zap.Error(err)) + } +} + +func interestTag(spaceId, pattern string) string { + return spaceId + "/" + pattern +} + +func wrapPublish(p *pubsubproto.Publish) *pubsubproto.PubSubMessage { + return &pubsubproto.PubSubMessage{Content: &pubsubproto.PubSubMessage_Publish{Publish: p}} +} + +func bytesEqual(a, b []byte) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/commonspace/pubsub/service_test.go b/commonspace/pubsub/service_test.go new file mode 100644 index 000000000..1dd416897 --- /dev/null +++ b/commonspace/pubsub/service_test.go @@ -0,0 +1,493 @@ +package pubsub + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/anyproto/any-sync/app" + "github.com/anyproto/any-sync/commonspace/object/accountdata" + "github.com/anyproto/any-sync/commonspace/pubsub/pubsubproto" + "github.com/anyproto/any-sync/net/peer" + "github.com/anyproto/any-sync/net/rpc/rpctest" + "github.com/anyproto/any-sync/testutil/accounttest" + "github.com/anyproto/any-sync/util/crypto" +) + +var testCtx = context.Background() + +const testSpace = "space1" + +// +// fakes +// + +type fakeMembership struct { + mu sync.Mutex + allowed map[string]bool // account id -> member +} + +func (f *fakeMembership) allow(accounts ...crypto.PubKey) { + f.mu.Lock() + defer f.mu.Unlock() + if f.allowed == nil { + f.allowed = make(map[string]bool) + } + for _, a := range accounts { + f.allowed[a.Account()] = true + } +} + +func (f *fakeMembership) CheckMember(_ context.Context, _ string, identity crypto.PubKey) error { + f.mu.Lock() + defer f.mu.Unlock() + if f.allowed[identity.Account()] { + return nil + } + return fmt.Errorf("not a member") +} + +type staticPeers struct { + mu sync.Mutex + peers []peer.Peer +} + +func (s *staticPeers) add(p peer.Peer) { + s.mu.Lock() + defer s.mu.Unlock() + s.peers = append(s.peers, p) +} + +func (s *staticPeers) replace(p peer.Peer) { + s.mu.Lock() + defer s.mu.Unlock() + s.peers = []peer.Peer{p} +} + +func (s *staticPeers) SpacePeers(_ context.Context, _ string) ([]peer.Peer, error) { + s.mu.Lock() + defer s.mu.Unlock() + return append([]peer.Peer(nil), s.peers...), nil +} + +type fakeRelay struct { + mu sync.Mutex + nodePeerIds map[string]bool + others []peer.Peer + forwardCalls atomic.Int32 +} + +func (f *fakeRelay) addNodePeer(peerId string) { + f.mu.Lock() + defer f.mu.Unlock() + if f.nodePeerIds == nil { + f.nodePeerIds = make(map[string]bool) + } + f.nodePeerIds[peerId] = true +} + +func (f *fakeRelay) addOther(p peer.Peer) { + f.mu.Lock() + defer f.mu.Unlock() + f.others = append(f.others, p) +} + +func (f *fakeRelay) IsResponsible(string) bool { return true } + +func (f *fakeRelay) IsResponsibleNode(_, peerId string) bool { + f.mu.Lock() + defer f.mu.Unlock() + return f.nodePeerIds[peerId] +} + +func (f *fakeRelay) OtherResponsiblePeers(_ context.Context, _ string) ([]peer.Peer, error) { + f.forwardCalls.Add(1) + f.mu.Lock() + defer f.mu.Unlock() + return append([]peer.Peer(nil), f.others...), nil +} + +// +// fixture +// + +type received struct { + topic string + account string + payload string +} + +type engineFx struct { + t *testing.T + name string + acc *accountdata.AccountKeys + svc *service + app *app.App + ts *rpctest.TestServer + membership *fakeMembership + peers *staticPeers + relay *fakeRelay + statuses chan *pubsubproto.Status + received chan received + ownedPeers []peer.Peer +} + +func (fx *engineFx) identity() crypto.PubKey { return fx.acc.SignKey.GetPublic() } + +func (fx *engineFx) finish() { + require.NoError(fx.t, fx.app.Close(testCtx)) + for _, p := range fx.ownedPeers { + _ = p.Close() + } +} + +func newEngineFx(t *testing.T, name string, membership *fakeMembership, relay *fakeRelay) *engineFx { + acc, err := accountdata.NewRandom() + require.NoError(t, err) + fx := &engineFx{ + t: t, + name: name, + acc: acc, + membership: membership, + relay: relay, + statuses: make(chan *pubsubproto.Status, 16), + received: make(chan received, 64), + } + deps := Deps{ + Membership: membership, + OnStatus: func(_ string, st *pubsubproto.Status) { fx.statuses <- st }, + // fast resync so reconnect tests don't wait the 20s default + Config: Config{ResyncInterval: 150 * time.Millisecond}, + } + if relay != nil { + deps.Relay = relay + } else { + fx.peers = &staticPeers{} + deps.Peers = fx.peers + } + fx.svc = New(deps).(*service) + + fx.app = new(app.App) + fx.app.Register(accounttest.NewWithAcc(acc)).Register(fx.svc) + require.NoError(t, fx.app.Start(testCtx)) + + fx.ts = rpctest.NewTestServer() + require.NoError(t, RegisterRpc(fx.ts.Mux, fx.svc)) + return fx +} + +func (fx *engineFx) handler() Handler { + return func(_, topic string, identity crypto.PubKey, payload []byte) { + fx.received <- received{topic: topic, account: identity.Account(), payload: string(payload)} + } +} + +// connect wires from -> to and returns from's peer handle for to. +// Both directions carry the respective remote's peerId and identity in ctx, +// mirroring what secureservice puts there in production. +func connect(t *testing.T, from, to *engineFx) peer.Peer { + fromIdentity, err := from.identity().Marshall() + require.NoError(t, err) + toIdentity, err := to.identity().Marshall() + require.NoError(t, err) + mcAtTo, mcAtFrom := rpctest.MultiConnPairWithClientServerIdentity( + from.acc.PeerId, to.acc.PeerId, fromIdentity, toIdentity) + pAtTo, err := peer.NewPeer(mcAtTo, to.ts) + require.NoError(t, err) + to.ownedPeers = append(to.ownedPeers, pAtTo) + pAtFrom, err := peer.NewPeer(mcAtFrom, from.ts) + require.NoError(t, err) + from.ownedPeers = append(from.ownedPeers, pAtFrom) + return pAtFrom +} + +// waitInterest polls the serving engine until topic matches remote interest, +// removing the subscribe/publish race inherent to fire-and-forget semantics. +func waitInterest(t *testing.T, serving *engineFx, topic string) { + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + serving.svc.remoteMu.Lock() + si := serving.svc.remote[testSpace] + var n int + if si != nil { + n = len(si.trie.Match(topic, nil)) + } + serving.svc.remoteMu.Unlock() + if n > 0 { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("interest for %s never registered on %s", topic, serving.name) +} + +func waitReceived(t *testing.T, fx *engineFx) received { + select { + case r := <-fx.received: + return r + case <-time.After(2 * time.Second): + t.Fatalf("%s: timeout waiting for message", fx.name) + return received{} + } +} + +func expectSilence(t *testing.T, fx *engineFx, d time.Duration) { + select { + case r := <-fx.received: + t.Fatalf("%s: unexpected message %+v", fx.name, r) + case <-time.After(d): + } +} + +func waitStatus(t *testing.T, fx *engineFx, code pubsubproto.ErrCodes) { + deadline := time.After(2 * time.Second) + for { + select { + case st := <-fx.statuses: + if st.Code == code { + return + } + case <-deadline: + t.Fatalf("%s: timeout waiting for status %s", fx.name, code) + } + } +} + +// +// tests +// + +// topology: clientA, clientC -> nodeB <-> nodeB2 <- clientA2 +type netFx struct { + nodeB, nodeB2, clientA, clientC, clientA2 *engineFx +} + +func newNetFx(t *testing.T) *netFx { + membership := &fakeMembership{} + relayB := &fakeRelay{} + relayB2 := &fakeRelay{} + + n := &netFx{ + nodeB: newEngineFx(t, "nodeB", membership, relayB), + nodeB2: newEngineFx(t, "nodeB2", membership, relayB2), + clientA: newEngineFx(t, "clientA", membership, nil), + clientC: newEngineFx(t, "clientC", membership, nil), + clientA2: newEngineFx(t, "clientA2", membership, nil), + } + membership.allow(n.clientA.identity(), n.clientC.identity(), n.clientA2.identity()) + + n.clientA.peers.add(connect(t, n.clientA, n.nodeB)) + n.clientC.peers.add(connect(t, n.clientC, n.nodeB)) + n.clientA2.peers.add(connect(t, n.clientA2, n.nodeB2)) + relayB.addOther(connect(t, n.nodeB, n.nodeB2)) + relayB2.addNodePeer(n.nodeB.acc.PeerId) + relayB.addNodePeer(n.nodeB2.acc.PeerId) + return n +} + +func (n *netFx) finish() { + for _, fx := range []*engineFx{n.clientA, n.clientC, n.clientA2, n.nodeB, n.nodeB2} { + fx.finish() + } +} + +func TestPubSubFanoutAndWildcards(t *testing.T) { + n := newNetFx(t) + defer n.finish() + + _, err := n.clientA.svc.Subscribe(testSpace, "chat/>", n.clientA.handler()) + require.NoError(t, err) + _, err = n.clientA.svc.Subscribe(testSpace, "acc/online/*", n.clientA.handler()) + require.NoError(t, err) + waitInterest(t, n.nodeB, "chat/room1/typing") + waitInterest(t, n.nodeB, "acc/online/"+n.clientC.identity().Account()) + + require.NoError(t, n.clientC.svc.Publish(testCtx, testSpace, "chat/room1/typing", []byte("tick"))) + r := waitReceived(t, n.clientA) + require.Equal(t, "chat/room1/typing", r.topic) + require.Equal(t, n.clientC.identity().Account(), r.account) + require.Equal(t, "tick", r.payload) + + ownTopic := "acc/online/" + n.clientC.identity().Account() + require.NoError(t, n.clientC.svc.Publish(testCtx, testSpace, ownTopic, []byte("on"))) + r = waitReceived(t, n.clientA) + require.Equal(t, ownTopic, r.topic) + + // no matching interest: fire-and-forget discards silently + require.NoError(t, n.clientC.svc.Publish(testCtx, testSpace, "other/topic", []byte("x"))) + expectSilence(t, n.clientA, 300*time.Millisecond) +} + +func TestPubSubNodeRelay(t *testing.T) { + n := newNetFx(t) + defer n.finish() + + _, err := n.clientA2.svc.Subscribe(testSpace, "chat/>", n.clientA2.handler()) + require.NoError(t, err) + waitInterest(t, n.nodeB2, "chat/x") + + require.NoError(t, n.clientC.svc.Publish(testCtx, testSpace, "chat/x", []byte("cross-node"))) + r := waitReceived(t, n.clientA2) + require.Equal(t, "chat/x", r.topic) + require.Equal(t, "cross-node", r.payload) + + // hop limit: B2 must not re-forward the relayed message + require.Equal(t, int32(0), n.nodeB2.relay.forwardCalls.Load()) + // B forwarded exactly once + require.Equal(t, int32(1), n.nodeB.relay.forwardCalls.Load()) +} + +func TestPubSubMembershipRejection(t *testing.T) { + n := newNetFx(t) + defer n.finish() + + outsider := newEngineFx(t, "outsider", n.nodeB.membership, nil) + defer outsider.finish() + outsider.peers.add(connect(t, outsider, n.nodeB)) + + _, err := outsider.svc.Subscribe(testSpace, "chat/>", outsider.handler()) + require.NoError(t, err) // local registration succeeds, the node rejects async + waitStatus(t, outsider, pubsubproto.ErrCodes_NotAMember) + + require.NoError(t, outsider.svc.Publish(testCtx, testSpace, "chat/x", []byte("spam"))) + waitStatus(t, outsider, pubsubproto.ErrCodes_NotAMember) +} + +func TestPubSubTopicOwnership(t *testing.T) { + n := newNetFx(t) + defer n.finish() + + victimTopic := "acc/online/" + n.clientA.identity().Account() + _, err := n.clientA.svc.Subscribe(testSpace, "acc/online/*", n.clientA.handler()) + require.NoError(t, err) + waitInterest(t, n.nodeB, victimTopic) + + // the client fails fast when publishing into someone else's self-owned topic + require.ErrorIs(t, n.clientC.svc.Publish(testCtx, testSpace, victimTopic, []byte("spoof")), + pubsubproto.ErrTopicNotOwned) + + // relay-side enforcement: a malicious client bypassing the local check is + // rejected at the node ingress and never fanned out + spoofed := &pubsubproto.Publish{ + SpaceId: testSpace, + Topic: victimTopic, + MsgId: testMsgId(776), + Payload: []byte("spoof"), + TimestampMilli: time.Now().UnixMilli(), + } + require.NoError(t, signPublish(n.clientC.acc.SignKey, spoofed)) + spoofCtx := peer.CtxWithPeerId(peer.CtxWithIdentity(testCtx, spoofed.Identity), n.clientC.acc.PeerId) + n.nodeB.svc.handlePublish(spoofCtx, n.clientC.acc.PeerId, spoofed) + expectSilence(t, n.clientA, 300*time.Millisecond) + + // receive-side enforcement: a forged message injected past the relay is dropped + forged := &pubsubproto.Publish{ + SpaceId: testSpace, + Topic: victimTopic, + MsgId: testMsgId(777), + Payload: []byte("forged"), + TimestampMilli: time.Now().UnixMilli(), + } + require.NoError(t, signPublish(n.clientC.acc.SignKey, forged)) + n.clientA.svc.receivePublish(testCtx, forged) + expectSilence(t, n.clientA, 300*time.Millisecond) +} + +func TestPubSubEchoSuppression(t *testing.T) { + n := newNetFx(t) + defer n.finish() + + _, err := n.clientA.svc.Subscribe(testSpace, "chat/>", n.clientA.handler()) + require.NoError(t, err) + waitInterest(t, n.nodeB, "chat/self") + + require.NoError(t, n.clientA.svc.Publish(testCtx, testSpace, "chat/self", []byte("echo?"))) + r := waitReceived(t, n.clientA) + require.Equal(t, "echo?", r.payload) + // the node echoes the message back to A's subscribed stream; dedup must drop it + expectSilence(t, n.clientA, 500*time.Millisecond) +} + +func TestPubSubStaleMessageDropped(t *testing.T) { + n := newNetFx(t) + defer n.finish() + + _, err := n.clientA.svc.Subscribe(testSpace, "chat/>", n.clientA.handler()) + require.NoError(t, err) + + // a validly-signed message with a timestamp far in the past is dropped even + // though its signature verifies — raising the replay bar after dedup eviction + stale := &pubsubproto.Publish{ + SpaceId: testSpace, + Topic: "chat/old", + MsgId: testMsgId(999), + Payload: []byte("replayed"), + TimestampMilli: time.Now().Add(-time.Hour).UnixMilli(), + } + require.NoError(t, signPublish(n.clientC.acc.SignKey, stale)) + n.clientA.svc.receivePublish(testCtx, stale) + expectSilence(t, n.clientA, 300*time.Millisecond) + + // a fresh message from the same sender is delivered + fresh := &pubsubproto.Publish{ + SpaceId: testSpace, + Topic: "chat/new", + MsgId: testMsgId(1000), + Payload: []byte("fresh"), + TimestampMilli: time.Now().UnixMilli(), + } + require.NoError(t, signPublish(n.clientC.acc.SignKey, fresh)) + n.clientA.svc.receivePublish(testCtx, fresh) + require.Equal(t, "fresh", waitReceived(t, n.clientA).payload) +} + +func TestPubSubDuplicatePathSuppression(t *testing.T) { + n := newNetFx(t) + defer n.finish() + + _, err := n.clientA.svc.Subscribe(testSpace, "chat/>", n.clientA.handler()) + require.NoError(t, err) + + p := &pubsubproto.Publish{ + SpaceId: testSpace, + Topic: "chat/dup", + MsgId: testMsgId(555), + Payload: []byte("once"), + TimestampMilli: time.Now().UnixMilli(), + } + require.NoError(t, signPublish(n.clientC.acc.SignKey, p)) + // the same message arrives twice (LAN path + node path) + n.clientA.svc.receivePublish(testCtx, p) + n.clientA.svc.receivePublish(testCtx, p) + r := waitReceived(t, n.clientA) + require.Equal(t, "once", r.payload) + expectSilence(t, n.clientA, 300*time.Millisecond) +} + +func TestPubSubUnsubscribe(t *testing.T) { + n := newNetFx(t) + defer n.finish() + + unsub, err := n.clientA.svc.Subscribe(testSpace, "chat/>", n.clientA.handler()) + require.NoError(t, err) + waitInterest(t, n.nodeB, "chat/x") + + unsub() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + n.nodeB.svc.remoteMu.Lock() + si := n.nodeB.svc.remote[testSpace] + n.nodeB.svc.remoteMu.Unlock() + if si == nil { + break + } + time.Sleep(10 * time.Millisecond) + } + require.NoError(t, n.clientC.svc.Publish(testCtx, testSpace, "chat/x", []byte("gone"))) + expectSilence(t, n.clientA, 300*time.Millisecond) +} diff --git a/commonspace/pubsub/sign.go b/commonspace/pubsub/sign.go new file mode 100644 index 000000000..9517dfccc --- /dev/null +++ b/commonspace/pubsub/sign.go @@ -0,0 +1,69 @@ +package pubsub + +import ( + "encoding/binary" + + "github.com/anyproto/any-sync/commonspace/pubsub/pubsubproto" + "github.com/anyproto/any-sync/util/crypto" +) + +const signPrefix = "anysync:pubsub:v1" + +// publishSignData builds the byte string covered by a Publish signature. +// Every variable-length field is length-prefixed so the encoding is unambiguous +// (plain concatenation would let field boundaries shift). The relayed flag is +// excluded because nodes mutate it in transit. +func publishSignData(p *pubsubproto.Publish) []byte { + size := len(signPrefix) + 4*4 + 8 + + len(p.SpaceId) + len(p.Topic) + len(p.MsgId) + len(p.KeyId) + len(p.Payload) + buf := make([]byte, 0, size) + buf = append(buf, signPrefix...) + for _, f := range [][]byte{[]byte(p.SpaceId), []byte(p.Topic), p.MsgId, []byte(p.KeyId)} { + buf = binary.LittleEndian.AppendUint32(buf, uint32(len(f))) + buf = append(buf, f...) + } + buf = binary.LittleEndian.AppendUint64(buf, uint64(p.TimestampMilli)) + buf = append(buf, p.Payload...) + return buf +} + +// signPublish stamps identity and signature on the message using the account key. +func signPublish(key crypto.PrivKey, p *pubsubproto.Publish) (err error) { + p.Identity, err = key.GetPublic().Marshall() + if err != nil { + return + } + p.Signature, err = key.Sign(publishSignData(p)) + return +} + +// identityOf unmarshals the sender's public key from the message without verifying +// the signature. Cheap: lets the receiver run membership/ownership filters before +// the expensive Ed25519 verify, so a forged-signature flood is shed cheaply. +func identityOf(p *pubsubproto.Publish) (crypto.PubKey, error) { + return crypto.UnmarshalEd25519PublicKeyProto(p.Identity) +} + +// verifySignature checks the message signature against an already-unmarshalled key. +func verifySignature(pubKey crypto.PubKey, p *pubsubproto.Publish) error { + ok, err := pubKey.Verify(publishSignData(p), p.Signature) + if err != nil { + return err + } + if !ok { + return pubsubproto.ErrInvalidMessage + } + return nil +} + +// verifyPublish unmarshals the identity and verifies the signature in one step. +func verifyPublish(p *pubsubproto.Publish) (crypto.PubKey, error) { + pubKey, err := identityOf(p) + if err != nil { + return nil, err + } + if err = verifySignature(pubKey, p); err != nil { + return nil, err + } + return pubKey, nil +} diff --git a/commonspace/pubsub/sign_test.go b/commonspace/pubsub/sign_test.go new file mode 100644 index 000000000..1019a91ba --- /dev/null +++ b/commonspace/pubsub/sign_test.go @@ -0,0 +1,82 @@ +package pubsub + +import ( + "crypto/rand" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/anyproto/any-sync/commonspace/pubsub/pubsubproto" + "github.com/anyproto/any-sync/util/crypto" +) + +func newTestPublish() *pubsubproto.Publish { + return &pubsubproto.Publish{ + SpaceId: "space1", + Topic: "chat/abc/typing", + MsgId: testMsgId(42), + KeyId: "key1", + Payload: []byte("hello"), + TimestampMilli: 1751800000000, + } +} + +func TestSignVerifyPublish(t *testing.T) { + priv, _, err := crypto.GenerateEd25519Key(rand.Reader) + require.NoError(t, err) + + p := newTestPublish() + require.NoError(t, signPublish(priv, p)) + + identity, err := verifyPublish(p) + require.NoError(t, err) + require.True(t, identity.Equals(priv.GetPublic())) + + // the relayed flag is excluded from the signature + p.Relayed = true + _, err = verifyPublish(p) + require.NoError(t, err) +} + +func TestVerifyPublishTampered(t *testing.T) { + priv, _, err := crypto.GenerateEd25519Key(rand.Reader) + require.NoError(t, err) + + fields := map[string]func(p *pubsubproto.Publish){ + "payload": func(p *pubsubproto.Publish) { p.Payload = []byte("evil") }, + "topic": func(p *pubsubproto.Publish) { p.Topic = "acc/online/victim" }, + "spaceId": func(p *pubsubproto.Publish) { p.SpaceId = "space2" }, + "msgId": func(p *pubsubproto.Publish) { p.MsgId = testMsgId(43) }, + "keyId": func(p *pubsubproto.Publish) { p.KeyId = "key2" }, + "ts": func(p *pubsubproto.Publish) { p.TimestampMilli++ }, + } + for name, tamper := range fields { + p := newTestPublish() + require.NoError(t, signPublish(priv, p)) + tamper(p) + _, err = verifyPublish(p) + require.Error(t, err, "tampered %s must fail verification", name) + } +} + +func TestVerifyPublishForeignIdentity(t *testing.T) { + priv1, _, err := crypto.GenerateEd25519Key(rand.Reader) + require.NoError(t, err) + _, pub2, err := crypto.GenerateEd25519Key(rand.Reader) + require.NoError(t, err) + + p := newTestPublish() + require.NoError(t, signPublish(priv1, p)) + // swap in another identity: signature no longer matches + p.Identity, err = pub2.Marshall() + require.NoError(t, err) + _, err = verifyPublish(p) + require.Error(t, err) +} + +func TestSignDataUnambiguous(t *testing.T) { + // shifting bytes between adjacent fields must change the signed data + p1 := &pubsubproto.Publish{SpaceId: "ab", Topic: "c"} + p2 := &pubsubproto.Publish{SpaceId: "a", Topic: "bc"} + require.NotEqual(t, publishSignData(p1), publishSignData(p2)) +} diff --git a/commonspace/pubsub/topic.go b/commonspace/pubsub/topic.go new file mode 100644 index 000000000..c1afa44a1 --- /dev/null +++ b/commonspace/pubsub/topic.go @@ -0,0 +1,112 @@ +package pubsub + +import ( + "strings" + + "github.com/anyproto/any-sync/commonspace/pubsub/pubsubproto" +) + +const ( + maxTopicLen = 256 + maxSegments = 16 + wildcardOne = "*" + wildcardTail = ">" + accNamespace = "acc" +) + +// splitTopic splits a topic into segments using a stack-allocated array for the +// common case. Leading, trailing and doubled separators produce empty segments, +// which validation rejects. +func splitTopic(topic string) []string { + var tsa [maxSegments]string + n := 0 + rest := topic + for n < maxSegments { + idx := strings.IndexByte(rest, '/') + if idx < 0 { + tsa[n] = rest + n++ + return tsa[:n:n] + } + tsa[n] = rest[:idx] + n++ + rest = rest[idx+1:] + } + // over maxSegments: return an over-length slice so validation rejects it + return append(tsa[:n:n], rest) +} + +// validateSegments applies the shared structural rules for topics and patterns. +func validateSegments(topic string, segs []string) error { + if len(topic) == 0 || len(topic) > maxTopicLen { + return pubsubproto.ErrInvalidTopic + } + if len(segs) > maxSegments { + return pubsubproto.ErrInvalidTopic + } + // canonical form: no leading/trailing separator, no empty segments + for _, s := range segs { + if s == "" { + return pubsubproto.ErrInvalidTopic + } + } + return nil +} + +// ValidateTopic checks a fully-qualified publish topic: canonical form, no wildcards. +func ValidateTopic(topic string) error { + segs := splitTopic(topic) + if err := validateSegments(topic, segs); err != nil { + return err + } + for _, s := range segs { + if strings.ContainsAny(s, "*>") { + return pubsubproto.ErrInvalidTopic + } + } + return nil +} + +// ValidatePattern checks a subscription pattern: canonical form, wildcards only as +// whole segments, '>' only in tail position. +func ValidatePattern(pattern string) error { + segs := splitTopic(pattern) + if err := validateSegments(pattern, segs); err != nil { + return err + } + for i, s := range segs { + switch s { + case wildcardOne: + continue + case wildcardTail: + if i != len(segs)-1 { + return pubsubproto.ErrInvalidTopic + } + default: + if strings.ContainsAny(s, "*>") { + return pubsubproto.ErrInvalidTopic + } + } + } + return nil +} + +// validateSpaceId rejects a spaceId that would break the "spaceId/pattern" tag +// encoding. spaceId must be non-empty and contain no '/'. +func validateSpaceId(spaceId string) error { + if spaceId == "" || strings.IndexByte(spaceId, '/') >= 0 { + return pubsubproto.ErrInvalidTopic + } + return nil +} + +// TopicOwner returns the account id that exclusively may publish to the topic, or "" +// if the topic is not in the self-owned acc/ namespace. The owner is the last segment. +// Assumes a validated fully-qualified topic. +func TopicOwner(topic string) string { + segs := splitTopic(topic) + if len(segs) < 2 || segs[0] != accNamespace { + return "" + } + return segs[len(segs)-1] +} diff --git a/commonspace/pubsub/topic_test.go b/commonspace/pubsub/topic_test.go new file mode 100644 index 000000000..45461cb19 --- /dev/null +++ b/commonspace/pubsub/topic_test.go @@ -0,0 +1,72 @@ +package pubsub + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestValidateTopic(t *testing.T) { + valid := []string{ + "presence", + "chat/abc/typing", + "acc/online/A5xyz", + strings.Repeat("a/", 15) + "a", // 16 segments + } + for _, topic := range valid { + require.NoError(t, ValidateTopic(topic), topic) + } + invalid := []string{ + "", + "/presence", + "presence/", + "a//b", + "chat/*/typing", + "chat/>", + "chat/ty*", + "chat/ty>pe", + strings.Repeat("a/", 16) + "a", // 17 segments + strings.Repeat("x", 257), + } + for _, topic := range invalid { + require.Error(t, ValidateTopic(topic), topic) + } +} + +func TestValidatePattern(t *testing.T) { + valid := []string{ + "presence", + "chat/*/typing", + "chat/>", + ">", + "*", + "acc/online/*", + "a/*/*/b", + } + for _, p := range valid { + require.NoError(t, ValidatePattern(p), p) + } + invalid := []string{ + "", + "/chat/*", + "chat/>/typing", // '>' not in tail position + "chat/ty*", // wildcard not a whole segment + "chat/*>", + "a//>", + } + for _, p := range invalid { + require.Error(t, ValidatePattern(p), p) + } +} + +func TestTopicOwner(t *testing.T) { + require.Equal(t, "A5xyz", TopicOwner("acc/online/A5xyz")) + require.Equal(t, "A5xyz", TopicOwner("acc/cursor/obj1/A5xyz")) + require.Equal(t, "", TopicOwner("presence")) + require.Equal(t, "", TopicOwner("chat/abc/typing")) + // bare "acc" has no owner segment + require.Equal(t, "", TopicOwner("acc")) + // non-acc first segment + require.Equal(t, "", TopicOwner("accounts/online/A5xyz")) +} diff --git a/commonspace/pubsub/trie.go b/commonspace/pubsub/trie.go new file mode 100644 index 000000000..ba91d28a3 --- /dev/null +++ b/commonspace/pubsub/trie.go @@ -0,0 +1,176 @@ +package pubsub + +// patternTrie is a per-space interest trie over '/'-separated topic segments, +// mirroring the NATS sublist shape: one level per segment, a literal-child map plus +// dedicated single-segment ('*') and tail ('>') wildcard slots per level. Terminals +// hold a refcount of subscribing streams so identical patterns from many streams +// share one node. Not goroutine-safe; the engine serializes access. +type patternTrie struct { + root *trieLevel + size int // number of live distinct patterns +} + +type trieLevel struct { + nodes map[string]*trieNode + pwc *trieNode // '*' + fwc *trieNode // '>' +} + +type trieNode struct { + next *trieLevel + pattern string // set iff this node terminates a pattern + refs int // subscriber refcount for the terminated pattern +} + +func newPatternTrie() *patternTrie { + return &patternTrie{root: &trieLevel{}} +} + +func (t *patternTrie) Len() int { + return t.size +} + +// Add registers one subscriber reference for the pattern (assumed validated). +// Returns true when the pattern is new to the trie (0 -> 1 transition). +func (t *patternTrie) Add(pattern string) bool { + segs := splitTopic(pattern) + level := t.root + var node *trieNode + for _, seg := range segs { + node = level.child(seg) + if node == nil { + node = &trieNode{} + level.setChild(seg, node) + } + if node.next == nil { + node.next = &trieLevel{} + } + level = node.next + } + node.pattern = pattern + node.refs++ + if node.refs == 1 { + t.size++ + return true + } + return false +} + +// Remove drops one subscriber reference; on the last reference the pattern is pruned. +// Returns true when the pattern was removed entirely (1 -> 0 transition). +func (t *patternTrie) Remove(pattern string) bool { + segs := splitTopic(pattern) + return t.remove(t.root, segs, pattern) +} + +func (t *patternTrie) remove(level *trieLevel, segs []string, pattern string) bool { + if len(segs) == 0 { + return false + } + node := level.child(segs[0]) + if node == nil { + return false + } + var removed bool + if len(segs) == 1 { + if node.refs == 0 { + return false + } + node.refs-- + if node.refs > 0 { + return false + } + node.pattern = "" + t.size-- + removed = true + } else { + if node.next == nil { + return false + } + removed = t.remove(node.next, segs[1:], pattern) + } + if node.refs == 0 && (node.next == nil || node.next.empty()) { + level.deleteChild(segs[0]) + } + return removed +} + +// Match walks the trie with the segments of a fully-qualified topic and appends +// every matching pattern to dst. Match order follows NATS matchLevel: the +// tail-wildcard terminal is collected at each level (it matches one-or-more +// remaining segments), then the '*' branch, then the literal branch. +func (t *patternTrie) Match(topic string, dst []string) []string { + segs := splitTopic(topic) + return matchLevel(t.root, segs, dst) +} + +func matchLevel(level *trieLevel, segs []string, dst []string) []string { + if level == nil || len(segs) == 0 { + return dst + } + if level.fwc != nil && level.fwc.refs > 0 { + dst = append(dst, level.fwc.pattern) + } + if level.pwc != nil { + dst = matchNode(level.pwc, segs[1:], dst) + } + if node := level.literal(segs[0]); node != nil { + dst = matchNode(node, segs[1:], dst) + } + return dst +} + +// matchNode resolves a node that consumed one segment against the remaining ones. +func matchNode(node *trieNode, rest []string, dst []string) []string { + if len(rest) == 0 { + if node.refs > 0 { + dst = append(dst, node.pattern) + } + return dst + } + return matchLevel(node.next, rest, dst) +} + +func (l *trieLevel) child(seg string) *trieNode { + switch seg { + case wildcardOne: + return l.pwc + case wildcardTail: + return l.fwc + default: + return l.nodes[seg] + } +} + +func (l *trieLevel) literal(seg string) *trieNode { + return l.nodes[seg] +} + +func (l *trieLevel) setChild(seg string, n *trieNode) { + switch seg { + case wildcardOne: + l.pwc = n + case wildcardTail: + l.fwc = n + default: + if l.nodes == nil { + l.nodes = make(map[string]*trieNode) + } + l.nodes[seg] = n + } +} + +func (l *trieLevel) deleteChild(seg string) { + switch seg { + case wildcardOne: + l.pwc = nil + case wildcardTail: + l.fwc = nil + default: + delete(l.nodes, seg) + } +} + +func (l *trieLevel) empty() bool { + return len(l.nodes) == 0 && l.pwc == nil && l.fwc == nil +} diff --git a/commonspace/pubsub/trie_test.go b/commonspace/pubsub/trie_test.go new file mode 100644 index 000000000..b721db666 --- /dev/null +++ b/commonspace/pubsub/trie_test.go @@ -0,0 +1,118 @@ +package pubsub + +import ( + "sort" + "testing" + + "github.com/stretchr/testify/require" +) + +func match(t *patternTrie, topic string) []string { + res := t.Match(topic, nil) + sort.Strings(res) + return res +} + +func TestTrieExactMatch(t *testing.T) { + tr := newPatternTrie() + require.True(t, tr.Add("presence")) + require.True(t, tr.Add("chat/abc/typing")) + + require.Equal(t, []string{"presence"}, match(tr, "presence")) + require.Equal(t, []string{"chat/abc/typing"}, match(tr, "chat/abc/typing")) + require.Empty(t, match(tr, "chat/abc")) + require.Empty(t, match(tr, "chat/abc/typing/extra")) + require.Empty(t, match(tr, "other")) +} + +func TestTrieSingleSegmentWildcard(t *testing.T) { + tr := newPatternTrie() + tr.Add("chat/*/typing") + + require.Equal(t, []string{"chat/*/typing"}, match(tr, "chat/abc/typing")) + require.Empty(t, match(tr, "chat/typing")) // '*' must consume one segment + require.Empty(t, match(tr, "chat/a/b/typing")) // '*' consumes exactly one + require.Empty(t, match(tr, "chat/abc/typing/late")) // trailing extra segment + + tr.Add("*") + require.Equal(t, []string{"*"}, match(tr, "presence")) + require.Empty(t, match(tr, "a/b")) +} + +func TestTrieTailWildcard(t *testing.T) { + tr := newPatternTrie() + tr.Add("chat/>") + + require.Equal(t, []string{"chat/>"}, match(tr, "chat/abc")) + require.Equal(t, []string{"chat/>"}, match(tr, "chat/a/b/c")) + require.Empty(t, match(tr, "chat")) // '>' requires at least one segment + + tr.Add(">") + require.Equal(t, []string{">"}, match(tr, "anything")) + require.Equal(t, []string{">", "chat/>"}, match(tr, "chat/x")) +} + +func TestTrieOverlappingPatterns(t *testing.T) { + tr := newPatternTrie() + tr.Add("chat/>") + tr.Add("chat/*/typing") + tr.Add("chat/abc/typing") + + require.Equal(t, + []string{"chat/*/typing", "chat/>", "chat/abc/typing"}, + match(tr, "chat/abc/typing")) + require.Equal(t, []string{"chat/>"}, match(tr, "chat/abc/presence")) +} + +func TestTrieAccWildcard(t *testing.T) { + tr := newPatternTrie() + tr.Add("acc/online/*") + require.Equal(t, []string{"acc/online/*"}, match(tr, "acc/online/A1")) + require.Equal(t, []string{"acc/online/*"}, match(tr, "acc/online/A2")) + require.Empty(t, match(tr, "acc/cursor/A1")) +} + +func TestTrieRefcounting(t *testing.T) { + tr := newPatternTrie() + require.True(t, tr.Add("chat/>")) + require.False(t, tr.Add("chat/>")) // second subscriber, same pattern + require.Equal(t, 1, tr.Len()) + + require.False(t, tr.Remove("chat/>")) // still one ref left + require.Equal(t, []string{"chat/>"}, match(tr, "chat/x")) + + require.True(t, tr.Remove("chat/>")) // last ref gone + require.Empty(t, match(tr, "chat/x")) + require.Equal(t, 0, tr.Len()) + + // removing a non-existent pattern is a no-op + require.False(t, tr.Remove("chat/>")) + require.False(t, tr.Remove("never/added")) +} + +func TestTriePruning(t *testing.T) { + tr := newPatternTrie() + tr.Add("a/b/c/d") + tr.Add("a/b/x") + require.True(t, tr.Remove("a/b/c/d")) + // sibling under the shared prefix still matches + require.Equal(t, []string{"a/b/x"}, match(tr, "a/b/x")) + require.Empty(t, match(tr, "a/b/c/d")) + + tr.Remove("a/b/x") + require.True(t, tr.root.empty(), "trie should be fully pruned") +} + +func TestTrieInteriorTerminal(t *testing.T) { + tr := newPatternTrie() + // a pattern that is a prefix of another + tr.Add("chat") + tr.Add("chat/abc") + require.Equal(t, []string{"chat"}, match(tr, "chat")) + require.Equal(t, []string{"chat/abc"}, match(tr, "chat/abc")) + + // removing the prefix pattern keeps the longer one intact + require.True(t, tr.Remove("chat")) + require.Empty(t, match(tr, "chat")) + require.Equal(t, []string{"chat/abc"}, match(tr, "chat/abc")) +} diff --git a/docs/stateless-pubsub/DESIGN.md b/docs/stateless-pubsub/DESIGN.md new file mode 100644 index 000000000..9ccbc0476 --- /dev/null +++ b/docs/stateless-pubsub/DESIGN.md @@ -0,0 +1,635 @@ +# Space-Scoped Stateless Pub/Sub over any-sync — Design + +Status: **DESIGN / SPEC** — resolves all open tensions from [RESEARCH.md](./RESEARCH.md). +Grounded against `any-sync@main`, `any-sync-node@main`, `anytype-heart@main` (July 2026). + +--- + +## 1. Summary + +A new ephemeral, fire-and-forget, at-most-once publish/subscribe channel scoped to a +space, carried over a **dedicated DRPC bidi stream** fully isolated from the sync engine. +Topics are plaintext `/`-separated hierarchies inside a space; subscriptions may use +NATS-style wildcards (`*` one segment, `>` trailing segments). Any space member +(Reader/Guest included) may publish and subscribe. Payloads are end-to-end encrypted with the space ReadKey and +signed with the sender's account key; relay nodes route ciphertext they cannot read. +Fan-out goes through the responsible sync nodes **and** directly to LAN-discovered peers, +with a small bounded msgId dedup cache on receivers. No message is ever persisted; the +only routing state is transient in-memory interest state (stream tags + a pattern trie) +that dies with the connection. + +### Resolved decisions + +| Question (RESEARCH.md §7) | Decision | +|---|---| +| Meaning of "stateless" | Sense (a): no message persistence anywhere. Transient in-memory interest tables (stream tags) allowed and used. | +| Stream reuse vs dedicated | Dedicated `PubSub` DRPC service + own stream; never touches `ObjectSyncStream` or the sync dispatch path (user steer, RESEARCH.md §4.4). | +| New service vs new RPC on SpaceSync | New service, own proto package — independent versioning, unknown-service fallback for old peers. | +| Topic model | `/`-separated segment hierarchy within a space. Subscriptions may use NATS-style wildcards: `*` matches exactly one segment, `>` matches one-or-more trailing segments (tail-only). Publishers must use fully-qualified topics. Matching runs on a bounded per-space pattern trie at the relay (§4.1). | +| Topic privacy | Plaintext to relays. Comparable exposure to today's plaintext `objectId`s on the sync path. | +| Subscribe permission | Any space member: `!NoPermissions()` (`commonspace/object/acl/list/models.go:90`). | +| Publish permission | Any space member. Attribution via per-message account-key signature; abuse contained by per-peer rate limits. Plus a reserved **self-owned namespace**: topics `acc/…/` accept publishes only from `accId` (§6.2). | +| Payload confidentiality | Encrypted client-side with current space ReadKey + `keyId` indirection (push-server pattern). Removed member loses access at next key rotation. | +| Authenticity | Per-message account-key signature, verified by receivers. Protects against a semi-trusted node forging/reattributing messages. | +| Topology | Publisher → its responsible node (+ direct LAN peers). Node relays once to the other responsible nodes (`relayed` flag, never re-forwarded → loop-free). | +| Duplicate suppression | Receiver-side bounded LRU keyed by `msgId` (duplicate paths exist by design: LAN + node). | +| Backpressure | Bounded queues, `TryAdd` drop-on-overflow end to end — the existing streampool discipline. Slow subscriber ⇒ dropped messages, never memory growth. | +| Queue groups (1-of-N) | Non-goal v1. | +| Catch-up / replay / retained messages | Non-goal, by definition of stateless. Reconnect ⇒ resubscribe ⇒ only new messages. | +| Presence lifecycle | Not in the protocol. v1 is a generic app API; presence (heartbeat/timeout/leave, Yjs-awareness style) is an app pattern on top (Appendix A). | + +### Requirements compliance (RESEARCH.md §8) + +- **R1 (on-protocol):** rides existing transports, secureservice handshake, DRPC mux, and a + second streampool instance. A dedicated sub-stream on the existing `MultiConn` — no new + transport or TLS handshake (`net/peer/peer.go:134`, `net/transport/transport.go:40-64`). +- **R2 (memory-effective):** per-stream bounded `mb.MB` queues with drop-on-overflow + (`net/streampool/stream.go:32-44`), interest state = stream tags + pattern trie, both + O(active subscriptions), fixed-size dedup LRU, per-peer publish token bucket, caps on + pattern count and topic/payload size. No path grows with message volume or offline + duration. +- **R3 (ACL-aware):** node gates subscribe *and* publish on space membership at the current + ACL head (a check that does **not** exist today on the sync path — this is new, additive + enforcement); confidentiality holds against the relay via ReadKey encryption; membership + removal cuts new traffic at key rotation and actively drops subscriptions (§6.4). + +--- + +## 2. Semantics contract + +- **At-most-once.** A publish reaching the relay is copied into bounded per-subscriber + queues; overflow drops. No acks, no retransmit, no ordering across publishers, no dedup + beyond the duplicate-path LRU. +- **Fire-and-forget.** A publish with no subscribers is discarded. Nothing is stored. +- **Decoupled.** Publishers don't know subscribers. Subscriber set is whoever holds a live + tagged stream at the instant of fan-out. +- **Subscribe is unacknowledged.** Success is silent (§3); interest takes effect when the + serving peer processes the frame, so messages published concurrently by others may be + missed. There is no "subscribed as of time T" guarantee — the same race NATS documents + for cluster-wide subscription visibility (nats-server#1142), inherent to at-most-once. + Apps needing a consistent starting state combine pub/sub with a snapshot read (e.g. + presence: subscribe, then announce yourself, which prompts others' next heartbeat). +- **Reconnect = clean slate.** Subscriptions die with the stream; the client re-subscribes + on reconnect and sees only new traffic. +- **Echo.** A publisher whose own interest set matches the topic receives its own message + back from the relay; the client suppresses these by pre-recording its own msgId in the + dedup ring at publish time, and delivers to local handlers via the normal dispatch queue + (bounded, drop-on-overflow — so local delivery is best-effort like everything else, not + a hard guarantee). Net effect = NATS `no_echo` semantics without a wire flag. (NATS + echoes by default with per-connection opt-out; we don't need the option because dedup + already exists.) +- **Delivery is best-effort, duplicates possible in theory** (LRU eviction under extreme + rates), so payload design must be idempotent/last-write-wins at the app level. In + practice the LRU makes duplicates vanishingly rare. + +--- + +## 3. Wire protocol + +New proto package in any-sync: `commonspace/pubsub/pubsubproto/protos/pubsub.proto` +(generated via the existing Makefile pipeline, `Makefile:23-32`). + +```proto +syntax = "proto3"; +package pubsub; + +service PubSub { + // One long-lived bidi stream per peer pair, multiplexing all spaces/topics. + rpc PubSubStream(stream PubSubMessage) returns (stream PubSubMessage); +} + +message PubSubMessage { + oneof content { + Subscribe subscribe = 1; + Unsubscribe unsubscribe = 2; + Publish publish = 3; + Status status = 4; + } +} + +// Delta semantics: adds topic patterns to the stream's interest set for spaceId. +// Patterns may contain wildcards: '*' (one segment), '>' (trailing segments, tail-only). +message Subscribe { + string spaceId = 1; + repeated string topics = 2; +} + +// Removes patterns (matched verbatim against the interest set, not expanded); +// empty topics = remove all patterns of spaceId. +message Unsubscribe { + string spaceId = 1; + repeated string topics = 2; +} + +message Publish { + string spaceId = 1; + string topic = 2; + bytes msgId = 3; // 16 random bytes, generated by publisher (dedup key) + string keyId = 4; // space ReadKey id used for payload; "" = plaintext (keyless spaces) + bytes payload = 5; // ciphertext (or plaintext iff keyId == "") + bytes identity = 6; // sender account pubkey (marshalled) + bytes signature = 7; // account-key sig, see §6.3 + int64 timestampMilli = 8; // sender wall clock, informational + bool relayed = 9; // set by a node when forwarding node→node; never re-forwarded; excluded from signature +} + +// Sent by the serving peer on rejected subscribe/publish. Success is silent. +message Status { + string spaceId = 1; + repeated string topics = 2; // echo of the offending request (topic of a publish goes here too) + ErrCodes code = 3; +} + +enum ErrCodes { + Ok = 0; + NotAMember = 1; // identity has no permissions in the space's ACL + NotResponsible = 2; // this node is not responsible for the space + RateLimited = 3; + TooManyTopics = 4; + InvalidMessage = 5; // malformed frame, oversized payload, identity mismatch, bad signature + TopicNotOwned = 6; // publish into acc/…/ by an identity other than accId + InvalidTopic = 7; // malformed topic/pattern: bad wildcard placement, reserved chars, non-canonical form + ErrorOffset = 800; // 100..700 are taken by existing proto packages +} +``` + +Constraints (enforced by the serving peer, values are config defaults — §9). A rejected +Subscribe/Publish gets a `Status` and is otherwise ignored — the stream **stays open** +(the NATS precedent: `maximum subscriptions exceeded` is an error reply, not a +disconnect): + +- topic: UTF-8, 1..256 bytes, `/`-separated segments (≤16 segments, no empty segments); + recommended segment charset alnum + `.-_` (matches the NATS guideline of ≤16 tokens / + ≤256 chars). Canonical form has **no leading `/`** + (`acc/x` and `/acc/x` would silently be different topics — rejected as `InvalidTopic`). +- **Wildcards (subscription patterns only):** `*` matches exactly one segment + (`chat/*/typing` ⇒ `chat/abc/typing`, not `chat/typing` or `chat/a/b/typing`); + `>` matches one-or-more trailing segments and is only valid as the final segment + (`chat/>` ⇒ everything under `chat/`). Wildcards must be complete segments (`chat/ty*` + is invalid). `*` and `>` are reserved characters everywhere else; a `Publish` topic + containing either is rejected — publishers always use fully-qualified topics. +- **Reserved self-owned namespace:** a topic whose first segment is `acc` (i.e. prefix + `acc/`) is publishable **only** by the account whose id equals the topic's *last* + segment — e.g. `acc/online/`, `acc/cursor/`. Subscribe remains open to + all members (including patterns such as `acc/online/*`). Violations ⇒ + `Status{TopicNotOwned}`. +- payload: ≤ 64 KiB. +- per-stream interest set: ≤ 100 patterns per space, ≤ 1000 total. +- `identity` in a `Publish` **must equal** the connection-context identity + (`net/peer/context.go:88`) when arriving from a client (`relayed == false`). + This binds attribution to the TLS/handshake-proven account without requiring the node + to verify the signature per message. + +Interest tag format inside the pool: `spaceId + "/" + pattern`, verbatim (spaceId +contains no `/`; first separator wins). Wildcard resolution happens in the pubsub +engine's matcher, not in the pool — see §4.1. + +--- + +## 4. Topology & relay rules + +``` + publisher client ──publish──▶ responsible node A ──relayed=true──▶ node B ──▶ its subscribers + │ │ node C ──▶ its subscribers + │ └──▶ A's local subscribers (tag fan-out) + └──────direct publish──▶ LAN peers (same space, discovered via mDNS) +``` + +Relay rules (complete): + +1. **Clients never forward.** A client receiving a `Publish` (from a node or a LAN peer) + delivers it locally only. +2. **A node forwards only client-originated messages** (`relayed == false` arriving on a + client stream): it stamps `relayed = true` and sends one copy to each *other* + responsible node for the space (same peer-resolution logic as + `any-sync-node/nodespace/peermanager/manager.go:87-103`), plus fans out to its local + subscribers via the tag index. +3. **A node never forwards `relayed == true`** — it only fans out locally. With + `ReplicationFactor = 3` (`nodeconf/nodeconf.go`), every message traverses at most + client → node → node, hop limit 2, loop-free without any dedup state on nodes. + (This is exactly the NATS cluster rule: "messages received from a route will only be + distributed to local clients" — a strict one-hop limit is how NATS full-mesh clusters + stay loop-free too.) +4. **A node rejects subscribe/publish for spaces it is not responsible for** + (`Status{NotResponsible}`) — mirrors `checkResponsible` + (`any-sync-node/nodespace/checks.go:14-24`). +5. **LAN peers are symmetric.** anytype-heart already runs a DRPC server for LAN peers + (`space/spacecore/rpchandler.go`) and unifies node + LAN peers in the per-space peer + manager (`space/spacecore/peermanager/manager.go:172-236`). Both sides run the same + pubsub component; a LAN peer's stream carries Subscribe frames like a node's does, and + publishes to LAN peers go direct. Clients apply the member-check on LAN subscribes the + same way nodes do (they hold the ACL). + +Duplicate paths are expected (a subscriber may get the same message from a LAN peer and +from its node). The **receiver** suppresses via a fixed-size LRU keyed by `msgId` +(default 4096 entries ≈ 64 KiB of ids). Publishers self-suppress echoes by msgId too +(§2, Echo). + +Client → node selection piggybacks on the existing responsible-peer choice +(`pool.GetOneOf(nodeIds)` — one node at a time, `manager.go:213`), so the pubsub stream +goes to the same node the client already syncs with. + +### 4.1 Interest matching (wildcards) + +The streampool tag index is exact-match (`streamIdsByTag`, +`net/streampool/streampool.go:76`), so the pubsub engine layers a matcher on top rather +than replacing the pool: + +- Each accepted subscription registers its **pattern string verbatim as the stream tag** + (`spaceId + "/" + pattern`) — the pool keeps doing stream bookkeeping (add/remove/GC + on stream close) exactly as today. +- In parallel, the engine maintains a **per-space segment trie** of live patterns, + mirroring the NATS sublist shape exactly (`server/sublist.go`): one level per segment, + `map[segment]*node` for literals plus two dedicated wildcard slots per level (`pwc` + for `*`, `fwc` for `>`), each terminal holding a refcount of subscribing streams. + Match order as in NATS `matchLevel`: at each level add `fwc` matches, branch through + `pwc`, hash-lookup the literal. +- On publish to concrete topic `T`: walk the trie with `T`'s segments — branching on the + literal edge, the `*` edge, and any terminal `>` edge — collecting every matching + pattern (O(segments × matched branches), segments ≤16). Then fan out once per matched + pattern tag, **deduplicating stream ids across patterns**: a stream subscribed to both + `chat/>` and `chat/*/typing` must receive one copy, not two. *Implemented* by teaching + `streampool.Broadcast` to dedup stream ids across the tag set it's given (it previously + collected per-tag with no cross-tag dedup) — so the engine passes all matched pattern + tags to one `Broadcast` call and the pool guarantees one copy per stream. +- Trie cleanup: refcount decrement on Unsubscribe; on stream close the engine reconciles + lazily — when a matched pattern's tag resolves to zero live streams, the pattern is + dropped from the trie. (Alternative: a stream-close callback from the pool; decided at + implementation time, both are bounded.) +- Exact-topic subscriptions are just patterns without wildcard segments — one code path. + +The trie is bounded by the same caps as the interest set (≤1000 patterns/stream, +≤100/space/stream), so relay memory stays O(active subscriptions), and matching cost is +paid only per publish within that space. + +**Deliberately no match-result cache.** NATS fronts its sublist with a 1024-entry +literal-subject → result cache, and its own issue history (nats-server#710, #941: <0.5% +hit rates, lock contention, latency spikes under sub/unsub churn) led to `NoCache` +sublists — which NATS uses for exactly our analog, the small per-connection permission +tries. Our tries are per-space (small) and ephemeral interest is churny (every +subscribe/unsubscribe would invalidate), so a direct walk of a ≤16-level trie beats a +cache we'd constantly flush. Revisit only with profiling evidence; the bounded +patch-on-insert design from NATS is the template if ever needed. + +--- + +## 5. What happens on each frame (serving peer) + +**Subscribe** — +resolve space ACL state (nodes: the space is hosted locally; clients/LAN: the open space); +check `PermissionsAtRecord(head, ctxIdentity)` is not `None`; validate patterns +(`InvalidTopic` on bad wildcard placement / reserved chars / non-canonical form); check +pattern-count caps; then register in the trie and `AddTagsCtx(ctx, spaceId+"/"+pattern...)`. +Reject ⇒ `Status`, no tag. + +**Unsubscribe** — `RemoveTagsCtx` + trie refcount decrement. No checks needed. + +**Publish** (from client stream) — +1. size/shape checks (a topic containing `*`/`>` ⇒ `InvalidTopic`); + `identity == ctxIdentity`; membership check against cached ACL state (map lookup); + self-owned-namespace check (topic `acc/…` ⇒ last segment must equal + `ctxIdentity.Account()` — one string compare); per-peer token bucket (§7). +2. Match the topic against the space's pattern trie (§4.1), dedup stream ids across + matched patterns, then `Broadcast(msg, matchedPatternTags...)` on the pubsub pool — + the existing tag-index fan-out (`net/streampool/streampool.go:360-377`), which + per-stream `TryAdd`s and drops on overflow. +3. If serving peer is a responsible node: stamp `relayed=true`, send to other responsible + nodes (lazy stream open via the pool's `Send` + PeerGetter). + +**Publish** (`relayed == true`, from a node stream) — +verify the sending peer is a responsible node for the space (peerId ∈ `NodeIds(spaceId)`); +fan out locally only (same trie match). The `identity == ctxIdentity` rule does not apply +(the forwarding node is not the author) — authenticity is the receiver's signature check +(§6.3). + +**Receive** (client) — dedup by msgId LRU → verify signature against `identity` → check +`identity` is a member at the local ACL head → if the topic is in the `acc/` namespace, +check its last segment equals `identity.Account()` (end-to-end enforcement: the signature +covers the topic, so not even a malicious relay can inject into someone else's self-owned +topic) → look up ReadKey by `keyId`, decrypt → dispatch to local topic handlers. Any +failure ⇒ drop + debug metric, never an error to the peer. + +--- + +## 6. Security model (R3) + +### 6.1 Threat model — the semi-trusted relay + +The node can: observe spaceIds, topics, sender identities, timing, sizes (accepted — +comparable to sync-path metadata today); drop, delay, reorder messages (accepted — +at-most-once contract). The node cannot: read payloads (no ReadKey); forge or reattribute +messages (signature); replay effectively. Replay defense is two-layer: the msgId dedup +ring catches the short window, and the receiver enforces a **signed-timestamp staleness +window** (`Config.MaxTimestampSkew`, default 5 min) so that even after the ring evicts an +id, a relay replaying an old signed frame is rejected on its stale timestamp. (A relay +cannot forge a fresh timestamp — it's covered by the signature.) + +### 6.2 Access control + +Both directions gate on **space membership at the current ACL head**, checked by the +serving peer from its local ACL copy — a cached in-memory lookup, no coordinator round +trip. This is *new* enforcement: today's sync path has none at subscribe time +(`any-sync-node/nodespace/rpchandler.go:329-331` accepts unchecked). Publish and +subscribe both require `!NoPermissions()`; no `CanWrite` requirement (decision: any +member publishes — presence/typing-style uses need Readers to emit). + +**Self-owned topics.** The `acc/` namespace (§3) adds a per-topic ownership rule on top +of membership: only the account named by the topic's last segment may publish there. +It is enforced twice — at the serving peer (cheap, because `identity == ctxIdentity` is +already bound by the handshake) and at every receiver (via the signature, which covers +the topic string). This gives apps spoof-proof per-account channels — e.g. +`acc/online/` — where consumers can trust the topic itself, not just the message +attribution. It is the in-protocol generalization of the push-server's "silent +self-channel" restriction (`anytype-push-server/push/push.go:140`), and matches the +proven NATS pattern for the same problem: per-identity subject prefixes (the +`_INBOX_.>` convention) rather than dynamic per-message grants. NATS violation +semantics also match ours: a permissions violation drops the message / rejects the +subscribe with an error and keeps the connection open — only authentication failures +disconnect. Wildcards make the +fan-in side cheap: one `acc/online/*` subscription covers every member's online topic, +and each received message is still individually ownership-checked against its concrete +topic and verified signature. + +### 6.3 Authenticity + +`signature = accountKey.Sign("anysync:pubsub:v1" | len‖spaceId | len‖topic | len‖msgId | +len‖keyId | le64(timestampMilli) | payload)`. Each variable-length field is length-prefixed +(le32) so field boundaries can't shift (e.g. `spaceId="ab",topic="c"` vs +`spaceId="a",topic="bc"` sign differently); `payload` trails unprefixed as the final field. +`relayed` is excluded (mutated in transit). Receivers verify; nodes don't need to +(attribution from clients is already bound by `identity == ctxIdentity`, and verifying +per-message on the relay buys little at real CPU cost). Ed25519 sign/verify is ~30-80 µs — +negligible at ephemeral-signal rates. + +Receivers run the cheap filters (local-interest match, membership, `acc/` ownership, +timestamp staleness) *before* the Ed25519 verify, and record the dedup ring only after +verify succeeds — so a relay/LAN-peer flood of forged-signature messages is shed cheaply +and cannot evict legitimate ids from the ring to reopen a replay window (§6.1). + +### 6.4 Confidentiality & membership change + +Payloads encrypt with `AclState.CurrentReadKey()` (`commonspace/object/acl/list/aclstate.go:170`), +carrying `CurrentReadKeyId()` as `keyId`. Receivers hold historical keys via the ACL, so +rotation mid-flight is safe. On member removal the existing rotation +(`aclrecordbuilder.go:761-844`) cuts decryption of new traffic automatically. Additionally, +the engine exposes `EvictMember(spaceId, identity)` — the node wires it to its ACL-update +hook (the `syncacl` updater, `commonspace/object/acl/syncacl/syncacl.go:53-56`, is the +precedent) to actively drop the removed identity's subscriptions: it strips that account's +per-stream tags (via `streampool.RemoveTagsById`, so delivery — which is tag-keyed — stops +even while the stream stays open) and decrements the match trie. Bounded work: one scan of +the streams subscribed to that space per ACL change. + +Spaces without a ReadKey (post-GO-7187 keyless/public spaces): `keyId = ""`, plaintext +payload, signature still required. + +--- + +## 7. Memory & abuse bounds (R2) + +| Resource | Bound | Mechanism | +|---|---|---| +| Outbound per-stream buffer | `queueSize` msgs (default 100) | `mb.MB` + `TryAdd` drop (`net/streampool/stream.go:39`) | +| Interest table | ≤1000 patterns/stream, ≤100/space/stream | reject with `TooManyTopics`; state dies with stream | +| Pattern trie (§4.1) | O(total live patterns × segments), segments ≤16 | same caps as interest table; lazily pruned when a pattern's tag has no live streams | +| Publish rate | token bucket per peer per stream (default 30 msg/s, burst 60) | checked in pubsub handler — **inside** the stream, because the RPC limiter only gates stream-open (`net/rpc/limiter/limiter.go:96-106`). Deliberate divergence from core NATS, which has no publish rate limiting (maintainers punt to network throughput) and instead disconnects slow consumers — acceptable for a trusted-client broker, not for our semi-trusted multi-tenant relays. We also drop rather than disconnect slow subscribers, which fits at-most-once ephemera | +| Payload size | ≤64 KiB | reject `InvalidMessage` | +| Dedup cache | fixed LRU, 4096 msgIds | evicts oldest, O(1) | +| Fan-out amplification | 1 upload → ≤2 node-node copies → N subscriber queues | node-side copy (publisher uploads once); `peerMessage.Copy()` pattern reuses the existing per-destination stamping (`stream.go:32-37`) | +| Idle streams | closed with the sub-connection; tags GC'd in `removeStream` (`streampool.go:431-446`) | existing | +| Zombie subscribers | stream in continuous queue-overflow for > 30 s is closed | NATS disconnects slow consumers outright to protect the system; we drop first (fits at-most-once), but a *persistently* full queue means a dead/wedged reader burning fan-out work — shed it and let the client reconnect fresh | + +No persistence, no unbounded map, no per-message allocation beyond the pooled message +structs (mirror `objectmessages` `sync.Pool` usage, `headupdate.go:13-38`). + +--- + +## 8. Component design per repo + +### 8.1 any-sync (this repo) + +1. **Generalize streampool for a second instance.** Extract a non-component constructor — + `streampool.NewPool(handler streamhandler.StreamHandler, cfg StreamConfig) Pool` — and + make the existing component (`streampool.go:27`, hard-bound to `streamhandler.CName` + at `streampool.go:98`) a thin wrapper. Backward compatible; the pubsub service embeds + its own pool with its own handler, queues, and tags. Sync flow control is untouched. +2. **`commonspace/pubsub/pubsubproto`** — proto + generated DRPC (Makefile pipeline). +3. **`commonspace/pubsub`** — the shared engine used by node, client, and LAN-server + sides alike: + - stream lifecycle: `OpenStream` to a peer / `ReadStream` for inbound (both feed the + private pool), resubscribe-on-reconnect using the `subscribeclient` watcher pattern + (`coordinator/subscribeclient/client.go:115-153`); + - interest handling: Subscribe/Unsubscribe → pattern validation + membership check + hook → per-space pattern trie (§4.1) + tags; + - publish path: validate → trie match + cross-pattern stream dedup → broadcast → + optional forward hook; + - receive path: dedup LRU → verify → decrypt → handler dispatch; + - pluggable interfaces so layering stays clean: + `MembershipChecker` (backed by `AclState`), `Crypto` (ReadKey encrypt/decrypt via the + space's `Acl()`), `Forwarder` (node-only), `RateLimiter`. + - public API: + `Publish(ctx, spaceId, topic string, payload []byte) error` (encrypt+sign+send) and + `Subscribe(spaceId, topic string, h Handler) (unsubscribe func())`, plus an error/ + status callback for surfaced `Status` frames. +4. **Reference wiring + tests** in the synctest style + (`commonspace/sync/synctest/`): multi-peer in-memory fixture proving fan-out, ACL + rejection, relay rules, drop-on-overflow, dedup. + +### 8.2 any-sync-node + +- Register `DRPCRegisterPubSub` next to SpaceSync (`nodespace/service.go:80`). +- `pubsubrelay` component: wires the shared engine with node deps — membership from the + hosted space's ACL, responsibility check from nodeconf, `Forwarder` resolving the other + responsible nodes (reuse `getResponsiblePeers` logic, + `nodespace/peermanager/manager.go:87-103`), rate-limiter config. +- ACL-update hook to evict removed members' tags (§6.4). + +### 8.3 anytype-heart (sketch — own design doc when we get there) + +- Register the pubsub client component in bootstrap next to streampool + (`core/anytype/bootstrap.go:263-281`); open streams to the space's responsible node and + LAN peers via the existing per-space peer manager; serve inbound LAN pubsub streams from + the existing client server (`space/spacecore/rpchandler.go`). +- Tie subscriptions to space open/close lifecycle; auto-resubscribe on + `rebuildResponsiblePeers`. +- Surface to apps: middleware commands (`PubsubPublish`, `PubsubSubscribe/Unsubscribe`) + emitting `pb.Event`s through the existing local event bus (`core/subscription/`), the + same delivery surface chat SSE uses. + +### 8.4 Compatibility & rollout + +- Old peers don't know the `PubSub` service → DRPC unknown-RPC error on stream open. + Client treats it as "pubsub unavailable on this peer", backs off (subscribeclient's + capped linear backoff), and retries opportunistically. No protoVersion bump required; + no coordinator/nodeconf changes (same node addresses, same responsibility mapping). +- Ship order: any-sync (lib) → any-sync-node deploy → heart. Until nodes deploy, LAN-only + pubsub still works between updated clients. + +--- + +## 9. Defaults (tunable via config) + +| Knob | Default | +|---|---| +| max payload | 64 KiB | +| max topic length | 256 B | +| max topics per stream | 1000 (100 per space) | +| publish rate per peer | 30 msg/s, burst 60 | +| per-stream write queue | 100 (client), 500 (node outbound) — match sync-side sizes | +| dedup LRU | 4096 msgIds | +| received-timestamp staleness window | 5 min (enforced at receive, `Config.MaxTimestampSkew`) | +| client interest resync interval | 20 s (`Config.ResyncInterval`) | +| pubsub stream peer TTL | 1 h (`Config.PeerTTL`) | + +## 10. Non-goals (v1) + +Queue groups (1-of-N); persistence, replay, retained messages, catch-up after reconnect; +delivery receipts/acks; cross-space topics (patterns never span spaces — `spaceId` is a +separate field, not a topic segment); protocol-level presence; WAN client↔client (only +LAN-discovered direct peers); interest propagation between nodes (a node always forwards +client publishes to the other responsible nodes, which drop them if nothing matches +locally — 2 bounded copies beats holding cross-node subscription state). + +On that last non-goal, NATS prior art maps cleanly onto our choice. NATS *clusters* do +propagate interest (RS+/RS-, refcounted per subject, advertised on the 0→1 transition, +withdrawn on N→0) because a cluster may span many servers and unnecessary fan-out is +expensive at that scale — the cost is every server holding the full cluster interest map +and an inherent propagation race (subscription visibility across the cluster is +asynchronous, nats-server#1142). NATS *gateways* (WAN) instead default to **optimistic +sends**: forward without interest knowledge, let the receiver reply "no interest", and +only switch to interest-only mode after ~1000 such rejections per account +(`server/gateway.go`, `defaultGatewayMaxRUnsubBeforeSwitch`). With a fixed fan-out of 2 +peer nodes per space and shared-space traffic being likely-relevant to all replicas, our +always-forward is the optimistic-send strategy at the scale where it wins. It also +sidesteps NATS's single biggest documented scaling pain — interest churn, where every +first-subscribe/last-unsubscribe is a cluster-wide broadcast plus a global client-cache +flush (nats-server#710/#941). If inter-node waste ever becomes measurable, the proven +*incremental* fix is the gateway one: a bounded per-topic no-interest map with a +switch-to-interest-only threshold — not full RS+/RS- interest replication (v2, not v1). + +## 11. Remaining open items + +1. Exact package/service naming bikeshed (`commonspace/pubsub` vs top-level `pubsub`; + service `PubSub` vs `SpacePubSub`). +2. Whether the node emits `Status{RateLimited}` per rejected publish or silently drops + after the first notice (flood of statuses is itself amplification — leaning: notify + once per window). +3. Metrics surface (per-topic counters are unbounded-cardinality; per-space is safe). + The private pool is now observable via `Deps.Metric` (`WithMetric(m, "pubsub")`); the + remaining work is the pubsub-specific counters below. + +## 12. Implementation status (v1 in any-sync) + +Implemented in `commonspace/pubsub` (+ `net/streampool` additions): full wire protocol, +flat+wildcard topic model with the NATS-sublist trie, ACL-gated subscribe/publish, signed +& encrypted payloads, node relay with the one-hop rule, LAN symmetry, echo/duplicate-path +suppression, per-peer publish rate limiting, and — after the multi-lens review — the +following hardening: + +- **Interest keyed by streamId, not peerId.** Serving-side interest lives in + `streams[streamId]` and the match trie refcounts *subscribing streams*; the close hook + carries the `streamId`. This fixes two review-found bugs: a reconnecting peer's fresh + stream no longer loses interest when the stale stream closes, and a stream that dies + mid-subscribe can't orphan interest (interest+tag are committed under one lock with + rollback if the stream vanished). Regression tests: `TestReconnectKeepsInterest`, + `TestStreamCloseDrainsInterest`. +- **Reconnect watcher.** A resync loop re-pushes local interest every `ResyncInterval` + and immediately on client-side stream close, and `OpenStream` sets `PeerTTL` so idle + pool GC doesn't silently reap a quiet subscriber. Without this a pure subscriber went + dead after the first drop. Test: `TestResyncRestoresDeliveryAfterDrop`. +- **`CloseSpace` / `EvictMember`.** Per-space teardown (a global service must release + closed-space state) and §6.4 active member eviction. Tests: `TestCloseSpaceDropsInterest`, + `TestEvictMemberStopsDelivery`. +- **Receive-path ordering + timestamp window + empty-identity guard** (§6.1/§6.3). +- **`Status.msgId`** for host-side rejection correlation; node publish shape errors return + `InvalidTopic` consistently with the client API; `spaceId` is validated to contain no `/`. + +Deferred (documented, not silent): + +- **Zombie shedding** (close a stream stuck in queue-overflow > 30 s, §7). Needs per-stream + drop stats surfaced from the pool; the drop-on-overflow bound already holds without it. +- **Subscribe-rate limiting / per-space lock sharding.** `remoteMu` is a single global + lock; a member can thrash subscribe/unsubscribe within the caps. Fine at expected scale; + shard or rate-limit if a multi-tenant relay shows contention. +- **Reader-level payload cap.** The 64 KiB cap is enforced after DRPC decode; enforcing it + at the stream reader (smaller `MaximumBufferSize`) needs per-stream buffer plumbing. +- **Rate-limiter size cap.** Per-peer buckets are time-GC'd but uncapped; bounded by + authenticated members in practice. +- **pubsub-specific metrics** (per-stream drop counter, slow-subscriber flag, one event per + overflow episode, close-reason strings) — §11.3. +- **`MembershipChecker` returning a permission level** rather than a bare member/not — a + one-way door kept simple for v1 (current-head `!NoPermissions()`). + +Downstream wiring (separate repos, per §8.2/§8.3): any-sync-node `pubsubrelay` component +(`Relay`/`Membership` from nodeconf + hosted ACL, `RegisterRpc`, `EvictMember` on ACL +change) and anytype-heart client (`Crypto` from the space ReadKey, `PeerProvider` from the +peer manager, `CloseSpace` on space unload, middleware commands). + Should follow the nats.go subscription observability contract: per-stream dropped + counter, a "slow subscriber" state flag, **one event per overflow episode** (not per + dropped message), plus a cumulative per-node counter (varz `slow_consumers` style) + and distinct close-reason strings (rate-limited vs zombie-shed vs transport error). + +--- + +## Appendix A — presence as an app pattern (non-normative recipe) + +Modeled on Yjs awareness (`y-protocols/awareness.js`), with deliberate corrections where +its semantics depend on a trusted relay. The critical difference: **y-websocket presence +relies on the server synthesizing `state:null` for a dead connection's clients** — +current y-websocket clients send *no* leave on tab close at all (the unload handler was +deliberately removed, yjs/y-websocket#165). Our relay cannot forge signed messages, so +that path does not exist here. + +**Entry & lifecycle.** Each device publishes its full presence entry +`{sessionId, clock, state|null}` on state change and as a heartbeat every ~15 s +(TTL/2); receivers expire an entry after ~30 s (TTL) without re-announce, measured by +**receiver-local receipt time** (immune to sender clock skew — Yjs's `lastUpdated` never +crosses the wire either). Full state per message, no diffs: every message stands alone, +so any at-most-once loss self-heals within one heartbeat, and per-message signing +composes cleanly. + +- **TTL expiry is the normative leave mechanism.** Explicit leave (`state=null`) is a + best-effort latency optimization on graceful shutdown. Worst-case ghost duration = + TTL. (Matrix `m.typing` runs entirely on refresh-or-expire; that is the baseline + guarantee.) +- **Key = `(accountId, sessionId)`, fresh random `sessionId` per app session** (Yjs: + new `clientID` per page load). An account with two devices is two entries; a rebooted + client never needs to out-clock its dead predecessor — the old entry just times out. + Never persist clocks across sessions. `accountId` comes from the verified message + `identity`, never from the payload — signing closes the identity-hijack hole Yjs + explicitly punts on (`PROTOCOL.md §6`), and Yjs's own-clientID `clock++` defense + becomes unnecessary. +- **Clock: bump before every publish** — state changes, heartbeats, leaves, and + reconnect re-announces alike, starting at 1. (Yjs leaves some paths un-bumped only to + interoperate with server-synthesized equal-clock nulls, and its un-bumped reconnect + re-announce causes a real invisibility race; with no synthesizing relay, always-bump + is strictly simpler and safer. Starting at 1 avoids the Yjs clock-0 trap where an + unknown client's first entry is dead on arrival.) +- **Accept rule:** accept iff `clock > knownClock`, or (`clock == knownClock` and + `state == null` and a live state exists) — equal-clock-null keeps leave idempotent + under duplicate/multi-path delivery. On removal (null or TTL), keep a + `sessionId → lastClock` tombstone for ≥ TTL so reordered pre-leave messages can't + resurrect a ghost. +- **Join snapshot:** a stateless relay cannot push the room state to a new joiner + (y-websocket's server does). Recipe: subscribe first, then announce yourself; peers + treat an unknown-session announce as a cue to re-announce early (with random jitter + ≤ heartbeat/2 to avoid an answer storm). Alternatively accept ≤ one heartbeat of join + blindness. +- **Fan-out hygiene:** never re-publish an applied remote update (Yjs's origin-blind + echo handler caused a documented N² message storm at ~20 users/room); keep state + small (identity + cursor); throttle high-frequency fields app-side (~10 Hz cursor + max, further coalesced by the publish token bucket); consider separate topics and + cadences for slow presence vs fast cursors. Idle-room budget is ≈ N²/heartbeat + deliveries — scale the heartbeat up for large spaces. + +Two topic layouts, both valid: + +- **Shared topic** `presence` (or `presence/{objectId}`): one subscription covers the + whole space; attribution comes from the verified `identity`. +- **Self-owned topics** `acc/online/`: spoof-proof per-account channels (§6.2). + Subscribe with `acc/online/*` to follow everyone at the cost of a single interest + entry, or with concrete topics to follow specific accounts. + +If sub-TTL leave latency ever matters, a v2 option is relay-emitted **unsigned transport +hints** ("subscriber stream closed") that clients may use only to shorten their local +expiry check for that peer — never as an authoritative removal; authority stays with +signed messages and TTL. diff --git a/docs/stateless-pubsub/RESEARCH.md b/docs/stateless-pubsub/RESEARCH.md new file mode 100644 index 000000000..cd714352c --- /dev/null +++ b/docs/stateless-pubsub/RESEARCH.md @@ -0,0 +1,250 @@ +# Stateless Pub/Sub in `any-sync` — Research Summary + +Status: **RESEARCH / PROBLEM-FRAMING** — deliberately contains **no chosen solution**. +Purpose: hand-off document for a spec author (fable). It frames the problem, surveys how +NATS and other modern systems do stateless pub/sub, maps the concrete `any-sync` substrate +(`file:line`-grounded against `main`), catalogs the prior art already in the repo, and +enumerates the open design tensions the spec must resolve. It does **not** pick a design. + +--- + +## 0. The ask (verbatim intent) + +> Users of the same space can **subscribe/publish topics within a space**. It must work +> **effectively over the any-sync protocol** (probably a DRPC stream), be **memory-effective**, +> and **respect the ACL access** of the user. + +Three hard requirements fall out of this, and every section below is oriented around them: + +1. **R1 — On-protocol.** Runs over the existing any-sync transport/DRPC machinery, not a side channel. +2. **R2 — Memory-effective.** Bounded, ephemeral footprint on both clients and relaying nodes; no unbounded buffers, no persistence. +3. **R3 — ACL-aware.** Publish/subscribe is gated by the space's access-control list and its encryption boundary. + +Plus the implicit scope word: **stateless** (Section 1 pins down what that actually means — it is ambiguous and the ambiguity matters). + +--- + +## 1. What "stateless pub/sub" means (and the ambiguity to resolve) + +Distilled from the reference systems (Sections 2–3), "stateless pub/sub" is the **core-NATS / Redis-pub-sub** family, characterized by: + +- **Fire-and-forget.** A publish with no live subscriber is simply discarded; it is never stored or replayed. +- **At-most-once delivery** (MQTT "QoS 0"). No acks, no retransmit, no dedup, no ordering guarantees across publishers. +- **No persistence.** No log, no durable queue, no retained "last message." (This is exactly what distinguishes it from any-sync's DAG trees, the KV store, and the coordinator Inbox — all of which *are* stateful.) +- **Full decoupling** of publishers and subscribers in space (don't know each other), time (needn't overlap beyond the instant of delivery), and synchronization (non-blocking). +- **Fan-out (1→N)** as the base pattern; optionally **load-balanced 1-of-N** ("queue groups"). + +**Ambiguity the spec MUST pin down — two independent axes of "stateless":** + +- **(a) Message statelessness** — no message is ever persisted (fire-and-forget). *All* systems in this family have this. +- **(b) Subscription/routing statelessness** — whether the *relay* holds any in-memory subscription-interest table. + - Core NATS is stateless in sense (a) but the **server does hold an in-memory interest graph** (subject → subscribers) to route efficiently. It is *not* stateless in sense (b). + - The fully-(b)-stateless alternative is "relay broadcasts everything, subscribers filter locally" — zero routing state but poor bandwidth-efficiency. + +This tension between (b) and **R2 (memory-effective)** / bandwidth-efficiency is one of the central design decisions and is called out again in Section 7. The word "stateless" in the ask most plausibly means (a) + *no durable/persistent* state, tolerating transient in-memory routing tables — but this must be confirmed, not assumed. + +--- + +## 2. Reference model: core NATS pub/sub + +(Sources: NATS docs — pubsub, subjects, queue groups; see Section 9.) + +**Subject-based addressing.** Publishers send to a **subject** (a string); subscribers register **interest** in subjects. "A subject is just a string the publisher and subscriber use to find each other." Messages carry `{subject, payload bytes, headers, optional reply-address}`. + +**Subject hierarchy + wildcards.** +- Subjects are dot-separated token hierarchies: `time.us.east.atlanta`. +- **`*`** matches exactly one token: `time.*.east` ⇒ `time.us.east`, `time.eu.east` (not `time.us.east.atlanta`). +- **`>`** matches one-or-more trailing tokens, tail-position only: `time.us.>` ⇒ everything under `time.us`. +- **Publishers must use fully-qualified subjects** (no wildcards); **only subscribers use wildcards.** +- Allowed chars: any Unicode except null, space, `.`, `*`, `>`; recommend alnum + `-`/`_`. `$`-prefixed reserved for system. Guideline ≤16 tokens, ≤256 chars. Max payload default 1 MB (server `max_payload`, cap 64 MB). + +**Delivery.** +- **Fan-out:** every interested subscriber gets a copy (1→N). +- **Queue groups:** subscribers sharing a queue name form a group; each message goes to **exactly one randomly-chosen** member — built-in load balancing + transparent scaling + "no-responders" signal. Queue-group names follow subject naming rules. +- **At-most-once.** Offline/disconnected subscriber ⇒ message lost. Messages with no subscribers are discarded. + +**What core NATS deliberately does NOT provide** (these are JetStream, a separate stateful layer): persistence, replay, guaranteed/at-least-once delivery, dedup, ordering, consumer cursors. Those are exactly the things a *stateless* design excludes. + +--- + +## 3. Landscape of modern pub/sub (comparison) + +| System | Topic model | Wildcards | Delivery | Persistence | Topology | Notable for us | +|---|---|---|---|---|---|---| +| **NATS core** | dot-hierarchy subjects | `*` (1 token), `>` (tail multi) | fan-out + queue-group (1-of-N) | none (JetStream is separate) | central broker(s), interest graph | the canonical model; subject wildcards; queue groups | +| **Redis pub/sub** | flat channels + patterns | `*`, `?`, `[..]` (glob) | fan-out | none | central server | simplest fire-and-forget; subscriber must be connected | +| **MQTT** | `/`-hierarchy topics | `+` (1 level), `#` (tail multi) | QoS 0/1/2 | retained msg + sessions ⇒ **stateful** | broker | wildcard syntax variant; "retained message" is the anti-pattern to avoid for stateless | +| **libp2p GossipSub** | flat topics | none | epidemic mesh fan-out; IHAVE/IWANT lazy-pull | none | **decentralized P2P mesh** (no broker) | closest to any-sync's decentralization ethos; mesh + fanout + peer-scoring for abuse resistance | +| **Yjs awareness** | one implicit channel/doc | n/a | state-based CRDT diff | ephemeral, auto-GC | transport-agnostic relay | **presence prototype**: `(clientID, clock, state\|null)`, `null`=offline, 15 s re-announce / 30 s timeout; kept *separate* from the CRDT doc | +| **Phoenix Channels** | `topic:subtopic` strings | none (exact) | fan-out; Presence built on PubSub | ephemeral | server (BEAM PubSub) | presence = minimal ephemeral metadata over pub/sub | +| **Matrix EDUs** | per-room | n/a | fan-out to room servers | ephemeral (EDU ≠ persistent event) | federated servers | typing/presence modeled as **Ephemeral Data Units**, explicitly distinct from the persistent event DAG | + +**Cross-cutting takeaways relevant to any-sync:** + +- **Topology is the fork in the road.** NATS/Redis/MQTT = central broker with an interest table. GossipSub = P2P mesh, no broker. any-sync sits *between*: clients reach a small set of **responsible sync nodes** (semi-trusted relays that store ciphertext they cannot read) and *may* also reach other clients directly (Section 4.6). The relay-through-node model is closest to a broker, but the broker is **untrusted for content** — which forces the encryption question (Section 6). +- **Presence is the killer stateless-pubsub use case** in local-first / collaborative systems (Yjs awareness, Phoenix Presence, Matrix EDUs), and it is *always* kept separate from the persistent CRDT/document layer. If presence/typing/cursors are a target use case, the Yjs awareness lifecycle (heartbeat + timeout + `null`-on-leave) is the reference to study, not NATS. +- **Wildcards cost the relay.** Subject-hierarchy matching (`*`/`>`, `+`/`#`) requires the relay to run a trie/interest-match per message. Flat topics (GossipSub, Redis channels, Phoenix) do not. This trades expressiveness against R2. + +--- + +## 4. The `any-sync` substrate (grounded map) + +All paths under `/Users/roma/anytype/any-sync`, line numbers vs `main`. **Scope caveat:** any-sync is a *library*. The concrete server-side `DRPCSpaceSyncServer` and the production `PeerManager`/`StreamHandler` live in downstream repos (`any-sync-node`, `anytype-heart`). This repo ships the interfaces, the client plumbing, and **reference implementations in test code** (`commonspace/spaceutils_test.go`, `commonspace/spacerpc_test.go`, `commonspace/sync/synctest/`). Every such boundary is flagged. + +### 4.1 Transport & DRPC (R1 foundation) + +- Three transports selected by address scheme: **Yamux** (TCP), **QUIC**, **WebTransport** — `net/transport/transport.go:24-28`. ALPN `"anysync"`; QUIC handshake at `net/transport/quic/quic.go:108-165`. +- Secure layer = libp2p-TLS + an app handshake that stamps `peerId`, account `identity`, versions into the connection context — `net/secureservice/secureservice.go:114-185`, ctx accessors `net/peer/context.go:33-106`. +- DRPC server is a `drpcmux` with a handler chain `limiter → metric → encoding` — `net/rpc/server/drpcserver.go:52-80`. Services register via generated `DRPCRegisterXxx(mux, impl)`. +- A bidirectional DRPC stream is obtained by `peer.AcquireDrpcConn` → generated stream method → `drpc.Stream{Send,Recv,MsgSend,MsgRecv}` — `net/peer/peer.go:134,270-297`. + +### 4.2 StreamPool — the fan-out core (R1 + R2) + +`net/streampool/streampool.go:51-69` — the central abstraction. It caches opened `drpc.Stream`s, **indexes them by peer and by tag**, opens them lazily, and pushes messages onto per-stream write queues: + +```go +type StreamPool interface { + app.ComponentRunnable + AddStream(stream drpc.Stream, queueSize int, tags ...string) error // outgoing + ReadStream(stream drpc.Stream, queueSize int, tags ...string) error // incoming, blocks reading + Send(ctx, msg drpc.Message, target PeerGetter) error // dial+send, async + SendById(ctx, msg drpc.Message, peerIds ...string) error // only if stream exists + Broadcast(ctx, msg drpc.Message, tags ...string) error // fan-out to all streams with tag + AddTagsCtx(ctx, tags ...string) error // subscribe a stream to tag(s) + RemoveTagsCtx(ctx, tags ...string) error // unsubscribe + Streams(tags ...string) []drpc.Stream +} +``` + +- Tag index: `streamIdsByTag map[string][]uint32` — `streampool.go:71-84`. In practice the tag is a **`spaceId`** (see 4.4). +- `Broadcast(msg, tags...)` writes to every stream carrying a listed tag — `streampool.go:360-377`. **This is the existing tag-keyed fan-out primitive.** +- `AddTagsCtx`/`RemoveTagsCtx` mutate a live stream's tag set at runtime — `streampool.go:379-429`. **This is the existing dynamic (un)subscribe hook.** +- **Memory-effectiveness levers (R2):** each stream has a **bounded** `mb.MB[drpc.Message]` queue (default size **100**), and `stream.write` uses `TryAdd` — **non-blocking, drops on overflow** — `net/streampool/stream.go:32-44`. Outbound dialing runs through a bounded `ExecPool` worker pool (`sendpool.go`). There is **no per-message persistence and no unbounded buffering** anywhere on this path. + +### 4.3 Message envelope & multiplexing + +- Generic space envelope `ObjectSyncMessage{spaceId, requestId, replyId, payload []byte, objectId, objectType}` — `commonspace/spacesyncproto/spacesync.pb.go:591-601`, proto at `spacesync.proto:100-108`. +- `objectId` **multiplexes many logical channels over one physical stream**; `objectType` enum is `{Tree=0, Acl=1, KeyValue=2}` — `spacesync.proto:349-353`. +- Wire wrapper `HeadUpdate` implements `drpc.Message` + a `peerMessage` tag interface (`SetPeerId`, `Copy`) so one message can be copied and stamped per destination during fan-out — `commonspace/sync/objectsync/objectmessages/headupdate.go:58-140`. Underlying `ObjectSyncMessage` is pooled via `sync.Pool` (`headupdate.go:13-38`). +- Encoding is protobuf (vtproto), optionally snappy-compressed, negotiated in handshake — `net/rpc/encoding/`. + +### 4.4 The existing space-level pub/sub (**most important prior art**) + +any-sync **already implements a coarse pub/sub where the "topic" is an entire `spaceId`**, over a single long-lived bidirectional stream: + +- **The stream:** `rpc ObjectSyncStream(stream ObjectSyncMessage) returns (stream ObjectSyncMessage)` — bidi, long-lived, one per (peer, connection) — `spacesync.proto:40`, server iface `spacesync_drpc.pb.go:245`. +- **The subscribe control frame:** `SpaceSubscription{ SpaceIds []string; Action }` with `SpaceSubscriptionAction { Subscribe=0, Unsubscribe=1 }` — `spacesync.proto:245-256`. It is carried **inside `ObjectSyncMessage.payload` with an empty `spaceId`**. +- **The wiring** (reference impl `commonspace/spaceutils_test.go:468-540`): on `OpenStream`, the client opens `ObjectSyncStream` and immediately `Send`s a `SpaceSubscription{Subscribe, [spaceId]}`. On the receive side, `HandleMessage` sees the empty-`spaceId` frame and calls `streamPool.AddTagsCtx(ctx, spaceIds...)` — tagging the stream so subsequent `Broadcast(msg, spaceId)` reaches it. Non-control frames route to `space.HandleMessage`. +- **Server side** registers the inbound stream into its own pool: `streamPool.ReadStream(stream, 100)` — `commonspace/spacerpc_test.go:170-172` — then pushes head-updates back down the same stream via `Broadcast(msg, spaceId)`. +- **Subscribe is also kicked during head-sync**: `diffsyncer.subscribe` builds `SpaceSubscription{Subscribe}` and sends it after a successful `SpacePush` — `commonspace/headsync/diffsyncer.go:275-293`. + +**Reading:** a stateless *topic* pub/sub is, structurally, a **refinement of this existing mechanism** to finer, ephemeral topics *within* a space — the same subscribe/unsubscribe control-frame pattern and the same tag-index fan-out. The delta is (i) a topic namespace below spaceId, (ii) an ephemeral message type that is **not** routed into the sync/DAG engine, and (iii) publish/subscribe ACL gating. + +> **⚠️ User steer (recorded concern) — do NOT reuse `ObjectSyncStream` itself.** `ObjectSyncStream` is an *upper-level* channel: every frame on it is routed into the sync engine (`space.HandleMessage` → `SyncService.HandleMessage` → per-`objectId` `multiqueue` → `objectSync.HandleHeadUpdate`, §4.5). Multiplexing ephemeral pub/sub onto that same stream **couples pub/sub to sync**: they share one bounded queue (size 100, drop-on-overflow), so a pub/sub burst can starve or drop sync (head-of-line blocking) and vice-versa; they share the sync dispatch/backpressure path; and it muddles fire-and-forget ephemera with DAG anti-entropy correctness. **Preferred direction: a *separate* DRPC stream on a *separate* sub-connection**, with its own tags, its own read/write loops, and its own bounded queues — fully isolated from sync flow control. This is cheap on the substrate: peers already multiplex many DRPC sub-streams over a single `MultiConn` (QUIC/yamux), and maintain a pool of reusable sub-connections — a "separate sub-connection" is another *multiplexed* stream, **not** a new transport/TLS handshake (`net/transport/transport.go:40-64` `MultiConn.Open`; `net/peer/peer.go:109-110,270-297` sub-conn pool; QUIC `MaxIncomingStreams=128` `net/transport/quic/quic.go:52`). What is reusable is the *pattern* (subscribe control-frame + `StreamPool` tag-index `Broadcast`) and the `StreamPool`/`StreamHandler` plumbing — instantiated as its own stream, not layered onto the sync stream. See Section 7, tension #2. + +### 4.5 Sync service dispatch + +- `SyncService.BroadcastMessage` → `peerManager.BroadcastMessage` — `commonspace/sync/sync.go:97-99`. +- `SyncService.HandleMessage` enqueues onto a **per-`objectId`** `multiqueue.MultiQueue` (size 100; **overflow silently dropped** via `mb.ErrOverflowed`) → `objectSync.HandleHeadUpdate` → object resolved by `objectId` — `sync.go:101-129`, `commonspace/sync/objectsync/synchandler.go:58-79`. +- Note the existing bounded-queue + drop-on-overflow discipline is already the R2 pattern; a pub/sub path would want the same. + +### 4.6 Node topology & who relays (R1 + topology decision) + +- Node roles: `tree`(=sync node), `consensus`, `file`, `coordinator`, `namingNode`, `paymentProcessingNode` — `nodeconf/config.go:20-30`. A "client" is any account not in the node list. +- **Responsible nodes via consistent hash:** `NodeIds(spaceId)` returns the tree nodes on the chash ring for that space; `ReplicationFactor = 3` ⇒ **3 responsible sync nodes per space** — `nodeconf/nodeconf.go:61-79,133-176`, `nodeconf/service.go:22-25`. Only `tree` nodes are ring members. +- **Space peer set = responsible sync nodes + directly-connected clients.** Sync nodes are the *always-reachable* members; client↔client is possible (one-to-one spaces, local discovery) but not guaranteed — `commonspace/peermanager/peermanager.go:17-34`, reference `commonspace/sync/synctest/testpeermanager.go:36-66`. +- **Implication:** the natural fan-out hub for a space is its responsible sync node(s). But those nodes are **semi-trusted relays that cannot read space content** — which is why R3/encryption (Section 6) is load-bearing, and why a broker-style design here is not a trusted broker. + +### 4.7 App framework (how a new component wires in) + +- Component registry with `Init(a)`/`Run(ctx)`/`Close(ctx)` + `Name()`; `MustComponent[T]` lookup; **per-space child app** (`ChildApp`) gives each space an isolated component graph — `app/app.go:34-52,133-207,226-280`; space graph built in `commonspace/spaceservice.go:188-260`. +- Adding a new DRPC service is a known, mechanical path (proto + `Makefile` generate line + `DRPCRegisterXxx` on server + client component + app registration) — `Makefile:23-32,47-60`; reference client component `coordinator/subscribeclient/client.go`. + +### 4.8 ACL / access control (R3) + +- **Permission ladder:** `None=0, Owner=1, Admin=2, Writer=3, Reader=4, Guest=5` — `commonspace/object/acl/aclrecordproto/aclrecord.pb.go:71-79`. Helpers: `CanWrite()` = Admin|Writer|Owner; **there is no `CanRead()`** — "can read" is expressed as `!NoPermissions()` (any non-`None`) — `commonspace/object/acl/list/models.go:79-152`. +- **Authorization is cryptographic and content-based, NOT a per-message live check:** + - **Writes** are enforced when a signed record/change is *validated on apply*: each change carries author `Identity` + signature, checked against `AclState.PermissionsAtRecord(aclHeadId, identity).CanWrite()` — `commonspace/object/tree/objecttree/objecttreevalidator.go:182-189`; KV analog `keyvaluestorage/storage.go:117`. + - **Reads** are enforced by **encryption**: content is encrypted with a per-space `ReadKey` handed (encrypted per member pubkey) only to members inside ACL records — `commonspace/object/acl/list/aclstate.go:50-59,152-195`; rotation on membership change `aclrecordbuilder.go:761-844`. +- **Identity vs peer:** device/peer key (libp2p, authenticates `peerId` at TLS) is distinct from the **account/identity key** (the ACL `Identity`). A node's inbound handshake uses `peerSignVerifier` to prove control of the account key, placing `identity` in the connection ctx — `net/secureservice/secureservice.go:107-174`, `credential.go:67-124`. +- **KEY FINDING for R3:** the shared sync layer performs **no per-message reader-authorization** and **no ACL check at message-handle time** — a grep across `commonspace/sync`, `commonspace/spacesyncproto`, `commonspace/headsync` finds no `Permissions/CanWrite/NoPermissions` usage. Membership is gated **at stream-open by the node** (that logic lives in `any-sync-node`), and content confidentiality relies on read-key **encryption** (non-members receive ciphertext they cannot decrypt). The coordinator-side `acl.AclService.Permissions(ctx, identity, spaceId)` exists for explicit checks — `acl/acl.go:172-180`. + +--- + +## 5. Prior art inside any-sync (what already exists to lean on or contrast) + +| Prior-art mechanism | Where | Relation to stateless pub/sub | +|---|---|---| +| **Space subscription over `ObjectSyncStream`** (`SpaceSubscription{Subscribe/Unsubscribe}` + `AddTagsCtx`) | `spacesync.proto:40,245-256`; `spaceutils_test.go:468-540` | **Direct precedent** — coarse pub/sub, topic == spaceId. A topic pub/sub generalizes this. | +| **`StreamPool.Broadcast(msg, tags...)`** tag-indexed fan-out | `net/streampool/streampool.go:360-377` | The reusable fan-out engine; topics could be additional tags. | +| **Coordinator `NotifySubscribe(req) → stream NotifySubscribeEvent`** | `coordinator.proto:57`, `coordinator/subscribeclient/{client,stream}.go` | Server-push subscription-stream pattern, but **coordinator-scoped** and fixed to enum event *types* (`InboxNewMessageEvent`, `NetworkConfigChangedEvent`) — not arbitrary space topics. Good template for a dedicated pub/sub service + auto-reconnect + `mb.MB` mailbox. | +| **Coordinator Inbox** (`InboxFetch` / `InboxAddMessage`, signed sender→receiver messages) | `coordinator.proto:51-54,434-473` | **Contrast / anti-pattern for "stateless":** this is *stateful* store-and-forward (persisted, fetch-by-offset, `hasMore`). Shows what stateless pub/sub deliberately is *not*. | +| **`mb.MB[T]` bounded mailbox** (`cheggaaa/mb/v3`) | `subscribeclient/stream.go:18`; stream queues `stream.go` | The idiomatic **memory-bounded** streaming buffer (R2) — bounded size, backpressure or drop. | +| **`multiqueue.MultiQueue` per-object sharded queue, drop-on-overflow** | `commonspace/sync/sync.go:101-129` | Existing R2 discipline for per-logical-channel inbound processing. | + +--- + +## 6. How R3 (ACL) specifically interacts with pub/sub — facts, not decisions + +The spec must resolve publish-permission and subscribe-permission against these substrate facts: + +- **Two distinct permissions are in play.** Publishing is a *write-like* action (`CanWrite()` ⇒ Admin/Writer/Owner). Subscribing/receiving is a *read-like* action (`!NoPermissions()` ⇒ any member incl. Reader/Guest). Presence/typing/cursors, however, are things a **Reader** plausibly should be allowed to *emit* — so "publish == CanWrite" may be too strict for the archetypal use case. **Open.** +- **No per-message ACL gate exists to reuse.** Enforcement today is either (a) node-side membership gating at stream/subscribe time, or (b) read-key encryption. A pub/sub design must choose one or both; there is no drop-in per-message reader check in the shared layer. +- **The relay node cannot be trusted with plaintext.** Consistent with the whole any-sync model, if payloads are encrypted with the space `ReadKey`, the relaying sync node routes ciphertext it cannot read — automatically enforcing read-confidentiality (non-members lack the key) at the cost of the node being unable to match on payload contents (fine) and potentially topic names (depends on whether topic strings are encrypted — **open**). +- **Key rotation on membership change is already handled** for stored content (`ReadKey` rotates, re-encrypted per remaining member — `aclrecordbuilder.go:761-844`). For *ephemeral* messages the question is whether pub/sub piggybacks the current `ReadKey` (so a removed member instantly loses the ability to decrypt new messages) or uses a separate ephemeral key. **Open.** +- **Publish authorization without a per-message check** implies either signing each ephemeral message (adds CPU + size — measure against R2) or relying on "only key-holders can produce decryptable messages" (confidentiality without authenticity — a receiver couldn't distinguish which member sent it, or prevent a Reader from spoofing). **Open trade-off.** + +--- + +## 7. Open design tensions the spec must resolve (NOT resolved here) + +Grouped by the requirement they stress. Each is a genuine fork with substrate consequences noted. + +**Topology & transport (R1)** +1. **Relay vs mesh.** Fan-out through the 3 responsible sync nodes (always reachable, broker-like, but untrusted-for-content) vs direct client↔client (P2P, GossipSub-like, not always reachable) vs hybrid. Substrate favors relay-through-node for reachability; mesh fits the decentralization ethos but has no guaranteed connectivity. +2. **Dedicated pub/sub stream on its own sub-connection vs reusing `ObjectSyncStream`.** **User steer (recorded, §4.4): do not reuse `ObjectSyncStream`** — it is upper-level and tied to the sync engine, so sharing it couples pub/sub and sync (shared bounded queue, head-of-line blocking, intertwined backpressure). The preferred direction is a **separate DRPC stream over a separate multiplexed sub-connection** (cheap: another sub-stream on the existing `MultiConn`, not a new handshake), reusing only the *pattern* (subscribe control-frame + `StreamPool` tag-index `Broadcast`) and the `StreamPool`/`StreamHandler` plumbing — not the sync stream. Remaining sub-decision for the spec: does the separate stream belong to a **new dedicated `pubsub` DRPC service** (à la coordinator `NotifySubscribe`, cleanest isolation of lifecycle/versioning) or a **new stream RPC added to the existing `SpaceSync` service** (fewer moving parts, same service registration)? Both are mechanically supported (Section 4.7); both keep pub/sub off the sync stream. + +**Topic model** +3. **Flat topics vs NATS-style hierarchy with wildcards** (`*`/`>` or `+`/`#`). Hierarchy+wildcards is expressive but forces the relay to run interest-matching per message (relay CPU/mem vs R2); flat topics map cleanly onto the existing tag index. If wildcards are wanted, the tag-index (`streamIdsByTag`, exact-match) is insufficient and a trie/matcher is required. +4. **Topic namespace & encryption of topic names.** Topics are scoped within a `spaceId`; are topic strings plaintext (relay can route on them but learns them) or derived/encrypted (relay routes on opaque handles)? Interacts with R3. + +**Statelessness & memory (R2)** +5. **Routing statelessness (Section 1 axis b).** Relay holds a topic→subscriber interest table (bandwidth-efficient, small transient state) vs relay broadcasts all space traffic and clients filter (zero routing state, wasteful). "Memory-effective" likely means the former with strictly-bounded tables, but confirm. +6. **Backpressure policy.** The substrate default is **drop-on-overflow** (`TryAdd`, `multiqueue` drop). For at-most-once stateless semantics that is coherent — but the spec should state it explicitly (slow subscriber ⇒ dropped messages, never memory growth). +7. **Fan-out amplification.** One publish × N subscribers × up to 3 relaying nodes. Where does the copy happen (node-side fan-out preferred so the publisher uploads once)? Bounds on N, message size, publish rate. + +**Delivery semantics** +8. **Plain fan-out only, or also queue-groups (1-of-N)?** Queue groups need group-membership state on the relay; likely out of scope for v1 but should be an explicit non-goal or goal. +9. **Presence lifecycle.** If presence/typing/cursors are in scope, adopt a Yjs-awareness-style **heartbeat + timeout + explicit-leave** (`null` state, ~15 s re-announce / ~30 s expiry) — otherwise "who is online" cannot be derived from fire-and-forget alone. Decide whether presence is a first-class feature or just an example payload. +10. **Reconnection.** Stateless ⇒ messages during a disconnect are lost by definition; on reconnect a subscriber re-subscribes and gets only new messages. Confirm no "catch-up" expectation (that would make it stateful). + +**ACL (R3)** — the four open items in Section 6 (publish vs subscribe permission level; encryption of payload/topic; ephemeral vs space `ReadKey`; per-message signing vs encryption-only). + +**Abuse resistance** +11. GossipSub-style peer scoring / rate-limiting is absent here; the substrate has a per-peer request rate-limit in `requestmanager` but nothing pub/sub-specific. Decide whether publish-rate limiting / anti-spam is in scope (a Reader flooding a topic). + +--- + +## 8. Success criteria the spec should be measured against + +- **R1:** rides existing transports + DRPC + StreamPool; no new side-channel; ideally reuses the `ObjectSyncStream`/tag machinery or cleanly mirrors the `NotifySubscribe` pattern. +- **R2:** per-connection and per-node footprint is **bounded and ephemeral** — bounded queues, drop (not buffer) on overflow, no persistence, transient routing tables sized O(active subscriptions). No path that grows memory with message volume or offline duration. +- **R3:** subscribe and publish are gated by ACL (membership + permission level), and confidentiality holds against the untrusted relay (encryption boundary preserved). A removed member loses access to new messages. +- **Semantics:** documented at-most-once, fire-and-forget, no ordering/dedup guarantees — matching the core-NATS/Redis family, explicitly *not* the stateful Inbox/DAG/KV families. + +--- + +## 9. Sources + +**any-sync (this repo, `main`)** — grounded `file:line` references inline throughout Section 4–6; key anchors: `net/streampool/streampool.go`, `commonspace/spacesyncproto/protos/spacesync.proto`, `commonspace/sync/sync.go`, `commonspace/object/acl/list/aclstate.go`, `nodeconf/nodeconf.go`, `coordinator/subscribeclient/`, `coordinator/coordinatorproto/protos/coordinator.proto`. + +**External:** +- NATS — Publish-Subscribe: https://docs.nats.io/nats-concepts/core-nats/pubsub +- NATS — Subjects & wildcards: https://docs.nats.io/nats-concepts/subjects +- NATS — Queue Groups: https://docs.nats.io/nats-concepts/core-nats/queue +- libp2p GossipSub (design, mesh/fanout, IHAVE/IWANT, peer scoring): https://github.com/libp2p/specs/tree/master/pubsub/gossipsub +- Yjs awareness protocol: https://github.com/yjs/y-protocols/blob/master/PROTOCOL.md and https://docs.yjs.dev/api/about-awareness +- Redis pub/sub: https://redis.io/docs/latest/develop/interact/pubsub/ +- MQTT topics/wildcards/QoS: https://mqtt.org/ (spec) +- Phoenix Channels & Presence: https://hexdocs.pm/phoenix/Phoenix.Channel.html , https://hexdocs.pm/phoenix/Phoenix.Presence.html +- Matrix ephemeral events (typing/presence EDUs): https://spec.matrix.org/ (server-server EDUs) diff --git a/net/streampool/context.go b/net/streampool/context.go index 724c172f8..2b12d7466 100644 --- a/net/streampool/context.go +++ b/net/streampool/context.go @@ -15,3 +15,12 @@ func streamCtx(ctx context.Context, streamId uint32, peerId string) context.Cont ctx = peer.CtxWithPeerId(ctx, peerId) return context.WithValue(ctx, streamCtxKeyStreamId, streamId) } + +// CtxStreamId returns the id of the stream that delivered the current message. +// It is set on the context passed to StreamHandler.HandleMessage, letting a +// handler key per-stream state (e.g. subscription interest) so that two streams +// from the same peer stay independent. +func CtxStreamId(ctx context.Context) (streamId uint32, ok bool) { + streamId, ok = ctx.Value(streamCtxKeyStreamId).(uint32) + return +} diff --git a/net/streampool/streampool.go b/net/streampool/streampool.go index d7a72c843..64622538e 100644 --- a/net/streampool/streampool.go +++ b/net/streampool/streampool.go @@ -33,6 +33,48 @@ func New() StreamPool { } } +// Option configures a standalone pool created with NewStreamPool. +type Option func(*streamPool) + +// WithStreamCloseHook registers a callback invoked after a stream is removed from +// the pool, outside the pool lock, with the closed stream's id, peerId and the +// tags it carried. The streamId lets a handler clean up per-stream state keyed on +// CtxStreamId even when several streams share a peerId. +func WithStreamCloseHook(hook func(streamId uint32, peerId string, tags []string)) Option { + return func(s *streamPool) { + s.closeHook = hook + } +} + +// WithMetric registers the standalone pool's prometheus metrics under the given +// prefix, so a service that owns a private pool (e.g. pubsub) is observable even +// though it never runs the app-component Init. +func WithMetric(m metric.Metric, prefix string) Option { + return func(s *streamPool) { + if m == nil { + return + } + s.metric = m + m.RegisterStreamPoolSyncMetric(s) + registerMetrics(m.Registry(), s, prefix) + } +} + +// NewStreamPool creates a standalone pool with explicit dependencies, for services +// that own a private pool (e.g. pubsub) instead of sharing the app-level component. +// The caller must not register it in the app and is responsible for calling +// Run(ctx) and Close(ctx); Init must not be called. +func NewStreamPool(handler streamhandler.StreamHandler, cfg StreamConfig, opts ...Option) StreamPool { + s := New().(*streamPool) + s.handler = handler + s.streamConfig = cfg + s.statService = debugstat.NewNoOp() + for _, opt := range opts { + opt(s) + } + return s +} + type configGetter interface { GetStreamConfig() StreamConfig } @@ -64,6 +106,9 @@ type StreamPool interface { AddTagsCtx(ctx context.Context, tags ...string) error // RemoveTagsCtx removes tags from stream, stream will be extracted from ctx RemoveTagsCtx(ctx context.Context, tags ...string) error + // RemoveTagsById removes tags from a specific stream by id, for callers that + // track streamId out of band. Missing streams and tags are ignored. + RemoveTagsById(streamId uint32, tags ...string) error // Streams gets all streams for specific tags Streams(tags ...string) (streams []drpc.Stream) } @@ -78,6 +123,7 @@ type streamPool struct { opening map[string]*openingProcess streamConfig StreamConfig dial *ExecPool + closeHook func(streamId uint32, peerId string, tags []string) mu sync.Mutex writeQueueSize int lastStreamId uint32 @@ -360,8 +406,19 @@ func (s *streamPool) openStream(ctx context.Context, p peer.Peer) *openingProces func (s *streamPool) Broadcast(ctx context.Context, msg drpc.Message, tags ...string) (err error) { s.mu.Lock() var streams []*stream + var seen map[uint32]struct{} + if len(tags) > 1 { + seen = make(map[uint32]struct{}) + } for _, tag := range tags { for _, streamId := range s.streamIdsByTag[tag] { + if seen != nil { + if _, ok := seen[streamId]; ok { + // a stream subscribed to several matching tags gets one copy + continue + } + seen[streamId] = struct{}{} + } streams = append(streams, s.streams[streamId]) } } @@ -428,12 +485,36 @@ func (s *streamPool) RemoveTagsCtx(ctx context.Context, tags ...string) error { return nil } -func (s *streamPool) removeStream(streamId uint32) { +func (s *streamPool) RemoveTagsById(streamId uint32, tags ...string) error { s.mu.Lock() defer s.mu.Unlock() + st, ok := s.streams[streamId] + if !ok { + return nil + } + var filtered = st.tags[:0] + var toRemove = make([]string, 0, len(tags)) + for _, t := range st.tags { + if slices.Contains(tags, t) { + toRemove = append(toRemove, t) + } else { + filtered = append(filtered, t) + } + } + st.tags = filtered + for _, t := range toRemove { + removeStream(s.streamIdsByTag, t, streamId) + } + return nil +} + +func (s *streamPool) removeStream(streamId uint32) { + s.mu.Lock() st := s.streams[streamId] if st == nil { + s.mu.Unlock() log.Fatal("removeStream: stream does not exist", zap.Uint32("streamId", streamId)) + return } removeStream(s.streamIdsByPeer, st.peerId, streamId) @@ -442,7 +523,15 @@ func (s *streamPool) removeStream(streamId uint32) { } delete(s.streams, streamId) - st.l.Debug("stream removed", zap.Strings("tags", st.tags)) + var closedTags []string + if s.closeHook != nil { + closedTags = slices.Clone(st.tags) + } + s.mu.Unlock() + st.l.Debug("stream removed", zap.Strings("tags", closedTags)) + if s.closeHook != nil { + s.closeHook(streamId, st.peerId, closedTags) + } } func (s *streamPool) Close(ctx context.Context) (err error) { From f2bca5695b29a80fa8c8d08c4048ae6e34426881 Mon Sep 17 00:00:00 2001 From: Roman Khafizianov Date: Mon, 6 Jul 2026 19:35:38 +0200 Subject: [PATCH 2/5] feat(pubsub): add RevalidateMembers for node ACL-change eviction Add Service.RevalidateMembers(spaceId, isMember) so a relay node can evict every subscriber whose account no longer passes the membership predicate in a single pass on an ACL change, refactoring EvictMember to share the core eviction path. --- commonspace/pubsub/reconnect_test.go | 34 ++++++++++++++++++++++++++++ commonspace/pubsub/service.go | 28 +++++++++++++++++++---- 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/commonspace/pubsub/reconnect_test.go b/commonspace/pubsub/reconnect_test.go index d2f351bb6..9465850ea 100644 --- a/commonspace/pubsub/reconnect_test.go +++ b/commonspace/pubsub/reconnect_test.go @@ -177,6 +177,40 @@ func TestEvictMemberStopsDelivery(t *testing.T) { expectSilence(t, n.clientA, 400*time.Millisecond) } +// TestRevalidateMembersEvictsNonMembers verifies the node-side ACL-change hook: +// RevalidateMembers evicts subscribers whose account no longer passes the member +// predicate while keeping current members subscribed. +func TestRevalidateMembersEvictsNonMembers(t *testing.T) { + n := newNetFx(t) + defer n.finish() + + _, err := n.clientA.svc.Subscribe(testSpace, "chat/>", n.clientA.handler()) + require.NoError(t, err) + _, err = n.clientC.svc.Subscribe(testSpace, "chat/>", n.clientC.handler()) + require.NoError(t, err) + waitMatchCount(t, n.nodeB, testSpace, "chat/x", 1) + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + n.nodeB.svc.remoteMu.Lock() + nstreams := len(n.nodeB.svc.streams) + n.nodeB.svc.remoteMu.Unlock() + if nstreams == 2 { + break + } + time.Sleep(10 * time.Millisecond) + } + + // only clientC remains a member; clientA is evicted in one pass + keep := n.clientC.identity().Account() + n.nodeB.svc.RevalidateMembers(testSpace, func(account string) bool { + return account == keep + }) + + require.NoError(t, n.clientC.svc.Publish(testCtx, testSpace, "chat/x", []byte("survivors"))) + require.Equal(t, "survivors", waitReceived(t, n.clientC).payload) + expectSilence(t, n.clientA, 400*time.Millisecond) +} + // TestCloseSpaceDropsInterest verifies CloseSpace tears down both local and // serving-side interest so a global service retains nothing for a closed space. func TestCloseSpaceDropsInterest(t *testing.T) { diff --git a/commonspace/pubsub/service.go b/commonspace/pubsub/service.go index 6a1489274..848b1fe74 100644 --- a/commonspace/pubsub/service.go +++ b/commonspace/pubsub/service.go @@ -51,6 +51,10 @@ type Service interface { // enforcing DESIGN §6.4 (active drop on ACL removal). Nodes wire it to their // ACL-update hook. No-op on clients (they hold no serving interest). EvictMember(spaceId string, identity crypto.PubKey) + // RevalidateMembers evicts every subscriber of the space whose account no + // longer passes isMember. Nodes call it on an ACL change to cut off all + // removed members in one pass. No-op on clients. + RevalidateMembers(spaceId string, isMember func(account string) bool) // HandleStream serves an inbound PubSubStream; blocks for the stream lifetime. HandleStream(stream drpc.Stream) error } @@ -374,14 +378,29 @@ func (s *service) CloseSpace(spaceId string) { func (s *service) EvictMember(spaceId string, identity crypto.PubKey) { account := identity.Account() + s.evictSpaceStreams(spaceId, func(strm *streamInterest) bool { + return strm.account == account + }) +} + +// RevalidateMembers evicts every subscriber of the space whose account no longer +// passes isMember. A node calls it on an ACL change to actively cut off removed +// members (DESIGN §6.4) rather than waiting for their streams to close. +func (s *service) RevalidateMembers(spaceId string, isMember func(account string) bool) { + s.evictSpaceStreams(spaceId, func(strm *streamInterest) bool { + return !isMember(strm.account) + }) +} + +// evictSpaceStreams drops the space interest of every stream matching evict, +// stripping its routing tags so delivery stops even while the stream stays open. +func (s *service) evictSpaceStreams(spaceId string, evict func(*streamInterest) bool) { s.remoteMu.Lock() + defer s.remoteMu.Unlock() si := s.remote[spaceId] for streamId, strm := range s.streams { - if strm.account != account { - continue - } spacePatterns := strm.bySpace[spaceId] - if len(spacePatterns) == 0 { + if len(spacePatterns) == 0 || !evict(strm) { continue } tags := make([]string, 0, len(spacePatterns)) @@ -401,7 +420,6 @@ func (s *service) EvictMember(spaceId string, identity crypto.PubKey) { if si != nil { s.pruneSpace(spaceId, si) } - s.remoteMu.Unlock() } func (s *service) sendInterest(spaceId string, patterns []string, subscribe bool) { From f05d5319cff2704de2097c7b63ac9174d590a323 Mon Sep 17 00:00:00 2001 From: Roman Khafizianov Date: Tue, 7 Jul 2026 08:57:52 +0200 Subject: [PATCH 3/5] fix(pubsub): address round-2 review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - WithMetric no longer registers the private pool into the shared single-slot sync metric (RegisterStreamPoolSyncMetric): that slot belongs to the app-level sync streampool, so a second pool there overwrote its OutgoingMsg telemetry and nulled it on Close. WithMetric now only registers the namespaced prometheus gauges — independent observability, no clobber. (F1, medium) - handleSubscribe cap rejection reports only the rejected patterns in the TooManyTopics status, not the whole subscribe batch. (F4) - removeStream always clones tags for its debug log. (F5) - drop the dead streamInterest.peerId field. (F6) - doc: dedup is a FIFO ring (not LRU); rate limiter is per-peer; note that Close blocks on in-flight handlers and that resync coverage is bounded by DialQueueSize. Adds TestCapExceededReportsOnlyRejected. The remaining review items were verified correct (lock ordering, interest+tag rollback, receive-path membership-before-verify, no double-decrement) — no change needed. --- commonspace/pubsub/reconnect_test.go | 35 +++++++++++++++++++++++++ commonspace/pubsub/service.go | 15 +++++------ docs/stateless-pubsub/DESIGN.md | 38 ++++++++++++++++++---------- net/streampool/streampool.go | 20 ++++++++------- 4 files changed, 78 insertions(+), 30 deletions(-) diff --git a/commonspace/pubsub/reconnect_test.go b/commonspace/pubsub/reconnect_test.go index 9465850ea..c29674a57 100644 --- a/commonspace/pubsub/reconnect_test.go +++ b/commonspace/pubsub/reconnect_test.go @@ -117,6 +117,41 @@ func TestStreamCloseDrainsInterest(t *testing.T) { require.Empty(t, node.svc.remote) } +// TestCapExceededReportsOnlyRejected verifies that when a subscribe exceeds the +// per-space pattern cap mid-batch, the TooManyTopics status echoes only the +// rejected patterns, and the ones accepted before the cap stay live. +func TestCapExceededReportsOnlyRejected(t *testing.T) { + membership := &fakeMembership{} + relay := &fakeRelay{} + node := newEngineFx(t, "node", membership, relay) + defer node.finish() + // tiny per-space cap so a 3-topic subscribe trips it after 2 + node.svc.cfg.MaxPatternsPerSpace = 2 + client := newEngineFx(t, "client", membership, nil) + defer client.finish() + membership.allow(client.identity()) + + s := rawStream(t, client, node) + require.NoError(t, s.Send(&pubsubproto.PubSubMessage{Content: &pubsubproto.PubSubMessage_Subscribe{ + Subscribe: &pubsubproto.Subscribe{SpaceId: testSpace, Topics: []string{"a", "b", "c"}}, + }})) + + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + msg, err := s.Recv() + require.NoError(t, err) + if st := msg.GetStatus(); st != nil { + require.Equal(t, pubsubproto.ErrCodes_TooManyTopics, st.Code) + require.Equal(t, []string{"c"}, st.Topics, "only the rejected topic is reported") + // the two accepted before the cap are live + require.Equal(t, 1, remoteMatchCount(node, testSpace, "a")) + require.Equal(t, 1, remoteMatchCount(node, testSpace, "b")) + return + } + } + t.Fatal("no TooManyTopics status received") +} + // TestInvalidSpaceIdRejected ensures a spaceId containing '/' (which would break // tag parsing) is rejected at subscribe with InvalidTopic. func TestInvalidSpaceIdRejected(t *testing.T) { diff --git a/commonspace/pubsub/service.go b/commonspace/pubsub/service.go index 848b1fe74..822a91dc3 100644 --- a/commonspace/pubsub/service.go +++ b/commonspace/pubsub/service.go @@ -103,7 +103,6 @@ type spaceInterest struct { // so a stream close can withdraw exactly its own contribution regardless of what // tags the pool recorded. type streamInterest struct { - peerId string account string // subscriber account id, for member eviction bySpace map[string]map[string]struct{} // spaceId -> patterns total int // total patterns across spaces on this stream @@ -545,7 +544,7 @@ func (s *service) handleSubscribe(ctx context.Context, peerId string, sub *pubsu } strm := s.streams[streamId] if strm == nil { - strm = &streamInterest{peerId: peerId, account: identity.Account(), bySpace: make(map[string]map[string]struct{})} + strm = &streamInterest{account: identity.Account(), bySpace: make(map[string]map[string]struct{})} s.streams[streamId] = strm } spacePatterns := strm.bySpace[sub.SpaceId] @@ -553,14 +552,14 @@ func (s *service) handleSubscribe(ctx context.Context, peerId string, sub *pubsu spacePatterns = make(map[string]struct{}) strm.bySpace[sub.SpaceId] = spacePatterns } - var accepted []string - var capExceeded bool - for _, pattern := range sub.Topics { + var accepted, rejected []string + for i, pattern := range sub.Topics { if _, exists := spacePatterns[pattern]; exists { continue } if len(spacePatterns) >= s.cfg.MaxPatternsPerSpace || strm.total >= s.cfg.MaxPatternsPerStream { - capExceeded = true + // cap hit: this pattern and every remaining one are rejected + rejected = sub.Topics[i:] break } spacePatterns[pattern] = struct{}{} @@ -585,8 +584,8 @@ func (s *service) handleSubscribe(ctx context.Context, peerId string, sub *pubsu } s.remoteMu.Unlock() - if capExceeded { - s.sendStatus(ctx, peerId, sub.SpaceId, sub.Topics, pubsubproto.ErrCodes_TooManyTopics) + if len(rejected) > 0 { + s.sendStatus(ctx, peerId, sub.SpaceId, rejected, pubsubproto.ErrCodes_TooManyTopics) } } diff --git a/docs/stateless-pubsub/DESIGN.md b/docs/stateless-pubsub/DESIGN.md index 9ccbc0476..b2f2a4bde 100644 --- a/docs/stateless-pubsub/DESIGN.md +++ b/docs/stateless-pubsub/DESIGN.md @@ -32,7 +32,7 @@ that dies with the connection. | Payload confidentiality | Encrypted client-side with current space ReadKey + `keyId` indirection (push-server pattern). Removed member loses access at next key rotation. | | Authenticity | Per-message account-key signature, verified by receivers. Protects against a semi-trusted node forging/reattributing messages. | | Topology | Publisher → its responsible node (+ direct LAN peers). Node relays once to the other responsible nodes (`relayed` flag, never re-forwarded → loop-free). | -| Duplicate suppression | Receiver-side bounded LRU keyed by `msgId` (duplicate paths exist by design: LAN + node). | +| Duplicate suppression | Receiver-side bounded FIFO ring keyed by `msgId` (duplicate paths exist by design: LAN + node). | | Backpressure | Bounded queues, `TryAdd` drop-on-overflow end to end — the existing streampool discipline. Slow subscriber ⇒ dropped messages, never memory growth. | | Queue groups (1-of-N) | Non-goal v1. | | Catch-up / replay / retained messages | Non-goal, by definition of stateless. Reconnect ⇒ resubscribe ⇒ only new messages. | @@ -45,7 +45,7 @@ that dies with the connection. transport or TLS handshake (`net/peer/peer.go:134`, `net/transport/transport.go:40-64`). - **R2 (memory-effective):** per-stream bounded `mb.MB` queues with drop-on-overflow (`net/streampool/stream.go:32-44`), interest state = stream tags + pattern trie, both - O(active subscriptions), fixed-size dedup LRU, per-peer publish token bucket, caps on + O(active subscriptions), fixed-size dedup FIFO ring, per-peer publish token bucket, caps on pattern count and topic/payload size. No path grows with message volume or offline duration. - **R3 (ACL-aware):** node gates subscribe *and* publish on space membership at the current @@ -59,7 +59,7 @@ that dies with the connection. - **At-most-once.** A publish reaching the relay is copied into bounded per-subscriber queues; overflow drops. No acks, no retransmit, no ordering across publishers, no dedup - beyond the duplicate-path LRU. + beyond the duplicate-path ring. - **Fire-and-forget.** A publish with no subscribers is discarded. Nothing is stored. - **Decoupled.** Publishers don't know subscribers. Subscriber set is whoever holds a live tagged stream at the instant of fan-out. @@ -78,9 +78,9 @@ that dies with the connection. a hard guarantee). Net effect = NATS `no_echo` semantics without a wire flag. (NATS echoes by default with per-connection opt-out; we don't need the option because dedup already exists.) -- **Delivery is best-effort, duplicates possible in theory** (LRU eviction under extreme +- **Delivery is best-effort, duplicates possible in theory** (ring eviction under extreme rates), so payload design must be idempotent/last-write-wins at the app level. In - practice the LRU makes duplicates vanishingly rare. + practice the ring makes duplicates vanishingly rare. --- @@ -221,7 +221,7 @@ Relay rules (complete): same way nodes do (they hold the ACL). Duplicate paths are expected (a subscriber may get the same message from a LAN peer and -from its node). The **receiver** suppresses via a fixed-size LRU keyed by `msgId` +from its node). The **receiver** suppresses via a fixed-size FIFO ring keyed by `msgId` (default 4096 entries ≈ 64 KiB of ids). Publishers self-suppress echoes by msgId too (§2, Echo). @@ -302,7 +302,7 @@ fan out locally only (same trie match). The `identity == ctxIdentity` rule does (the forwarding node is not the author) — authenticity is the receiver's signature check (§6.3). -**Receive** (client) — dedup by msgId LRU → verify signature against `identity` → check +**Receive** (client) — dedup by msgId ring → verify signature against `identity` → check `identity` is a member at the local ACL head → if the topic is in the `acc/` namespace, check its last segment equals `identity.Account()` (end-to-end enforcement: the signature covers the topic, so not even a malicious relay can inject into someone else's self-owned @@ -391,9 +391,9 @@ payload, signature still required. | Outbound per-stream buffer | `queueSize` msgs (default 100) | `mb.MB` + `TryAdd` drop (`net/streampool/stream.go:39`) | | Interest table | ≤1000 patterns/stream, ≤100/space/stream | reject with `TooManyTopics`; state dies with stream | | Pattern trie (§4.1) | O(total live patterns × segments), segments ≤16 | same caps as interest table; lazily pruned when a pattern's tag has no live streams | -| Publish rate | token bucket per peer per stream (default 30 msg/s, burst 60) | checked in pubsub handler — **inside** the stream, because the RPC limiter only gates stream-open (`net/rpc/limiter/limiter.go:96-106`). Deliberate divergence from core NATS, which has no publish rate limiting (maintainers punt to network throughput) and instead disconnects slow consumers — acceptable for a trusted-client broker, not for our semi-trusted multi-tenant relays. We also drop rather than disconnect slow subscribers, which fits at-most-once ephemera | +| Publish rate | token bucket **per peer** (default 30 msg/s, burst 60) | checked in pubsub handler — **inside** the stream, because the RPC limiter only gates stream-open (`net/rpc/limiter/limiter.go:96-106`). One bucket per peer (all of a peer's streams share it — stricter than per-stream, so a peer can't multiply its budget by opening streams). Deliberate divergence from core NATS, which has no publish rate limiting (maintainers punt to network throughput) and instead disconnects slow consumers — acceptable for a trusted-client broker, not for our semi-trusted multi-tenant relays. We also drop rather than disconnect slow subscribers, which fits at-most-once ephemera | | Payload size | ≤64 KiB | reject `InvalidMessage` | -| Dedup cache | fixed LRU, 4096 msgIds | evicts oldest, O(1) | +| Dedup cache | fixed FIFO ring, 4096 msgIds | evicts oldest, O(1) | | Fan-out amplification | 1 upload → ≤2 node-node copies → N subscriber queues | node-side copy (publisher uploads once); `peerMessage.Copy()` pattern reuses the existing per-destination stamping (`stream.go:32-37`) | | Idle streams | closed with the sub-connection; tags GC'd in `removeStream` (`streampool.go:431-446`) | existing | | Zombie subscribers | stream in continuous queue-overflow for > 30 s is closed | NATS disconnects slow consumers outright to protect the system; we drop first (fits at-most-once), but a *persistently* full queue means a dead/wedged reader burning fan-out work — shed it and let the client reconnect fresh | @@ -422,7 +422,7 @@ structs (mirror `objectmessages` `sync.Pool` usage, `headupdate.go:13-38`). hook → per-space pattern trie (§4.1) + tags; - publish path: validate → trie match + cross-pattern stream dedup → broadcast → optional forward hook; - - receive path: dedup LRU → verify → decrypt → handler dispatch; + - receive path: dedup ring → verify → decrypt → handler dispatch; - pluggable interfaces so layering stays clean: `MembershipChecker` (backed by `AclState`), `Crypto` (ReadKey encrypt/decrypt via the space's `Acl()`), `Forwarder` (node-only), `RateLimiter`. @@ -475,7 +475,7 @@ structs (mirror `objectmessages` `sync.Pool` usage, `headupdate.go:13-38`). | max topics per stream | 1000 (100 per space) | | publish rate per peer | 30 msg/s, burst 60 | | per-stream write queue | 100 (client), 500 (node outbound) — match sync-side sizes | -| dedup LRU | 4096 msgIds | +| dedup ring | 4096 msgIds | | received-timestamp staleness window | 5 min (enforced at receive, `Config.MaxTimestampSkew`) | | client interest resync interval | 20 s (`Config.ResyncInterval`) | | pubsub stream peer TTL | 1 h (`Config.PeerTTL`) | @@ -514,8 +514,11 @@ switch-to-interest-only threshold — not full RS+/RS- interest replication (v2, after the first notice (flood of statuses is itself amplification — leaning: notify once per window). 3. Metrics surface (per-topic counters are unbounded-cardinality; per-space is safe). - The private pool is now observable via `Deps.Metric` (`WithMetric(m, "pubsub")`); the - remaining work is the pubsub-specific counters below. + The private pool is observable via `Deps.Metric` (`WithMetric(m, "pubsub")`), which + registers namespaced prometheus gauges (stream/tag/dial counts). It deliberately does + **not** register into the shared single-slot sync-metric (`RegisterStreamPoolSyncMetric`) + — that belongs to the app-level sync streampool, and a second pool there would overwrite + and, on Close, null it. The remaining work is the pubsub-specific counters below. ## 12. Implementation status (v1 in any-sync) @@ -558,6 +561,15 @@ Deferred (documented, not silent): overflow episode, close-reason strings) — §11.3. - **`MembershipChecker` returning a permission level** rather than a bare member/not — a one-way door kept simple for v1 (current-head `!NoPermissions()`). +- **Close blocks on in-flight dispatch handlers.** `Close` waits for the dispatch loop to + drain; a handler that violates the "must not block" contract wedges shutdown (no timeout). + Acceptable given the contract; a stuck app handler is an app bug, and adding a Close + timeout would mask it. +- **Resync coverage for many-space clients.** Each resync pass fires one dial task per local + space through the bounded dial queue (`DialQueueSize`, default 100); a client with more + spaces than that drops the overflow for that tick and restores their interest on a later + tick (self-correcting, no leak — re-sends are idempotent). Heart should size + `DialQueueSize` to its expected open-space count. Downstream wiring (separate repos, per §8.2/§8.3): any-sync-node `pubsubrelay` component (`Relay`/`Membership` from nodeconf + hosted ACL, `RegisterRpc`, `EvictMember` on ACL diff --git a/net/streampool/streampool.go b/net/streampool/streampool.go index 64622538e..fff35bb88 100644 --- a/net/streampool/streampool.go +++ b/net/streampool/streampool.go @@ -46,16 +46,21 @@ func WithStreamCloseHook(hook func(streamId uint32, peerId string, tags []string } } -// WithMetric registers the standalone pool's prometheus metrics under the given -// prefix, so a service that owns a private pool (e.g. pubsub) is observable even -// though it never runs the app-component Init. +// WithMetric registers the standalone pool's prometheus gauges (stream_count, +// tag_count, dial_queue) under the given namespace prefix, so a service that owns +// a private pool (e.g. pubsub) is observable even though it never runs the +// app-component Init. +// +// It deliberately does NOT call RegisterStreamPoolSyncMetric: that feeds a single +// shared slot on the metric component meant for the one app-level sync streampool, +// and registering a second pool there would overwrite (and, on Close, null) the +// sync pool's OutgoingMsg telemetry. The namespaced gauges below give the private +// pool independent observability without touching that slot. func WithMetric(m metric.Metric, prefix string) Option { return func(s *streamPool) { if m == nil { return } - s.metric = m - m.RegisterStreamPoolSyncMetric(s) registerMetrics(m.Registry(), s, prefix) } } @@ -523,10 +528,7 @@ func (s *streamPool) removeStream(streamId uint32) { } delete(s.streams, streamId) - var closedTags []string - if s.closeHook != nil { - closedTags = slices.Clone(st.tags) - } + closedTags := slices.Clone(st.tags) s.mu.Unlock() st.l.Debug("stream removed", zap.Strings("tags", closedTags)) if s.closeHook != nil { From 5f4384519df623907e3be287f7127878647eb938 Mon Sep 17 00:00:00 2001 From: Roman Khafizianov Date: Wed, 15 Jul 2026 20:41:11 +0200 Subject: [PATCH 4/5] docs(pubsub): consolidate stateless-pubsub design+research into one reference Replace docs/stateless-pubsub/{DESIGN,RESEARCH}.md with a single as-built docs/stateless-pubsub.md: present-tense reference for the shipped v1 plus a compact background/rationale distilled from the research. Corrects drift against the code (SyncInterest/RevalidateMembers, NewStreamPool, Unexpected=0 zero value, receive-path ordering) and drops the now-resolved open items. --- docs/stateless-pubsub.md | 750 ++++++++++++++++++++++++++++++ docs/stateless-pubsub/DESIGN.md | 647 -------------------------- docs/stateless-pubsub/RESEARCH.md | 250 ---------- 3 files changed, 750 insertions(+), 897 deletions(-) create mode 100644 docs/stateless-pubsub.md delete mode 100644 docs/stateless-pubsub/DESIGN.md delete mode 100644 docs/stateless-pubsub/RESEARCH.md diff --git a/docs/stateless-pubsub.md b/docs/stateless-pubsub.md new file mode 100644 index 000000000..ab10564b9 --- /dev/null +++ b/docs/stateless-pubsub.md @@ -0,0 +1,750 @@ +# Space-Scoped Stateless Pub/Sub over any-sync + +Status: **IMPLEMENTED (v1)** — shipped in this repo as `commonspace/pubsub` (engine) plus +additions to `net/streampool`. This document is the as-built reference: it describes what +the code does, why the design is shaped this way, and what is deliberately left for +downstream repos or a later version. Grounded against the current tree +(`commonspace/pubsub/`, `net/streampool/`). + +Downstream wiring — the any-sync-node relay component and the anytype-heart client — lives +in separate repos and is tracked in [§13](#13-implementation-status). + +--- + +## 1. Summary + +An ephemeral, fire-and-forget, at-most-once publish/subscribe channel scoped to a space, +carried over a **dedicated DRPC bidi stream** fully isolated from the sync engine. Topics +are plaintext `/`-separated hierarchies inside a space; subscriptions may use NATS-style +wildcards (`*` one segment, `>` trailing segments). Any space member (Reader/Guest +included) may publish and subscribe. Payloads are end-to-end encrypted with the space +ReadKey and signed with the sender's account key; relay nodes route ciphertext they cannot +read. Fan-out goes through the responsible sync nodes **and** directly to LAN-discovered +peers, with a small bounded msgId dedup cache on receivers. No message is ever persisted; +the only routing state is transient in-memory interest state (stream tags + a pattern trie) +that dies with the connection. + +The engine (`commonspace/pubsub.Service`, `CName = "common.commonspace.pubsub"`) is a single +component used by **both** sides: nodes wire it with a `Relay` (relay role), clients wire it +with a `PeerProvider` (client role). One long-lived `PubSubStream` per peer pair multiplexes +all spaces and topics. + +### Design decisions + +| Question | Decision | +|---|---| +| Meaning of "stateless" | No message persistence anywhere. Transient in-memory interest tables (stream tags + trie) are allowed and used. | +| Stream reuse vs dedicated | Dedicated `PubSub` DRPC service + its own streampool instance; never touches `ObjectSyncStream` or the sync dispatch path. | +| New service vs new RPC on SpaceSync | New service, own proto package — independent versioning, unknown-service fallback for old peers. | +| Topic model | `/`-separated segment hierarchy within a space. Subscriptions may use NATS-style wildcards: `*` matches exactly one segment, `>` matches one-or-more trailing segments (tail-only). Publishers must use fully-qualified topics. Matching runs on a bounded per-space pattern trie at the relay ([§5.1](#51-interest-matching-wildcards)). | +| Topic privacy | Plaintext to relays. Comparable exposure to today's plaintext `objectId`s on the sync path. | +| Subscribe permission | Any space member: `!NoPermissions()` (`commonspace/object/acl/list/models.go:90`). | +| Publish permission | Any space member. Attribution via per-message account-key signature; abuse contained by per-peer rate limits. Plus a reserved **self-owned namespace**: topics `acc/…/` accept publishes only from `accId` ([§7.2](#72-access-control)). | +| Payload confidentiality | Encrypted client-side with the current space ReadKey + `keyId` indirection. Removed member loses access at next key rotation. | +| Authenticity | Per-message account-key signature, verified by receivers. Protects against a semi-trusted node forging or reattributing messages. | +| Topology | Publisher → its responsible node (+ direct LAN peers). Node relays once to the other responsible nodes (`relayed` flag, never re-forwarded → loop-free). | +| Duplicate suppression | Receiver-side bounded FIFO ring keyed by `msgId` (duplicate paths exist by design: LAN + node). | +| Backpressure | Bounded queues, `TryAdd` drop-on-overflow end to end — the existing streampool discipline. Slow subscriber ⇒ dropped messages, never memory growth. | +| Queue groups (1-of-N) | Non-goal v1. | +| Catch-up / replay / retained messages | Non-goal, by definition of stateless. Reconnect ⇒ resubscribe ⇒ only new messages. | +| Presence lifecycle | Not in the protocol. v1 is a generic app API; presence (heartbeat/timeout/leave, Yjs-awareness style) is an app pattern on top ([Appendix A](#appendix-a--presence-as-an-app-pattern-non-normative)). | + +### Requirements compliance + +The feature was specified against three hard requirements from the original ask ("users of +the same space can subscribe/publish topics within a space; work effectively over the +any-sync protocol; be memory-effective; respect ACL access"): + +- **R1 (on-protocol):** rides existing transports, the secureservice handshake, the DRPC + mux, and a second streampool instance. A dedicated sub-stream on the existing `MultiConn` + — no new transport or TLS handshake. +- **R2 (memory-effective):** per-stream bounded `mb.MB` queues with drop-on-overflow + (`net/streampool/stream.go:32-44`), interest state = stream tags + pattern trie (both + O(active subscriptions)), a fixed-size dedup FIFO ring, a per-peer publish token bucket, + and caps on pattern count and topic/payload size. No path grows with message volume or + offline duration. +- **R3 (ACL-aware):** the serving peer gates subscribe *and* publish on space membership at + the current ACL head (new, additive enforcement — the sync path has no such check at + subscribe time); confidentiality holds against the relay via ReadKey encryption; membership + removal cuts new traffic at key rotation and actively drops subscriptions ([§7.4](#74-confidentiality--membership-change)). + +--- + +## 2. Background & rationale + +This section distills the research that shaped the design, for readers who want the *why* +without the original problem-framing document. + +### 2.1 Where this sits in the pub/sub landscape + +"Stateless pub/sub" here means the **core-NATS / Redis-pub-sub** family: fire-and-forget, +at-most-once, no persistence, full decoupling of publishers and subscribers. The relay may +hold a small *transient* in-memory interest table (as core NATS does) — that is not +"stateful" in the sense that matters; only durable message state is excluded. + +| System | Topic model | Wildcards | Delivery | Persistence | Notable for us | +|---|---|---|---|---|---| +| **NATS core** | dot-hierarchy subjects | `*` (1 token), `>` (tail multi) | fan-out + queue-group (1-of-N) | none (JetStream is separate) | the canonical model; subject wildcards; sublist trie | +| **Redis pub/sub** | flat channels + patterns | `*`, `?`, `[..]` glob | fan-out | none | simplest fire-and-forget; subscriber must be connected | +| **MQTT** | `/`-hierarchy topics | `+` (1 level), `#` (tail multi) | QoS 0/1/2 | retained msg + sessions ⇒ *stateful* | wildcard syntax variant; retained message is the anti-pattern to avoid | +| **libp2p GossipSub** | flat topics | none | epidemic mesh fan-out | none | closest to any-sync's decentralization ethos; peer-scoring for abuse | +| **Yjs awareness** | one implicit channel/doc | n/a | state-based CRDT diff | ephemeral, auto-GC | the presence reference: `(clientID, clock, state\|null)`, re-announce/timeout | +| **Phoenix Channels** | `topic:subtopic` strings | none | fan-out; Presence on top | ephemeral | presence = minimal ephemeral metadata over pub/sub | +| **Matrix EDUs** | per-room | n/a | fan-out to room servers | ephemeral | typing/presence as Ephemeral Data Units, distinct from the persistent event DAG | + +Three cross-cutting lessons drove the design: + +- **Topology is the fork.** Central-broker systems keep an interest table; GossipSub is a + brokerless mesh. any-sync sits between: clients reach a small set of **responsible sync + nodes** (semi-trusted relays that store ciphertext they cannot read) and *may* also reach + other clients directly (LAN). The relay-through-node model is broker-like, but the broker + is **untrusted for content** — which forces the encryption + signing model in [§7](#7-security-model). +- **Presence is the killer use case**, and it is *always* kept separate from the persistent + document layer (Yjs awareness, Phoenix Presence, Matrix EDUs). v1 keeps presence out of + the protocol and provides it as an app recipe ([Appendix A](#appendix-a--presence-as-an-app-pattern-non-normative)). +- **Wildcards cost the relay** a per-message trie walk; flat topics do not. We accept that + cost for expressiveness, bounded by per-space pattern caps ([§5.1](#51-interest-matching-wildcards)). + +### 2.2 The any-sync substrate we build on + +The load-bearing facts about this codebase that the design leans on: + +- **StreamPool is the fan-out core.** `net/streampool` caches `drpc.Stream`s, indexes them + by peer and by **tag**, opens them lazily, and pushes messages onto **bounded** per-stream + write queues (`mb.MB`, `TryAdd`, drop-on-overflow — `net/streampool/stream.go:32-44`). + `Broadcast(msg, tags...)` is the existing tag-keyed fan-out primitive; `AddTagsCtx` / + `RemoveTagsCtx` are the existing dynamic (un)subscribe hooks. Pub/sub instantiates its own + pool rather than layering onto the sync pool. +- **Responsible nodes via consistent hash.** `NodeIds(spaceId)` returns the tree nodes on + the chash ring for a space; `ReplicationFactor = 3` ⇒ up to 3 responsible sync nodes per + space. These are the always-reachable fan-out hubs. +- **ACL is cryptographic, not a per-message live check.** Writes are enforced when a signed + record is validated on apply; reads are enforced by **encryption** — content is encrypted + with a per-space `ReadKey` handed only to members, and the key rotates on membership + change. The permission ladder is `None=0, Owner=1, Admin=2, Writer=3, Reader=4, Guest=5`; + there is no `CanRead()` — "can read" is `!NoPermissions()`. The shared sync layer performs + **no** per-message reader-authorization, so pub/sub's ACL gating at subscribe/publish time + is new, additive enforcement. +- **Don't reuse `ObjectSyncStream`.** Every frame on it is routed into the sync engine and + shares one bounded queue; multiplexing ephemeral pub/sub there would couple the two + (head-of-line blocking, intertwined backpressure) and muddle fire-and-forget ephemera with + DAG anti-entropy. A separate multiplexed DRPC stream is cheap — another sub-stream on the + existing `MultiConn`, not a new handshake — so pub/sub gets its own stream, tags, and + queues, fully isolated from sync flow control. + +### 2.3 Why the big forks landed where they did + +- **Dedicated service, not a new SpaceSync RPC** — independent lifecycle and versioning; + old peers that don't know `PubSub` fail the stream open cleanly and the client treats the + peer as pubsub-unavailable. +- **Relay-through-node, not pure mesh** — nodes are the only always-reachable members; mesh + fits the ethos but has no guaranteed connectivity. LAN peers are used *in addition*, not + instead. +- **Per-message signing on top of encryption** — encryption alone gives confidentiality but + not authenticity: a Reader (or the relay) could spoof or reattribute a message that other + members can still decrypt. Ed25519 sign/verify (~30–80 µs) is negligible at + ephemeral-signal rates and buys spoof-proof attribution, which the self-owned `acc/` + namespace then builds on. +- **No queue groups, no interest propagation between nodes in v1** — see [§14](#14-non-goals-v1). + +--- + +## 3. Semantics contract + +- **At-most-once.** A publish reaching the relay is copied into bounded per-subscriber + queues; overflow drops. No acks, no retransmit, no ordering across publishers, no dedup + beyond the duplicate-path ring. +- **Fire-and-forget.** A publish with no subscribers is discarded. Nothing is stored. +- **Decoupled.** Publishers don't know subscribers. The subscriber set is whoever holds a + live tagged stream at the instant of fan-out. +- **Subscribe is unacknowledged.** Success is silent ([§6](#6-what-happens-on-each-frame-serving-peer)); interest takes effect when the serving + peer processes the frame, so messages published concurrently by others may be missed. + There is no "subscribed as of time T" guarantee — the same race NATS documents for + cluster-wide subscription visibility, inherent to at-most-once. Apps needing a consistent + starting state combine pub/sub with a snapshot read (e.g. presence: subscribe, then + announce yourself, which prompts others' next heartbeat). +- **Reconnect = clean slate.** Subscriptions die with the stream; the client re-subscribes on + reconnect (via the resync loop, [§9](#9-lifecycle--reconnect)) and sees only new traffic. +- **Echo.** A publisher whose own interest set matches the topic would receive its own + message back from the relay. The client suppresses this by pre-recording its own `msgId` in + the dedup ring at publish time and delivering to local handlers directly (bounded, + drop-on-overflow dispatch queue — so local delivery is best-effort too). Net effect = NATS + `no_echo` semantics without a wire flag. +- **Duplicates possible in theory** (ring eviction under extreme rates), so payload design + must be idempotent / last-write-wins at the app level. In practice the ring makes + duplicates vanishingly rare. + +--- + +## 4. Wire protocol + +Proto package `commonspace/pubsub/pubsubproto/protos/pubsub.proto` (generated via the +Makefile pipeline): + +```proto +syntax = "proto3"; +package pubsub; + +service PubSub { + // One long-lived bidi stream per peer pair, multiplexing all spaces/topics. + rpc PubSubStream(stream PubSubMessage) returns (stream PubSubMessage); +} + +message PubSubMessage { + oneof content { + Subscribe subscribe = 1; + Unsubscribe unsubscribe = 2; + Publish publish = 3; + Status status = 4; + } +} + +// Delta semantics: adds topic patterns to the stream's interest set for spaceId. +// Patterns may contain wildcards: '*' (one segment), '>' (trailing segments, tail-only). +message Subscribe { + string spaceId = 1; + repeated string topics = 2; +} + +// Removes patterns (matched verbatim against the interest set, not expanded); +// empty topics = remove all patterns of spaceId. +message Unsubscribe { + string spaceId = 1; + repeated string topics = 2; +} + +message Publish { + string spaceId = 1; + string topic = 2; // fully-qualified; wildcards not allowed + bytes msgId = 3; // 16 random bytes, generated by publisher (dedup key) + string keyId = 4; // space ReadKey id used for payload; "" = plaintext (keyless spaces) + bytes payload = 5; // ciphertext (or plaintext iff keyId == "") + bytes identity = 6; // sender account pubkey (marshalled) + bytes signature = 7; // account-key sig, see §7.3 + int64 timestampMilli = 8; // sender wall clock, informational + bool relayed = 9; // set by a node when forwarding node→node; never re-forwarded; excluded from signature +} + +// Sent by the serving peer on a rejected subscribe/publish. Success is silent. +message Status { + string spaceId = 1; + repeated string topics = 2; // echo of the offending request (topic of a publish goes here too) + ErrCodes code = 3; + bytes msgId = 4; // echoes a rejected publish's id for correlation; empty for subscribe rejections +} + +enum ErrCodes { + Unexpected = 0; // zero value: a zeroed Status reads as an error, not success + NotAMember = 1; // identity has no permissions in the space's ACL + NotResponsible = 2; // the serving node is not responsible for the space + RateLimited = 3; + TooManyTopics = 4; + InvalidMessage = 5; // malformed frame, oversized payload, identity mismatch, bad signature + TopicNotOwned = 6; // publish into acc/…/ by an identity other than accId + InvalidTopic = 7; // malformed topic/pattern: bad wildcard placement, reserved chars, non-canonical form + ErrorOffset = 1100; // core bands 100..900 are full; pubsub takes 1100 (see docs/rpc-error-offsets.md) +} +``` + +Constraints enforced by the serving peer (config defaults, [§10](#10-configuration--defaults)). A rejected +Subscribe/Publish gets a `Status` and is otherwise ignored — the stream **stays open** (the +NATS precedent: a limit violation is an error reply, not a disconnect): + +- **topic:** UTF-8, 1..256 bytes, `/`-separated segments (≤16 segments, no empty segments); + recommended segment charset alnum + `.-_`. Canonical form has **no leading `/`** (`acc/x` + and `/acc/x` would silently differ — rejected as `InvalidTopic`). The 256-byte limit is a + compile-time constant (`topic.go`), not a config knob. +- **wildcards (subscription patterns only):** `*` matches exactly one segment + (`chat/*/typing` ⇒ `chat/abc/typing`, not `chat/typing` or `chat/a/b/typing`); `>` matches + one-or-more trailing segments and is valid only as the final segment (`chat/>` ⇒ everything + under `chat/`). Wildcards must be complete segments (`chat/ty*` is invalid). A `Publish` + topic containing `*` or `>` is rejected — publishers always use fully-qualified topics. +- **reserved self-owned namespace:** a topic whose first segment is `acc` is publishable + **only** by the account whose id equals the topic's *last* segment — e.g. + `acc/online/`, `acc/cursor/`. Subscribe stays open to all members (including + patterns like `acc/online/*`). Violations ⇒ `Status{TopicNotOwned}`. +- **payload:** ≤ 64 KiB (enforced after DRPC decode). +- **per-stream interest set:** ≤ 100 patterns per space, ≤ 1000 total. +- **identity in a Publish must equal the connection-context identity** (`net/peer/context.go:88`) + when arriving from a client (`relayed == false`). This binds attribution to the + handshake-proven account without the node verifying the signature per message. + +Interest tag format inside the pool: `spaceId + "/" + pattern`, verbatim (spaceId contains no +`/`; first separator wins). Wildcard resolution happens in the pubsub engine's matcher, not +in the pool ([§5.1](#51-interest-matching-wildcards)). + +--- + +## 5. Topology & relay rules + +``` + publisher client ──publish──▶ responsible node A ──relayed=true──▶ node B ──▶ its subscribers + │ │ node C ──▶ its subscribers + │ └──▶ A's local subscribers (tag fan-out) + └──────direct publish──▶ LAN peers (same space, discovered via mDNS) +``` + +1. **Clients never forward.** A client receiving a `Publish` (from a node or a LAN peer) + delivers it locally only. +2. **A node forwards only client-originated messages** (`relayed == false` arriving on a + client stream): it stamps `relayed = true` and sends one copy to each *other* responsible + node for the space, plus fans out to its local subscribers via the tag index. +3. **A node never forwards `relayed == true`** — it only fans out locally, and only if the + sending peer is itself a responsible node for the space (`Relay.IsResponsibleNode`). With + `ReplicationFactor = 3`, every message traverses at most client → node → node, hop limit + 2, loop-free without any dedup state on nodes. (This is the NATS cluster rule: messages + from a route are distributed only to local clients.) +4. **A node rejects subscribe/publish for spaces it is not responsible for** + (`Status{NotResponsible}`, via `Relay.IsResponsible`). +5. **LAN peers are symmetric.** Both sides run the same pubsub component; a LAN peer's stream + carries Subscribe frames like a node's does, and publishes to LAN peers go direct. Clients + apply the member-check on LAN subscribes the same way nodes do (they hold the ACL). + +Duplicate paths are expected (a subscriber may get the same message from a LAN peer and from +its node). The **receiver** suppresses via a fixed-size FIFO ring keyed by `msgId` (default +4096 entries). Publishers self-suppress echoes by msgId too ([§3](#3-semantics-contract), Echo). + +Client → node selection piggybacks on the host's existing responsible-peer choice (one node +at a time), so the pubsub stream goes to the same node the client already syncs with. + +### 5.1 Interest matching (wildcards) + +The streampool tag index is exact-match (`streamIdsByTag`, `net/streampool/streampool.go`), +so the pubsub engine layers a matcher on top rather than replacing the pool: + +- Each accepted subscription registers its **pattern string verbatim as the stream tag** + (`spaceId + "/" + pattern`) — the pool keeps doing stream bookkeeping (add/remove/GC on + stream close) exactly as today. +- In parallel, the engine maintains a **per-space segment trie** of live patterns, mirroring + the NATS sublist shape (`trie.go`): one level per segment, a `map[segment]` for literals + plus two dedicated wildcard slots per level (`pwc` for `*`, `fwc` for `>`), each terminal + holding a refcount of subscribing streams. Match order follows NATS `matchLevel`: at each + level add `fwc` matches, branch through `pwc`, hash-lookup the literal. +- On publish to concrete topic `T`: walk the trie with `T`'s segments, collecting every + matching pattern (O(segments × matched branches), segments ≤ 16). Then fan out once per + matched pattern tag, **deduplicating stream ids across patterns**: a stream subscribed to + both `chat/>` and `chat/*/typing` receives one copy, not two. `streampool.Broadcast` dedups + stream ids across the tag set it is given (it builds a `seen` set when handed more than one + tag — `net/streampool/streampool.go:411-439`), so the engine passes all matched pattern + tags to a single `Broadcast` call and the pool guarantees one copy per stream. +- Trie cleanup: refcount decrement on Unsubscribe; on stream close the engine reconciles via + the pool's `WithStreamCloseHook` callback (`net/streampool/streampool.go:43`), keyed by + `streamId`, dropping the stream's patterns from the trie. +- Exact-topic subscriptions are just patterns without wildcard segments — one code path. + +The trie is bounded by the same caps as the interest set (≤ 1000 patterns/stream, ≤ 100/space/ +stream), so relay memory stays O(active subscriptions), and matching cost is paid only per +publish within that space. + +**Deliberately no match-result cache.** NATS fronts its sublist with a literal-subject → +result cache, but its own history (`<0.5%` hit rates, lock contention, latency spikes under +sub/unsub churn) led to `NoCache` sublists for exactly our analog: small, churny, +per-connection tries. Our tries are per-space (small) and ephemeral interest is churny, so a +direct walk of a ≤ 16-level trie beats a cache we'd constantly flush. Revisit only with +profiling evidence. + +--- + +## 6. What happens on each frame (serving peer) + +**Subscribe** — validate `spaceId` (non-empty, no `/`); resolve space ACL state; reject with +`Status{NotResponsible}` if the node isn't responsible; check membership +(`MembershipChecker.CheckMember`); validate patterns (`InvalidTopic` on bad wildcard +placement / reserved chars / non-canonical form); check pattern-count caps; then register in +the trie and add the tags to the stream. Interest and tag are committed under one lock with +rollback if the stream vanished mid-subscribe. Reject ⇒ `Status`, no tag. + +**Unsubscribe** — remove tags + trie refcount decrement. No checks needed. + +**Publish (from a client stream, `relayed == false`)** — +1. Size/shape checks (a topic containing `*`/`>` ⇒ `InvalidTopic`); `identity == ctxIdentity` + with a non-empty guard; responsibility check; membership check against cached ACL state; + self-owned-namespace check (topic `acc/…` ⇒ last segment must equal the sender's account — + one string compare); per-peer token bucket ([§8](#8-memory--abuse-bounds)). A rejected publish returns a + `Status` (rate-limited publishes are notified per rejection, not once per window). +2. Match the topic against the space's pattern trie ([§5.1](#51-interest-matching-wildcards)), dedup stream ids across + matched patterns, then `Broadcast` on the pubsub pool — the existing tag-index fan-out, + which per-stream `TryAdd`s and drops on overflow. +3. Stamp `relayed = true` and send to the other responsible nodes (`Relay.OtherResponsiblePeers`, + lazy stream open via the pool). + +**Publish (`relayed == true`, from a node stream)** — verify the sending peer is a +responsible node for the space (`Relay.IsResponsibleNode`); fan out locally only (same trie +match). The `identity == ctxIdentity` rule does not apply (the forwarding node is not the +author); authenticity is the receiver's signature check ([§7.3](#73-authenticity)). + +**Receive (client)** — the order is cheap-filters-first, verify, then dedup: +local-interest match → membership (`identity` is a member at the local ACL head) → if the +topic is in the `acc/` namespace, check its last segment equals `identity.Account()` +(end-to-end: the signature covers the topic, so not even a malicious relay can inject into +someone else's self-owned topic) → timestamp staleness → **Ed25519 signature verify** → +record `msgId` in the dedup ring → resolve ReadKey by `keyId`, decrypt → dispatch to local +topic handlers on the bounded dispatch queue. Any failure ⇒ drop + debug metric, never an +error to the peer. Recording the ring only *after* verify means a forged-signature flood is +shed cheaply and cannot evict legitimate ids to reopen a replay window. + +--- + +## 7. Security model + +### 7.1 Threat model — the semi-trusted relay + +The node **can**: observe spaceIds, topics, sender identities, timing, sizes (accepted — +comparable to sync-path metadata today); drop, delay, reorder messages (accepted — +at-most-once contract). The node **cannot**: read payloads (no ReadKey); forge or reattribute +messages (signature); replay effectively. Replay defense is two-layer: the `msgId` dedup ring +catches the short window, and the receiver enforces a **signed-timestamp staleness window** +(`Config.MaxTimestampSkew`, default 5 min) so that even after the ring evicts an id, a relay +replaying an old signed frame is rejected on its stale timestamp. A relay cannot forge a fresh +timestamp — it is covered by the signature. + +### 7.2 Access control + +Both directions gate on **space membership at the current ACL head**, checked by the serving +peer from its local ACL copy — a cached in-memory lookup, no coordinator round trip. This is +*new* enforcement: today's sync path has none at subscribe time. Publish and subscribe both +require `!NoPermissions()`; there is no `CanWrite` requirement (any member publishes — +presence/typing-style uses need Readers to emit). + +**Self-owned topics.** The `acc/` namespace adds a per-topic ownership rule on top of +membership: only the account named by the topic's last segment may publish there. It is +enforced in **three** places — the publisher's own `Publish` call fails fast before sending; +the serving peer rejects it (cheap, because `identity == ctxIdentity` is bound by the +handshake); and every receiver re-checks it (via the signature, which covers the topic +string). This gives apps spoof-proof per-account channels — e.g. `acc/online/` — where +consumers can trust the topic itself, not just the message attribution. It generalizes the +push-server's "silent self-channel" restriction and matches the proven NATS pattern +(per-identity subject prefixes rather than dynamic per-message grants). Wildcards make the +fan-in side cheap: one `acc/online/*` subscription covers every member's online topic, and +each received message is still individually ownership-checked against its concrete topic and +verified signature. + +### 7.3 Authenticity + +``` +signature = accountKey.Sign( + "anysync:pubsub:v1" | le32‖spaceId | le32‖topic | le32‖msgId | le32‖keyId | + le64(timestampMilli) | payload) +``` + +Each variable-length field is length-prefixed (le32) so field boundaries can't shift (e.g. +`spaceId="ab",topic="c"` vs `spaceId="a",topic="bc"` sign differently); `payload` trails +unprefixed as the final field. `relayed` is excluded (mutated in transit). Receivers verify; +nodes don't need to (attribution from clients is already bound by `identity == ctxIdentity`, +and verifying per-message on the relay buys little at real CPU cost). Ed25519 sign/verify is +~30–80 µs — negligible at ephemeral-signal rates. + +Receivers run the cheap filters (local-interest match, membership, `acc/` ownership, timestamp +staleness) *before* the Ed25519 verify, and record the dedup ring only after verify succeeds +([§6](#6-what-happens-on-each-frame-serving-peer)) — so a relay/LAN-peer flood of forged-signature messages is shed cheaply and +cannot evict legitimate ids from the ring to reopen a replay window ([§7.1](#71-threat-model--the-semi-trusted-relay)). + +### 7.4 Confidentiality & membership change + +Payloads encrypt with the space's current ReadKey, carrying its key id as `keyId`. Receivers +hold historical keys via the ACL, so rotation mid-flight is safe. On member removal the +existing ACL key rotation cuts decryption of new traffic automatically. Additionally the +engine exposes two active-eviction entry points that nodes wire to their ACL-update hook: + +- **`RevalidateMembers(spaceId, isMember func(account string) bool)`** — the ACL-change hook. + On an ACL update the node calls it once; it evicts *every* subscriber of the space whose + account no longer passes `isMember`, in a single pass. +- **`EvictMember(spaceId, identity)`** — the single-identity variant, for when exactly one + account is being removed. + +Both strip the target's per-stream tags (via `streampool.RemoveTagsById`, so delivery — +which is tag-keyed — stops even while the stream stays open) and decrement the match trie. +Bounded work: one scan of the streams subscribed to that space per ACL change. + +Spaces without a ReadKey (keyless/public spaces): `keyId = ""`, plaintext payload, signature +still required. A nil `Crypto` in `Deps` selects this mode. + +--- + +## 8. Memory & abuse bounds + +| Resource | Bound | Mechanism | +|---|---|---| +| Outbound per-stream buffer | `WriteQueueSize` msgs (default 100) | `mb.MB` + `TryAdd` drop (`net/streampool/stream.go:39`) | +| Interest table | ≤ 1000 patterns/stream, ≤ 100/space/stream | reject with `TooManyTopics`; state dies with stream | +| Pattern trie ([§5.1](#51-interest-matching-wildcards)) | O(total live patterns × segments), segments ≤ 16 | same caps as interest table; pruned via stream-close hook | +| Publish rate | token bucket **per peer** (default 30 msg/s, burst 60) | checked in the pubsub handler — *inside* the stream, because the RPC limiter only gates stream-open. One bucket per peer (all of a peer's streams share it), so opening more streams can't multiply the budget. Deliberate divergence from core NATS, which has no publish rate limiting; acceptable for trusted clients but not for semi-trusted multi-tenant relays | +| Local dispatch queue | `DispatchQueueSize` msgs (default 100) | `mb.MB`, drop-on-overflow — local handler delivery is best-effort | +| Payload size | ≤ 64 KiB | reject `InvalidMessage` (after DRPC decode) | +| Dedup cache | fixed FIFO ring, 4096 msgIds | evicts oldest, O(1) | +| Fan-out amplification | 1 upload → ≤ 2 node-node copies → N subscriber queues | node-side copy (publisher uploads once); per-destination stamping reuses the streampool copy path | +| Idle streams | closed with the sub-connection; tags GC'd on stream removal; `PeerTTL` keeps a quiet subscriber's peer from idle-GC reaping | existing pool behavior + `Config.PeerTTL` | + +No persistence, no unbounded map on the message path, no per-message allocation beyond pooled +message structs. + +--- + +## 9. Lifecycle & reconnect + +Because streampool opens streams lazily and never re-sends interest on a fresh stream, a pure +subscriber would go silent after its stream drops (a routine event: idle pool GC reaps quiet +streams). The engine handles this: + +- **`SyncInterest(ctx, spaceId)`** re-sends all local interest for a space to its current + peers. Hosts call it after (re)connecting to a space's peers (e.g. on + `rebuildResponsiblePeers`). +- A **resync loop** (client role only) re-pushes all local interest every + `Config.ResyncInterval` (default 20 s) and immediately on a client-side stream close. + Re-sends are idempotent (the serving side dedups per stream) and bounded by the local + subscription set. Each pass fires one dial task per local space through the bounded dial + queue (`DialQueueSize`); a client with more open spaces than that drops the overflow for a + tick and restores it on a later tick (self-correcting, no leak). +- **`CloseSpace(spaceId)`** drops all local and serving-side interest for a space and + withdraws local interest from its peers. Hosts call it on space unload so a global service + doesn't retain closed-space state. + +Interest on the serving side is keyed by **streamId, not peerId**: two streams from the same +peer are independent, so a reconnecting peer's fresh stream keeps its interest when the stale +stream closes, and a stream that dies mid-subscribe can't orphan interest. + +--- + +## 10. Configuration & defaults + +`pubsub.Config` (zero values take the defaults below, via `Config.withDefaults()`): + +| Knob | Field | Default | +|---|---|---| +| max payload | `MaxPayloadSize` | 64 KiB | +| max patterns per stream | `MaxPatternsPerStream` | 1000 | +| max patterns per space | `MaxPatternsPerSpace` | 100 | +| publish rate per peer | `PublishRps` / `PublishBurst` | 30 msg/s / burst 60 | +| per-stream write queue | `WriteQueueSize` | 100 (both client and node) | +| local dispatch queue | `DispatchQueueSize` | 100 | +| dedup ring | `DedupSize` | 4096 msgIds | +| received-timestamp staleness window | `MaxTimestampSkew` | 5 min | +| client interest resync interval | `ResyncInterval` | 20 s | +| pubsub stream peer TTL | `PeerTTL` | 1 h | +| outbound dial pool | `DialQueueWorkers` / `DialQueueSize` | 4 workers / 100 queued | + +Max topic length (256 B) and the signature prefix are compile-time constants, not config +fields. + +--- + +## 11. Public API & package layout + +### 11.1 `commonspace/pubsub` (the engine) + +`Service` is the single component used by both node and client hosts: + +```go +type Service interface { + app.ComponentRunnable + + // Publish encrypts, signs and fire-and-forgets payload to the topic within the space. + Publish(ctx context.Context, spaceId, topic string, payload []byte) error + + // Subscribe registers a local handler for a pattern and pushes the interest to the + // space's peers. The returned func unregisters and unsubscribes. + Subscribe(spaceId, pattern string, h Handler) (unsubscribe func(), err error) + + // SyncInterest (re)sends all local interest for the space to its current peers; + // call after (re)connecting to a space's peers. + SyncInterest(ctx context.Context, spaceId string) error + + // CloseSpace drops all local and serving-side interest for the space and withdraws + // the local interest from its peers. + CloseSpace(spaceId string) + + // EvictMember drops all serving-side interest of one identity in a space (§7.4). + EvictMember(spaceId string, identity crypto.PubKey) + + // RevalidateMembers evicts every subscriber of the space whose account no longer + // passes isMember — the ACL-change hook (§7.4). + RevalidateMembers(spaceId string, isMember func(account string) bool) + + // HandleStream serves an inbound PubSubStream; blocks for the stream lifetime. + HandleStream(stream drpc.Stream) error +} + +func New(deps Deps) Service + +// Handler receives a decrypted, signature-verified message on a subscribed topic. +// Handlers run on a bounded dispatch queue and MUST NOT block. +type Handler func(spaceId, topic string, identity crypto.PubKey, payload []byte) +``` + +`Deps` carries the pluggable pieces the host wires in: + +| Field | Role | Node | Client | +|---|---|---|---| +| `Membership MembershipChecker` | gate subscribe/publish on ACL membership | ✓ | ✓ (LAN serving) | +| `Crypto Crypto` | ReadKey encrypt/decrypt; nil ⇒ plaintext (keyless spaces) | — | ✓ | +| `Peers PeerProvider` | resolve a space's peers (responsible node + LAN) | — | ✓ | +| `Relay Relay` | responsibility + other-node resolution; nil on clients | ✓ | — | +| `OnStatus StatusHandler` | observe rejection `Status` frames from serving peers | optional | optional | +| `Metric metric.Metric` | register the private pool's prometheus gauges | optional | optional | +| `Config Config` | bounds ([§10](#10-configuration--defaults)) | ✓ | ✓ | + +`RegisterRpc(mux, service)` registers the `PubSub` DRPC service on a node's mux. + +### 11.2 `net/streampool` additions + +The pubsub engine reuses the streampool but needs a **second, independent instance**, so the +pool grew a few seams (all backward compatible; the sync-side pool is untouched): + +- **`NewStreamPool(handler streamhandler.StreamHandler, cfg StreamConfig, opts ...Option) StreamPool`** + — a non-component constructor. The existing component `New()` remains the base; the pubsub + service embeds its own pool with its own handler, queues, and tags. +- **`WithStreamCloseHook(func(streamId uint32, peerId string, tags []string))`** — a + post-removal callback the engine uses to reconcile trie interest on stream close, keyed by + `streamId`. +- **`WithMetric(m, prefix)`** — registers namespaced gauges (the pubsub pool uses prefix + `"pubsub"`), deliberately *not* the shared single-slot sync metric. +- **`RemoveTagsById(streamId, tags...)`** — removes tags from a specific stream, used by + member eviction ([§7.4](#74-confidentiality--membership-change)) to stop delivery while the stream stays open. +- **`Broadcast` cross-tag dedup** — when handed more than one tag, `Broadcast` builds a `seen` + set so a stream matching multiple patterns receives one copy. + +--- + +## 12. Tests + +The engine ships with a multi-peer in-memory fixture (in the synctest style) covering fan-out, +ACL rejection, relay rules, drop-on-overflow, dedup, echo suppression, ownership, and the +lifecycle edge cases. Notable regression tests: + +- `TestReconnectKeepsInterest`, `TestStreamCloseDrainsInterest` — interest keyed by streamId. +- `TestResyncRestoresDeliveryAfterDrop` — resync loop revives a dropped subscriber. +- `TestCloseSpaceDropsInterest`, `TestEvictMemberStopsDelivery`, + `TestRevalidateMembersEvictsNonMembers` — teardown and active eviction. +- `TestPubSubTopicOwnership`, `TestPubSubEchoSuppression` — `acc/` ownership at all layers, + echo self-suppression. + +Plus focused unit tests: `trie_test.go` (wildcard matching), `dedup_test.go` (ring), +`sign_test.go` (signature format), `topic_test.go` (validation). + +--- + +## 13. Implementation status + +**Done (this repo, `commonspace/pubsub` + `net/streampool`):** full wire protocol, flat + +wildcard topic model with the NATS-sublist trie, ACL-gated subscribe/publish, signed & +encrypted payloads, node relay with the one-hop rule, LAN symmetry, echo/duplicate-path +suppression, per-peer publish rate limiting, receive-path filter ordering + timestamp +staleness window + empty-identity guard, streamId-keyed interest with reconnect resync, and +active member eviction (`EvictMember` / `RevalidateMembers`). + +**Deferred (documented, not silent):** + +- **Zombie shedding** — close a stream stuck in queue-overflow for a sustained window. Needs + per-stream drop stats from the pool; the drop-on-overflow bound already holds without it. +- **Subscribe-rate limiting / per-space lock sharding.** Serving-side interest is guarded by a + single global mutex; a member can thrash subscribe/unsubscribe within the caps. Fine at + expected scale; shard or rate-limit if a multi-tenant relay shows contention. +- **Reader-level payload cap.** The 64 KiB cap is enforced after DRPC decode; enforcing it at + the stream reader (smaller buffer) needs per-stream buffer plumbing. +- **Rate-limiter size cap.** Per-peer buckets are time-GC'd but uncapped; bounded by + authenticated members in practice. +- **Pubsub-specific metrics** — per-stream drop counter, slow-subscriber flag, one event per + overflow episode, close-reason strings. The private pool is already observable via + `Deps.Metric`; these are the additional counters. Per-topic counters are deliberately not + added (unbounded cardinality); per-space is safe. +- **`MembershipChecker` returning a permission level** rather than a bare member/not — a + one-way door kept simple for v1 (current-head `!NoPermissions()`). +- **`Close` blocks on in-flight dispatch handlers** with no timeout; a handler that violates + the "must not block" contract wedges shutdown. Acceptable given the contract. + +**Downstream wiring (separate repos, not in this repo):** + +- **any-sync-node** — a `pubsubrelay` component: register `PubSub` next to SpaceSync via + `RegisterRpc`; wire `Relay`/`Membership` from nodeconf + the hosted space's ACL; call + `RevalidateMembers` (or `EvictMember`) from the ACL-update hook. +- **anytype-heart** — the client component: wire `Crypto` from the space ReadKey, `Peers` + from the per-space peer manager, `SyncInterest` on `rebuildResponsiblePeers`, `CloseSpace` + on space unload; serve inbound LAN pubsub streams from the existing client server; surface + to apps via middleware commands (`PubsubPublish`, `PubsubSubscribe`/`Unsubscribe`) emitting + `pb.Event`s through the local event bus. This side gets its own design doc. + +**Compatibility & rollout.** Old peers don't know the `PubSub` service → DRPC unknown-RPC +error on stream open; the client treats the peer as "pubsub unavailable", backs off, and +retries opportunistically. No protoVersion bump, no coordinator/nodeconf changes. Ship order: +any-sync (this lib) → any-sync-node deploy → heart. Until nodes deploy, LAN-only pubsub still +works between updated clients. + +--- + +## 14. Non-goals (v1) + +Queue groups (1-of-N); persistence, replay, retained messages, catch-up after reconnect; +delivery receipts/acks; cross-space topics (patterns never span spaces — `spaceId` is a +separate field, not a topic segment); protocol-level presence; WAN client↔client (only +LAN-discovered direct peers); interest propagation between nodes (a node always forwards +client publishes to the other responsible nodes, which drop them if nothing matches locally — +2 bounded copies beats holding cross-node subscription state). + +On that last non-goal, the NATS prior art maps cleanly onto our choice. NATS *clusters* +propagate interest (refcounted per subject) because a cluster may span many servers and +unnecessary fan-out is expensive at that scale — at the cost of every server holding the full +cluster interest map and an inherent propagation race. NATS *gateways* (WAN) instead default +to **optimistic sends**: forward without interest knowledge, let the receiver reply "no +interest", and switch to interest-only mode only after ~1000 rejections per account. With a +fixed fan-out of 2 peer nodes per space and shared-space traffic being likely-relevant to all +replicas, our always-forward is the optimistic-send strategy at the scale where it wins, and +it sidesteps NATS's biggest documented scaling pain (interest churn = a cluster-wide broadcast +plus a global cache flush per first-subscribe/last-unsubscribe). If inter-node waste ever +becomes measurable, the proven incremental fix is the gateway one: a bounded per-topic +no-interest map with a switch-to-interest-only threshold — not full interest replication. + +--- + +## Appendix A — presence as an app pattern (non-normative) + +Presence is deliberately **not** in the protocol. This is the recommended app recipe, modeled +on Yjs awareness with corrections where its semantics depend on a trusted relay. The critical +difference: y-websocket presence relies on the *server synthesizing* `state:null` for a dead +connection's clients. Our relay cannot forge signed messages, so that path does not exist +here — TTL expiry is the normative leave mechanism. + +**Entry & lifecycle.** Each device publishes its full presence entry +`{sessionId, clock, state|null}` on state change and as a heartbeat every ~15 s (TTL/2); +receivers expire an entry after ~30 s (TTL) without re-announce, measured by +**receiver-local receipt time** (immune to sender clock skew). Full state per message, no +diffs: every message stands alone, so any at-most-once loss self-heals within one heartbeat, +and per-message signing composes cleanly. + +- **TTL expiry is the normative leave.** Explicit leave (`state=null`) is a best-effort + latency optimization on graceful shutdown. Worst-case ghost duration = TTL. (Matrix + `m.typing` runs entirely on refresh-or-expire; that is the baseline.) +- **Key = `(accountId, sessionId)`, fresh random `sessionId` per app session.** An account + with two devices is two entries; a rebooted client never needs to out-clock its dead + predecessor — the old entry just times out. Never persist clocks across sessions. + `accountId` comes from the verified message `identity`, never from the payload — signing + closes the identity-hijack hole Yjs punts on. +- **Clock: bump before every publish** — state changes, heartbeats, leaves, reconnect + re-announces alike, starting at 1. (Starting at 1 avoids the Yjs clock-0 trap where an + unknown client's first entry is dead on arrival.) +- **Accept rule:** accept iff `clock > knownClock`, or (`clock == knownClock` and + `state == null` and a live state exists) — equal-clock-null keeps leave idempotent under + duplicate/multi-path delivery. On removal, keep a `sessionId → lastClock` tombstone for + ≥ TTL so reordered pre-leave messages can't resurrect a ghost. +- **Join snapshot:** a stateless relay cannot push room state to a new joiner. Recipe: + subscribe first, then announce yourself; peers treat an unknown-session announce as a cue to + re-announce early (with random jitter ≤ heartbeat/2 to avoid an answer storm). Alternatively + accept ≤ one heartbeat of join blindness. +- **Fan-out hygiene:** never re-publish an applied remote update (Yjs's origin-blind echo + handler caused a documented N² storm at ~20 users/room); keep state small (identity + + cursor); throttle high-frequency fields app-side (~10 Hz cursor max, further coalesced by + the publish token bucket); consider separate topics and cadences for slow presence vs fast + cursors. Idle-room budget is ≈ N²/heartbeat deliveries — scale the heartbeat up for large + spaces. + +Two topic layouts, both valid: + +- **Shared topic** `presence` (or `presence/{objectId}`): one subscription covers the whole + space; attribution comes from the verified `identity`. +- **Self-owned topics** `acc/online/`: spoof-proof per-account channels ([§7.2](#72-access-control)). + Subscribe with `acc/online/*` to follow everyone at the cost of a single interest entry, or + with concrete topics to follow specific accounts. + +If sub-TTL leave latency ever matters, a v2 option is relay-emitted **unsigned transport +hints** ("subscriber stream closed") that clients may use only to shorten their local expiry +check for that peer — never as an authoritative removal; authority stays with signed messages +and TTL. diff --git a/docs/stateless-pubsub/DESIGN.md b/docs/stateless-pubsub/DESIGN.md deleted file mode 100644 index b2f2a4bde..000000000 --- a/docs/stateless-pubsub/DESIGN.md +++ /dev/null @@ -1,647 +0,0 @@ -# Space-Scoped Stateless Pub/Sub over any-sync — Design - -Status: **DESIGN / SPEC** — resolves all open tensions from [RESEARCH.md](./RESEARCH.md). -Grounded against `any-sync@main`, `any-sync-node@main`, `anytype-heart@main` (July 2026). - ---- - -## 1. Summary - -A new ephemeral, fire-and-forget, at-most-once publish/subscribe channel scoped to a -space, carried over a **dedicated DRPC bidi stream** fully isolated from the sync engine. -Topics are plaintext `/`-separated hierarchies inside a space; subscriptions may use -NATS-style wildcards (`*` one segment, `>` trailing segments). Any space member -(Reader/Guest included) may publish and subscribe. Payloads are end-to-end encrypted with the space ReadKey and -signed with the sender's account key; relay nodes route ciphertext they cannot read. -Fan-out goes through the responsible sync nodes **and** directly to LAN-discovered peers, -with a small bounded msgId dedup cache on receivers. No message is ever persisted; the -only routing state is transient in-memory interest state (stream tags + a pattern trie) -that dies with the connection. - -### Resolved decisions - -| Question (RESEARCH.md §7) | Decision | -|---|---| -| Meaning of "stateless" | Sense (a): no message persistence anywhere. Transient in-memory interest tables (stream tags) allowed and used. | -| Stream reuse vs dedicated | Dedicated `PubSub` DRPC service + own stream; never touches `ObjectSyncStream` or the sync dispatch path (user steer, RESEARCH.md §4.4). | -| New service vs new RPC on SpaceSync | New service, own proto package — independent versioning, unknown-service fallback for old peers. | -| Topic model | `/`-separated segment hierarchy within a space. Subscriptions may use NATS-style wildcards: `*` matches exactly one segment, `>` matches one-or-more trailing segments (tail-only). Publishers must use fully-qualified topics. Matching runs on a bounded per-space pattern trie at the relay (§4.1). | -| Topic privacy | Plaintext to relays. Comparable exposure to today's plaintext `objectId`s on the sync path. | -| Subscribe permission | Any space member: `!NoPermissions()` (`commonspace/object/acl/list/models.go:90`). | -| Publish permission | Any space member. Attribution via per-message account-key signature; abuse contained by per-peer rate limits. Plus a reserved **self-owned namespace**: topics `acc/…/` accept publishes only from `accId` (§6.2). | -| Payload confidentiality | Encrypted client-side with current space ReadKey + `keyId` indirection (push-server pattern). Removed member loses access at next key rotation. | -| Authenticity | Per-message account-key signature, verified by receivers. Protects against a semi-trusted node forging/reattributing messages. | -| Topology | Publisher → its responsible node (+ direct LAN peers). Node relays once to the other responsible nodes (`relayed` flag, never re-forwarded → loop-free). | -| Duplicate suppression | Receiver-side bounded FIFO ring keyed by `msgId` (duplicate paths exist by design: LAN + node). | -| Backpressure | Bounded queues, `TryAdd` drop-on-overflow end to end — the existing streampool discipline. Slow subscriber ⇒ dropped messages, never memory growth. | -| Queue groups (1-of-N) | Non-goal v1. | -| Catch-up / replay / retained messages | Non-goal, by definition of stateless. Reconnect ⇒ resubscribe ⇒ only new messages. | -| Presence lifecycle | Not in the protocol. v1 is a generic app API; presence (heartbeat/timeout/leave, Yjs-awareness style) is an app pattern on top (Appendix A). | - -### Requirements compliance (RESEARCH.md §8) - -- **R1 (on-protocol):** rides existing transports, secureservice handshake, DRPC mux, and a - second streampool instance. A dedicated sub-stream on the existing `MultiConn` — no new - transport or TLS handshake (`net/peer/peer.go:134`, `net/transport/transport.go:40-64`). -- **R2 (memory-effective):** per-stream bounded `mb.MB` queues with drop-on-overflow - (`net/streampool/stream.go:32-44`), interest state = stream tags + pattern trie, both - O(active subscriptions), fixed-size dedup FIFO ring, per-peer publish token bucket, caps on - pattern count and topic/payload size. No path grows with message volume or offline - duration. -- **R3 (ACL-aware):** node gates subscribe *and* publish on space membership at the current - ACL head (a check that does **not** exist today on the sync path — this is new, additive - enforcement); confidentiality holds against the relay via ReadKey encryption; membership - removal cuts new traffic at key rotation and actively drops subscriptions (§6.4). - ---- - -## 2. Semantics contract - -- **At-most-once.** A publish reaching the relay is copied into bounded per-subscriber - queues; overflow drops. No acks, no retransmit, no ordering across publishers, no dedup - beyond the duplicate-path ring. -- **Fire-and-forget.** A publish with no subscribers is discarded. Nothing is stored. -- **Decoupled.** Publishers don't know subscribers. Subscriber set is whoever holds a live - tagged stream at the instant of fan-out. -- **Subscribe is unacknowledged.** Success is silent (§3); interest takes effect when the - serving peer processes the frame, so messages published concurrently by others may be - missed. There is no "subscribed as of time T" guarantee — the same race NATS documents - for cluster-wide subscription visibility (nats-server#1142), inherent to at-most-once. - Apps needing a consistent starting state combine pub/sub with a snapshot read (e.g. - presence: subscribe, then announce yourself, which prompts others' next heartbeat). -- **Reconnect = clean slate.** Subscriptions die with the stream; the client re-subscribes - on reconnect and sees only new traffic. -- **Echo.** A publisher whose own interest set matches the topic receives its own message - back from the relay; the client suppresses these by pre-recording its own msgId in the - dedup ring at publish time, and delivers to local handlers via the normal dispatch queue - (bounded, drop-on-overflow — so local delivery is best-effort like everything else, not - a hard guarantee). Net effect = NATS `no_echo` semantics without a wire flag. (NATS - echoes by default with per-connection opt-out; we don't need the option because dedup - already exists.) -- **Delivery is best-effort, duplicates possible in theory** (ring eviction under extreme - rates), so payload design must be idempotent/last-write-wins at the app level. In - practice the ring makes duplicates vanishingly rare. - ---- - -## 3. Wire protocol - -New proto package in any-sync: `commonspace/pubsub/pubsubproto/protos/pubsub.proto` -(generated via the existing Makefile pipeline, `Makefile:23-32`). - -```proto -syntax = "proto3"; -package pubsub; - -service PubSub { - // One long-lived bidi stream per peer pair, multiplexing all spaces/topics. - rpc PubSubStream(stream PubSubMessage) returns (stream PubSubMessage); -} - -message PubSubMessage { - oneof content { - Subscribe subscribe = 1; - Unsubscribe unsubscribe = 2; - Publish publish = 3; - Status status = 4; - } -} - -// Delta semantics: adds topic patterns to the stream's interest set for spaceId. -// Patterns may contain wildcards: '*' (one segment), '>' (trailing segments, tail-only). -message Subscribe { - string spaceId = 1; - repeated string topics = 2; -} - -// Removes patterns (matched verbatim against the interest set, not expanded); -// empty topics = remove all patterns of spaceId. -message Unsubscribe { - string spaceId = 1; - repeated string topics = 2; -} - -message Publish { - string spaceId = 1; - string topic = 2; - bytes msgId = 3; // 16 random bytes, generated by publisher (dedup key) - string keyId = 4; // space ReadKey id used for payload; "" = plaintext (keyless spaces) - bytes payload = 5; // ciphertext (or plaintext iff keyId == "") - bytes identity = 6; // sender account pubkey (marshalled) - bytes signature = 7; // account-key sig, see §6.3 - int64 timestampMilli = 8; // sender wall clock, informational - bool relayed = 9; // set by a node when forwarding node→node; never re-forwarded; excluded from signature -} - -// Sent by the serving peer on rejected subscribe/publish. Success is silent. -message Status { - string spaceId = 1; - repeated string topics = 2; // echo of the offending request (topic of a publish goes here too) - ErrCodes code = 3; -} - -enum ErrCodes { - Ok = 0; - NotAMember = 1; // identity has no permissions in the space's ACL - NotResponsible = 2; // this node is not responsible for the space - RateLimited = 3; - TooManyTopics = 4; - InvalidMessage = 5; // malformed frame, oversized payload, identity mismatch, bad signature - TopicNotOwned = 6; // publish into acc/…/ by an identity other than accId - InvalidTopic = 7; // malformed topic/pattern: bad wildcard placement, reserved chars, non-canonical form - ErrorOffset = 800; // 100..700 are taken by existing proto packages -} -``` - -Constraints (enforced by the serving peer, values are config defaults — §9). A rejected -Subscribe/Publish gets a `Status` and is otherwise ignored — the stream **stays open** -(the NATS precedent: `maximum subscriptions exceeded` is an error reply, not a -disconnect): - -- topic: UTF-8, 1..256 bytes, `/`-separated segments (≤16 segments, no empty segments); - recommended segment charset alnum + `.-_` (matches the NATS guideline of ≤16 tokens / - ≤256 chars). Canonical form has **no leading `/`** - (`acc/x` and `/acc/x` would silently be different topics — rejected as `InvalidTopic`). -- **Wildcards (subscription patterns only):** `*` matches exactly one segment - (`chat/*/typing` ⇒ `chat/abc/typing`, not `chat/typing` or `chat/a/b/typing`); - `>` matches one-or-more trailing segments and is only valid as the final segment - (`chat/>` ⇒ everything under `chat/`). Wildcards must be complete segments (`chat/ty*` - is invalid). `*` and `>` are reserved characters everywhere else; a `Publish` topic - containing either is rejected — publishers always use fully-qualified topics. -- **Reserved self-owned namespace:** a topic whose first segment is `acc` (i.e. prefix - `acc/`) is publishable **only** by the account whose id equals the topic's *last* - segment — e.g. `acc/online/`, `acc/cursor/`. Subscribe remains open to - all members (including patterns such as `acc/online/*`). Violations ⇒ - `Status{TopicNotOwned}`. -- payload: ≤ 64 KiB. -- per-stream interest set: ≤ 100 patterns per space, ≤ 1000 total. -- `identity` in a `Publish` **must equal** the connection-context identity - (`net/peer/context.go:88`) when arriving from a client (`relayed == false`). - This binds attribution to the TLS/handshake-proven account without requiring the node - to verify the signature per message. - -Interest tag format inside the pool: `spaceId + "/" + pattern`, verbatim (spaceId -contains no `/`; first separator wins). Wildcard resolution happens in the pubsub -engine's matcher, not in the pool — see §4.1. - ---- - -## 4. Topology & relay rules - -``` - publisher client ──publish──▶ responsible node A ──relayed=true──▶ node B ──▶ its subscribers - │ │ node C ──▶ its subscribers - │ └──▶ A's local subscribers (tag fan-out) - └──────direct publish──▶ LAN peers (same space, discovered via mDNS) -``` - -Relay rules (complete): - -1. **Clients never forward.** A client receiving a `Publish` (from a node or a LAN peer) - delivers it locally only. -2. **A node forwards only client-originated messages** (`relayed == false` arriving on a - client stream): it stamps `relayed = true` and sends one copy to each *other* - responsible node for the space (same peer-resolution logic as - `any-sync-node/nodespace/peermanager/manager.go:87-103`), plus fans out to its local - subscribers via the tag index. -3. **A node never forwards `relayed == true`** — it only fans out locally. With - `ReplicationFactor = 3` (`nodeconf/nodeconf.go`), every message traverses at most - client → node → node, hop limit 2, loop-free without any dedup state on nodes. - (This is exactly the NATS cluster rule: "messages received from a route will only be - distributed to local clients" — a strict one-hop limit is how NATS full-mesh clusters - stay loop-free too.) -4. **A node rejects subscribe/publish for spaces it is not responsible for** - (`Status{NotResponsible}`) — mirrors `checkResponsible` - (`any-sync-node/nodespace/checks.go:14-24`). -5. **LAN peers are symmetric.** anytype-heart already runs a DRPC server for LAN peers - (`space/spacecore/rpchandler.go`) and unifies node + LAN peers in the per-space peer - manager (`space/spacecore/peermanager/manager.go:172-236`). Both sides run the same - pubsub component; a LAN peer's stream carries Subscribe frames like a node's does, and - publishes to LAN peers go direct. Clients apply the member-check on LAN subscribes the - same way nodes do (they hold the ACL). - -Duplicate paths are expected (a subscriber may get the same message from a LAN peer and -from its node). The **receiver** suppresses via a fixed-size FIFO ring keyed by `msgId` -(default 4096 entries ≈ 64 KiB of ids). Publishers self-suppress echoes by msgId too -(§2, Echo). - -Client → node selection piggybacks on the existing responsible-peer choice -(`pool.GetOneOf(nodeIds)` — one node at a time, `manager.go:213`), so the pubsub stream -goes to the same node the client already syncs with. - -### 4.1 Interest matching (wildcards) - -The streampool tag index is exact-match (`streamIdsByTag`, -`net/streampool/streampool.go:76`), so the pubsub engine layers a matcher on top rather -than replacing the pool: - -- Each accepted subscription registers its **pattern string verbatim as the stream tag** - (`spaceId + "/" + pattern`) — the pool keeps doing stream bookkeeping (add/remove/GC - on stream close) exactly as today. -- In parallel, the engine maintains a **per-space segment trie** of live patterns, - mirroring the NATS sublist shape exactly (`server/sublist.go`): one level per segment, - `map[segment]*node` for literals plus two dedicated wildcard slots per level (`pwc` - for `*`, `fwc` for `>`), each terminal holding a refcount of subscribing streams. - Match order as in NATS `matchLevel`: at each level add `fwc` matches, branch through - `pwc`, hash-lookup the literal. -- On publish to concrete topic `T`: walk the trie with `T`'s segments — branching on the - literal edge, the `*` edge, and any terminal `>` edge — collecting every matching - pattern (O(segments × matched branches), segments ≤16). Then fan out once per matched - pattern tag, **deduplicating stream ids across patterns**: a stream subscribed to both - `chat/>` and `chat/*/typing` must receive one copy, not two. *Implemented* by teaching - `streampool.Broadcast` to dedup stream ids across the tag set it's given (it previously - collected per-tag with no cross-tag dedup) — so the engine passes all matched pattern - tags to one `Broadcast` call and the pool guarantees one copy per stream. -- Trie cleanup: refcount decrement on Unsubscribe; on stream close the engine reconciles - lazily — when a matched pattern's tag resolves to zero live streams, the pattern is - dropped from the trie. (Alternative: a stream-close callback from the pool; decided at - implementation time, both are bounded.) -- Exact-topic subscriptions are just patterns without wildcard segments — one code path. - -The trie is bounded by the same caps as the interest set (≤1000 patterns/stream, -≤100/space/stream), so relay memory stays O(active subscriptions), and matching cost is -paid only per publish within that space. - -**Deliberately no match-result cache.** NATS fronts its sublist with a 1024-entry -literal-subject → result cache, and its own issue history (nats-server#710, #941: <0.5% -hit rates, lock contention, latency spikes under sub/unsub churn) led to `NoCache` -sublists — which NATS uses for exactly our analog, the small per-connection permission -tries. Our tries are per-space (small) and ephemeral interest is churny (every -subscribe/unsubscribe would invalidate), so a direct walk of a ≤16-level trie beats a -cache we'd constantly flush. Revisit only with profiling evidence; the bounded -patch-on-insert design from NATS is the template if ever needed. - ---- - -## 5. What happens on each frame (serving peer) - -**Subscribe** — -resolve space ACL state (nodes: the space is hosted locally; clients/LAN: the open space); -check `PermissionsAtRecord(head, ctxIdentity)` is not `None`; validate patterns -(`InvalidTopic` on bad wildcard placement / reserved chars / non-canonical form); check -pattern-count caps; then register in the trie and `AddTagsCtx(ctx, spaceId+"/"+pattern...)`. -Reject ⇒ `Status`, no tag. - -**Unsubscribe** — `RemoveTagsCtx` + trie refcount decrement. No checks needed. - -**Publish** (from client stream) — -1. size/shape checks (a topic containing `*`/`>` ⇒ `InvalidTopic`); - `identity == ctxIdentity`; membership check against cached ACL state (map lookup); - self-owned-namespace check (topic `acc/…` ⇒ last segment must equal - `ctxIdentity.Account()` — one string compare); per-peer token bucket (§7). -2. Match the topic against the space's pattern trie (§4.1), dedup stream ids across - matched patterns, then `Broadcast(msg, matchedPatternTags...)` on the pubsub pool — - the existing tag-index fan-out (`net/streampool/streampool.go:360-377`), which - per-stream `TryAdd`s and drops on overflow. -3. If serving peer is a responsible node: stamp `relayed=true`, send to other responsible - nodes (lazy stream open via the pool's `Send` + PeerGetter). - -**Publish** (`relayed == true`, from a node stream) — -verify the sending peer is a responsible node for the space (peerId ∈ `NodeIds(spaceId)`); -fan out locally only (same trie match). The `identity == ctxIdentity` rule does not apply -(the forwarding node is not the author) — authenticity is the receiver's signature check -(§6.3). - -**Receive** (client) — dedup by msgId ring → verify signature against `identity` → check -`identity` is a member at the local ACL head → if the topic is in the `acc/` namespace, -check its last segment equals `identity.Account()` (end-to-end enforcement: the signature -covers the topic, so not even a malicious relay can inject into someone else's self-owned -topic) → look up ReadKey by `keyId`, decrypt → dispatch to local topic handlers. Any -failure ⇒ drop + debug metric, never an error to the peer. - ---- - -## 6. Security model (R3) - -### 6.1 Threat model — the semi-trusted relay - -The node can: observe spaceIds, topics, sender identities, timing, sizes (accepted — -comparable to sync-path metadata today); drop, delay, reorder messages (accepted — -at-most-once contract). The node cannot: read payloads (no ReadKey); forge or reattribute -messages (signature); replay effectively. Replay defense is two-layer: the msgId dedup -ring catches the short window, and the receiver enforces a **signed-timestamp staleness -window** (`Config.MaxTimestampSkew`, default 5 min) so that even after the ring evicts an -id, a relay replaying an old signed frame is rejected on its stale timestamp. (A relay -cannot forge a fresh timestamp — it's covered by the signature.) - -### 6.2 Access control - -Both directions gate on **space membership at the current ACL head**, checked by the -serving peer from its local ACL copy — a cached in-memory lookup, no coordinator round -trip. This is *new* enforcement: today's sync path has none at subscribe time -(`any-sync-node/nodespace/rpchandler.go:329-331` accepts unchecked). Publish and -subscribe both require `!NoPermissions()`; no `CanWrite` requirement (decision: any -member publishes — presence/typing-style uses need Readers to emit). - -**Self-owned topics.** The `acc/` namespace (§3) adds a per-topic ownership rule on top -of membership: only the account named by the topic's last segment may publish there. -It is enforced twice — at the serving peer (cheap, because `identity == ctxIdentity` is -already bound by the handshake) and at every receiver (via the signature, which covers -the topic string). This gives apps spoof-proof per-account channels — e.g. -`acc/online/` — where consumers can trust the topic itself, not just the message -attribution. It is the in-protocol generalization of the push-server's "silent -self-channel" restriction (`anytype-push-server/push/push.go:140`), and matches the -proven NATS pattern for the same problem: per-identity subject prefixes (the -`_INBOX_.>` convention) rather than dynamic per-message grants. NATS violation -semantics also match ours: a permissions violation drops the message / rejects the -subscribe with an error and keeps the connection open — only authentication failures -disconnect. Wildcards make the -fan-in side cheap: one `acc/online/*` subscription covers every member's online topic, -and each received message is still individually ownership-checked against its concrete -topic and verified signature. - -### 6.3 Authenticity - -`signature = accountKey.Sign("anysync:pubsub:v1" | len‖spaceId | len‖topic | len‖msgId | -len‖keyId | le64(timestampMilli) | payload)`. Each variable-length field is length-prefixed -(le32) so field boundaries can't shift (e.g. `spaceId="ab",topic="c"` vs -`spaceId="a",topic="bc"` sign differently); `payload` trails unprefixed as the final field. -`relayed` is excluded (mutated in transit). Receivers verify; nodes don't need to -(attribution from clients is already bound by `identity == ctxIdentity`, and verifying -per-message on the relay buys little at real CPU cost). Ed25519 sign/verify is ~30-80 µs — -negligible at ephemeral-signal rates. - -Receivers run the cheap filters (local-interest match, membership, `acc/` ownership, -timestamp staleness) *before* the Ed25519 verify, and record the dedup ring only after -verify succeeds — so a relay/LAN-peer flood of forged-signature messages is shed cheaply -and cannot evict legitimate ids from the ring to reopen a replay window (§6.1). - -### 6.4 Confidentiality & membership change - -Payloads encrypt with `AclState.CurrentReadKey()` (`commonspace/object/acl/list/aclstate.go:170`), -carrying `CurrentReadKeyId()` as `keyId`. Receivers hold historical keys via the ACL, so -rotation mid-flight is safe. On member removal the existing rotation -(`aclrecordbuilder.go:761-844`) cuts decryption of new traffic automatically. Additionally, -the engine exposes `EvictMember(spaceId, identity)` — the node wires it to its ACL-update -hook (the `syncacl` updater, `commonspace/object/acl/syncacl/syncacl.go:53-56`, is the -precedent) to actively drop the removed identity's subscriptions: it strips that account's -per-stream tags (via `streampool.RemoveTagsById`, so delivery — which is tag-keyed — stops -even while the stream stays open) and decrements the match trie. Bounded work: one scan of -the streams subscribed to that space per ACL change. - -Spaces without a ReadKey (post-GO-7187 keyless/public spaces): `keyId = ""`, plaintext -payload, signature still required. - ---- - -## 7. Memory & abuse bounds (R2) - -| Resource | Bound | Mechanism | -|---|---|---| -| Outbound per-stream buffer | `queueSize` msgs (default 100) | `mb.MB` + `TryAdd` drop (`net/streampool/stream.go:39`) | -| Interest table | ≤1000 patterns/stream, ≤100/space/stream | reject with `TooManyTopics`; state dies with stream | -| Pattern trie (§4.1) | O(total live patterns × segments), segments ≤16 | same caps as interest table; lazily pruned when a pattern's tag has no live streams | -| Publish rate | token bucket **per peer** (default 30 msg/s, burst 60) | checked in pubsub handler — **inside** the stream, because the RPC limiter only gates stream-open (`net/rpc/limiter/limiter.go:96-106`). One bucket per peer (all of a peer's streams share it — stricter than per-stream, so a peer can't multiply its budget by opening streams). Deliberate divergence from core NATS, which has no publish rate limiting (maintainers punt to network throughput) and instead disconnects slow consumers — acceptable for a trusted-client broker, not for our semi-trusted multi-tenant relays. We also drop rather than disconnect slow subscribers, which fits at-most-once ephemera | -| Payload size | ≤64 KiB | reject `InvalidMessage` | -| Dedup cache | fixed FIFO ring, 4096 msgIds | evicts oldest, O(1) | -| Fan-out amplification | 1 upload → ≤2 node-node copies → N subscriber queues | node-side copy (publisher uploads once); `peerMessage.Copy()` pattern reuses the existing per-destination stamping (`stream.go:32-37`) | -| Idle streams | closed with the sub-connection; tags GC'd in `removeStream` (`streampool.go:431-446`) | existing | -| Zombie subscribers | stream in continuous queue-overflow for > 30 s is closed | NATS disconnects slow consumers outright to protect the system; we drop first (fits at-most-once), but a *persistently* full queue means a dead/wedged reader burning fan-out work — shed it and let the client reconnect fresh | - -No persistence, no unbounded map, no per-message allocation beyond the pooled message -structs (mirror `objectmessages` `sync.Pool` usage, `headupdate.go:13-38`). - ---- - -## 8. Component design per repo - -### 8.1 any-sync (this repo) - -1. **Generalize streampool for a second instance.** Extract a non-component constructor — - `streampool.NewPool(handler streamhandler.StreamHandler, cfg StreamConfig) Pool` — and - make the existing component (`streampool.go:27`, hard-bound to `streamhandler.CName` - at `streampool.go:98`) a thin wrapper. Backward compatible; the pubsub service embeds - its own pool with its own handler, queues, and tags. Sync flow control is untouched. -2. **`commonspace/pubsub/pubsubproto`** — proto + generated DRPC (Makefile pipeline). -3. **`commonspace/pubsub`** — the shared engine used by node, client, and LAN-server - sides alike: - - stream lifecycle: `OpenStream` to a peer / `ReadStream` for inbound (both feed the - private pool), resubscribe-on-reconnect using the `subscribeclient` watcher pattern - (`coordinator/subscribeclient/client.go:115-153`); - - interest handling: Subscribe/Unsubscribe → pattern validation + membership check - hook → per-space pattern trie (§4.1) + tags; - - publish path: validate → trie match + cross-pattern stream dedup → broadcast → - optional forward hook; - - receive path: dedup ring → verify → decrypt → handler dispatch; - - pluggable interfaces so layering stays clean: - `MembershipChecker` (backed by `AclState`), `Crypto` (ReadKey encrypt/decrypt via the - space's `Acl()`), `Forwarder` (node-only), `RateLimiter`. - - public API: - `Publish(ctx, spaceId, topic string, payload []byte) error` (encrypt+sign+send) and - `Subscribe(spaceId, topic string, h Handler) (unsubscribe func())`, plus an error/ - status callback for surfaced `Status` frames. -4. **Reference wiring + tests** in the synctest style - (`commonspace/sync/synctest/`): multi-peer in-memory fixture proving fan-out, ACL - rejection, relay rules, drop-on-overflow, dedup. - -### 8.2 any-sync-node - -- Register `DRPCRegisterPubSub` next to SpaceSync (`nodespace/service.go:80`). -- `pubsubrelay` component: wires the shared engine with node deps — membership from the - hosted space's ACL, responsibility check from nodeconf, `Forwarder` resolving the other - responsible nodes (reuse `getResponsiblePeers` logic, - `nodespace/peermanager/manager.go:87-103`), rate-limiter config. -- ACL-update hook to evict removed members' tags (§6.4). - -### 8.3 anytype-heart (sketch — own design doc when we get there) - -- Register the pubsub client component in bootstrap next to streampool - (`core/anytype/bootstrap.go:263-281`); open streams to the space's responsible node and - LAN peers via the existing per-space peer manager; serve inbound LAN pubsub streams from - the existing client server (`space/spacecore/rpchandler.go`). -- Tie subscriptions to space open/close lifecycle; auto-resubscribe on - `rebuildResponsiblePeers`. -- Surface to apps: middleware commands (`PubsubPublish`, `PubsubSubscribe/Unsubscribe`) - emitting `pb.Event`s through the existing local event bus (`core/subscription/`), the - same delivery surface chat SSE uses. - -### 8.4 Compatibility & rollout - -- Old peers don't know the `PubSub` service → DRPC unknown-RPC error on stream open. - Client treats it as "pubsub unavailable on this peer", backs off (subscribeclient's - capped linear backoff), and retries opportunistically. No protoVersion bump required; - no coordinator/nodeconf changes (same node addresses, same responsibility mapping). -- Ship order: any-sync (lib) → any-sync-node deploy → heart. Until nodes deploy, LAN-only - pubsub still works between updated clients. - ---- - -## 9. Defaults (tunable via config) - -| Knob | Default | -|---|---| -| max payload | 64 KiB | -| max topic length | 256 B | -| max topics per stream | 1000 (100 per space) | -| publish rate per peer | 30 msg/s, burst 60 | -| per-stream write queue | 100 (client), 500 (node outbound) — match sync-side sizes | -| dedup ring | 4096 msgIds | -| received-timestamp staleness window | 5 min (enforced at receive, `Config.MaxTimestampSkew`) | -| client interest resync interval | 20 s (`Config.ResyncInterval`) | -| pubsub stream peer TTL | 1 h (`Config.PeerTTL`) | - -## 10. Non-goals (v1) - -Queue groups (1-of-N); persistence, replay, retained messages, catch-up after reconnect; -delivery receipts/acks; cross-space topics (patterns never span spaces — `spaceId` is a -separate field, not a topic segment); protocol-level presence; WAN client↔client (only -LAN-discovered direct peers); interest propagation between nodes (a node always forwards -client publishes to the other responsible nodes, which drop them if nothing matches -locally — 2 bounded copies beats holding cross-node subscription state). - -On that last non-goal, NATS prior art maps cleanly onto our choice. NATS *clusters* do -propagate interest (RS+/RS-, refcounted per subject, advertised on the 0→1 transition, -withdrawn on N→0) because a cluster may span many servers and unnecessary fan-out is -expensive at that scale — the cost is every server holding the full cluster interest map -and an inherent propagation race (subscription visibility across the cluster is -asynchronous, nats-server#1142). NATS *gateways* (WAN) instead default to **optimistic -sends**: forward without interest knowledge, let the receiver reply "no interest", and -only switch to interest-only mode after ~1000 such rejections per account -(`server/gateway.go`, `defaultGatewayMaxRUnsubBeforeSwitch`). With a fixed fan-out of 2 -peer nodes per space and shared-space traffic being likely-relevant to all replicas, our -always-forward is the optimistic-send strategy at the scale where it wins. It also -sidesteps NATS's single biggest documented scaling pain — interest churn, where every -first-subscribe/last-unsubscribe is a cluster-wide broadcast plus a global client-cache -flush (nats-server#710/#941). If inter-node waste ever becomes measurable, the proven -*incremental* fix is the gateway one: a bounded per-topic no-interest map with a -switch-to-interest-only threshold — not full RS+/RS- interest replication (v2, not v1). - -## 11. Remaining open items - -1. Exact package/service naming bikeshed (`commonspace/pubsub` vs top-level `pubsub`; - service `PubSub` vs `SpacePubSub`). -2. Whether the node emits `Status{RateLimited}` per rejected publish or silently drops - after the first notice (flood of statuses is itself amplification — leaning: notify - once per window). -3. Metrics surface (per-topic counters are unbounded-cardinality; per-space is safe). - The private pool is observable via `Deps.Metric` (`WithMetric(m, "pubsub")`), which - registers namespaced prometheus gauges (stream/tag/dial counts). It deliberately does - **not** register into the shared single-slot sync-metric (`RegisterStreamPoolSyncMetric`) - — that belongs to the app-level sync streampool, and a second pool there would overwrite - and, on Close, null it. The remaining work is the pubsub-specific counters below. - -## 12. Implementation status (v1 in any-sync) - -Implemented in `commonspace/pubsub` (+ `net/streampool` additions): full wire protocol, -flat+wildcard topic model with the NATS-sublist trie, ACL-gated subscribe/publish, signed -& encrypted payloads, node relay with the one-hop rule, LAN symmetry, echo/duplicate-path -suppression, per-peer publish rate limiting, and — after the multi-lens review — the -following hardening: - -- **Interest keyed by streamId, not peerId.** Serving-side interest lives in - `streams[streamId]` and the match trie refcounts *subscribing streams*; the close hook - carries the `streamId`. This fixes two review-found bugs: a reconnecting peer's fresh - stream no longer loses interest when the stale stream closes, and a stream that dies - mid-subscribe can't orphan interest (interest+tag are committed under one lock with - rollback if the stream vanished). Regression tests: `TestReconnectKeepsInterest`, - `TestStreamCloseDrainsInterest`. -- **Reconnect watcher.** A resync loop re-pushes local interest every `ResyncInterval` - and immediately on client-side stream close, and `OpenStream` sets `PeerTTL` so idle - pool GC doesn't silently reap a quiet subscriber. Without this a pure subscriber went - dead after the first drop. Test: `TestResyncRestoresDeliveryAfterDrop`. -- **`CloseSpace` / `EvictMember`.** Per-space teardown (a global service must release - closed-space state) and §6.4 active member eviction. Tests: `TestCloseSpaceDropsInterest`, - `TestEvictMemberStopsDelivery`. -- **Receive-path ordering + timestamp window + empty-identity guard** (§6.1/§6.3). -- **`Status.msgId`** for host-side rejection correlation; node publish shape errors return - `InvalidTopic` consistently with the client API; `spaceId` is validated to contain no `/`. - -Deferred (documented, not silent): - -- **Zombie shedding** (close a stream stuck in queue-overflow > 30 s, §7). Needs per-stream - drop stats surfaced from the pool; the drop-on-overflow bound already holds without it. -- **Subscribe-rate limiting / per-space lock sharding.** `remoteMu` is a single global - lock; a member can thrash subscribe/unsubscribe within the caps. Fine at expected scale; - shard or rate-limit if a multi-tenant relay shows contention. -- **Reader-level payload cap.** The 64 KiB cap is enforced after DRPC decode; enforcing it - at the stream reader (smaller `MaximumBufferSize`) needs per-stream buffer plumbing. -- **Rate-limiter size cap.** Per-peer buckets are time-GC'd but uncapped; bounded by - authenticated members in practice. -- **pubsub-specific metrics** (per-stream drop counter, slow-subscriber flag, one event per - overflow episode, close-reason strings) — §11.3. -- **`MembershipChecker` returning a permission level** rather than a bare member/not — a - one-way door kept simple for v1 (current-head `!NoPermissions()`). -- **Close blocks on in-flight dispatch handlers.** `Close` waits for the dispatch loop to - drain; a handler that violates the "must not block" contract wedges shutdown (no timeout). - Acceptable given the contract; a stuck app handler is an app bug, and adding a Close - timeout would mask it. -- **Resync coverage for many-space clients.** Each resync pass fires one dial task per local - space through the bounded dial queue (`DialQueueSize`, default 100); a client with more - spaces than that drops the overflow for that tick and restores their interest on a later - tick (self-correcting, no leak — re-sends are idempotent). Heart should size - `DialQueueSize` to its expected open-space count. - -Downstream wiring (separate repos, per §8.2/§8.3): any-sync-node `pubsubrelay` component -(`Relay`/`Membership` from nodeconf + hosted ACL, `RegisterRpc`, `EvictMember` on ACL -change) and anytype-heart client (`Crypto` from the space ReadKey, `PeerProvider` from the -peer manager, `CloseSpace` on space unload, middleware commands). - Should follow the nats.go subscription observability contract: per-stream dropped - counter, a "slow subscriber" state flag, **one event per overflow episode** (not per - dropped message), plus a cumulative per-node counter (varz `slow_consumers` style) - and distinct close-reason strings (rate-limited vs zombie-shed vs transport error). - ---- - -## Appendix A — presence as an app pattern (non-normative recipe) - -Modeled on Yjs awareness (`y-protocols/awareness.js`), with deliberate corrections where -its semantics depend on a trusted relay. The critical difference: **y-websocket presence -relies on the server synthesizing `state:null` for a dead connection's clients** — -current y-websocket clients send *no* leave on tab close at all (the unload handler was -deliberately removed, yjs/y-websocket#165). Our relay cannot forge signed messages, so -that path does not exist here. - -**Entry & lifecycle.** Each device publishes its full presence entry -`{sessionId, clock, state|null}` on state change and as a heartbeat every ~15 s -(TTL/2); receivers expire an entry after ~30 s (TTL) without re-announce, measured by -**receiver-local receipt time** (immune to sender clock skew — Yjs's `lastUpdated` never -crosses the wire either). Full state per message, no diffs: every message stands alone, -so any at-most-once loss self-heals within one heartbeat, and per-message signing -composes cleanly. - -- **TTL expiry is the normative leave mechanism.** Explicit leave (`state=null`) is a - best-effort latency optimization on graceful shutdown. Worst-case ghost duration = - TTL. (Matrix `m.typing` runs entirely on refresh-or-expire; that is the baseline - guarantee.) -- **Key = `(accountId, sessionId)`, fresh random `sessionId` per app session** (Yjs: - new `clientID` per page load). An account with two devices is two entries; a rebooted - client never needs to out-clock its dead predecessor — the old entry just times out. - Never persist clocks across sessions. `accountId` comes from the verified message - `identity`, never from the payload — signing closes the identity-hijack hole Yjs - explicitly punts on (`PROTOCOL.md §6`), and Yjs's own-clientID `clock++` defense - becomes unnecessary. -- **Clock: bump before every publish** — state changes, heartbeats, leaves, and - reconnect re-announces alike, starting at 1. (Yjs leaves some paths un-bumped only to - interoperate with server-synthesized equal-clock nulls, and its un-bumped reconnect - re-announce causes a real invisibility race; with no synthesizing relay, always-bump - is strictly simpler and safer. Starting at 1 avoids the Yjs clock-0 trap where an - unknown client's first entry is dead on arrival.) -- **Accept rule:** accept iff `clock > knownClock`, or (`clock == knownClock` and - `state == null` and a live state exists) — equal-clock-null keeps leave idempotent - under duplicate/multi-path delivery. On removal (null or TTL), keep a - `sessionId → lastClock` tombstone for ≥ TTL so reordered pre-leave messages can't - resurrect a ghost. -- **Join snapshot:** a stateless relay cannot push the room state to a new joiner - (y-websocket's server does). Recipe: subscribe first, then announce yourself; peers - treat an unknown-session announce as a cue to re-announce early (with random jitter - ≤ heartbeat/2 to avoid an answer storm). Alternatively accept ≤ one heartbeat of join - blindness. -- **Fan-out hygiene:** never re-publish an applied remote update (Yjs's origin-blind - echo handler caused a documented N² message storm at ~20 users/room); keep state - small (identity + cursor); throttle high-frequency fields app-side (~10 Hz cursor - max, further coalesced by the publish token bucket); consider separate topics and - cadences for slow presence vs fast cursors. Idle-room budget is ≈ N²/heartbeat - deliveries — scale the heartbeat up for large spaces. - -Two topic layouts, both valid: - -- **Shared topic** `presence` (or `presence/{objectId}`): one subscription covers the - whole space; attribution comes from the verified `identity`. -- **Self-owned topics** `acc/online/`: spoof-proof per-account channels (§6.2). - Subscribe with `acc/online/*` to follow everyone at the cost of a single interest - entry, or with concrete topics to follow specific accounts. - -If sub-TTL leave latency ever matters, a v2 option is relay-emitted **unsigned transport -hints** ("subscriber stream closed") that clients may use only to shorten their local -expiry check for that peer — never as an authoritative removal; authority stays with -signed messages and TTL. diff --git a/docs/stateless-pubsub/RESEARCH.md b/docs/stateless-pubsub/RESEARCH.md deleted file mode 100644 index cd714352c..000000000 --- a/docs/stateless-pubsub/RESEARCH.md +++ /dev/null @@ -1,250 +0,0 @@ -# Stateless Pub/Sub in `any-sync` — Research Summary - -Status: **RESEARCH / PROBLEM-FRAMING** — deliberately contains **no chosen solution**. -Purpose: hand-off document for a spec author (fable). It frames the problem, surveys how -NATS and other modern systems do stateless pub/sub, maps the concrete `any-sync` substrate -(`file:line`-grounded against `main`), catalogs the prior art already in the repo, and -enumerates the open design tensions the spec must resolve. It does **not** pick a design. - ---- - -## 0. The ask (verbatim intent) - -> Users of the same space can **subscribe/publish topics within a space**. It must work -> **effectively over the any-sync protocol** (probably a DRPC stream), be **memory-effective**, -> and **respect the ACL access** of the user. - -Three hard requirements fall out of this, and every section below is oriented around them: - -1. **R1 — On-protocol.** Runs over the existing any-sync transport/DRPC machinery, not a side channel. -2. **R2 — Memory-effective.** Bounded, ephemeral footprint on both clients and relaying nodes; no unbounded buffers, no persistence. -3. **R3 — ACL-aware.** Publish/subscribe is gated by the space's access-control list and its encryption boundary. - -Plus the implicit scope word: **stateless** (Section 1 pins down what that actually means — it is ambiguous and the ambiguity matters). - ---- - -## 1. What "stateless pub/sub" means (and the ambiguity to resolve) - -Distilled from the reference systems (Sections 2–3), "stateless pub/sub" is the **core-NATS / Redis-pub-sub** family, characterized by: - -- **Fire-and-forget.** A publish with no live subscriber is simply discarded; it is never stored or replayed. -- **At-most-once delivery** (MQTT "QoS 0"). No acks, no retransmit, no dedup, no ordering guarantees across publishers. -- **No persistence.** No log, no durable queue, no retained "last message." (This is exactly what distinguishes it from any-sync's DAG trees, the KV store, and the coordinator Inbox — all of which *are* stateful.) -- **Full decoupling** of publishers and subscribers in space (don't know each other), time (needn't overlap beyond the instant of delivery), and synchronization (non-blocking). -- **Fan-out (1→N)** as the base pattern; optionally **load-balanced 1-of-N** ("queue groups"). - -**Ambiguity the spec MUST pin down — two independent axes of "stateless":** - -- **(a) Message statelessness** — no message is ever persisted (fire-and-forget). *All* systems in this family have this. -- **(b) Subscription/routing statelessness** — whether the *relay* holds any in-memory subscription-interest table. - - Core NATS is stateless in sense (a) but the **server does hold an in-memory interest graph** (subject → subscribers) to route efficiently. It is *not* stateless in sense (b). - - The fully-(b)-stateless alternative is "relay broadcasts everything, subscribers filter locally" — zero routing state but poor bandwidth-efficiency. - -This tension between (b) and **R2 (memory-effective)** / bandwidth-efficiency is one of the central design decisions and is called out again in Section 7. The word "stateless" in the ask most plausibly means (a) + *no durable/persistent* state, tolerating transient in-memory routing tables — but this must be confirmed, not assumed. - ---- - -## 2. Reference model: core NATS pub/sub - -(Sources: NATS docs — pubsub, subjects, queue groups; see Section 9.) - -**Subject-based addressing.** Publishers send to a **subject** (a string); subscribers register **interest** in subjects. "A subject is just a string the publisher and subscriber use to find each other." Messages carry `{subject, payload bytes, headers, optional reply-address}`. - -**Subject hierarchy + wildcards.** -- Subjects are dot-separated token hierarchies: `time.us.east.atlanta`. -- **`*`** matches exactly one token: `time.*.east` ⇒ `time.us.east`, `time.eu.east` (not `time.us.east.atlanta`). -- **`>`** matches one-or-more trailing tokens, tail-position only: `time.us.>` ⇒ everything under `time.us`. -- **Publishers must use fully-qualified subjects** (no wildcards); **only subscribers use wildcards.** -- Allowed chars: any Unicode except null, space, `.`, `*`, `>`; recommend alnum + `-`/`_`. `$`-prefixed reserved for system. Guideline ≤16 tokens, ≤256 chars. Max payload default 1 MB (server `max_payload`, cap 64 MB). - -**Delivery.** -- **Fan-out:** every interested subscriber gets a copy (1→N). -- **Queue groups:** subscribers sharing a queue name form a group; each message goes to **exactly one randomly-chosen** member — built-in load balancing + transparent scaling + "no-responders" signal. Queue-group names follow subject naming rules. -- **At-most-once.** Offline/disconnected subscriber ⇒ message lost. Messages with no subscribers are discarded. - -**What core NATS deliberately does NOT provide** (these are JetStream, a separate stateful layer): persistence, replay, guaranteed/at-least-once delivery, dedup, ordering, consumer cursors. Those are exactly the things a *stateless* design excludes. - ---- - -## 3. Landscape of modern pub/sub (comparison) - -| System | Topic model | Wildcards | Delivery | Persistence | Topology | Notable for us | -|---|---|---|---|---|---|---| -| **NATS core** | dot-hierarchy subjects | `*` (1 token), `>` (tail multi) | fan-out + queue-group (1-of-N) | none (JetStream is separate) | central broker(s), interest graph | the canonical model; subject wildcards; queue groups | -| **Redis pub/sub** | flat channels + patterns | `*`, `?`, `[..]` (glob) | fan-out | none | central server | simplest fire-and-forget; subscriber must be connected | -| **MQTT** | `/`-hierarchy topics | `+` (1 level), `#` (tail multi) | QoS 0/1/2 | retained msg + sessions ⇒ **stateful** | broker | wildcard syntax variant; "retained message" is the anti-pattern to avoid for stateless | -| **libp2p GossipSub** | flat topics | none | epidemic mesh fan-out; IHAVE/IWANT lazy-pull | none | **decentralized P2P mesh** (no broker) | closest to any-sync's decentralization ethos; mesh + fanout + peer-scoring for abuse resistance | -| **Yjs awareness** | one implicit channel/doc | n/a | state-based CRDT diff | ephemeral, auto-GC | transport-agnostic relay | **presence prototype**: `(clientID, clock, state\|null)`, `null`=offline, 15 s re-announce / 30 s timeout; kept *separate* from the CRDT doc | -| **Phoenix Channels** | `topic:subtopic` strings | none (exact) | fan-out; Presence built on PubSub | ephemeral | server (BEAM PubSub) | presence = minimal ephemeral metadata over pub/sub | -| **Matrix EDUs** | per-room | n/a | fan-out to room servers | ephemeral (EDU ≠ persistent event) | federated servers | typing/presence modeled as **Ephemeral Data Units**, explicitly distinct from the persistent event DAG | - -**Cross-cutting takeaways relevant to any-sync:** - -- **Topology is the fork in the road.** NATS/Redis/MQTT = central broker with an interest table. GossipSub = P2P mesh, no broker. any-sync sits *between*: clients reach a small set of **responsible sync nodes** (semi-trusted relays that store ciphertext they cannot read) and *may* also reach other clients directly (Section 4.6). The relay-through-node model is closest to a broker, but the broker is **untrusted for content** — which forces the encryption question (Section 6). -- **Presence is the killer stateless-pubsub use case** in local-first / collaborative systems (Yjs awareness, Phoenix Presence, Matrix EDUs), and it is *always* kept separate from the persistent CRDT/document layer. If presence/typing/cursors are a target use case, the Yjs awareness lifecycle (heartbeat + timeout + `null`-on-leave) is the reference to study, not NATS. -- **Wildcards cost the relay.** Subject-hierarchy matching (`*`/`>`, `+`/`#`) requires the relay to run a trie/interest-match per message. Flat topics (GossipSub, Redis channels, Phoenix) do not. This trades expressiveness against R2. - ---- - -## 4. The `any-sync` substrate (grounded map) - -All paths under `/Users/roma/anytype/any-sync`, line numbers vs `main`. **Scope caveat:** any-sync is a *library*. The concrete server-side `DRPCSpaceSyncServer` and the production `PeerManager`/`StreamHandler` live in downstream repos (`any-sync-node`, `anytype-heart`). This repo ships the interfaces, the client plumbing, and **reference implementations in test code** (`commonspace/spaceutils_test.go`, `commonspace/spacerpc_test.go`, `commonspace/sync/synctest/`). Every such boundary is flagged. - -### 4.1 Transport & DRPC (R1 foundation) - -- Three transports selected by address scheme: **Yamux** (TCP), **QUIC**, **WebTransport** — `net/transport/transport.go:24-28`. ALPN `"anysync"`; QUIC handshake at `net/transport/quic/quic.go:108-165`. -- Secure layer = libp2p-TLS + an app handshake that stamps `peerId`, account `identity`, versions into the connection context — `net/secureservice/secureservice.go:114-185`, ctx accessors `net/peer/context.go:33-106`. -- DRPC server is a `drpcmux` with a handler chain `limiter → metric → encoding` — `net/rpc/server/drpcserver.go:52-80`. Services register via generated `DRPCRegisterXxx(mux, impl)`. -- A bidirectional DRPC stream is obtained by `peer.AcquireDrpcConn` → generated stream method → `drpc.Stream{Send,Recv,MsgSend,MsgRecv}` — `net/peer/peer.go:134,270-297`. - -### 4.2 StreamPool — the fan-out core (R1 + R2) - -`net/streampool/streampool.go:51-69` — the central abstraction. It caches opened `drpc.Stream`s, **indexes them by peer and by tag**, opens them lazily, and pushes messages onto per-stream write queues: - -```go -type StreamPool interface { - app.ComponentRunnable - AddStream(stream drpc.Stream, queueSize int, tags ...string) error // outgoing - ReadStream(stream drpc.Stream, queueSize int, tags ...string) error // incoming, blocks reading - Send(ctx, msg drpc.Message, target PeerGetter) error // dial+send, async - SendById(ctx, msg drpc.Message, peerIds ...string) error // only if stream exists - Broadcast(ctx, msg drpc.Message, tags ...string) error // fan-out to all streams with tag - AddTagsCtx(ctx, tags ...string) error // subscribe a stream to tag(s) - RemoveTagsCtx(ctx, tags ...string) error // unsubscribe - Streams(tags ...string) []drpc.Stream -} -``` - -- Tag index: `streamIdsByTag map[string][]uint32` — `streampool.go:71-84`. In practice the tag is a **`spaceId`** (see 4.4). -- `Broadcast(msg, tags...)` writes to every stream carrying a listed tag — `streampool.go:360-377`. **This is the existing tag-keyed fan-out primitive.** -- `AddTagsCtx`/`RemoveTagsCtx` mutate a live stream's tag set at runtime — `streampool.go:379-429`. **This is the existing dynamic (un)subscribe hook.** -- **Memory-effectiveness levers (R2):** each stream has a **bounded** `mb.MB[drpc.Message]` queue (default size **100**), and `stream.write` uses `TryAdd` — **non-blocking, drops on overflow** — `net/streampool/stream.go:32-44`. Outbound dialing runs through a bounded `ExecPool` worker pool (`sendpool.go`). There is **no per-message persistence and no unbounded buffering** anywhere on this path. - -### 4.3 Message envelope & multiplexing - -- Generic space envelope `ObjectSyncMessage{spaceId, requestId, replyId, payload []byte, objectId, objectType}` — `commonspace/spacesyncproto/spacesync.pb.go:591-601`, proto at `spacesync.proto:100-108`. -- `objectId` **multiplexes many logical channels over one physical stream**; `objectType` enum is `{Tree=0, Acl=1, KeyValue=2}` — `spacesync.proto:349-353`. -- Wire wrapper `HeadUpdate` implements `drpc.Message` + a `peerMessage` tag interface (`SetPeerId`, `Copy`) so one message can be copied and stamped per destination during fan-out — `commonspace/sync/objectsync/objectmessages/headupdate.go:58-140`. Underlying `ObjectSyncMessage` is pooled via `sync.Pool` (`headupdate.go:13-38`). -- Encoding is protobuf (vtproto), optionally snappy-compressed, negotiated in handshake — `net/rpc/encoding/`. - -### 4.4 The existing space-level pub/sub (**most important prior art**) - -any-sync **already implements a coarse pub/sub where the "topic" is an entire `spaceId`**, over a single long-lived bidirectional stream: - -- **The stream:** `rpc ObjectSyncStream(stream ObjectSyncMessage) returns (stream ObjectSyncMessage)` — bidi, long-lived, one per (peer, connection) — `spacesync.proto:40`, server iface `spacesync_drpc.pb.go:245`. -- **The subscribe control frame:** `SpaceSubscription{ SpaceIds []string; Action }` with `SpaceSubscriptionAction { Subscribe=0, Unsubscribe=1 }` — `spacesync.proto:245-256`. It is carried **inside `ObjectSyncMessage.payload` with an empty `spaceId`**. -- **The wiring** (reference impl `commonspace/spaceutils_test.go:468-540`): on `OpenStream`, the client opens `ObjectSyncStream` and immediately `Send`s a `SpaceSubscription{Subscribe, [spaceId]}`. On the receive side, `HandleMessage` sees the empty-`spaceId` frame and calls `streamPool.AddTagsCtx(ctx, spaceIds...)` — tagging the stream so subsequent `Broadcast(msg, spaceId)` reaches it. Non-control frames route to `space.HandleMessage`. -- **Server side** registers the inbound stream into its own pool: `streamPool.ReadStream(stream, 100)` — `commonspace/spacerpc_test.go:170-172` — then pushes head-updates back down the same stream via `Broadcast(msg, spaceId)`. -- **Subscribe is also kicked during head-sync**: `diffsyncer.subscribe` builds `SpaceSubscription{Subscribe}` and sends it after a successful `SpacePush` — `commonspace/headsync/diffsyncer.go:275-293`. - -**Reading:** a stateless *topic* pub/sub is, structurally, a **refinement of this existing mechanism** to finer, ephemeral topics *within* a space — the same subscribe/unsubscribe control-frame pattern and the same tag-index fan-out. The delta is (i) a topic namespace below spaceId, (ii) an ephemeral message type that is **not** routed into the sync/DAG engine, and (iii) publish/subscribe ACL gating. - -> **⚠️ User steer (recorded concern) — do NOT reuse `ObjectSyncStream` itself.** `ObjectSyncStream` is an *upper-level* channel: every frame on it is routed into the sync engine (`space.HandleMessage` → `SyncService.HandleMessage` → per-`objectId` `multiqueue` → `objectSync.HandleHeadUpdate`, §4.5). Multiplexing ephemeral pub/sub onto that same stream **couples pub/sub to sync**: they share one bounded queue (size 100, drop-on-overflow), so a pub/sub burst can starve or drop sync (head-of-line blocking) and vice-versa; they share the sync dispatch/backpressure path; and it muddles fire-and-forget ephemera with DAG anti-entropy correctness. **Preferred direction: a *separate* DRPC stream on a *separate* sub-connection**, with its own tags, its own read/write loops, and its own bounded queues — fully isolated from sync flow control. This is cheap on the substrate: peers already multiplex many DRPC sub-streams over a single `MultiConn` (QUIC/yamux), and maintain a pool of reusable sub-connections — a "separate sub-connection" is another *multiplexed* stream, **not** a new transport/TLS handshake (`net/transport/transport.go:40-64` `MultiConn.Open`; `net/peer/peer.go:109-110,270-297` sub-conn pool; QUIC `MaxIncomingStreams=128` `net/transport/quic/quic.go:52`). What is reusable is the *pattern* (subscribe control-frame + `StreamPool` tag-index `Broadcast`) and the `StreamPool`/`StreamHandler` plumbing — instantiated as its own stream, not layered onto the sync stream. See Section 7, tension #2. - -### 4.5 Sync service dispatch - -- `SyncService.BroadcastMessage` → `peerManager.BroadcastMessage` — `commonspace/sync/sync.go:97-99`. -- `SyncService.HandleMessage` enqueues onto a **per-`objectId`** `multiqueue.MultiQueue` (size 100; **overflow silently dropped** via `mb.ErrOverflowed`) → `objectSync.HandleHeadUpdate` → object resolved by `objectId` — `sync.go:101-129`, `commonspace/sync/objectsync/synchandler.go:58-79`. -- Note the existing bounded-queue + drop-on-overflow discipline is already the R2 pattern; a pub/sub path would want the same. - -### 4.6 Node topology & who relays (R1 + topology decision) - -- Node roles: `tree`(=sync node), `consensus`, `file`, `coordinator`, `namingNode`, `paymentProcessingNode` — `nodeconf/config.go:20-30`. A "client" is any account not in the node list. -- **Responsible nodes via consistent hash:** `NodeIds(spaceId)` returns the tree nodes on the chash ring for that space; `ReplicationFactor = 3` ⇒ **3 responsible sync nodes per space** — `nodeconf/nodeconf.go:61-79,133-176`, `nodeconf/service.go:22-25`. Only `tree` nodes are ring members. -- **Space peer set = responsible sync nodes + directly-connected clients.** Sync nodes are the *always-reachable* members; client↔client is possible (one-to-one spaces, local discovery) but not guaranteed — `commonspace/peermanager/peermanager.go:17-34`, reference `commonspace/sync/synctest/testpeermanager.go:36-66`. -- **Implication:** the natural fan-out hub for a space is its responsible sync node(s). But those nodes are **semi-trusted relays that cannot read space content** — which is why R3/encryption (Section 6) is load-bearing, and why a broker-style design here is not a trusted broker. - -### 4.7 App framework (how a new component wires in) - -- Component registry with `Init(a)`/`Run(ctx)`/`Close(ctx)` + `Name()`; `MustComponent[T]` lookup; **per-space child app** (`ChildApp`) gives each space an isolated component graph — `app/app.go:34-52,133-207,226-280`; space graph built in `commonspace/spaceservice.go:188-260`. -- Adding a new DRPC service is a known, mechanical path (proto + `Makefile` generate line + `DRPCRegisterXxx` on server + client component + app registration) — `Makefile:23-32,47-60`; reference client component `coordinator/subscribeclient/client.go`. - -### 4.8 ACL / access control (R3) - -- **Permission ladder:** `None=0, Owner=1, Admin=2, Writer=3, Reader=4, Guest=5` — `commonspace/object/acl/aclrecordproto/aclrecord.pb.go:71-79`. Helpers: `CanWrite()` = Admin|Writer|Owner; **there is no `CanRead()`** — "can read" is expressed as `!NoPermissions()` (any non-`None`) — `commonspace/object/acl/list/models.go:79-152`. -- **Authorization is cryptographic and content-based, NOT a per-message live check:** - - **Writes** are enforced when a signed record/change is *validated on apply*: each change carries author `Identity` + signature, checked against `AclState.PermissionsAtRecord(aclHeadId, identity).CanWrite()` — `commonspace/object/tree/objecttree/objecttreevalidator.go:182-189`; KV analog `keyvaluestorage/storage.go:117`. - - **Reads** are enforced by **encryption**: content is encrypted with a per-space `ReadKey` handed (encrypted per member pubkey) only to members inside ACL records — `commonspace/object/acl/list/aclstate.go:50-59,152-195`; rotation on membership change `aclrecordbuilder.go:761-844`. -- **Identity vs peer:** device/peer key (libp2p, authenticates `peerId` at TLS) is distinct from the **account/identity key** (the ACL `Identity`). A node's inbound handshake uses `peerSignVerifier` to prove control of the account key, placing `identity` in the connection ctx — `net/secureservice/secureservice.go:107-174`, `credential.go:67-124`. -- **KEY FINDING for R3:** the shared sync layer performs **no per-message reader-authorization** and **no ACL check at message-handle time** — a grep across `commonspace/sync`, `commonspace/spacesyncproto`, `commonspace/headsync` finds no `Permissions/CanWrite/NoPermissions` usage. Membership is gated **at stream-open by the node** (that logic lives in `any-sync-node`), and content confidentiality relies on read-key **encryption** (non-members receive ciphertext they cannot decrypt). The coordinator-side `acl.AclService.Permissions(ctx, identity, spaceId)` exists for explicit checks — `acl/acl.go:172-180`. - ---- - -## 5. Prior art inside any-sync (what already exists to lean on or contrast) - -| Prior-art mechanism | Where | Relation to stateless pub/sub | -|---|---|---| -| **Space subscription over `ObjectSyncStream`** (`SpaceSubscription{Subscribe/Unsubscribe}` + `AddTagsCtx`) | `spacesync.proto:40,245-256`; `spaceutils_test.go:468-540` | **Direct precedent** — coarse pub/sub, topic == spaceId. A topic pub/sub generalizes this. | -| **`StreamPool.Broadcast(msg, tags...)`** tag-indexed fan-out | `net/streampool/streampool.go:360-377` | The reusable fan-out engine; topics could be additional tags. | -| **Coordinator `NotifySubscribe(req) → stream NotifySubscribeEvent`** | `coordinator.proto:57`, `coordinator/subscribeclient/{client,stream}.go` | Server-push subscription-stream pattern, but **coordinator-scoped** and fixed to enum event *types* (`InboxNewMessageEvent`, `NetworkConfigChangedEvent`) — not arbitrary space topics. Good template for a dedicated pub/sub service + auto-reconnect + `mb.MB` mailbox. | -| **Coordinator Inbox** (`InboxFetch` / `InboxAddMessage`, signed sender→receiver messages) | `coordinator.proto:51-54,434-473` | **Contrast / anti-pattern for "stateless":** this is *stateful* store-and-forward (persisted, fetch-by-offset, `hasMore`). Shows what stateless pub/sub deliberately is *not*. | -| **`mb.MB[T]` bounded mailbox** (`cheggaaa/mb/v3`) | `subscribeclient/stream.go:18`; stream queues `stream.go` | The idiomatic **memory-bounded** streaming buffer (R2) — bounded size, backpressure or drop. | -| **`multiqueue.MultiQueue` per-object sharded queue, drop-on-overflow** | `commonspace/sync/sync.go:101-129` | Existing R2 discipline for per-logical-channel inbound processing. | - ---- - -## 6. How R3 (ACL) specifically interacts with pub/sub — facts, not decisions - -The spec must resolve publish-permission and subscribe-permission against these substrate facts: - -- **Two distinct permissions are in play.** Publishing is a *write-like* action (`CanWrite()` ⇒ Admin/Writer/Owner). Subscribing/receiving is a *read-like* action (`!NoPermissions()` ⇒ any member incl. Reader/Guest). Presence/typing/cursors, however, are things a **Reader** plausibly should be allowed to *emit* — so "publish == CanWrite" may be too strict for the archetypal use case. **Open.** -- **No per-message ACL gate exists to reuse.** Enforcement today is either (a) node-side membership gating at stream/subscribe time, or (b) read-key encryption. A pub/sub design must choose one or both; there is no drop-in per-message reader check in the shared layer. -- **The relay node cannot be trusted with plaintext.** Consistent with the whole any-sync model, if payloads are encrypted with the space `ReadKey`, the relaying sync node routes ciphertext it cannot read — automatically enforcing read-confidentiality (non-members lack the key) at the cost of the node being unable to match on payload contents (fine) and potentially topic names (depends on whether topic strings are encrypted — **open**). -- **Key rotation on membership change is already handled** for stored content (`ReadKey` rotates, re-encrypted per remaining member — `aclrecordbuilder.go:761-844`). For *ephemeral* messages the question is whether pub/sub piggybacks the current `ReadKey` (so a removed member instantly loses the ability to decrypt new messages) or uses a separate ephemeral key. **Open.** -- **Publish authorization without a per-message check** implies either signing each ephemeral message (adds CPU + size — measure against R2) or relying on "only key-holders can produce decryptable messages" (confidentiality without authenticity — a receiver couldn't distinguish which member sent it, or prevent a Reader from spoofing). **Open trade-off.** - ---- - -## 7. Open design tensions the spec must resolve (NOT resolved here) - -Grouped by the requirement they stress. Each is a genuine fork with substrate consequences noted. - -**Topology & transport (R1)** -1. **Relay vs mesh.** Fan-out through the 3 responsible sync nodes (always reachable, broker-like, but untrusted-for-content) vs direct client↔client (P2P, GossipSub-like, not always reachable) vs hybrid. Substrate favors relay-through-node for reachability; mesh fits the decentralization ethos but has no guaranteed connectivity. -2. **Dedicated pub/sub stream on its own sub-connection vs reusing `ObjectSyncStream`.** **User steer (recorded, §4.4): do not reuse `ObjectSyncStream`** — it is upper-level and tied to the sync engine, so sharing it couples pub/sub and sync (shared bounded queue, head-of-line blocking, intertwined backpressure). The preferred direction is a **separate DRPC stream over a separate multiplexed sub-connection** (cheap: another sub-stream on the existing `MultiConn`, not a new handshake), reusing only the *pattern* (subscribe control-frame + `StreamPool` tag-index `Broadcast`) and the `StreamPool`/`StreamHandler` plumbing — not the sync stream. Remaining sub-decision for the spec: does the separate stream belong to a **new dedicated `pubsub` DRPC service** (à la coordinator `NotifySubscribe`, cleanest isolation of lifecycle/versioning) or a **new stream RPC added to the existing `SpaceSync` service** (fewer moving parts, same service registration)? Both are mechanically supported (Section 4.7); both keep pub/sub off the sync stream. - -**Topic model** -3. **Flat topics vs NATS-style hierarchy with wildcards** (`*`/`>` or `+`/`#`). Hierarchy+wildcards is expressive but forces the relay to run interest-matching per message (relay CPU/mem vs R2); flat topics map cleanly onto the existing tag index. If wildcards are wanted, the tag-index (`streamIdsByTag`, exact-match) is insufficient and a trie/matcher is required. -4. **Topic namespace & encryption of topic names.** Topics are scoped within a `spaceId`; are topic strings plaintext (relay can route on them but learns them) or derived/encrypted (relay routes on opaque handles)? Interacts with R3. - -**Statelessness & memory (R2)** -5. **Routing statelessness (Section 1 axis b).** Relay holds a topic→subscriber interest table (bandwidth-efficient, small transient state) vs relay broadcasts all space traffic and clients filter (zero routing state, wasteful). "Memory-effective" likely means the former with strictly-bounded tables, but confirm. -6. **Backpressure policy.** The substrate default is **drop-on-overflow** (`TryAdd`, `multiqueue` drop). For at-most-once stateless semantics that is coherent — but the spec should state it explicitly (slow subscriber ⇒ dropped messages, never memory growth). -7. **Fan-out amplification.** One publish × N subscribers × up to 3 relaying nodes. Where does the copy happen (node-side fan-out preferred so the publisher uploads once)? Bounds on N, message size, publish rate. - -**Delivery semantics** -8. **Plain fan-out only, or also queue-groups (1-of-N)?** Queue groups need group-membership state on the relay; likely out of scope for v1 but should be an explicit non-goal or goal. -9. **Presence lifecycle.** If presence/typing/cursors are in scope, adopt a Yjs-awareness-style **heartbeat + timeout + explicit-leave** (`null` state, ~15 s re-announce / ~30 s expiry) — otherwise "who is online" cannot be derived from fire-and-forget alone. Decide whether presence is a first-class feature or just an example payload. -10. **Reconnection.** Stateless ⇒ messages during a disconnect are lost by definition; on reconnect a subscriber re-subscribes and gets only new messages. Confirm no "catch-up" expectation (that would make it stateful). - -**ACL (R3)** — the four open items in Section 6 (publish vs subscribe permission level; encryption of payload/topic; ephemeral vs space `ReadKey`; per-message signing vs encryption-only). - -**Abuse resistance** -11. GossipSub-style peer scoring / rate-limiting is absent here; the substrate has a per-peer request rate-limit in `requestmanager` but nothing pub/sub-specific. Decide whether publish-rate limiting / anti-spam is in scope (a Reader flooding a topic). - ---- - -## 8. Success criteria the spec should be measured against - -- **R1:** rides existing transports + DRPC + StreamPool; no new side-channel; ideally reuses the `ObjectSyncStream`/tag machinery or cleanly mirrors the `NotifySubscribe` pattern. -- **R2:** per-connection and per-node footprint is **bounded and ephemeral** — bounded queues, drop (not buffer) on overflow, no persistence, transient routing tables sized O(active subscriptions). No path that grows memory with message volume or offline duration. -- **R3:** subscribe and publish are gated by ACL (membership + permission level), and confidentiality holds against the untrusted relay (encryption boundary preserved). A removed member loses access to new messages. -- **Semantics:** documented at-most-once, fire-and-forget, no ordering/dedup guarantees — matching the core-NATS/Redis family, explicitly *not* the stateful Inbox/DAG/KV families. - ---- - -## 9. Sources - -**any-sync (this repo, `main`)** — grounded `file:line` references inline throughout Section 4–6; key anchors: `net/streampool/streampool.go`, `commonspace/spacesyncproto/protos/spacesync.proto`, `commonspace/sync/sync.go`, `commonspace/object/acl/list/aclstate.go`, `nodeconf/nodeconf.go`, `coordinator/subscribeclient/`, `coordinator/coordinatorproto/protos/coordinator.proto`. - -**External:** -- NATS — Publish-Subscribe: https://docs.nats.io/nats-concepts/core-nats/pubsub -- NATS — Subjects & wildcards: https://docs.nats.io/nats-concepts/subjects -- NATS — Queue Groups: https://docs.nats.io/nats-concepts/core-nats/queue -- libp2p GossipSub (design, mesh/fanout, IHAVE/IWANT, peer scoring): https://github.com/libp2p/specs/tree/master/pubsub/gossipsub -- Yjs awareness protocol: https://github.com/yjs/y-protocols/blob/master/PROTOCOL.md and https://docs.yjs.dev/api/about-awareness -- Redis pub/sub: https://redis.io/docs/latest/develop/interact/pubsub/ -- MQTT topics/wildcards/QoS: https://mqtt.org/ (spec) -- Phoenix Channels & Presence: https://hexdocs.pm/phoenix/Phoenix.Channel.html , https://hexdocs.pm/phoenix/Phoenix.Presence.html -- Matrix ephemeral events (typing/presence EDUs): https://spec.matrix.org/ (server-server EDUs) From 8be97eee0f274c1d29110dae6c98c29a98c06654 Mon Sep 17 00:00:00 2001 From: Roman Khafizianov Date: Wed, 15 Jul 2026 20:41:31 +0200 Subject: [PATCH 5/5] fix(pubsub): move rpc error offset 800 -> 1100 800 now belongs to filesyncv2 (commonfile/fileproto/fileprotov2) on the v0.13.x base; the any-sync core <1000 offset range is fully allocated, so pubsub takes the next free offset, 1100. Hand-edited the generated descriptor (the pinned protoc-gen plugins are Linux binaries and won't run locally; regenerating with local plugins introduced unrelated drpc/vtproto version drift). Verified the rawDesc varint round-trips to 1100 -> "ErrorOffset". Record the allocation in docs/rpc-error-offsets.md (brought in from v0.13.x) and reconcile its "core < 1000" rule with the now-exhausted core range. --- .../pubsub/pubsubproto/protos/pubsub.proto | 4 +- commonspace/pubsub/pubsubproto/pubsub.pb.go | 24 ++-- docs/rpc-error-offsets.md | 111 ++++++++++++++++++ 3 files changed, 126 insertions(+), 13 deletions(-) create mode 100644 docs/rpc-error-offsets.md diff --git a/commonspace/pubsub/pubsubproto/protos/pubsub.proto b/commonspace/pubsub/pubsubproto/protos/pubsub.proto index ecc49672c..ba563feb4 100644 --- a/commonspace/pubsub/pubsubproto/protos/pubsub.proto +++ b/commonspace/pubsub/pubsubproto/protos/pubsub.proto @@ -24,7 +24,9 @@ enum ErrCodes { TopicNotOwned = 6; // InvalidTopic - malformed topic or pattern: bad wildcard placement, reserved chars, non-canonical form InvalidTopic = 7; - ErrorOffset = 800; + // ErrorOffset - error-code band for this proto package; see docs/rpc-error-offsets.md + // (100..900 are the any-sync core bands; pubsub uses 1100) + ErrorOffset = 1100; } // PubSubMessage is the single frame type carried by PubSubStream diff --git a/commonspace/pubsub/pubsubproto/pubsub.pb.go b/commonspace/pubsub/pubsubproto/pubsub.pb.go index ac1d45856..37141f5c8 100644 --- a/commonspace/pubsub/pubsubproto/pubsub.pb.go +++ b/commonspace/pubsub/pubsubproto/pubsub.pb.go @@ -39,21 +39,21 @@ const ( ErrCodes_TopicNotOwned ErrCodes = 6 // InvalidTopic - malformed topic or pattern: bad wildcard placement, reserved chars, non-canonical form ErrCodes_InvalidTopic ErrCodes = 7 - ErrCodes_ErrorOffset ErrCodes = 800 + ErrCodes_ErrorOffset ErrCodes = 1100 ) // Enum value maps for ErrCodes. var ( ErrCodes_name = map[int32]string{ - 0: "Unexpected", - 1: "NotAMember", - 2: "NotResponsible", - 3: "RateLimited", - 4: "TooManyTopics", - 5: "InvalidMessage", - 6: "TopicNotOwned", - 7: "InvalidTopic", - 800: "ErrorOffset", + 0: "Unexpected", + 1: "NotAMember", + 2: "NotResponsible", + 3: "RateLimited", + 4: "TooManyTopics", + 5: "InvalidMessage", + 6: "TopicNotOwned", + 7: "InvalidTopic", + 1100: "ErrorOffset", } ErrCodes_value = map[string]int32{ "Unexpected": 0, @@ -64,7 +64,7 @@ var ( "InvalidMessage": 5, "TopicNotOwned": 6, "InvalidTopic": 7, - "ErrorOffset": 800, + "ErrorOffset": 1100, } ) @@ -550,7 +550,7 @@ const file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDesc = "" + "\x0eInvalidMessage\x10\x05\x12\x11\n" + "\rTopicNotOwned\x10\x06\x12\x10\n" + "\fInvalidTopic\x10\a\x12\x10\n" + - "\vErrorOffset\x10\xa0\x062J\n" + + "\vErrorOffset\x10\xcc\x082J\n" + "\x06PubSub\x12@\n" + "\fPubSubStream\x12\x15.pubsub.PubSubMessage\x1a\x15.pubsub.PubSubMessage(\x010\x01B Z\x1ecommonspace/pubsub/pubsubprotob\x06proto3" diff --git a/docs/rpc-error-offsets.md b/docs/rpc-error-offsets.md new file mode 100644 index 000000000..84e9d17c0 --- /dev/null +++ b/docs/rpc-error-offsets.md @@ -0,0 +1,111 @@ +# RPC Error Code Offsets + +## Overview + +any-sync maps typed RPC errors onto numeric [dRPC](https://storj.io/drpc) error codes +through a single **process-global** registry in +[`net/rpc/rpcerr`](../net/rpc/rpcerr/registry.go). + +```go +// net/rpc/rpcerr/registry.go +var errsMap = make(map[uint64]error) // global, one per process + +func RegisterErr(err error, code uint64) error { + if e, ok := errsMap[code]; ok { + panic(fmt.Errorf("attempt to register error with existing code: %d ...", code)) + } + // ... +} + +type ErrGroup int64 +func (g ErrGroup) Register(err error, code uint64) error { + return RegisterErr(err, uint64(g)+code) // registers at offset+code +} +``` + +Each proto/service declares an `ErrorOffset` constant and registers its errors +through `rpcerr.ErrGroup(ErrorOffset)`; a concrete error's wire code is +`ErrorOffset + localCode`. + +### Why this needs coordinating across repos + +`errsMap` is **global to the running binary**, and registration happens in +`var`/`init` blocks at import time. A node binary routinely links **any-sync +plus one or more downstream repos** (e.g. `any-sync-node` also registers +`nodesync` errors). If any two groups ever register the same `offset + code`, +the program **panics at startup** — before any request is served. + +So offsets are a shared namespace across the whole anyproto ecosystem, not just +within one repo. **This file is the source of truth for who owns which offset.** + +## Allocation rules + +1. **Offsets are multiples of 100.** Each group owns the band + `[offset, offset+99]`. +2. **Local codes must stay in `0..99`** so a group never bleeds into the next + band. (The `ErrorOffset` enum value itself is only the base — it is never + registered as a code.) +3. **Local code `0` is the group's `Unexpected`/fallback** (registers at + `offset+0`), mirroring `filesync.ErrCodes.Unexpected = 0`. +4. **any-sync core historically used `< 1000`.** That range is now fully + allocated (100–900), so new **core** groups take the next free `>= 1000` + offset (e.g. `pubsub` at 1100). Downstream repos that import any-sync also + use **`>= 1000`**; the two now share that space, so always claim the next + free offset from the tables below and add a row. +5. When you add a group, **pick the next free offset and add a row below.** + +### Globally reserved low codes + +The base registry claims two codes directly (outside any group), so **no group +may use offset `0`**: + +| Code | Name | Source | +|------|------|--------| +| 1 | `Unexpected` | `net/rpc/rpcerr/registry.go` | +| 2 | `Closed` | `net/rpc/rpcerr/registry.go` | + +## Reserved offsets + +### any-sync (core, `< 1000`) + +| Offset | Band | Proto / service | Package | +|--------|------|-----------------|---------| +| 100 | 100–199 | `spacesync` | `commonspace/spacesyncproto` | +| 200 | 200–299 | `filesync` (File, v1) | `commonfile/fileproto` | +| 300 | 300–399 | `coordinator` | `coordinator/coordinatorproto` | +| 400 | 400–499 | `treechange` | `commonspace/object/tree/treechangeproto` | +| 500 | 500–599 | `consensus` | `consensus/consensusproto` | +| 600 | 600–699 | `payment` | `paymentservice/paymentserviceproto` | +| 700 | 700–799 | `limiter` | `net/rpc/limiter/limiterproto` | +| 800 | 800–899 | `filesyncv2` (FileV2, v2 broker) | `commonfile/fileproto/fileprotov2` | +| 900 | 900–999 | `filesyncp2p` (FileP2P, p2p file transfer) | `commonfile/fileproto/filep2p` | + +The `< 1000` range is now full; further **core** groups continue in the +`>= 1000` tables below (see `pubsub` at 1100). + +### Core (any-sync) & downstream repos (`>= 1000`) + +Downstream groups live in other repositories but share this global registry +whenever their binary also links any-sync. Core any-sync groups appear here too +once the `< 1000` range is exhausted. + +| Offset | Band | Proto / service | Repo · package | +|--------|------|-----------------|----------------| +| 1000 | 1000–1099 | `nodesync` | `any-sync-node` · `nodesync/nodesyncproto` | +| 1100 | 1100–1199 | `pubsub` (space-scoped stateless pub/sub) | any-sync **(core)** · `commonspace/pubsub/pubsubproto` | +| 1200 | 1200–1299 | `push` | `anytype-push-server` · `pushclient/pushapi` | +| 1300 | 1300–1399 | `bobrik` | `bobrik-clusterctl` · `bobrikclient/bobrikapi` | +| 1400+ | — | _free_ | — | + +## Adding a new group + +1. Choose the next free offset from the tables above (core: next free `< 1000`; + downstream: next free `>= 1000`). +2. In the `.proto`, add `ErrorOffset = ;` to the service's `ErrCodes` + enum, with local codes `0..N` (`0` = `Unexpected`). +3. Register in a Go errors file via `rpcerr.ErrGroup(_ErrorOffset)` + (see `commonfile/fileproto/fileprotov2/fileprotov2err/fileprotov2err.go` for + the current template). +4. **Add a row to the correct table in this file.** +5. Build and run any test — a colliding offset panics at init, so a green test + run proves the new band is clear.