diff --git a/Makefile b/Makefile index 5fdf6cfa3..16fa75fb0 100644 --- a/Makefile +++ b/Makefile @@ -47,12 +47,15 @@ proto: $(call generate_drpc,$(PKGMAP),commonspace/spacesyncproto/protos) $(call generate_drpc,$(PKGMAP),commonspace/clientspaceproto/protos) $(call generate_drpc,$(PKGMAP),commonfile/fileproto/protos) + $(call generate_drpc,,commonfile/fileproto/fileprotov2/protos) + $(call generate_drpc,,commonfile/fileproto/filep2p/protos) $(call generate_drpc,$(PKGMAP),net/streampool/testservice/protos) $(call generate_drpc,$(PKGMAP),net/endtoendtest/testpeer/testproto/protos) $(call generate_drpc,,net/secureservice/handshake/handshakeproto/protos) $(call generate_drpc,,net/rpc/limiter/limiterproto/protos) + $(call generate_drpc,,commonspace/pubsub/pubsubproto/protos) $(call generate_drpc,$(PKGMAP),coordinator/coordinatorproto/protos) $(call generate_drpc,,consensus/consensusproto/protos) $(call generate_drpc,,identityrepo/identityrepoproto/protos) diff --git a/app/ldiff/diff.go b/app/ldiff/diff.go index d5781b4c6..da719aaea 100644 --- a/app/ldiff/diff.go +++ b/app/ldiff/diff.go @@ -1,7 +1,7 @@ // Package ldiff provides a container of elements with fixed id and changeable content. // Diff can calculate the difference with another diff container (you can make it remote) with minimum hops and traffic. // -//go:generate mockgen -destination mock_ldiff/mock_ldiff.go github.com/anyproto/any-sync/app/ldiff Diff,Remote,DiffContainer +//go:generate mockgen -destination mock_ldiff/mock_ldiff.go github.com/anyproto/any-sync/app/ldiff Diff,Remote package ldiff import ( diff --git a/app/ldiff/diffcontainer.go b/app/ldiff/diffcontainer.go deleted file mode 100644 index c39fbd3ca..000000000 --- a/app/ldiff/diffcontainer.go +++ /dev/null @@ -1,72 +0,0 @@ -package ldiff - -import ( - "context" - "encoding/hex" - - "github.com/zeebo/blake3" -) - -type RemoteTypeChecker interface { - DiffTypeCheck(ctx context.Context, diffContainer DiffContainer) (needsSync bool, diff Diff, err error) -} - -type DiffContainer interface { - DiffTypeCheck(ctx context.Context, typeChecker RemoteTypeChecker) (needsSync bool, diff Diff, err error) - OldDiff() Diff - NewDiff() Diff - RemoveId(id string) error -} - -type diffContainer struct { - newDiff Diff - oldDiff Diff -} - -type Hasher struct { - hasher *blake3.Hasher -} - -func (h *Hasher) HashId(id string) string { - h.hasher.Reset() - h.hasher.WriteString(id) - return hex.EncodeToString(h.hasher.Sum(nil)) -} - -func NewHasher() *Hasher { - return &Hasher{hashersPool.Get().(*blake3.Hasher)} -} - -func ReleaseHasher(hasher *Hasher) { - hashersPool.Put(hasher.hasher) -} - -func (d *diffContainer) NewDiff() Diff { - return d.newDiff -} - -func (d *diffContainer) OldDiff() Diff { - return d.oldDiff -} - -func (d *diffContainer) Set(elements ...Element) { - d.newDiff.Set(elements...) - d.oldDiff.Set(elements...) -} - -func (d *diffContainer) RemoveId(id string) error { - _ = d.newDiff.RemoveId(id) - _ = d.oldDiff.RemoveId(id) - return nil -} - -func (d *diffContainer) DiffTypeCheck(ctx context.Context, typeChecker RemoteTypeChecker) (needsSync bool, diff Diff, err error) { - return typeChecker.DiffTypeCheck(ctx, d) -} - -func NewDiffContainer(new, old Diff) DiffContainer { - return &diffContainer{ - newDiff: new, - oldDiff: old, - } -} diff --git a/app/ldiff/hasher.go b/app/ldiff/hasher.go new file mode 100644 index 000000000..c0ec71a59 --- /dev/null +++ b/app/ldiff/hasher.go @@ -0,0 +1,25 @@ +package ldiff + +import ( + "encoding/hex" + + "github.com/zeebo/blake3" +) + +type Hasher struct { + hasher *blake3.Hasher +} + +func (h *Hasher) HashId(id string) string { + h.hasher.Reset() + h.hasher.WriteString(id) + return hex.EncodeToString(h.hasher.Sum(nil)) +} + +func NewHasher() *Hasher { + return &Hasher{hashersPool.Get().(*blake3.Hasher)} +} + +func ReleaseHasher(hasher *Hasher) { + hashersPool.Put(hasher.hasher) +} diff --git a/app/ldiff/mock_ldiff/mock_ldiff.go b/app/ldiff/mock_ldiff/mock_ldiff.go index 31e80ddb3..b4113335d 100644 --- a/app/ldiff/mock_ldiff/mock_ldiff.go +++ b/app/ldiff/mock_ldiff/mock_ldiff.go @@ -1,9 +1,9 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/anyproto/any-sync/app/ldiff (interfaces: Diff,Remote,DiffContainer) +// Source: github.com/anyproto/any-sync/app/ldiff (interfaces: Diff,Remote) // // Generated by this command: // -// mockgen -destination mock_ldiff/mock_ldiff.go github.com/anyproto/any-sync/app/ldiff Diff,Remote,DiffContainer +// mockgen -destination mock_ldiff/mock_ldiff.go github.com/anyproto/any-sync/app/ldiff Diff,Remote // // Package mock_ldiff is a generated GoMock package. @@ -227,85 +227,3 @@ func (mr *MockRemoteMockRecorder) Ranges(ctx, ranges, resBuf any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Ranges", reflect.TypeOf((*MockRemote)(nil).Ranges), ctx, ranges, resBuf) } - -// MockDiffContainer is a mock of DiffContainer interface. -type MockDiffContainer struct { - ctrl *gomock.Controller - recorder *MockDiffContainerMockRecorder - isgomock struct{} -} - -// MockDiffContainerMockRecorder is the mock recorder for MockDiffContainer. -type MockDiffContainerMockRecorder struct { - mock *MockDiffContainer -} - -// NewMockDiffContainer creates a new mock instance. -func NewMockDiffContainer(ctrl *gomock.Controller) *MockDiffContainer { - mock := &MockDiffContainer{ctrl: ctrl} - mock.recorder = &MockDiffContainerMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockDiffContainer) EXPECT() *MockDiffContainerMockRecorder { - return m.recorder -} - -// DiffTypeCheck mocks base method. -func (m *MockDiffContainer) DiffTypeCheck(ctx context.Context, typeChecker ldiff.RemoteTypeChecker) (bool, ldiff.Diff, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DiffTypeCheck", ctx, typeChecker) - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(ldiff.Diff) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// DiffTypeCheck indicates an expected call of DiffTypeCheck. -func (mr *MockDiffContainerMockRecorder) DiffTypeCheck(ctx, typeChecker any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiffTypeCheck", reflect.TypeOf((*MockDiffContainer)(nil).DiffTypeCheck), ctx, typeChecker) -} - -// NewDiff mocks base method. -func (m *MockDiffContainer) NewDiff() ldiff.Diff { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "NewDiff") - ret0, _ := ret[0].(ldiff.Diff) - return ret0 -} - -// NewDiff indicates an expected call of NewDiff. -func (mr *MockDiffContainerMockRecorder) NewDiff() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewDiff", reflect.TypeOf((*MockDiffContainer)(nil).NewDiff)) -} - -// OldDiff mocks base method. -func (m *MockDiffContainer) OldDiff() ldiff.Diff { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "OldDiff") - ret0, _ := ret[0].(ldiff.Diff) - return ret0 -} - -// OldDiff indicates an expected call of OldDiff. -func (mr *MockDiffContainerMockRecorder) OldDiff() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OldDiff", reflect.TypeOf((*MockDiffContainer)(nil).OldDiff)) -} - -// RemoveId mocks base method. -func (m *MockDiffContainer) RemoveId(id string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RemoveId", id) - ret0, _ := ret[0].(error) - return ret0 -} - -// RemoveId indicates an expected call of RemoveId. -func (mr *MockDiffContainerMockRecorder) RemoveId(id any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveId", reflect.TypeOf((*MockDiffContainer)(nil).RemoveId), id) -} diff --git a/app/ocache/entry.go b/app/ocache/entry.go index 87b20db59..102125ac1 100644 --- a/app/ocache/entry.go +++ b/app/ocache/entry.go @@ -23,10 +23,16 @@ type entry struct { lastUsage time.Time load chan struct{} loadErr error - value Object - close chan struct{} - mx sync.Mutex - cancel context.CancelFunc + // loadAborted marks a failed load whose OWN context was already done + // when the loadFunc returned — the load was killed (its first caller + // went away, or the cache is closing), not refused by the loadFunc. + // Written by oCache.load before the load channel closes; read only + // after <-load, like loadErr. + loadAborted bool + value Object + close chan struct{} + mx sync.Mutex + cancel context.CancelFunc } func newEntry(id string, value Object, state entryState) *entry { @@ -97,7 +103,10 @@ func (e *entry) waitClose(ctx context.Context, id string) (res bool, err error) } } -func (e *entry) setClosing(wait bool) (prevState, curState entryState) { +// setClosing transitions the entry to closing. With wait it blocks until another +// closer is done with it, bounded by ctx: that closer may be inside a TryClose +// that waits on an unresponsive peer. +func (e *entry) setClosing(ctx context.Context, wait bool) (prevState, curState entryState, err error) { e.mx.Lock() prevState = e.state curState = e.state @@ -112,7 +121,14 @@ func (e *entry) setClosing(wait bool) (prevState, curState entryState) { if !wait { return } - <-waitCh + select { + case <-waitCh: + case <-ctx.Done(): + e.mx.Lock() + curState = e.state + e.mx.Unlock() + return prevState, curState, ctx.Err() + } e.mx.Lock() } if e.state != entryStateClosed { diff --git a/app/ocache/ocache.go b/app/ocache/ocache.go index 7c63ec58f..7d2acfbfb 100644 --- a/app/ocache/ocache.go +++ b/app/ocache/ocache.go @@ -21,6 +21,8 @@ var ( var ( defaultTTL = time.Minute defaultGC = 20 * time.Second + // bounds Close against entries the gc is already closing + closeTimeout = 10 * time.Second ) var log = logger.NewNamed("ocache") @@ -56,6 +58,8 @@ func New(loadFunc LoadFunc, opts ...Option) OCache { gc: defaultGC, closeCh: make(chan struct{}), log: log.Sugar(), + + closeTimeout: closeTimeout, } for _, o := range opts { if o != nil { @@ -119,40 +123,72 @@ type oCache struct { closeCh chan struct{} log *zap.SugaredLogger metrics *metrics + + closeTimeout time.Duration } +// maxLoadRetries bounds Get's re-attempts after an aborted load (see the +// retry in Get). Each retry is a fresh load — the failed entry is deleted +// — so the bound only matters under a storm of loads that keep getting +// killed mid-flight. +const maxLoadRetries = 3 + func (c *oCache) Get(ctx context.Context, id string) (value Object, err error) { var ( - e *entry - ok bool - load bool + counted bool + retries int ) -Load: - c.mu.Lock() - if c.closed { + for { + c.mu.Lock() + if c.closed { + c.mu.Unlock() + return nil, ErrClosed + } + e, ok := c.data[id] + load := false + if !ok { + e = newEntry(id, nil, entryStateLoading) + load = true + c.data[id] = e + } + e.lastUsage = time.Now() c.mu.Unlock() - return nil, ErrClosed - } - if e, ok = c.data[id]; !ok { - e = newEntry(id, nil, entryStateLoading) - load = true - c.data[id] = e - } - e.lastUsage = time.Now() - c.mu.Unlock() - reload, err := e.waitClose(ctx, id) - if err != nil { - return nil, err - } - if reload { - goto Load - } - c.metricsGet(!load) - if load { - c.load(ctx, id, e) - return e.value, e.loadErr + reload, err := e.waitClose(ctx, id) + if err != nil { + return nil, err + } + if reload { + continue + } + if !counted { + c.metricsGet(!load) + counted = true + } + if load { + c.load(ctx, id, e) + value, err = e.value, e.loadErr + } else { + value, err = e.waitLoad(ctx, id) + } + // A load runs under the context of whichever caller arrived first + // and its result is shared with every concurrent waiter. If that + // first caller goes away mid-load (its request finished, its + // per-round timeout fired), the load is killed with an error that + // has nothing to do with the waiters — retry instead of surfacing + // it: the failed entry is already deleted, so the retry starts a + // fresh load owned by a live context. loadAborted distinguishes a + // killed load from a loadFunc that failed on its own (an internal + // dial timeout returns context.DeadlineExceeded with the load's ctx + // still alive — that is a verdict, not an abort, and is never + // retried). The ctx.Err() check both scopes the retry to callers + // that are still alive and proves waitLoad returned via the load + // channel, making the loadAborted read safe. + if err != nil && ctx.Err() == nil && e.loadAborted && retries < maxLoadRetries { + retries++ + continue + } + return value, err } - return e.waitLoad(ctx, id) } func (c *oCache) Pick(ctx context.Context, id string) (value Object, err error) { @@ -172,6 +208,10 @@ func (c *oCache) load(ctx context.Context, id string, e *entry) { ctx, cancel := context.WithCancel(ctx) e.setCancel(cancel) value, err := c.loadFunc(ctx, id) + // Read before cancel(): a done ctx here means the load was killed + // (first caller gone, or cancelLoad on cache close) rather than the + // loadFunc failing on its own — recorded so Get's waiters can retry. + aborted := ctx.Err() != nil cancel() c.mu.Lock() @@ -181,6 +221,7 @@ func (c *oCache) load(ctx context.Context, id string, e *entry) { } if err != nil { e.loadErr = err + e.loadAborted = aborted delete(c.data, id) } else { e.value = value @@ -215,10 +256,19 @@ func (c *oCache) closeAndDelete(e *entry) { } func (c *oCache) remove(ctx context.Context, e *entry) (ok bool, err error) { - if _, err = e.waitLoad(ctx, e.id); err != nil { + return c.removeCtx(ctx, ctx, e) +} + +// loadCtx bounds waiting for an in-flight load, closingCtx bounds waiting for +// another closer to release the entry +func (c *oCache) removeCtx(loadCtx, closingCtx context.Context, e *entry) (ok bool, err error) { + if _, err = e.waitLoad(loadCtx, e.id); err != nil { + return false, err + } + _, curState, err := e.setClosing(closingCtx, true) + if err != nil { return false, err } - _, curState := e.setClosing(true) if curState == entryStateClosing { ok = true err = e.value.Close() @@ -263,7 +313,7 @@ func (c *oCache) TryRemove(id string) (ok bool, err error) { c.mu.Unlock() - prevState, _ := e.setClosing(false) + prevState, _, _ := e.setClosing(context.Background(), false) if prevState == entryStateClosing || prevState == entryStateClosed { return false, nil } @@ -357,7 +407,7 @@ func (c *oCache) GC() { c.mu.Unlock() closedNum := 0 for _, e := range toClose { - prevState, _ := e.setClosing(false) + prevState, _, _ := e.setClosing(context.Background(), false) if prevState == entryStateClosing || prevState == entryStateClosed { continue } @@ -396,8 +446,14 @@ func (c *oCache) Close() (err error) { toClose = append(toClose, e) } c.mu.Unlock() + // one deadline for the whole pass, spent only on entries another closer holds: + // that closer can be a gc stuck in TryClose on an unresponsive peer. Loads are + // already cancelled above, and value.Close takes no ctx, so uncontended entries + // still close normally once the deadline has passed. + closingCtx, cancel := context.WithTimeout(context.Background(), c.closeTimeout) + defer cancel() for _, e := range toClose { - if _, err := c.remove(context.Background(), e); err != nil && err != ErrNotExists { + if _, err := c.removeCtx(context.Background(), closingCtx, e); err != nil && err != ErrNotExists { c.log.With("object_id", e.id).Warnf("cache close: object close error: %v", err) } } diff --git a/app/ocache/ocache_test.go b/app/ocache/ocache_test.go index 4d3ed7917..96821b634 100644 --- a/app/ocache/ocache_test.go +++ b/app/ocache/ocache_test.go @@ -152,6 +152,102 @@ func TestOCache_Get(t *testing.T) { }) } +func TestOCache_GetForeignCtxRetry(t *testing.T) { + t.Run("waiter survives first caller's cancellation", func(t *testing.T) { + // The first caller owns the load's context; every concurrent Get + // shares the load's result. Pre-fix, cancelling the first caller + // failed the waiter with a context error it did not cause. + var ( + started = make(chan struct{}, 2) + release = make(chan struct{}) + calls atomic.Int32 + ) + c := New(func(ctx context.Context, id string) (Object, error) { + calls.Add(1) + started <- struct{}{} + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-release: + return &testObject{name: id}, nil + } + }) + ownerCtx, cancelOwner := context.WithCancel(context.Background()) + ownerErr := make(chan error, 1) + go func() { + _, err := c.Get(ownerCtx, "id") + ownerErr <- err + }() + <-started // the owner's load is in flight + waiterErr := make(chan error, 1) + go func() { + val, err := c.Get(context.Background(), "id") + if err == nil && val == nil { + err = errors.New("nil value") + } + waiterErr <- err + }() + time.Sleep(time.Millisecond * 10) // let the waiter join the in-flight load + cancelOwner() + require.Equal(t, context.Canceled, <-ownerErr, "the owner's own cancellation is its error") + <-started // the waiter retried: a fresh load, owned by its live ctx + close(release) + require.NoError(t, <-waiterErr, "a live waiter must not inherit the owner's cancellation") + assert.Equal(t, int32(2), calls.Load()) + assert.NoError(t, c.Close()) + }) + t.Run("no retry when the loadFunc fails with its own internal timeout", func(t *testing.T) { + // A loadFunc-internal deadline (e.g. a transport dial timeout) + // returns a context error while the load's own ctx is alive — + // that is a verdict about the resource, not a killed load, and + // must surface immediately instead of multiplying dial attempts. + var calls atomic.Int32 + c := New(func(ctx context.Context, id string) (Object, error) { + calls.Add(1) + return nil, fmt.Errorf("dial: %w", context.DeadlineExceeded) + }) + _, err := c.Get(context.Background(), "id") + require.ErrorIs(t, err, context.DeadlineExceeded) + assert.Equal(t, int32(1), calls.Load()) + assert.NoError(t, c.Close()) + }) + t.Run("retry is bounded under repeated aborts", func(t *testing.T) { + // White-box: a pre-poisoned entry (failed, aborted, never removed + // from the map — unlike a real load) makes every retry observe the + // same aborted result, so Get must give up at maxLoadRetries + // instead of spinning. + c := New(func(ctx context.Context, id string) (Object, error) { + return &testObject{name: id}, nil + }).(*oCache) + e := newEntry("id", nil, entryStateLoading) + e.loadErr = context.Canceled + e.loadAborted = true + close(e.load) + c.mu.Lock() + c.data["id"] = e + c.mu.Unlock() + _, err := c.Get(context.Background(), "id") + require.ErrorIs(t, err, context.Canceled) + c.mu.Lock() + delete(c.data, "id") + c.mu.Unlock() + assert.NoError(t, c.Close()) + }) + t.Run("no retry when the caller's own ctx is done", func(t *testing.T) { + var calls atomic.Int32 + c := New(func(ctx context.Context, id string) (Object, error) { + calls.Add(1) + return nil, ctx.Err() + }) + cctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := c.Get(cctx, "id") + require.ErrorIs(t, err, context.Canceled) + assert.Equal(t, int32(1), calls.Load()) + assert.NoError(t, c.Close()) + }) +} + func TestOCache_GC(t *testing.T) { t.Run("test gc expired object", func(t *testing.T) { c := New(func(ctx context.Context, id string) (value Object, err error) { @@ -487,7 +583,10 @@ func TestOCacheCancelWhenRemove(t *testing.T) { time.Sleep(time.Millisecond * 10) c.Close() <-stopLoad - require.Equal(t, context.Canceled, err) + // Close cancels the in-flight load; the caller's retry then observes + // the closed cache and reports ErrClosed — the actual reason — rather + // than the load's raw context.Canceled. + require.Equal(t, ErrClosed, err) } func TestOCacheFuzzy(t *testing.T) { @@ -748,3 +847,65 @@ func TestOCache_RemoveBusyRevertDoubleClose(t *testing.T) { } } } + +// An entry another closer already owns must not stall cache close: that closer +// can be a gc sitting in TryClose against an unresponsive peer. +func TestOCache_CloseBoundedByOtherCloser(t *testing.T) { + c := New(func(ctx context.Context, id string) (Object, error) { + return NewTestObject(id, true, nil), nil + }) + oc := c.(*oCache) + oc.closeTimeout = 20 * time.Millisecond + _, err := c.Get(ctx, "id") + require.NoError(t, err) + + oc.mu.Lock() + e := oc.data["id"] + oc.mu.Unlock() + _, curState, err := e.setClosing(ctx, false) + require.NoError(t, err) + require.Equal(t, entryState(entryStateClosing), curState) + + done := make(chan struct{}) + go func() { + _ = c.Close() + close(done) + }() + select { + case <-done: + case <-time.After(time.Second): + require.Fail(t, "Close blocked behind an entry owned by another closer") + } +} + +// The close deadline is spent only on entries another closer holds: healthy +// entries must still be closed after it has expired. +func TestOCache_CloseClosesHealthyEntriesAfterDeadline(t *testing.T) { + var objects []*testObject + c := New(func(ctx context.Context, id string) (Object, error) { + o := NewTestObject(id, true, nil) + objects = append(objects, o) + return o, nil + }) + oc := c.(*oCache) + oc.closeTimeout = time.Millisecond + for i := 0; i < 50; i++ { + _, err := c.Get(ctx, fmt.Sprint(i)) + require.NoError(t, err) + } + // one entry is held by another closer and will eat the whole deadline + oc.mu.Lock() + held := oc.data["0"] + oc.mu.Unlock() + _, _, err := held.setClosing(ctx, false) + require.NoError(t, err) + + require.NoError(t, c.Close()) + var notClosed []string + for _, o := range objects { + if o.name != "0" && !o.closeCalled { + notClosed = append(notClosed, o.name) + } + } + require.Empty(t, notClosed, "entries nobody else holds must still be closed after the deadline") +} diff --git a/app/olddiff/diff.go b/app/olddiff/diff.go deleted file mode 100644 index 11f1bb226..000000000 --- a/app/olddiff/diff.go +++ /dev/null @@ -1,322 +0,0 @@ -package olddiff - -import ( - "bytes" - "context" - "encoding/hex" - "errors" - "math" - "sync" - - "github.com/cespare/xxhash" - "github.com/huandu/skiplist" - "github.com/zeebo/blake3" - - "github.com/anyproto/any-sync/app/ldiff" - "github.com/anyproto/any-sync/commonspace/spacesyncproto" -) - -// New creates precalculated Diff container -// -// divideFactor - means how many hashes you want to ask for once -// -// it must be 2 or greater -// normal value usually between 4 and 64 -// -// compareThreshold - means the maximum count of elements remote diff will send directly -// -// if elements under range will be more - remote diff will send only hash -// it must be 1 or greater -// normal value between 8 and 64 -// -// Less threshold and divideFactor - less traffic but more requests -func New(divideFactor, compareThreshold int) ldiff.Diff { - return newDiff(divideFactor, compareThreshold) -} - -func newDiff(divideFactor, compareThreshold int) ldiff.Diff { - if divideFactor < 2 { - divideFactor = 2 - } - if compareThreshold < 1 { - compareThreshold = 1 - } - d := &diff{ - divideFactor: divideFactor, - compareThreshold: compareThreshold, - } - d.sl = skiplist.New(d) - d.ranges = newHashRanges(divideFactor, compareThreshold, d.sl) - d.ranges.dirty[d.ranges.topRange] = struct{}{} - d.ranges.recalculateHashes() - return d -} - -var hashersPool = &sync.Pool{ - New: func() any { - return blake3.New() - }, -} - -var ErrElementNotFound = errors.New("ldiff: element not found") - -type element struct { - ldiff.Element - hash uint64 -} - -// Diff contains elements and can compare it with Remote diff -type diff struct { - sl *skiplist.SkipList - divideFactor int - compareThreshold int - ranges *hashRanges - mu sync.RWMutex -} - -// Compare implements skiplist interface -func (d *diff) Compare(lhs, rhs interface{}) int { - lhe := lhs.(*element) - rhe := rhs.(*element) - if lhe.Id == rhe.Id { - return 0 - } - if lhe.hash > rhe.hash { - return 1 - } else if lhe.hash < rhe.hash { - return -1 - } - if lhe.Id > rhe.Id { - return 1 - } else { - return -1 - } -} - -// CalcScore implements skiplist interface -func (d *diff) CalcScore(key interface{}) float64 { - return 0 -} - -// Set adds or update element in container -func (d *diff) Set(elements ...ldiff.Element) { - d.mu.Lock() - defer d.mu.Unlock() - for _, e := range elements { - hash := xxhash.Sum64([]byte(e.Id)) - el := &element{Element: e, hash: hash} - d.sl.Remove(el) - d.sl.Set(el, nil) - d.ranges.addElement(hash) - } - d.ranges.recalculateHashes() -} - -func (d *diff) Ids() (ids []string) { - d.mu.RLock() - defer d.mu.RUnlock() - - ids = make([]string, 0, d.sl.Len()) - - cur := d.sl.Front() - for cur != nil { - el := cur.Key().(*element).Element - ids = append(ids, el.Id) - cur = cur.Next() - } - return -} - -func (d *diff) Len() int { - d.mu.RLock() - defer d.mu.RUnlock() - return d.sl.Len() -} - -func (d *diff) DiffType() spacesyncproto.DiffType { - return spacesyncproto.DiffType_V1 -} - -func (d *diff) Elements() (elements []ldiff.Element) { - d.mu.RLock() - defer d.mu.RUnlock() - - elements = make([]ldiff.Element, 0, d.sl.Len()) - - cur := d.sl.Front() - for cur != nil { - el := cur.Key().(*element).Element - elements = append(elements, el) - cur = cur.Next() - } - return -} - -func (d *diff) Element(id string) (ldiff.Element, error) { - d.mu.RLock() - defer d.mu.RUnlock() - el := d.sl.Get(&element{Element: ldiff.Element{Id: id}, hash: xxhash.Sum64([]byte(id))}) - if el == nil { - return ldiff.Element{}, ErrElementNotFound - } - if e, ok := el.Key().(*element); ok { - return e.Element, nil - } - return ldiff.Element{}, ErrElementNotFound -} - -func (d *diff) Hash() string { - d.mu.RLock() - defer d.mu.RUnlock() - return hex.EncodeToString(d.ranges.hash()) -} - -// RemoveId removes element by id -func (d *diff) RemoveId(id string) error { - d.mu.Lock() - defer d.mu.Unlock() - hash := xxhash.Sum64([]byte(id)) - el := &element{Element: ldiff.Element{ - Id: id, - }, hash: hash} - if d.sl.Remove(el) == nil { - return ErrElementNotFound - } - d.ranges.removeElement(hash) - d.ranges.recalculateHashes() - return nil -} - -func (d *diff) getRange(r ldiff.Range) (rr ldiff.RangeResult) { - rng := d.ranges.getRange(r.From, r.To) - // if we have the division for this range - if rng != nil { - rr.Hash = rng.hash - rr.Count = rng.elements - if !r.Elements && rng.isDivided { - return - } - } - - el := d.sl.Find(&element{hash: r.From}) - rr.Elements = make([]ldiff.Element, 0, d.divideFactor) - for el != nil && el.Key().(*element).hash <= r.To { - elem := el.Key().(*element).Element - el = el.Next() - rr.Elements = append(rr.Elements, elem) - } - rr.Count = len(rr.Elements) - return -} - -// Ranges calculates given ranges and return results -func (d *diff) Ranges(ctx context.Context, ranges []ldiff.Range, resBuf []ldiff.RangeResult) (results []ldiff.RangeResult, err error) { - d.mu.RLock() - defer d.mu.RUnlock() - - results = resBuf[:0] - for _, r := range ranges { - results = append(results, d.getRange(r)) - } - return -} - -type diffCtx struct { - newIds, changedIds, removedIds []string - - toSend, prepare []ldiff.Range - myRes, otherRes []ldiff.RangeResult -} - -var errMismatched = errors.New("query and results mismatched") - -// Diff makes diff with remote container -func (d *diff) Diff(ctx context.Context, dl ldiff.Remote) (newIds, changedIds, removedIds []string, err error) { - dctx := &diffCtx{} - dctx.toSend = append(dctx.toSend, ldiff.Range{ - From: 0, - To: math.MaxUint64, - }) - for len(dctx.toSend) > 0 { - select { - case <-ctx.Done(): - err = ctx.Err() - return - default: - } - if dctx.otherRes, err = dl.Ranges(ctx, dctx.toSend, dctx.otherRes); err != nil { - return - } - if dctx.myRes, err = d.Ranges(ctx, dctx.toSend, dctx.myRes); err != nil { - return - } - if len(dctx.otherRes) != len(dctx.toSend) || len(dctx.myRes) != len(dctx.toSend) { - err = errMismatched - return - } - for i, r := range dctx.toSend { - d.compareResults(dctx, r, dctx.myRes[i], dctx.otherRes[i]) - } - dctx.toSend, dctx.prepare = dctx.prepare, dctx.toSend - dctx.prepare = dctx.prepare[:0] - } - return dctx.newIds, dctx.changedIds, dctx.removedIds, nil -} - -func (d *diff) compareResults(dctx *diffCtx, r ldiff.Range, myRes, otherRes ldiff.RangeResult) { - // both hash equals - do nothing - if bytes.Equal(myRes.Hash, otherRes.Hash) { - return - } - - // other has elements - if len(otherRes.Elements) == otherRes.Count { - if len(myRes.Elements) == myRes.Count { - d.compareElements(dctx, myRes.Elements, otherRes.Elements) - } else { - r.Elements = true - d.compareElements(dctx, d.getRange(r).Elements, otherRes.Elements) - } - return - } - // request all elements from other, because we don't have enough - if len(myRes.Elements) == myRes.Count { - r.Elements = true - dctx.prepare = append(dctx.prepare, r) - return - } - rangeTuples := genTupleRanges(r.From, r.To, d.divideFactor) - for _, tuple := range rangeTuples { - dctx.prepare = append(dctx.prepare, ldiff.Range{From: tuple.from, To: tuple.to}) - } - return -} - -func (d *diff) compareElements(dctx *diffCtx, my, other []ldiff.Element) { - find := func(list []ldiff.Element, targetEl ldiff.Element) (has, eq bool) { - for _, el := range list { - if el.Id == targetEl.Id { - return true, el.Head == targetEl.Head - } - } - return false, false - } - - for _, el := range my { - has, eq := find(other, el) - if !has { - dctx.removedIds = append(dctx.removedIds, el.Id) - continue - } else { - if !eq { - dctx.changedIds = append(dctx.changedIds, el.Id) - } - } - } - - for _, el := range other { - if has, _ := find(my, el); !has { - dctx.newIds = append(dctx.newIds, el.Id) - } - } -} diff --git a/app/olddiff/diff_test.go b/app/olddiff/diff_test.go deleted file mode 100644 index bdc5c3671..000000000 --- a/app/olddiff/diff_test.go +++ /dev/null @@ -1,428 +0,0 @@ -package olddiff - -import ( - "context" - "fmt" - "math" - "sort" - "testing" - - "github.com/google/uuid" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "golang.org/x/exp/slices" - - "github.com/anyproto/any-sync/app/ldiff" -) - -func TestDiff_fillRange(t *testing.T) { - d := New(4, 4).(*diff) - for i := 0; i < 10; i++ { - el := ldiff.Element{ - Id: fmt.Sprint(i), - Head: fmt.Sprint("h", i), - } - d.Set(el) - } - t.Log(d.sl.Len()) - - t.Run("elements", func(t *testing.T) { - r := ldiff.Range{From: 0, To: math.MaxUint64} - res := d.getRange(r) - assert.NotNil(t, res.Hash) - assert.Equal(t, res.Count, 10) - }) -} - -func TestDiff_Diff(t *testing.T) { - ctx := context.Background() - t.Run("basic", func(t *testing.T) { - d1 := New(16, 16) - d2 := New(16, 16) - for i := 0; i < 1000; i++ { - id := fmt.Sprint(i) - head := uuid.NewString() - d1.Set(ldiff.Element{ - Id: id, - Head: head, - }) - d2.Set(ldiff.Element{ - Id: id, - Head: head, - }) - } - - newIds, changedIds, removedIds, err := d1.Diff(ctx, d2) - require.NoError(t, err) - assert.Len(t, newIds, 0) - assert.Len(t, changedIds, 0) - assert.Len(t, removedIds, 0) - - d2.Set(ldiff.Element{ - Id: "newD1", - Head: "newD1", - }) - d2.Set(ldiff.Element{ - Id: "1", - Head: "changed", - }) - require.NoError(t, d2.RemoveId("0")) - - newIds, changedIds, removedIds, err = d1.Diff(ctx, d2) - require.NoError(t, err) - assert.Len(t, newIds, 1) - assert.Len(t, changedIds, 1) - assert.Len(t, removedIds, 1) - }) - t.Run("complex", func(t *testing.T) { - d1 := New(16, 128) - d2 := New(16, 128) - length := 10000 - for i := 0; i < length; i++ { - id := fmt.Sprint(i) - head := uuid.NewString() - d1.Set(ldiff.Element{ - Id: id, - Head: head, - }) - } - - newIds, changedIds, removedIds, err := d1.Diff(ctx, d2) - require.NoError(t, err) - assert.Len(t, newIds, 0) - assert.Len(t, changedIds, 0) - assert.Len(t, removedIds, length) - - for i := 0; i < length; i++ { - id := fmt.Sprint(i) - head := uuid.NewString() - d2.Set(ldiff.Element{ - Id: id, - Head: head, - }) - } - - newIds, changedIds, removedIds, err = d1.Diff(ctx, d2) - require.NoError(t, err) - assert.Len(t, newIds, 0) - assert.Len(t, changedIds, length) - assert.Len(t, removedIds, 0) - - for i := 0; i < length; i++ { - id := fmt.Sprint(i) - head := uuid.NewString() - d2.Set(ldiff.Element{ - Id: id, - Head: head, - }) - } - - res, err := d1.Ranges( - context.Background(), - []ldiff.Range{{From: 0, To: math.MaxUint64, Elements: true}}, - nil) - require.NoError(t, err) - require.Len(t, res, 1) - for i, el := range res[0].Elements { - if i < length/2 { - continue - } - id := el.Id - head := el.Head - d2.Set(ldiff.Element{ - Id: id, - Head: head, - }) - } - - newIds, changedIds, removedIds, err = d1.Diff(ctx, d2) - require.NoError(t, err) - assert.Len(t, newIds, 0) - assert.Len(t, changedIds, length/2) - assert.Len(t, removedIds, 0) - }) - t.Run("empty", func(t *testing.T) { - d1 := New(16, 16) - d2 := New(16, 16) - newIds, changedIds, removedIds, err := d1.Diff(ctx, d2) - require.NoError(t, err) - assert.Len(t, newIds, 0) - assert.Len(t, changedIds, 0) - assert.Len(t, removedIds, 0) - }) - t.Run("one empty", func(t *testing.T) { - d1 := New(4, 4) - d2 := New(4, 4) - length := 10000 - for i := 0; i < length; i++ { - d2.Set(ldiff.Element{ - Id: fmt.Sprint(i), - Head: uuid.NewString(), - }) - } - newIds, changedIds, removedIds, err := d1.Diff(ctx, d2) - require.NoError(t, err) - assert.Len(t, newIds, length) - assert.Len(t, changedIds, 0) - assert.Len(t, removedIds, 0) - }) - t.Run("not intersecting", func(t *testing.T) { - d1 := New(16, 16) - d2 := New(16, 16) - length := 10000 - for i := 0; i < length; i++ { - d1.Set(ldiff.Element{ - Id: fmt.Sprint(i), - Head: uuid.NewString(), - }) - } - for i := length; i < length*2; i++ { - d2.Set(ldiff.Element{ - Id: fmt.Sprint(i), - Head: uuid.NewString(), - }) - } - newIds, changedIds, removedIds, err := d1.Diff(ctx, d2) - require.NoError(t, err) - assert.Len(t, newIds, length) - assert.Len(t, changedIds, 0) - assert.Len(t, removedIds, length) - }) - t.Run("context cancel", func(t *testing.T) { - d1 := New(4, 4) - d2 := New(4, 4) - for i := 0; i < 10; i++ { - d2.Set(ldiff.Element{ - Id: fmt.Sprint(i), - Head: uuid.NewString(), - }) - } - var cancel func() - ctx, cancel = context.WithCancel(ctx) - cancel() - _, _, _, err := d1.Diff(ctx, d2) - assert.ErrorIs(t, err, context.Canceled) - }) -} - -func BenchmarkDiff_Ranges(b *testing.B) { - d := New(16, 16) - for i := 0; i < 10000; i++ { - id := fmt.Sprint(i) - head := uuid.NewString() - d.Set(ldiff.Element{ - Id: id, - Head: head, - }) - } - ctx := context.Background() - b.ResetTimer() - b.ReportAllocs() - var resBuf []ldiff.RangeResult - var ranges = []ldiff.Range{{From: 0, To: math.MaxUint64}} - for i := 0; i < b.N; i++ { - d.Ranges(ctx, ranges, resBuf) - resBuf = resBuf[:0] - } -} - -func TestDiff_Hash(t *testing.T) { - d := New(16, 16) - h1 := d.Hash() - assert.NotEmpty(t, h1) - d.Set(ldiff.Element{Id: "1"}) - h2 := d.Hash() - assert.NotEmpty(t, h2) - assert.NotEqual(t, h1, h2) -} - -func TestDiff_Element(t *testing.T) { - d := New(16, 16) - for i := 0; i < 10; i++ { - d.Set(ldiff.Element{Id: fmt.Sprint("id", i), Head: fmt.Sprint("head", i)}) - } - _, err := d.Element("not found") - assert.Equal(t, ErrElementNotFound, err) - - el, err := d.Element("id5") - require.NoError(t, err) - assert.Equal(t, "head5", el.Head) - - d.Set(ldiff.Element{"id5", "otherHead"}) - el, err = d.Element("id5") - require.NoError(t, err) - assert.Equal(t, "otherHead", el.Head) -} - -func TestDiff_Ids(t *testing.T) { - d := New(16, 16) - var ids []string - for i := 0; i < 10; i++ { - id := fmt.Sprint("id", i) - d.Set(ldiff.Element{Id: id, Head: fmt.Sprint("head", i)}) - ids = append(ids, id) - } - gotIds := d.Ids() - sort.Strings(gotIds) - assert.Equal(t, ids, gotIds) - assert.Equal(t, len(ids), d.Len()) -} - -func TestDiff_Elements(t *testing.T) { - d := New(16, 16) - var els []ldiff.Element - for i := 0; i < 10; i++ { - id := fmt.Sprint("id", i) - el := ldiff.Element{Id: id, Head: fmt.Sprint("head", i)} - d.Set(el) - els = append(els, el) - } - gotEls := d.Elements() - sort.Slice(gotEls, func(i, j int) bool { - return gotEls[i].Id < gotEls[j].Id - }) - assert.Equal(t, els, gotEls) -} - -func TestRangesAddRemove(t *testing.T) { - length := 10000 - divideFactor := 4 - compareThreshold := 4 - addTwice := func() string { - d := New(divideFactor, compareThreshold) - var els []ldiff.Element - for i := 0; i < length; i++ { - if i < length/20 { - continue - } - els = append(els, ldiff.Element{ - Id: fmt.Sprint(i), - Head: fmt.Sprint("h", i), - }) - } - d.Set(els...) - els = els[:0] - for i := 0; i < length/20; i++ { - els = append(els, ldiff.Element{ - Id: fmt.Sprint(i), - Head: fmt.Sprint("h", i), - }) - } - d.Set(els...) - return d.Hash() - } - addOnce := func() string { - d := New(divideFactor, compareThreshold) - var els []ldiff.Element - for i := 0; i < length; i++ { - els = append(els, ldiff.Element{ - Id: fmt.Sprint(i), - Head: fmt.Sprint("h", i), - }) - } - d.Set(els...) - return d.Hash() - } - addRemove := func() string { - d := New(divideFactor, compareThreshold) - var els []ldiff.Element - for i := 0; i < length; i++ { - els = append(els, ldiff.Element{ - Id: fmt.Sprint(i), - Head: fmt.Sprint("h", i), - }) - } - d.Set(els...) - for i := 0; i < length/20; i++ { - err := d.RemoveId(fmt.Sprint(i)) - require.NoError(t, err) - } - els = els[:0] - for i := 0; i < length/20; i++ { - els = append(els, ldiff.Element{ - Id: fmt.Sprint(i), - Head: fmt.Sprint("h", i), - }) - } - d.Set(els...) - return d.Hash() - } - require.Equal(t, addTwice(), addOnce(), addRemove()) -} - -func printBestParams() { - numTests := 10 - length := 100000 - calcParams := func(divideFactor, compareThreshold, length int) (total, maxLevel, avgLevel, zeroEls int) { - d := New(divideFactor, compareThreshold).(*diff) - var els []ldiff.Element - for i := 0; i < length; i++ { - els = append(els, ldiff.Element{ - Id: uuid.NewString(), - Head: uuid.NewString(), - }) - } - d.Set(els...) - for _, rng := range d.ranges.ranges { - if rng.elements == 0 { - zeroEls++ - } - if rng.level > maxLevel { - maxLevel = rng.level - } - avgLevel += rng.level - } - total = len(d.ranges.ranges) - avgLevel = avgLevel / total - return - } - type result struct { - divFactor, compThreshold, numRanges, maxLevel, avgLevel, zeroEls int - } - sf := func(i, j result) int { - if i.numRanges < j.numRanges { - return -1 - } else if i.numRanges == j.numRanges { - return 0 - } else { - return 1 - } - } - var results []result - for divFactor := 0; divFactor < 6; divFactor++ { - df := 1 << divFactor - for compThreshold := 0; compThreshold < 10; compThreshold++ { - ct := 1 << compThreshold - fmt.Println("starting, df:", df, "ct:", ct) - var rngs []result - for i := 0; i < numTests; i++ { - total, maxLevel, avgLevel, zeroEls := calcParams(df, ct, length) - rngs = append(rngs, result{ - divFactor: df, - compThreshold: ct, - numRanges: total, - maxLevel: maxLevel, - avgLevel: avgLevel, - zeroEls: zeroEls, - }) - } - slices.SortFunc(rngs, sf) - ranges := rngs[len(rngs)/2] - results = append(results, ranges) - } - } - slices.SortFunc(results, sf) - fmt.Println(results) - // 100000 - [{16 512 273 2 1 0} {4 512 341 4 3 0} {2 512 511 8 7 0} {1 512 511 8 7 0} - // {8 256 585 3 2 0} {8 512 585 3 2 0} {1 256 1023 9 8 0} {2 256 1023 9 8 0} - // {32 256 1057 2 1 0} {32 512 1057 2 1 0} {32 128 1089 3 1 0} {4 256 1365 5 4 0} - // {4 128 1369 6 4 0} {2 128 2049 11 9 0} {1 128 2049 11 9 0} {1 64 4157 12 10 0} - // {2 64 4159 12 10 0} {16 128 4369 3 2 0} {16 64 4369 3 2 0} {16 256 4369 3 2 0} - // {8 64 4681 4 3 0} {8 128 4681 4 3 0} {4 64 5461 6 5 0} {4 32 6389 7 5 0} - // {8 32 6505 5 4 17} {16 32 8033 4 3 374} {2 32 8619 13 11 0} {1 32 8621 13 11 0} - // {2 16 17837 15 12 0} {1 16 17847 15 12 0} {4 16 21081 8 6 22} {32 64 33825 3 2 1578} - // {32 32 33825 3 2 1559} {32 16 33825 3 2 1518} {8 16 35881 5 4 1313} {16 16 66737 4 3 13022}] - // 1000000 - [{8 256 11753 5 4 0}] - // 1000000 - [{16 128 69905 4 3 0}] - // 1000000 - [{32 256 33825 3 2 0}] -} diff --git a/app/olddiff/hashrange.go b/app/olddiff/hashrange.go deleted file mode 100644 index cfc58976f..000000000 --- a/app/olddiff/hashrange.go +++ /dev/null @@ -1,223 +0,0 @@ -package olddiff - -import ( - "math" - - "github.com/huandu/skiplist" - "github.com/zeebo/blake3" - "golang.org/x/exp/slices" -) - -type hashRange struct { - from, to uint64 - parent *hashRange - isDivided bool - elements int - level int - hash []byte -} - -type rangeTuple struct { - from, to uint64 -} - -type hashRanges struct { - ranges map[rangeTuple]*hashRange - topRange *hashRange - sl *skiplist.SkipList - dirty map[*hashRange]struct{} - divideFactor int - compareThreshold int -} - -func newHashRanges(divideFactor, compareThreshold int, sl *skiplist.SkipList) *hashRanges { - h := &hashRanges{ - ranges: make(map[rangeTuple]*hashRange), - dirty: make(map[*hashRange]struct{}), - divideFactor: divideFactor, - compareThreshold: compareThreshold, - sl: sl, - } - h.topRange = &hashRange{ - from: 0, - to: math.MaxUint64, - isDivided: true, - level: 0, - } - h.ranges[rangeTuple{from: 0, to: math.MaxUint64}] = h.topRange - h.makeBottomRanges(h.topRange) - return h -} - -func (h *hashRanges) hash() []byte { - return h.topRange.hash -} - -func (h *hashRanges) addElement(elHash uint64) { - rng := h.topRange - rng.elements++ - for rng.isDivided { - rng = h.getBottomRange(rng, elHash) - rng.elements++ - } - h.dirty[rng] = struct{}{} - if rng.elements > h.compareThreshold { - rng.isDivided = true - h.makeBottomRanges(rng) - } - if rng.parent != nil { - if _, ok := h.dirty[rng.parent]; ok { - delete(h.dirty, rng.parent) - } - } -} - -func (h *hashRanges) removeElement(elHash uint64) { - rng := h.topRange - rng.elements-- - for rng.isDivided { - rng = h.getBottomRange(rng, elHash) - rng.elements-- - } - parent := rng.parent - if parent.elements <= h.compareThreshold && parent != h.topRange { - ranges := genTupleRanges(parent.from, parent.to, h.divideFactor) - for _, tuple := range ranges { - child := h.ranges[tuple] - delete(h.ranges, tuple) - delete(h.dirty, child) - } - parent.isDivided = false - h.dirty[parent] = struct{}{} - } else { - h.dirty[rng] = struct{}{} - } -} - -func (h *hashRanges) recalculateHashes() { - for len(h.dirty) > 0 { - var slDirty []*hashRange - for rng := range h.dirty { - slDirty = append(slDirty, rng) - } - slices.SortFunc(slDirty, func(a, b *hashRange) int { - if a.level < b.level { - return -1 - } else if a.level > b.level { - return 1 - } else { - return 0 - } - }) - for _, rng := range slDirty { - if rng.isDivided { - rng.hash = h.calcDividedHash(rng) - } else { - rng.hash, rng.elements = h.calcElementsHash(rng.from, rng.to) - } - delete(h.dirty, rng) - if rng.parent != nil { - h.dirty[rng.parent] = struct{}{} - } - } - } -} - -func (h *hashRanges) getRange(from, to uint64) *hashRange { - return h.ranges[rangeTuple{from: from, to: to}] -} - -func (h *hashRanges) getBottomRange(rng *hashRange, elHash uint64) *hashRange { - df := uint64(h.divideFactor) - perRange := (rng.to - rng.from) / df - align := ((rng.to-rng.from)%df + 1) % df - if align == 0 { - perRange++ - } - bucket := (elHash - rng.from) / perRange - tuple := rangeTuple{from: rng.from + bucket*perRange, to: rng.from - 1 + (bucket+1)*perRange} - if bucket == df-1 { - tuple.to += align - } - return h.ranges[tuple] -} - -func (h *hashRanges) makeBottomRanges(rng *hashRange) { - ranges := genTupleRanges(rng.from, rng.to, h.divideFactor) - for _, tuple := range ranges { - newRange := h.makeRange(tuple, rng) - h.ranges[tuple] = newRange - if newRange.elements > h.compareThreshold { - if _, ok := h.dirty[rng]; ok { - delete(h.dirty, rng) - } - h.dirty[newRange] = struct{}{} - newRange.isDivided = true - h.makeBottomRanges(newRange) - } - } -} - -func (h *hashRanges) makeRange(tuple rangeTuple, parent *hashRange) *hashRange { - newRange := &hashRange{ - from: tuple.from, - to: tuple.to, - parent: parent, - } - hash, els := h.calcElementsHash(tuple.from, tuple.to) - newRange.hash = hash - newRange.level = parent.level + 1 - newRange.elements = els - return newRange -} - -func (h *hashRanges) calcDividedHash(rng *hashRange) (hash []byte) { - hasher := hashersPool.Get().(*blake3.Hasher) - defer hashersPool.Put(hasher) - hasher.Reset() - ranges := genTupleRanges(rng.from, rng.to, h.divideFactor) - for _, tuple := range ranges { - child := h.ranges[tuple] - hasher.Write(child.hash) - } - hash = hasher.Sum(nil) - return -} - -func genTupleRanges(from, to uint64, divideFactor int) (prepare []rangeTuple) { - df := uint64(divideFactor) - perRange := (to - from) / df - align := ((to-from)%df + 1) % df - if align == 0 { - perRange++ - } - var j = from - for i := 0; i < divideFactor; i++ { - if i == divideFactor-1 { - perRange += align - } - prepare = append(prepare, rangeTuple{from: j, to: j + perRange - 1}) - j += perRange - } - return -} - -func (h *hashRanges) calcElementsHash(from, to uint64) (hash []byte, els int) { - hasher := hashersPool.Get().(*blake3.Hasher) - defer hashersPool.Put(hasher) - hasher.Reset() - - el := h.sl.Find(&element{hash: from}) - for el != nil && el.Key().(*element).hash <= to { - elem := el.Key().(*element).Element - el = el.Next() - - hasher.WriteString(elem.Id) - hasher.WriteString(elem.Head) - els++ - } - if els != 0 { - hash = hasher.Sum(nil) - } - return -} diff --git a/commonfile/fileproto/filep2p/filep2p.pb.go b/commonfile/fileproto/filep2p/filep2p.pb.go new file mode 100644 index 000000000..b65a36c25 --- /dev/null +++ b/commonfile/fileproto/filep2p/filep2p.pb.go @@ -0,0 +1,498 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v3.14.0 +// source: commonfile/fileproto/filep2p/protos/filep2p.proto + +package filep2p + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// ErrCodes are WHOLE-RPC (drpc-level) errors, mirroring v1 filesync.ErrCodes +// and v2's fileprotov2.ErrCodes. Wired through rpcerr.ErrGroup(ErrorOffset) in +// the filep2perr package. Offset 900 is the next free rpcerr block (v1=200, +// v2=800). Zero value Unexpected=0 registers at offset+0 so an uncoded fault +// reads as Unexpected. +type ErrCodes int32 + +const ( + ErrCodes_Unexpected ErrCodes = 0 // unclassified server fault (registry fallback) + ErrCodes_Forbidden ErrCodes = 1 // connection identity may not read this space + ErrCodes_InvalidRequest ErrCodes = 2 // malformed: empty spaceId, bad rootCid, bad range + ErrCodes_FileNotFound ErrCodes = 3 // server does not hold this (spaceId, rootCid) in full + ErrCodes_ErrorOffset ErrCodes = 900 // rpcerr.ErrGroup base (constant, never an error) +) + +// Enum value maps for ErrCodes. +var ( + ErrCodes_name = map[int32]string{ + 0: "Unexpected", + 1: "Forbidden", + 2: "InvalidRequest", + 3: "FileNotFound", + 900: "ErrorOffset", + } + ErrCodes_value = map[string]int32{ + "Unexpected": 0, + "Forbidden": 1, + "InvalidRequest": 2, + "FileNotFound": 3, + "ErrorOffset": 900, + } +) + +func (x ErrCodes) Enum() *ErrCodes { + p := new(ErrCodes) + *p = x + return p +} + +func (x ErrCodes) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ErrCodes) Descriptor() protoreflect.EnumDescriptor { + return file_commonfile_fileproto_filep2p_protos_filep2p_proto_enumTypes[0].Descriptor() +} + +func (ErrCodes) Type() protoreflect.EnumType { + return &file_commonfile_fileproto_filep2p_protos_filep2p_proto_enumTypes[0] +} + +func (x ErrCodes) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ErrCodes.Descriptor instead. +func (ErrCodes) EnumDescriptor() ([]byte, []int) { + return file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDescGZIP(), []int{0} +} + +type Availability int32 + +const ( + Availability_None Availability = 0 // server has no block of this file + Availability_Partial Availability = 1 // server has some but not all blocks (cannot ObjectRead yet) + Availability_Full Availability = 2 // server has every block; ObjectRead will serve +) + +// Enum value maps for Availability. +var ( + Availability_name = map[int32]string{ + 0: "None", + 1: "Partial", + 2: "Full", + } + Availability_value = map[string]int32{ + "None": 0, + "Partial": 1, + "Full": 2, + } +) + +func (x Availability) Enum() *Availability { + p := new(Availability) + *p = x + return p +} + +func (x Availability) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (Availability) Descriptor() protoreflect.EnumDescriptor { + return file_commonfile_fileproto_filep2p_protos_filep2p_proto_enumTypes[1].Descriptor() +} + +func (Availability) Type() protoreflect.EnumType { + return &file_commonfile_fileproto_filep2p_protos_filep2p_proto_enumTypes[1] +} + +func (x Availability) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use Availability.Descriptor instead. +func (Availability) EnumDescriptor() ([]byte, []int) { + return file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDescGZIP(), []int{1} +} + +type FileCheckRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + SpaceId string `protobuf:"bytes,1,opt,name=spaceId,proto3" json:"spaceId,omitempty"` // auth scope + RootCids [][]byte `protobuf:"bytes,2,rep,name=rootCids,proto3" json:"rootCids,omitempty"` // files to report on (result match key) + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileCheckRequest) Reset() { + *x = FileCheckRequest{} + mi := &file_commonfile_fileproto_filep2p_protos_filep2p_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileCheckRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileCheckRequest) ProtoMessage() {} + +func (x *FileCheckRequest) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_filep2p_protos_filep2p_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileCheckRequest.ProtoReflect.Descriptor instead. +func (*FileCheckRequest) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDescGZIP(), []int{0} +} + +func (x *FileCheckRequest) GetSpaceId() string { + if x != nil { + return x.SpaceId + } + return "" +} + +func (x *FileCheckRequest) GetRootCids() [][]byte { + if x != nil { + return x.RootCids + } + return nil +} + +type FileCheckResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Files []*FileAvailability `protobuf:"bytes,1,rep,name=files,proto3" json:"files,omitempty"` // one per requested rootCid, keyed by rootCid + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileCheckResponse) Reset() { + *x = FileCheckResponse{} + mi := &file_commonfile_fileproto_filep2p_protos_filep2p_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileCheckResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileCheckResponse) ProtoMessage() {} + +func (x *FileCheckResponse) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_filep2p_protos_filep2p_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileCheckResponse.ProtoReflect.Descriptor instead. +func (*FileCheckResponse) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDescGZIP(), []int{1} +} + +func (x *FileCheckResponse) GetFiles() []*FileAvailability { + if x != nil { + return x.Files + } + return nil +} + +type FileAvailability struct { + state protoimpl.MessageState `protogen:"open.v1"` + RootCid []byte `protobuf:"bytes,1,opt,name=rootCid,proto3" json:"rootCid,omitempty"` // echoes the requested rootCid (match key) + Have Availability `protobuf:"varint,2,opt,name=have,proto3,enum=filesyncp2p.Availability" json:"have,omitempty"` // how much of the file the server holds + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileAvailability) Reset() { + *x = FileAvailability{} + mi := &file_commonfile_fileproto_filep2p_protos_filep2p_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileAvailability) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileAvailability) ProtoMessage() {} + +func (x *FileAvailability) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_filep2p_protos_filep2p_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileAvailability.ProtoReflect.Descriptor instead. +func (*FileAvailability) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDescGZIP(), []int{2} +} + +func (x *FileAvailability) GetRootCid() []byte { + if x != nil { + return x.RootCid + } + return nil +} + +func (x *FileAvailability) GetHave() Availability { + if x != nil { + return x.Have + } + return Availability_None +} + +type ObjectReadRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + SpaceId string `protobuf:"bytes,1,opt,name=spaceId,proto3" json:"spaceId,omitempty"` // auth scope + RootCid []byte `protobuf:"bytes,2,opt,name=rootCid,proto3" json:"rootCid,omitempty"` // file root — selects the CAR object to read from + Offset uint64 `protobuf:"varint,3,opt,name=offset,proto3" json:"offset,omitempty"` // start byte within the object + Length uint32 `protobuf:"varint,4,opt,name=length,proto3" json:"length,omitempty"` // number of bytes requested from offset + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ObjectReadRequest) Reset() { + *x = ObjectReadRequest{} + mi := &file_commonfile_fileproto_filep2p_protos_filep2p_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ObjectReadRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ObjectReadRequest) ProtoMessage() {} + +func (x *ObjectReadRequest) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_filep2p_protos_filep2p_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ObjectReadRequest.ProtoReflect.Descriptor instead. +func (*ObjectReadRequest) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDescGZIP(), []int{3} +} + +func (x *ObjectReadRequest) GetSpaceId() string { + if x != nil { + return x.SpaceId + } + return "" +} + +func (x *ObjectReadRequest) GetRootCid() []byte { + if x != nil { + return x.RootCid + } + return nil +} + +func (x *ObjectReadRequest) GetOffset() uint64 { + if x != nil { + return x.Offset + } + return 0 +} + +func (x *ObjectReadRequest) GetLength() uint32 { + if x != nil { + return x.Length + } + return 0 +} + +type ObjectReadResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` // the requested bytes (short only at object end) + TotalSize uint64 `protobuf:"varint,2,opt,name=totalSize,proto3" json:"totalSize,omitempty"` // whole-object size (like HTTP Content-Range total) + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ObjectReadResponse) Reset() { + *x = ObjectReadResponse{} + mi := &file_commonfile_fileproto_filep2p_protos_filep2p_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ObjectReadResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ObjectReadResponse) ProtoMessage() {} + +func (x *ObjectReadResponse) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_filep2p_protos_filep2p_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ObjectReadResponse.ProtoReflect.Descriptor instead. +func (*ObjectReadResponse) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDescGZIP(), []int{4} +} + +func (x *ObjectReadResponse) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +func (x *ObjectReadResponse) GetTotalSize() uint64 { + if x != nil { + return x.TotalSize + } + return 0 +} + +var File_commonfile_fileproto_filep2p_protos_filep2p_proto protoreflect.FileDescriptor + +const file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDesc = "" + + "\n" + + "1commonfile/fileproto/filep2p/protos/filep2p.proto\x12\vfilesyncp2p\"H\n" + + "\x10FileCheckRequest\x12\x18\n" + + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x12\x1a\n" + + "\brootCids\x18\x02 \x03(\fR\brootCids\"H\n" + + "\x11FileCheckResponse\x123\n" + + "\x05files\x18\x01 \x03(\v2\x1d.filesyncp2p.FileAvailabilityR\x05files\"[\n" + + "\x10FileAvailability\x12\x18\n" + + "\arootCid\x18\x01 \x01(\fR\arootCid\x12-\n" + + "\x04have\x18\x02 \x01(\x0e2\x19.filesyncp2p.AvailabilityR\x04have\"w\n" + + "\x11ObjectReadRequest\x12\x18\n" + + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x12\x18\n" + + "\arootCid\x18\x02 \x01(\fR\arootCid\x12\x16\n" + + "\x06offset\x18\x03 \x01(\x04R\x06offset\x12\x16\n" + + "\x06length\x18\x04 \x01(\rR\x06length\"F\n" + + "\x12ObjectReadResponse\x12\x12\n" + + "\x04data\x18\x01 \x01(\fR\x04data\x12\x1c\n" + + "\ttotalSize\x18\x02 \x01(\x04R\ttotalSize*a\n" + + "\bErrCodes\x12\x0e\n" + + "\n" + + "Unexpected\x10\x00\x12\r\n" + + "\tForbidden\x10\x01\x12\x12\n" + + "\x0eInvalidRequest\x10\x02\x12\x10\n" + + "\fFileNotFound\x10\x03\x12\x10\n" + + "\vErrorOffset\x10\x84\a*/\n" + + "\fAvailability\x12\b\n" + + "\x04None\x10\x00\x12\v\n" + + "\aPartial\x10\x01\x12\b\n" + + "\x04Full\x10\x022\xa4\x01\n" + + "\aFileP2P\x12J\n" + + "\tFileCheck\x12\x1d.filesyncp2p.FileCheckRequest\x1a\x1e.filesyncp2p.FileCheckResponse\x12M\n" + + "\n" + + "ObjectRead\x12\x1e.filesyncp2p.ObjectReadRequest\x1a\x1f.filesyncp2p.ObjectReadResponseB\x1eZ\x1ccommonfile/fileproto/filep2pb\x06proto3" + +var ( + file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDescOnce sync.Once + file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDescData []byte +) + +func file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDescGZIP() []byte { + file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDescOnce.Do(func() { + file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDesc), len(file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDesc))) + }) + return file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDescData +} + +var file_commonfile_fileproto_filep2p_protos_filep2p_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_commonfile_fileproto_filep2p_protos_filep2p_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_commonfile_fileproto_filep2p_protos_filep2p_proto_goTypes = []any{ + (ErrCodes)(0), // 0: filesyncp2p.ErrCodes + (Availability)(0), // 1: filesyncp2p.Availability + (*FileCheckRequest)(nil), // 2: filesyncp2p.FileCheckRequest + (*FileCheckResponse)(nil), // 3: filesyncp2p.FileCheckResponse + (*FileAvailability)(nil), // 4: filesyncp2p.FileAvailability + (*ObjectReadRequest)(nil), // 5: filesyncp2p.ObjectReadRequest + (*ObjectReadResponse)(nil), // 6: filesyncp2p.ObjectReadResponse +} +var file_commonfile_fileproto_filep2p_protos_filep2p_proto_depIdxs = []int32{ + 4, // 0: filesyncp2p.FileCheckResponse.files:type_name -> filesyncp2p.FileAvailability + 1, // 1: filesyncp2p.FileAvailability.have:type_name -> filesyncp2p.Availability + 2, // 2: filesyncp2p.FileP2P.FileCheck:input_type -> filesyncp2p.FileCheckRequest + 5, // 3: filesyncp2p.FileP2P.ObjectRead:input_type -> filesyncp2p.ObjectReadRequest + 3, // 4: filesyncp2p.FileP2P.FileCheck:output_type -> filesyncp2p.FileCheckResponse + 6, // 5: filesyncp2p.FileP2P.ObjectRead:output_type -> filesyncp2p.ObjectReadResponse + 4, // [4:6] is the sub-list for method output_type + 2, // [2:4] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_commonfile_fileproto_filep2p_protos_filep2p_proto_init() } +func file_commonfile_fileproto_filep2p_protos_filep2p_proto_init() { + if File_commonfile_fileproto_filep2p_protos_filep2p_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDesc), len(file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDesc)), + NumEnums: 2, + NumMessages: 5, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_commonfile_fileproto_filep2p_protos_filep2p_proto_goTypes, + DependencyIndexes: file_commonfile_fileproto_filep2p_protos_filep2p_proto_depIdxs, + EnumInfos: file_commonfile_fileproto_filep2p_protos_filep2p_proto_enumTypes, + MessageInfos: file_commonfile_fileproto_filep2p_protos_filep2p_proto_msgTypes, + }.Build() + File_commonfile_fileproto_filep2p_protos_filep2p_proto = out.File + file_commonfile_fileproto_filep2p_protos_filep2p_proto_goTypes = nil + file_commonfile_fileproto_filep2p_protos_filep2p_proto_depIdxs = nil +} diff --git a/commonfile/fileproto/filep2p/filep2p_drpc.pb.go b/commonfile/fileproto/filep2p/filep2p_drpc.pb.go new file mode 100644 index 000000000..b85a10985 --- /dev/null +++ b/commonfile/fileproto/filep2p/filep2p_drpc.pb.go @@ -0,0 +1,154 @@ +// Code generated by protoc-gen-go-drpc. DO NOT EDIT. +// protoc-gen-go-drpc version: v1.0.0 +// source: commonfile/fileproto/filep2p/protos/filep2p.proto + +package filep2p + +import ( + context "context" + errors "errors" + drpc1 "github.com/planetscale/vtprotobuf/codec/drpc" + drpc "storj.io/drpc" + drpcerr "storj.io/drpc/drpcerr" +) + +type drpcEncoding_File_commonfile_fileproto_filep2p_protos_filep2p_proto struct{} + +func (drpcEncoding_File_commonfile_fileproto_filep2p_protos_filep2p_proto) Marshal(msg drpc.Message) ([]byte, error) { + return drpc1.Marshal(msg) +} + +func (drpcEncoding_File_commonfile_fileproto_filep2p_protos_filep2p_proto) Unmarshal(buf []byte, msg drpc.Message) error { + return drpc1.Unmarshal(buf, msg) +} + +func (drpcEncoding_File_commonfile_fileproto_filep2p_protos_filep2p_proto) JSONMarshal(msg drpc.Message) ([]byte, error) { + return drpc1.JSONMarshal(msg) +} + +func (drpcEncoding_File_commonfile_fileproto_filep2p_protos_filep2p_proto) JSONUnmarshal(buf []byte, msg drpc.Message) error { + return drpc1.JSONUnmarshal(buf, msg) +} + +type DRPCFileP2PClient interface { + DRPCConn() drpc.Conn + + FileCheck(ctx context.Context, in *FileCheckRequest) (*FileCheckResponse, error) + ObjectRead(ctx context.Context, in *ObjectReadRequest) (*ObjectReadResponse, error) +} + +type drpcFileP2PClient struct { + cc drpc.Conn +} + +func NewDRPCFileP2PClient(cc drpc.Conn) DRPCFileP2PClient { + return &drpcFileP2PClient{cc} +} + +func (c *drpcFileP2PClient) DRPCConn() drpc.Conn { return c.cc } + +func (c *drpcFileP2PClient) FileCheck(ctx context.Context, in *FileCheckRequest) (*FileCheckResponse, error) { + out := new(FileCheckResponse) + err := c.cc.Invoke(ctx, "/filesyncp2p.FileP2P/FileCheck", drpcEncoding_File_commonfile_fileproto_filep2p_protos_filep2p_proto{}, in, out) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *drpcFileP2PClient) ObjectRead(ctx context.Context, in *ObjectReadRequest) (*ObjectReadResponse, error) { + out := new(ObjectReadResponse) + err := c.cc.Invoke(ctx, "/filesyncp2p.FileP2P/ObjectRead", drpcEncoding_File_commonfile_fileproto_filep2p_protos_filep2p_proto{}, in, out) + if err != nil { + return nil, err + } + return out, nil +} + +type DRPCFileP2PServer interface { + FileCheck(context.Context, *FileCheckRequest) (*FileCheckResponse, error) + ObjectRead(context.Context, *ObjectReadRequest) (*ObjectReadResponse, error) +} + +type DRPCFileP2PUnimplementedServer struct{} + +func (s *DRPCFileP2PUnimplementedServer) FileCheck(context.Context, *FileCheckRequest) (*FileCheckResponse, error) { + return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) +} + +func (s *DRPCFileP2PUnimplementedServer) ObjectRead(context.Context, *ObjectReadRequest) (*ObjectReadResponse, error) { + return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) +} + +type DRPCFileP2PDescription struct{} + +func (DRPCFileP2PDescription) NumMethods() int { return 2 } + +func (DRPCFileP2PDescription) Method(n int) (string, drpc.Encoding, drpc.Receiver, interface{}, bool) { + switch n { + case 0: + return "/filesyncp2p.FileP2P/FileCheck", drpcEncoding_File_commonfile_fileproto_filep2p_protos_filep2p_proto{}, + func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) { + return srv.(DRPCFileP2PServer). + FileCheck( + ctx, + in1.(*FileCheckRequest), + ) + }, DRPCFileP2PServer.FileCheck, true + case 1: + return "/filesyncp2p.FileP2P/ObjectRead", drpcEncoding_File_commonfile_fileproto_filep2p_protos_filep2p_proto{}, + func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) { + return srv.(DRPCFileP2PServer). + ObjectRead( + ctx, + in1.(*ObjectReadRequest), + ) + }, DRPCFileP2PServer.ObjectRead, true + default: + return "", nil, nil, nil, false + } +} + +func DRPCRegisterFileP2P(mux drpc.Mux, impl DRPCFileP2PServer) error { + return mux.Register(impl, DRPCFileP2PDescription{}) +} + +type DRPCFileP2P_FileCheckStream interface { + drpc.Stream + SendAndClose(*FileCheckResponse) error +} + +type drpcFileP2P_FileCheckStream struct { + drpc.Stream +} + +func (x *drpcFileP2P_FileCheckStream) GetStream() drpc.Stream { + return x.Stream +} + +func (x *drpcFileP2P_FileCheckStream) SendAndClose(m *FileCheckResponse) error { + if err := x.MsgSend(m, drpcEncoding_File_commonfile_fileproto_filep2p_protos_filep2p_proto{}); err != nil { + return err + } + return x.CloseSend() +} + +type DRPCFileP2P_ObjectReadStream interface { + drpc.Stream + SendAndClose(*ObjectReadResponse) error +} + +type drpcFileP2P_ObjectReadStream struct { + drpc.Stream +} + +func (x *drpcFileP2P_ObjectReadStream) GetStream() drpc.Stream { + return x.Stream +} + +func (x *drpcFileP2P_ObjectReadStream) SendAndClose(m *ObjectReadResponse) error { + if err := x.MsgSend(m, drpcEncoding_File_commonfile_fileproto_filep2p_protos_filep2p_proto{}); err != nil { + return err + } + return x.CloseSend() +} diff --git a/commonfile/fileproto/filep2p/filep2p_vtproto.pb.go b/commonfile/fileproto/filep2p/filep2p_vtproto.pb.go new file mode 100644 index 000000000..0ebf30138 --- /dev/null +++ b/commonfile/fileproto/filep2p/filep2p_vtproto.pb.go @@ -0,0 +1,918 @@ +// Code generated by protoc-gen-go-vtproto. DO NOT EDIT. +// protoc-gen-go-vtproto version: v0.6.0 +// source: commonfile/fileproto/filep2p/protos/filep2p.proto + +package filep2p + +import ( + fmt "fmt" + protohelpers "github.com/planetscale/vtprotobuf/protohelpers" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + io "io" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +func (m *FileCheckRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FileCheckRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *FileCheckRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.RootCids) > 0 { + for iNdEx := len(m.RootCids) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.RootCids[iNdEx]) + copy(dAtA[i:], m.RootCids[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.RootCids[iNdEx]))) + i-- + dAtA[i] = 0x12 + } + } + if len(m.SpaceId) > 0 { + i -= len(m.SpaceId) + copy(dAtA[i:], m.SpaceId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *FileCheckResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FileCheckResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *FileCheckResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Files) > 0 { + for iNdEx := len(m.Files) - 1; iNdEx >= 0; iNdEx-- { + size, err := m.Files[iNdEx].MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *FileAvailability) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FileAvailability) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *FileAvailability) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.Have != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Have)) + i-- + dAtA[i] = 0x10 + } + if len(m.RootCid) > 0 { + i -= len(m.RootCid) + copy(dAtA[i:], m.RootCid) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.RootCid))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *ObjectReadRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ObjectReadRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *ObjectReadRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.Length != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Length)) + i-- + dAtA[i] = 0x20 + } + if m.Offset != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Offset)) + i-- + dAtA[i] = 0x18 + } + if len(m.RootCid) > 0 { + i -= len(m.RootCid) + copy(dAtA[i:], m.RootCid) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.RootCid))) + i-- + dAtA[i] = 0x12 + } + if len(m.SpaceId) > 0 { + i -= len(m.SpaceId) + copy(dAtA[i:], m.SpaceId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *ObjectReadResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ObjectReadResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *ObjectReadResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.TotalSize != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.TotalSize)) + i-- + dAtA[i] = 0x10 + } + if len(m.Data) > 0 { + i -= len(m.Data) + copy(dAtA[i:], m.Data) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Data))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *FileCheckRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.SpaceId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if len(m.RootCids) > 0 { + for _, b := range m.RootCids { + l = len(b) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *FileCheckResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Files) > 0 { + for _, e := range m.Files { + l = e.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *FileAvailability) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.RootCid) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.Have != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Have)) + } + n += len(m.unknownFields) + return n +} + +func (m *ObjectReadRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.SpaceId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.RootCid) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.Offset != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Offset)) + } + if m.Length != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Length)) + } + n += len(m.unknownFields) + return n +} + +func (m *ObjectReadResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Data) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.TotalSize != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.TotalSize)) + } + n += len(m.unknownFields) + return n +} + +func (m *FileCheckRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FileCheckRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FileCheckRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RootCids", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RootCids = append(m.RootCids, make([]byte, postIndex-iNdEx)) + copy(m.RootCids[len(m.RootCids)-1], dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *FileCheckResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FileCheckResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FileCheckResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Files", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Files = append(m.Files, &FileAvailability{}) + if err := m.Files[len(m.Files)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *FileAvailability) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FileAvailability: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FileAvailability: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RootCid", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RootCid = append(m.RootCid[:0], dAtA[iNdEx:postIndex]...) + if m.RootCid == nil { + m.RootCid = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Have", wireType) + } + m.Have = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Have |= Availability(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ObjectReadRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ObjectReadRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ObjectReadRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RootCid", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RootCid = append(m.RootCid[:0], dAtA[iNdEx:postIndex]...) + if m.RootCid == nil { + m.RootCid = []byte{} + } + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Offset", wireType) + } + m.Offset = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Offset |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Length", wireType) + } + m.Length = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Length |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ObjectReadResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ObjectReadResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ObjectReadResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) + if m.Data == nil { + m.Data = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field TotalSize", wireType) + } + m.TotalSize = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.TotalSize |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} diff --git a/commonfile/fileproto/filep2p/filep2perr/filep2perr.go b/commonfile/fileproto/filep2p/filep2perr/filep2perr.go new file mode 100644 index 000000000..c6e8bf10a --- /dev/null +++ b/commonfile/fileproto/filep2p/filep2perr/filep2perr.go @@ -0,0 +1,35 @@ +package filep2perr + +import ( + "fmt" + + "github.com/anyproto/any-sync/commonfile/fileproto/filep2p" + "github.com/anyproto/any-sync/net/rpc/rpcerr" +) + +// errGroup registers the peer-to-peer file WHOLE-RPC (drpc-level) errors in the +// offset-900 block, distinct from v1 filesync's 200 and v2's 800. Mirrors the +// commonfile/fileproto/fileprotoerr and fileprotov2err packages. +var ( + errGroup = rpcerr.ErrGroup(filep2p.ErrCodes_ErrorOffset) + ErrUnexpected = errGroup.Register(fmt.Errorf("unexpected p2p file error"), uint64(filep2p.ErrCodes_Unexpected)) + ErrForbidden = errGroup.Register(fmt.Errorf("forbidden"), uint64(filep2p.ErrCodes_Forbidden)) + ErrInvalidRequest = errGroup.Register(fmt.Errorf("invalid request"), uint64(filep2p.ErrCodes_InvalidRequest)) + ErrFileNotFound = errGroup.Register(fmt.Errorf("file not found"), uint64(filep2p.ErrCodes_FileNotFound)) +) + +// FromCode maps a whole-RPC ErrCodes value to its registered drpc error so +// handlers can uniformly `return filep2perr.FromCode(code)` at the RPC +// boundary. Unknown/zero codes fall back to ErrUnexpected. +func FromCode(code filep2p.ErrCodes) error { + switch code { + case filep2p.ErrCodes_Forbidden: + return ErrForbidden + case filep2p.ErrCodes_InvalidRequest: + return ErrInvalidRequest + case filep2p.ErrCodes_FileNotFound: + return ErrFileNotFound + default: + return ErrUnexpected + } +} diff --git a/commonfile/fileproto/filep2p/filep2perr/filep2perr_test.go b/commonfile/fileproto/filep2p/filep2perr/filep2perr_test.go new file mode 100644 index 000000000..30605323a --- /dev/null +++ b/commonfile/fileproto/filep2p/filep2perr/filep2perr_test.go @@ -0,0 +1,32 @@ +package filep2perr + +import ( + "testing" + + // Import the neighbouring error groups so this test binary registers + // all of them together: rpcerr panics at init if two groups claim the + // same offset+code, so a green run proves the 900 band does not + // collide with v1 (200) / v2 (800) or anything else linked here. + _ "github.com/anyproto/any-sync/commonfile/fileproto/fileprotoerr" + _ "github.com/anyproto/any-sync/commonfile/fileproto/fileprotov2/fileprotov2err" + + "github.com/anyproto/any-sync/commonfile/fileproto/filep2p" +) + +func TestFromCodeMapsEveryCode(t *testing.T) { + cases := map[filep2p.ErrCodes]error{ + filep2p.ErrCodes_Unexpected: ErrUnexpected, + filep2p.ErrCodes_Forbidden: ErrForbidden, + filep2p.ErrCodes_InvalidRequest: ErrInvalidRequest, + filep2p.ErrCodes_FileNotFound: ErrFileNotFound, + } + for code, want := range cases { + if got := FromCode(code); got != want { + t.Fatalf("FromCode(%v) = %v, want %v", code, got, want) + } + } + // Unknown codes fall back to Unexpected. + if got := FromCode(filep2p.ErrCodes(999)); got != ErrUnexpected { + t.Fatalf("FromCode(unknown) = %v, want ErrUnexpected", got) + } +} diff --git a/commonfile/fileproto/filep2p/protos/filep2p.proto b/commonfile/fileproto/filep2p/protos/filep2p.proto new file mode 100644 index 000000000..4d79fd0f4 --- /dev/null +++ b/commonfile/fileproto/filep2p/protos/filep2p.proto @@ -0,0 +1,92 @@ +syntax = "proto3"; +package filesyncp2p; + +option go_package = "commonfile/fileproto/filep2p"; + +// FileP2P is the peer-to-peer file transfer service. It is served +// READ-ONLY by a client to other LAN peers so a device can fetch a file's +// encrypted CAR object directly from a nearby peer that already holds it, +// instead of the file-node / object-storage path — enabling local-network +// file sync, offline (no reachable node) reads, and durability takeover +// (a peer re-uploading a file whose creator never made it durable). +// +// drpc route prefix is "/filesyncp2p.FileP2P/..." — DISTINCT from v1 +// "/filesync.File/..." and v2 "/filesyncv2.FileV2/...", so all three can be +// mounted on one drpc mux. Names are load-bearing: method paths derive from +// `package filesyncp2p` + `service FileP2P`. +// +// Model: a client stores one immutable, content-addressed CARv2 object per +// file, byte-identical to the object the S3/public-read path serves. So a peer +// is simply an ALTERNATE SOURCE of that same object: ObjectRead returns object +// byte RANGES exactly like an HTTP Range GET, and the fetcher's existing +// seed/verify path (parse header+index, then verify every block against its +// cid) is reused unchanged. Because a partial local copy has holes, ObjectRead +// is served only by peers that hold the file in FULL; FileCheck reports which. +// +// Conventions: +// * requests are ROOT-SCOPED (rootCid) — the store is one CAR per root, keyed +// by rootCid, with no global block-CID index; +// * identity is NEVER a request field — it is the authenticated connection; +// authorization (may this peer read this space) is the server's job; +// * object bytes are CIPHERTEXT (the file layer is encrypted before the DAG +// is built), so serving them discloses no plaintext; the FETCHER verifies +// every block against its cid (nothing on this wire does). +service FileP2P { + // FileCheck reports, per requested rootCid, whether the server holds the + // file fully, partially, or not at all — so a client can pick a FULL peer + // to ObjectRead from without first knowing the object geometry. + rpc FileCheck(FileCheckRequest) returns (FileCheckResponse); + // ObjectRead returns a byte range of the file's CARv2 object. Mirrors an + // HTTP Range GET: totalSize echoes the whole-object size (for the head + // probe / index-tail read). Served only when the file is held in full. + rpc ObjectRead(ObjectReadRequest) returns (ObjectReadResponse); +} + +// ErrCodes are WHOLE-RPC (drpc-level) errors, mirroring v1 filesync.ErrCodes +// and v2's fileprotov2.ErrCodes. Wired through rpcerr.ErrGroup(ErrorOffset) in +// the filep2perr package. Offset 900 is the next free rpcerr block (v1=200, +// v2=800). Zero value Unexpected=0 registers at offset+0 so an uncoded fault +// reads as Unexpected. +enum ErrCodes { + Unexpected = 0; // unclassified server fault (registry fallback) + Forbidden = 1; // connection identity may not read this space + InvalidRequest = 2; // malformed: empty spaceId, bad rootCid, bad range + FileNotFound = 3; // server does not hold this (spaceId, rootCid) in full + ErrorOffset = 900; // rpcerr.ErrGroup base (constant, never an error) +} + +// ---- FileCheck (availability) --------------------------------------------- + +message FileCheckRequest { + string spaceId = 1; // auth scope + repeated bytes rootCids = 2; // files to report on (result match key) +} + +message FileCheckResponse { + repeated FileAvailability files = 1; // one per requested rootCid, keyed by rootCid +} + +message FileAvailability { + bytes rootCid = 1; // echoes the requested rootCid (match key) + Availability have = 2; // how much of the file the server holds +} + +enum Availability { + None = 0; // server has no block of this file + Partial = 1; // server has some but not all blocks (cannot ObjectRead yet) + Full = 2; // server has every block; ObjectRead will serve +} + +// ---- ObjectRead (ranged CAR bytes) ---------------------------------------- + +message ObjectReadRequest { + string spaceId = 1; // auth scope + bytes rootCid = 2; // file root — selects the CAR object to read from + uint64 offset = 3; // start byte within the object + uint32 length = 4; // number of bytes requested from offset +} + +message ObjectReadResponse { + bytes data = 1; // the requested bytes (short only at object end) + uint64 totalSize = 2; // whole-object size (like HTTP Content-Range total) +} diff --git a/commonfile/fileproto/fileprotov2/fileprotov2err/fileprotov2err.go b/commonfile/fileproto/fileprotov2/fileprotov2err/fileprotov2err.go new file mode 100644 index 000000000..6905073e5 --- /dev/null +++ b/commonfile/fileproto/fileprotov2/fileprotov2err/fileprotov2err.go @@ -0,0 +1,40 @@ +package fileprotov2err + +import ( + "fmt" + + "github.com/anyproto/any-sync/commonfile/fileproto/fileprotov2" + "github.com/anyproto/any-sync/net/rpc/rpcerr" +) + +// errGroup registers the v2 WHOLE-RPC (drpc-level) errors in the offset-800 +// block, distinct from v1 filesync's 200 block. Only the cross-cutting +// ErrCodes values are registered here; per-item ErrCode outcomes travel in the +// response body and are NOT registered with rpcerr. Mirrors the v1 +// commonfile/fileproto/fileprotoerr package. +var ( + errGroup = rpcerr.ErrGroup(fileprotov2.ErrCodes_ErrorOffset) + ErrUnexpected = errGroup.Register(fmt.Errorf("unexpected filenode-v2 error"), uint64(fileprotov2.ErrCodes_Unexpected)) + ErrForbidden = errGroup.Register(fmt.Errorf("forbidden"), uint64(fileprotov2.ErrCodes_Forbidden)) + ErrInvalidRequest = errGroup.Register(fmt.Errorf("invalid request"), uint64(fileprotov2.ErrCodes_InvalidRequest)) + ErrQuerySizeExceeded = errGroup.Register(fmt.Errorf("query size exceeded"), uint64(fileprotov2.ErrCodes_QuerySizeExceeded)) + ErrNotResponsible = errGroup.Register(fmt.Errorf("node is not responsible for the space"), uint64(fileprotov2.ErrCodes_NotResponsible)) +) + +// FromCode maps a whole-RPC ErrCodes value to its registered drpc error so +// handlers can uniformly `return fileprotov2err.FromCode(code)` at the RPC +// boundary. Unknown/zero codes fall back to ErrUnexpected. +func FromCode(code fileprotov2.ErrCodes) error { + switch code { + case fileprotov2.ErrCodes_Forbidden: + return ErrForbidden + case fileprotov2.ErrCodes_InvalidRequest: + return ErrInvalidRequest + case fileprotov2.ErrCodes_QuerySizeExceeded: + return ErrQuerySizeExceeded + case fileprotov2.ErrCodes_NotResponsible: + return ErrNotResponsible + default: + return ErrUnexpected + } +} diff --git a/commonfile/fileproto/fileprotov2/filev2.pb.go b/commonfile/fileproto/fileprotov2/filev2.pb.go new file mode 100644 index 000000000..fd252132d --- /dev/null +++ b/commonfile/fileproto/fileprotov2/filev2.pb.go @@ -0,0 +1,1432 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v3.21.12 +// source: commonfile/fileproto/fileprotov2/protos/filev2.proto + +package fileprotov2 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// ErrCodes are WHOLE-RPC (drpc-level) errors, mirroring v1 filesync.ErrCodes. +// Wired through rpcerr.ErrGroup(ErrCodes_ErrorOffset) in the fileprotov2err +// package. Returned as a NON-nil drpc error ONLY for auth / malformed / +// batch-too-large / misrouted. Never used to report the outcome of an +// individual item (that is the per-item ErrCode below). Zero value +// Unexpected=0 registers at offset+0 (=800), so an uncoded server fault reads +// as Unexpected — exactly as v1 does at 200. Offset 800 is the only free +// rpcerr block (v1=200, 100/300/400/500/600/700 are taken). +type ErrCodes int32 + +const ( + ErrCodes_Unexpected ErrCodes = 0 // unclassified server fault (registry fallback) + ErrCodes_Forbidden ErrCodes = 1 // connection identity is not permitted at all + ErrCodes_InvalidRequest ErrCodes = 2 // malformed: empty spaceId, empty items, bad rootCid bytes + ErrCodes_QuerySizeExceeded ErrCodes = 3 // batch too large (too many items in one request) + ErrCodes_NotResponsible ErrCodes = 4 // retryable routing signal: this node is not in the space's chash pair + ErrCodes_ErrorOffset ErrCodes = 800 // rpcerr.ErrGroup base for v2 (constant, never an error) +) + +// Enum value maps for ErrCodes. +var ( + ErrCodes_name = map[int32]string{ + 0: "Unexpected", + 1: "Forbidden", + 2: "InvalidRequest", + 3: "QuerySizeExceeded", + 4: "NotResponsible", + 800: "ErrorOffset", + } + ErrCodes_value = map[string]int32{ + "Unexpected": 0, + "Forbidden": 1, + "InvalidRequest": 2, + "QuerySizeExceeded": 3, + "NotResponsible": 4, + "ErrorOffset": 800, + } +) + +func (x ErrCodes) Enum() *ErrCodes { + p := new(ErrCodes) + *p = x + return p +} + +func (x ErrCodes) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ErrCodes) Descriptor() protoreflect.EnumDescriptor { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_enumTypes[0].Descriptor() +} + +func (ErrCodes) Type() protoreflect.EnumType { + return &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_enumTypes[0] +} + +func (x ErrCodes) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ErrCodes.Descriptor instead. +func (ErrCodes) EnumDescriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{0} +} + +// ErrCode is the PER-ITEM, in-body business outcome (err == nil at the RPC +// layer). Flat status enum with an explicit success sentinel Ok=0. The typed +// payload on a *Result is valid ONLY when code == Ok. Deliberately a SEPARATE +// enum from ErrCodes: its zero value means success (not error), it never flows +// through rpcerr, and its value short-names are kept disjoint from ErrCodes so +// proto3's package-level enum-value scoping stays clean. Servers MUST set +// `code` explicitly on every result (Ok is the proto3 default, so an unset +// code would be misread as a success with an empty payload). +type ErrCode int32 + +const ( + ErrCode_Ok ErrCode = 0 // item succeeded; the typed payload is populated + ErrCode_ErrUnexpected ErrCode = 1 // internal error handling this single item + ErrCode_ErrCidNotFound ErrCode = 2 // sign/download: no such stored file for this rootCid + ErrCode_ErrForbidden ErrCode = 3 // identity may not access this item's space/file + ErrCode_ErrLimitExceeded ErrCode = 4 // upload rejected: space/account storage limit reached + ErrCode_ErrSpaceNotFound ErrCode = 5 // quota-info: unknown/deleted space id + ErrCode_ErrNotResponsible ErrCode = 6 // retryable routing signal: this node is not in this item's space chash pair +) + +// Enum value maps for ErrCode. +var ( + ErrCode_name = map[int32]string{ + 0: "Ok", + 1: "ErrUnexpected", + 2: "ErrCidNotFound", + 3: "ErrForbidden", + 4: "ErrLimitExceeded", + 5: "ErrSpaceNotFound", + 6: "ErrNotResponsible", + } + ErrCode_value = map[string]int32{ + "Ok": 0, + "ErrUnexpected": 1, + "ErrCidNotFound": 2, + "ErrForbidden": 3, + "ErrLimitExceeded": 4, + "ErrSpaceNotFound": 5, + "ErrNotResponsible": 6, + } +) + +func (x ErrCode) Enum() *ErrCode { + p := new(ErrCode) + *p = x + return p +} + +func (x ErrCode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ErrCode) Descriptor() protoreflect.EnumDescriptor { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_enumTypes[1].Descriptor() +} + +func (ErrCode) Type() protoreflect.EnumType { + return &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_enumTypes[1] +} + +func (x ErrCode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ErrCode.Descriptor instead. +func (ErrCode) EnumDescriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{1} +} + +type UploadRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + SpaceId string `protobuf:"bytes,1,opt,name=spaceId,proto3" json:"spaceId,omitempty"` // shared space for every item (auth scope) + Items []*UploadRequestItem `protobuf:"bytes,2,rep,name=items,proto3" json:"items,omitempty"` // files to obtain an upload target for + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *UploadRequest) Reset() { + *x = UploadRequest{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *UploadRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UploadRequest) ProtoMessage() {} + +func (x *UploadRequest) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UploadRequest.ProtoReflect.Descriptor instead. +func (*UploadRequest) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{0} +} + +func (x *UploadRequest) GetSpaceId() string { + if x != nil { + return x.SpaceId + } + return "" +} + +func (x *UploadRequest) GetItems() []*UploadRequestItem { + if x != nil { + return x.Items + } + return nil +} + +type UploadRequestItem struct { + state protoimpl.MessageState `protogen:"open.v1"` + RootCid []byte `protobuf:"bytes,1,opt,name=rootCid,proto3" json:"rootCid,omitempty"` // CID of the file root (result match key) + Size uint64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"` // declared total byte size of the file + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *UploadRequestItem) Reset() { + *x = UploadRequestItem{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *UploadRequestItem) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UploadRequestItem) ProtoMessage() {} + +func (x *UploadRequestItem) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UploadRequestItem.ProtoReflect.Descriptor instead. +func (*UploadRequestItem) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{1} +} + +func (x *UploadRequestItem) GetRootCid() []byte { + if x != nil { + return x.RootCid + } + return nil +} + +func (x *UploadRequestItem) GetSize() uint64 { + if x != nil { + return x.Size + } + return 0 +} + +type UploadResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Results []*UploadResult `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` // one per requested item, keyed by rootCid + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *UploadResponse) Reset() { + *x = UploadResponse{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *UploadResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UploadResponse) ProtoMessage() {} + +func (x *UploadResponse) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UploadResponse.ProtoReflect.Descriptor instead. +func (*UploadResponse) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{2} +} + +func (x *UploadResponse) GetResults() []*UploadResult { + if x != nil { + return x.Results + } + return nil +} + +type UploadResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + RootCid []byte `protobuf:"bytes,1,opt,name=rootCid,proto3" json:"rootCid,omitempty"` // echoes UploadRequestItem.rootCid (match key) + Code ErrCode `protobuf:"varint,2,opt,name=code,proto3,enum=filesyncv2.ErrCode" json:"code,omitempty"` // per-item outcome; `upload` valid only when Ok + Upload *PresignedUpload `protobuf:"bytes,3,opt,name=upload,proto3" json:"upload,omitempty"` // set ONLY when code == Ok + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *UploadResult) Reset() { + *x = UploadResult{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *UploadResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UploadResult) ProtoMessage() {} + +func (x *UploadResult) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UploadResult.ProtoReflect.Descriptor instead. +func (*UploadResult) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{3} +} + +func (x *UploadResult) GetRootCid() []byte { + if x != nil { + return x.RootCid + } + return nil +} + +func (x *UploadResult) GetCode() ErrCode { + if x != nil { + return x.Code + } + return ErrCode_Ok +} + +func (x *UploadResult) GetUpload() *PresignedUpload { + if x != nil { + return x.Upload + } + return nil +} + +// PresignedUpload is an OPAQUE presigned upload target. No S3 vendor structs +// leak into the shared proto: the client POSTs `fields` + the object body to +// `url` verbatim. Modeled as a presigned POST form (fields are ordered +// key/value pairs rather than a map) so the signing order is deterministic and +// reproducible. +type PresignedUpload struct { + state protoimpl.MessageState `protogen:"open.v1"` + Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` // presigned POST endpoint + Fields []*PresignField `protobuf:"bytes,2,rep,name=fields,proto3" json:"fields,omitempty"` // multipart form fields to send verbatim, in order + MaxContentLength uint64 `protobuf:"varint,3,opt,name=maxContentLength,proto3" json:"maxContentLength,omitempty"` // hard upper bound the presign enforces (bytes) + Expiry int64 `protobuf:"varint,4,opt,name=expiry,proto3" json:"expiry,omitempty"` // unix seconds; POST must complete before this + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PresignedUpload) Reset() { + *x = PresignedUpload{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PresignedUpload) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PresignedUpload) ProtoMessage() {} + +func (x *PresignedUpload) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PresignedUpload.ProtoReflect.Descriptor instead. +func (*PresignedUpload) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{4} +} + +func (x *PresignedUpload) GetUrl() string { + if x != nil { + return x.Url + } + return "" +} + +func (x *PresignedUpload) GetFields() []*PresignField { + if x != nil { + return x.Fields + } + return nil +} + +func (x *PresignedUpload) GetMaxContentLength() uint64 { + if x != nil { + return x.MaxContentLength + } + return 0 +} + +func (x *PresignedUpload) GetExpiry() int64 { + if x != nil { + return x.Expiry + } + return 0 +} + +type PresignField struct { + state protoimpl.MessageState `protogen:"open.v1"` + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PresignField) Reset() { + *x = PresignField{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PresignField) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PresignField) ProtoMessage() {} + +func (x *PresignField) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PresignField.ProtoReflect.Descriptor instead. +func (*PresignField) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{5} +} + +func (x *PresignField) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *PresignField) GetValue() string { + if x != nil { + return x.Value + } + return "" +} + +type RequestSignRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + SpaceId string `protobuf:"bytes,1,opt,name=spaceId,proto3" json:"spaceId,omitempty"` // shared space for every rootCid (auth scope) + RootCids [][]byte `protobuf:"bytes,2,rep,name=rootCids,proto3" json:"rootCids,omitempty"` // files to obtain a durable-custody receipt for + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RequestSignRequest) Reset() { + *x = RequestSignRequest{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RequestSignRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RequestSignRequest) ProtoMessage() {} + +func (x *RequestSignRequest) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RequestSignRequest.ProtoReflect.Descriptor instead. +func (*RequestSignRequest) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{6} +} + +func (x *RequestSignRequest) GetSpaceId() string { + if x != nil { + return x.SpaceId + } + return "" +} + +func (x *RequestSignRequest) GetRootCids() [][]byte { + if x != nil { + return x.RootCids + } + return nil +} + +type RequestSignResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Results []*RequestSignResult `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` // one per rootCid, keyed by rootCid + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RequestSignResponse) Reset() { + *x = RequestSignResponse{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RequestSignResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RequestSignResponse) ProtoMessage() {} + +func (x *RequestSignResponse) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RequestSignResponse.ProtoReflect.Descriptor instead. +func (*RequestSignResponse) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{7} +} + +func (x *RequestSignResponse) GetResults() []*RequestSignResult { + if x != nil { + return x.Results + } + return nil +} + +type RequestSignResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + RootCid []byte `protobuf:"bytes,1,opt,name=rootCid,proto3" json:"rootCid,omitempty"` // echoes the requested rootCid (match key) + Code ErrCode `protobuf:"varint,2,opt,name=code,proto3,enum=filesyncv2.ErrCode" json:"code,omitempty"` // per-item outcome; `receipt` valid only when Ok + Receipt *NetworkSignReceipt `protobuf:"bytes,3,opt,name=receipt,proto3" json:"receipt,omitempty"` // set ONLY when code == Ok + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RequestSignResult) Reset() { + *x = RequestSignResult{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RequestSignResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RequestSignResult) ProtoMessage() {} + +func (x *RequestSignResult) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RequestSignResult.ProtoReflect.Descriptor instead. +func (*RequestSignResult) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{8} +} + +func (x *RequestSignResult) GetRootCid() []byte { + if x != nil { + return x.RootCid + } + return nil +} + +func (x *RequestSignResult) GetCode() ErrCode { + if x != nil { + return x.Code + } + return ErrCode_Ok +} + +func (x *RequestSignResult) GetReceipt() *NetworkSignReceipt { + if x != nil { + return x.Receipt + } + return nil +} + +// NetworkSignReceipt mirrors coordinator SpaceReceiptWithSignature: an opaque +// protobuf-encoded payload plus the network-key signature over it. The payload +// schema is intentionally not defined here (opaque blob forwarded verbatim). +type NetworkSignReceipt struct { + state protoimpl.MessageState `protogen:"open.v1"` + ReceiptPayload []byte `protobuf:"bytes,1,opt,name=receiptPayload,proto3" json:"receiptPayload,omitempty"` // protobuf-encoded signed receipt payload (opaque) + Signature []byte `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` // network key signature over receiptPayload + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *NetworkSignReceipt) Reset() { + *x = NetworkSignReceipt{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *NetworkSignReceipt) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NetworkSignReceipt) ProtoMessage() {} + +func (x *NetworkSignReceipt) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NetworkSignReceipt.ProtoReflect.Descriptor instead. +func (*NetworkSignReceipt) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{9} +} + +func (x *NetworkSignReceipt) GetReceiptPayload() []byte { + if x != nil { + return x.ReceiptPayload + } + return nil +} + +func (x *NetworkSignReceipt) GetSignature() []byte { + if x != nil { + return x.Signature + } + return nil +} + +type RequestDownloadRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + SpaceId string `protobuf:"bytes,1,opt,name=spaceId,proto3" json:"spaceId,omitempty"` // shared space for every rootCid (auth scope) + RootCids [][]byte `protobuf:"bytes,2,rep,name=rootCids,proto3" json:"rootCids,omitempty"` // files to obtain a signed GET target for + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RequestDownloadRequest) Reset() { + *x = RequestDownloadRequest{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RequestDownloadRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RequestDownloadRequest) ProtoMessage() {} + +func (x *RequestDownloadRequest) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RequestDownloadRequest.ProtoReflect.Descriptor instead. +func (*RequestDownloadRequest) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{10} +} + +func (x *RequestDownloadRequest) GetSpaceId() string { + if x != nil { + return x.SpaceId + } + return "" +} + +func (x *RequestDownloadRequest) GetRootCids() [][]byte { + if x != nil { + return x.RootCids + } + return nil +} + +type RequestDownloadResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Results []*RequestDownloadResult `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` // one per rootCid, keyed by rootCid + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RequestDownloadResponse) Reset() { + *x = RequestDownloadResponse{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RequestDownloadResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RequestDownloadResponse) ProtoMessage() {} + +func (x *RequestDownloadResponse) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RequestDownloadResponse.ProtoReflect.Descriptor instead. +func (*RequestDownloadResponse) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{11} +} + +func (x *RequestDownloadResponse) GetResults() []*RequestDownloadResult { + if x != nil { + return x.Results + } + return nil +} + +type RequestDownloadResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + RootCid []byte `protobuf:"bytes,1,opt,name=rootCid,proto3" json:"rootCid,omitempty"` // echoes the requested rootCid (match key) + Code ErrCode `protobuf:"varint,2,opt,name=code,proto3,enum=filesyncv2.ErrCode" json:"code,omitempty"` // per-item outcome; `download` valid only when Ok + Download *SignedDownload `protobuf:"bytes,3,opt,name=download,proto3" json:"download,omitempty"` // set ONLY when code == Ok + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RequestDownloadResult) Reset() { + *x = RequestDownloadResult{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RequestDownloadResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RequestDownloadResult) ProtoMessage() {} + +func (x *RequestDownloadResult) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RequestDownloadResult.ProtoReflect.Descriptor instead. +func (*RequestDownloadResult) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{12} +} + +func (x *RequestDownloadResult) GetRootCid() []byte { + if x != nil { + return x.RootCid + } + return nil +} + +func (x *RequestDownloadResult) GetCode() ErrCode { + if x != nil { + return x.Code + } + return ErrCode_Ok +} + +func (x *RequestDownloadResult) GetDownload() *SignedDownload { + if x != nil { + return x.Download + } + return nil +} + +// SignedDownload is an OPAQUE signed GET target (kept as a message, symmetric +// with PresignedUpload, so download can grow response headers etc. without a +// wire break on the result envelope). +type SignedDownload struct { + state protoimpl.MessageState `protogen:"open.v1"` + Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` // opaque signed GET url + Expiry int64 `protobuf:"varint,2,opt,name=expiry,proto3" json:"expiry,omitempty"` // unix seconds the url is valid until + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SignedDownload) Reset() { + *x = SignedDownload{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SignedDownload) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SignedDownload) ProtoMessage() {} + +func (x *SignedDownload) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SignedDownload.ProtoReflect.Descriptor instead. +func (*SignedDownload) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{13} +} + +func (x *SignedDownload) GetUrl() string { + if x != nil { + return x.Url + } + return "" +} + +func (x *SignedDownload) GetExpiry() int64 { + if x != nil { + return x.Expiry + } + return 0 +} + +type SpaceInfoRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + SpaceIds []string `protobuf:"bytes,1,rep,name=spaceIds,proto3" json:"spaceIds,omitempty"` // batch of spaces to report on (result match key) + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SpaceInfoRequest) Reset() { + *x = SpaceInfoRequest{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SpaceInfoRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SpaceInfoRequest) ProtoMessage() {} + +func (x *SpaceInfoRequest) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[14] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SpaceInfoRequest.ProtoReflect.Descriptor instead. +func (*SpaceInfoRequest) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{14} +} + +func (x *SpaceInfoRequest) GetSpaceIds() []string { + if x != nil { + return x.SpaceIds + } + return nil +} + +type SpaceInfoResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Results []*SpaceInfoResult `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` // one per requested space, keyed by spaceId + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SpaceInfoResponse) Reset() { + *x = SpaceInfoResponse{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SpaceInfoResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SpaceInfoResponse) ProtoMessage() {} + +func (x *SpaceInfoResponse) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SpaceInfoResponse.ProtoReflect.Descriptor instead. +func (*SpaceInfoResponse) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{15} +} + +func (x *SpaceInfoResponse) GetResults() []*SpaceInfoResult { + if x != nil { + return x.Results + } + return nil +} + +type SpaceInfoResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + SpaceId string `protobuf:"bytes,1,opt,name=spaceId,proto3" json:"spaceId,omitempty"` // echoes the requested spaceId (match key) + Code ErrCode `protobuf:"varint,2,opt,name=code,proto3,enum=filesyncv2.ErrCode" json:"code,omitempty"` // per-item outcome; `quota` valid only when Ok + Quota *SpaceQuota `protobuf:"bytes,3,opt,name=quota,proto3" json:"quota,omitempty"` // set ONLY when code == Ok + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SpaceInfoResult) Reset() { + *x = SpaceInfoResult{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SpaceInfoResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SpaceInfoResult) ProtoMessage() {} + +func (x *SpaceInfoResult) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SpaceInfoResult.ProtoReflect.Descriptor instead. +func (*SpaceInfoResult) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{16} +} + +func (x *SpaceInfoResult) GetSpaceId() string { + if x != nil { + return x.SpaceId + } + return "" +} + +func (x *SpaceInfoResult) GetCode() ErrCode { + if x != nil { + return x.Code + } + return ErrCode_Ok +} + +func (x *SpaceInfoResult) GetQuota() *SpaceQuota { + if x != nil { + return x.Quota + } + return nil +} + +type SpaceQuota struct { + state protoimpl.MessageState `protogen:"open.v1"` + LimitBytes uint64 `protobuf:"varint,1,opt,name=limitBytes,proto3" json:"limitBytes,omitempty"` // effective storage limit for the requester in this space + TotalUsageBytes uint64 `protobuf:"varint,2,opt,name=totalUsageBytes,proto3" json:"totalUsageBytes,omitempty"` // counted usage == durableUsageBytes + inflightUsageBytes + DurableUsageBytes uint64 `protobuf:"varint,3,opt,name=durableUsageBytes,proto3" json:"durableUsageBytes,omitempty"` // bytes fully committed to durable storage + InflightUsageBytes uint64 `protobuf:"varint,4,opt,name=inflightUsageBytes,proto3" json:"inflightUsageBytes,omitempty"` // bytes reserved by presigned/not-yet-committed uploads + FilesCount uint64 `protobuf:"varint,5,opt,name=filesCount,proto3" json:"filesCount,omitempty"` // number of file roots stored in the space + // account-pool view for the requesting identity (files are charged to + // their uploader; all the identity's spaces share one pool). Zero when + // the node has no coordinator-backed limits (self-hosted fallback). + AccountLimitBytes uint64 `protobuf:"varint,6,opt,name=accountLimitBytes,proto3" json:"accountLimitBytes,omitempty"` // the identity's total file-storage cap + AccountUsageBytes uint64 `protobuf:"varint,7,opt,name=accountUsageBytes,proto3" json:"accountUsageBytes,omitempty"` // aggregated durable usage across the identity's spaces + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SpaceQuota) Reset() { + *x = SpaceQuota{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SpaceQuota) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SpaceQuota) ProtoMessage() {} + +func (x *SpaceQuota) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[17] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SpaceQuota.ProtoReflect.Descriptor instead. +func (*SpaceQuota) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{17} +} + +func (x *SpaceQuota) GetLimitBytes() uint64 { + if x != nil { + return x.LimitBytes + } + return 0 +} + +func (x *SpaceQuota) GetTotalUsageBytes() uint64 { + if x != nil { + return x.TotalUsageBytes + } + return 0 +} + +func (x *SpaceQuota) GetDurableUsageBytes() uint64 { + if x != nil { + return x.DurableUsageBytes + } + return 0 +} + +func (x *SpaceQuota) GetInflightUsageBytes() uint64 { + if x != nil { + return x.InflightUsageBytes + } + return 0 +} + +func (x *SpaceQuota) GetFilesCount() uint64 { + if x != nil { + return x.FilesCount + } + return 0 +} + +func (x *SpaceQuota) GetAccountLimitBytes() uint64 { + if x != nil { + return x.AccountLimitBytes + } + return 0 +} + +func (x *SpaceQuota) GetAccountUsageBytes() uint64 { + if x != nil { + return x.AccountUsageBytes + } + return 0 +} + +// InfoRequest is deliberately empty: Info is static node metadata, +// extensible later (capabilities, limits) without a wire break. +type InfoRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *InfoRequest) Reset() { + *x = InfoRequest{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *InfoRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*InfoRequest) ProtoMessage() {} + +func (x *InfoRequest) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[18] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use InfoRequest.ProtoReflect.Descriptor instead. +func (*InfoRequest) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{18} +} + +type InfoResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // publicReadBaseUrl, when non-empty, lets clients construct durable object + // URLs as {publicReadBaseUrl}/blob/{spaceId}/{rootCid} with zero broker + // round-trips; clients cache it. Empty = no public read: use RequestDownload. + PublicReadBaseUrl string `protobuf:"bytes,1,opt,name=publicReadBaseUrl,proto3" json:"publicReadBaseUrl,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *InfoResponse) Reset() { + *x = InfoResponse{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *InfoResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*InfoResponse) ProtoMessage() {} + +func (x *InfoResponse) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[19] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use InfoResponse.ProtoReflect.Descriptor instead. +func (*InfoResponse) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{19} +} + +func (x *InfoResponse) GetPublicReadBaseUrl() string { + if x != nil { + return x.PublicReadBaseUrl + } + return "" +} + +var File_commonfile_fileproto_fileprotov2_protos_filev2_proto protoreflect.FileDescriptor + +const file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDesc = "" + + "\n" + + "4commonfile/fileproto/fileprotov2/protos/filev2.proto\x12\n" + + "filesyncv2\"^\n" + + "\rUploadRequest\x12\x18\n" + + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x123\n" + + "\x05items\x18\x02 \x03(\v2\x1d.filesyncv2.UploadRequestItemR\x05items\"A\n" + + "\x11UploadRequestItem\x12\x18\n" + + "\arootCid\x18\x01 \x01(\fR\arootCid\x12\x12\n" + + "\x04size\x18\x02 \x01(\x04R\x04size\"D\n" + + "\x0eUploadResponse\x122\n" + + "\aresults\x18\x01 \x03(\v2\x18.filesyncv2.UploadResultR\aresults\"\x86\x01\n" + + "\fUploadResult\x12\x18\n" + + "\arootCid\x18\x01 \x01(\fR\arootCid\x12'\n" + + "\x04code\x18\x02 \x01(\x0e2\x13.filesyncv2.ErrCodeR\x04code\x123\n" + + "\x06upload\x18\x03 \x01(\v2\x1b.filesyncv2.PresignedUploadR\x06upload\"\x99\x01\n" + + "\x0fPresignedUpload\x12\x10\n" + + "\x03url\x18\x01 \x01(\tR\x03url\x120\n" + + "\x06fields\x18\x02 \x03(\v2\x18.filesyncv2.PresignFieldR\x06fields\x12*\n" + + "\x10maxContentLength\x18\x03 \x01(\x04R\x10maxContentLength\x12\x16\n" + + "\x06expiry\x18\x04 \x01(\x03R\x06expiry\"6\n" + + "\fPresignField\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value\"J\n" + + "\x12RequestSignRequest\x12\x18\n" + + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x12\x1a\n" + + "\brootCids\x18\x02 \x03(\fR\brootCids\"N\n" + + "\x13RequestSignResponse\x127\n" + + "\aresults\x18\x01 \x03(\v2\x1d.filesyncv2.RequestSignResultR\aresults\"\x90\x01\n" + + "\x11RequestSignResult\x12\x18\n" + + "\arootCid\x18\x01 \x01(\fR\arootCid\x12'\n" + + "\x04code\x18\x02 \x01(\x0e2\x13.filesyncv2.ErrCodeR\x04code\x128\n" + + "\areceipt\x18\x03 \x01(\v2\x1e.filesyncv2.NetworkSignReceiptR\areceipt\"Z\n" + + "\x12NetworkSignReceipt\x12&\n" + + "\x0ereceiptPayload\x18\x01 \x01(\fR\x0ereceiptPayload\x12\x1c\n" + + "\tsignature\x18\x02 \x01(\fR\tsignature\"N\n" + + "\x16RequestDownloadRequest\x12\x18\n" + + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x12\x1a\n" + + "\brootCids\x18\x02 \x03(\fR\brootCids\"V\n" + + "\x17RequestDownloadResponse\x12;\n" + + "\aresults\x18\x01 \x03(\v2!.filesyncv2.RequestDownloadResultR\aresults\"\x92\x01\n" + + "\x15RequestDownloadResult\x12\x18\n" + + "\arootCid\x18\x01 \x01(\fR\arootCid\x12'\n" + + "\x04code\x18\x02 \x01(\x0e2\x13.filesyncv2.ErrCodeR\x04code\x126\n" + + "\bdownload\x18\x03 \x01(\v2\x1a.filesyncv2.SignedDownloadR\bdownload\":\n" + + "\x0eSignedDownload\x12\x10\n" + + "\x03url\x18\x01 \x01(\tR\x03url\x12\x16\n" + + "\x06expiry\x18\x02 \x01(\x03R\x06expiry\".\n" + + "\x10SpaceInfoRequest\x12\x1a\n" + + "\bspaceIds\x18\x01 \x03(\tR\bspaceIds\"J\n" + + "\x11SpaceInfoResponse\x125\n" + + "\aresults\x18\x01 \x03(\v2\x1b.filesyncv2.SpaceInfoResultR\aresults\"\x82\x01\n" + + "\x0fSpaceInfoResult\x12\x18\n" + + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x12'\n" + + "\x04code\x18\x02 \x01(\x0e2\x13.filesyncv2.ErrCodeR\x04code\x12,\n" + + "\x05quota\x18\x03 \x01(\v2\x16.filesyncv2.SpaceQuotaR\x05quota\"\xb0\x02\n" + + "\n" + + "SpaceQuota\x12\x1e\n" + + "\n" + + "limitBytes\x18\x01 \x01(\x04R\n" + + "limitBytes\x12(\n" + + "\x0ftotalUsageBytes\x18\x02 \x01(\x04R\x0ftotalUsageBytes\x12,\n" + + "\x11durableUsageBytes\x18\x03 \x01(\x04R\x11durableUsageBytes\x12.\n" + + "\x12inflightUsageBytes\x18\x04 \x01(\x04R\x12inflightUsageBytes\x12\x1e\n" + + "\n" + + "filesCount\x18\x05 \x01(\x04R\n" + + "filesCount\x12,\n" + + "\x11accountLimitBytes\x18\x06 \x01(\x04R\x11accountLimitBytes\x12,\n" + + "\x11accountUsageBytes\x18\a \x01(\x04R\x11accountUsageBytes\"\r\n" + + "\vInfoRequest\"<\n" + + "\fInfoResponse\x12,\n" + + "\x11publicReadBaseUrl\x18\x01 \x01(\tR\x11publicReadBaseUrl*z\n" + + "\bErrCodes\x12\x0e\n" + + "\n" + + "Unexpected\x10\x00\x12\r\n" + + "\tForbidden\x10\x01\x12\x12\n" + + "\x0eInvalidRequest\x10\x02\x12\x15\n" + + "\x11QuerySizeExceeded\x10\x03\x12\x12\n" + + "\x0eNotResponsible\x10\x04\x12\x10\n" + + "\vErrorOffset\x10\xa0\x06*\x8d\x01\n" + + "\aErrCode\x12\x06\n" + + "\x02Ok\x10\x00\x12\x11\n" + + "\rErrUnexpected\x10\x01\x12\x12\n" + + "\x0eErrCidNotFound\x10\x02\x12\x10\n" + + "\fErrForbidden\x10\x03\x12\x14\n" + + "\x10ErrLimitExceeded\x10\x04\x12\x14\n" + + "\x10ErrSpaceNotFound\x10\x05\x12\x15\n" + + "\x11ErrNotResponsible\x10\x062\xfa\x02\n" + + "\x06FileV2\x12?\n" + + "\x06Upload\x12\x19.filesyncv2.UploadRequest\x1a\x1a.filesyncv2.UploadResponse\x12N\n" + + "\vRequestSign\x12\x1e.filesyncv2.RequestSignRequest\x1a\x1f.filesyncv2.RequestSignResponse\x12Z\n" + + "\x0fRequestDownload\x12\".filesyncv2.RequestDownloadRequest\x1a#.filesyncv2.RequestDownloadResponse\x12H\n" + + "\tSpaceInfo\x12\x1c.filesyncv2.SpaceInfoRequest\x1a\x1d.filesyncv2.SpaceInfoResponse\x129\n" + + "\x04Info\x12\x17.filesyncv2.InfoRequest\x1a\x18.filesyncv2.InfoResponseB\"Z commonfile/fileproto/fileprotov2b\x06proto3" + +var ( + file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescOnce sync.Once + file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescData []byte +) + +func file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP() []byte { + file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescOnce.Do(func() { + file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDesc), len(file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDesc))) + }) + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescData +} + +var file_commonfile_fileproto_fileprotov2_protos_filev2_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes = make([]protoimpl.MessageInfo, 20) +var file_commonfile_fileproto_fileprotov2_protos_filev2_proto_goTypes = []any{ + (ErrCodes)(0), // 0: filesyncv2.ErrCodes + (ErrCode)(0), // 1: filesyncv2.ErrCode + (*UploadRequest)(nil), // 2: filesyncv2.UploadRequest + (*UploadRequestItem)(nil), // 3: filesyncv2.UploadRequestItem + (*UploadResponse)(nil), // 4: filesyncv2.UploadResponse + (*UploadResult)(nil), // 5: filesyncv2.UploadResult + (*PresignedUpload)(nil), // 6: filesyncv2.PresignedUpload + (*PresignField)(nil), // 7: filesyncv2.PresignField + (*RequestSignRequest)(nil), // 8: filesyncv2.RequestSignRequest + (*RequestSignResponse)(nil), // 9: filesyncv2.RequestSignResponse + (*RequestSignResult)(nil), // 10: filesyncv2.RequestSignResult + (*NetworkSignReceipt)(nil), // 11: filesyncv2.NetworkSignReceipt + (*RequestDownloadRequest)(nil), // 12: filesyncv2.RequestDownloadRequest + (*RequestDownloadResponse)(nil), // 13: filesyncv2.RequestDownloadResponse + (*RequestDownloadResult)(nil), // 14: filesyncv2.RequestDownloadResult + (*SignedDownload)(nil), // 15: filesyncv2.SignedDownload + (*SpaceInfoRequest)(nil), // 16: filesyncv2.SpaceInfoRequest + (*SpaceInfoResponse)(nil), // 17: filesyncv2.SpaceInfoResponse + (*SpaceInfoResult)(nil), // 18: filesyncv2.SpaceInfoResult + (*SpaceQuota)(nil), // 19: filesyncv2.SpaceQuota + (*InfoRequest)(nil), // 20: filesyncv2.InfoRequest + (*InfoResponse)(nil), // 21: filesyncv2.InfoResponse +} +var file_commonfile_fileproto_fileprotov2_protos_filev2_proto_depIdxs = []int32{ + 3, // 0: filesyncv2.UploadRequest.items:type_name -> filesyncv2.UploadRequestItem + 5, // 1: filesyncv2.UploadResponse.results:type_name -> filesyncv2.UploadResult + 1, // 2: filesyncv2.UploadResult.code:type_name -> filesyncv2.ErrCode + 6, // 3: filesyncv2.UploadResult.upload:type_name -> filesyncv2.PresignedUpload + 7, // 4: filesyncv2.PresignedUpload.fields:type_name -> filesyncv2.PresignField + 10, // 5: filesyncv2.RequestSignResponse.results:type_name -> filesyncv2.RequestSignResult + 1, // 6: filesyncv2.RequestSignResult.code:type_name -> filesyncv2.ErrCode + 11, // 7: filesyncv2.RequestSignResult.receipt:type_name -> filesyncv2.NetworkSignReceipt + 14, // 8: filesyncv2.RequestDownloadResponse.results:type_name -> filesyncv2.RequestDownloadResult + 1, // 9: filesyncv2.RequestDownloadResult.code:type_name -> filesyncv2.ErrCode + 15, // 10: filesyncv2.RequestDownloadResult.download:type_name -> filesyncv2.SignedDownload + 18, // 11: filesyncv2.SpaceInfoResponse.results:type_name -> filesyncv2.SpaceInfoResult + 1, // 12: filesyncv2.SpaceInfoResult.code:type_name -> filesyncv2.ErrCode + 19, // 13: filesyncv2.SpaceInfoResult.quota:type_name -> filesyncv2.SpaceQuota + 2, // 14: filesyncv2.FileV2.Upload:input_type -> filesyncv2.UploadRequest + 8, // 15: filesyncv2.FileV2.RequestSign:input_type -> filesyncv2.RequestSignRequest + 12, // 16: filesyncv2.FileV2.RequestDownload:input_type -> filesyncv2.RequestDownloadRequest + 16, // 17: filesyncv2.FileV2.SpaceInfo:input_type -> filesyncv2.SpaceInfoRequest + 20, // 18: filesyncv2.FileV2.Info:input_type -> filesyncv2.InfoRequest + 4, // 19: filesyncv2.FileV2.Upload:output_type -> filesyncv2.UploadResponse + 9, // 20: filesyncv2.FileV2.RequestSign:output_type -> filesyncv2.RequestSignResponse + 13, // 21: filesyncv2.FileV2.RequestDownload:output_type -> filesyncv2.RequestDownloadResponse + 17, // 22: filesyncv2.FileV2.SpaceInfo:output_type -> filesyncv2.SpaceInfoResponse + 21, // 23: filesyncv2.FileV2.Info:output_type -> filesyncv2.InfoResponse + 19, // [19:24] is the sub-list for method output_type + 14, // [14:19] is the sub-list for method input_type + 14, // [14:14] is the sub-list for extension type_name + 14, // [14:14] is the sub-list for extension extendee + 0, // [0:14] is the sub-list for field type_name +} + +func init() { file_commonfile_fileproto_fileprotov2_protos_filev2_proto_init() } +func file_commonfile_fileproto_fileprotov2_protos_filev2_proto_init() { + if File_commonfile_fileproto_fileprotov2_protos_filev2_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDesc), len(file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDesc)), + NumEnums: 2, + NumMessages: 20, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_commonfile_fileproto_fileprotov2_protos_filev2_proto_goTypes, + DependencyIndexes: file_commonfile_fileproto_fileprotov2_protos_filev2_proto_depIdxs, + EnumInfos: file_commonfile_fileproto_fileprotov2_protos_filev2_proto_enumTypes, + MessageInfos: file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes, + }.Build() + File_commonfile_fileproto_fileprotov2_protos_filev2_proto = out.File + file_commonfile_fileproto_fileprotov2_protos_filev2_proto_goTypes = nil + file_commonfile_fileproto_fileprotov2_protos_filev2_proto_depIdxs = nil +} diff --git a/commonfile/fileproto/fileprotov2/filev2_drpc.pb.go b/commonfile/fileproto/fileprotov2/filev2_drpc.pb.go new file mode 100644 index 000000000..9da77b7e2 --- /dev/null +++ b/commonfile/fileproto/fileprotov2/filev2_drpc.pb.go @@ -0,0 +1,266 @@ +// Code generated by protoc-gen-go-drpc. DO NOT EDIT. +// protoc-gen-go-drpc version: v0.0.34 +// source: commonfile/fileproto/fileprotov2/protos/filev2.proto + +package fileprotov2 + +import ( + context "context" + errors "errors" + drpc1 "github.com/planetscale/vtprotobuf/codec/drpc" + drpc "storj.io/drpc" + drpcerr "storj.io/drpc/drpcerr" +) + +type drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto struct{} + +func (drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto) Marshal(msg drpc.Message) ([]byte, error) { + return drpc1.Marshal(msg) +} + +func (drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto) Unmarshal(buf []byte, msg drpc.Message) error { + return drpc1.Unmarshal(buf, msg) +} + +func (drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto) JSONMarshal(msg drpc.Message) ([]byte, error) { + return drpc1.JSONMarshal(msg) +} + +func (drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto) JSONUnmarshal(buf []byte, msg drpc.Message) error { + return drpc1.JSONUnmarshal(buf, msg) +} + +type DRPCFileV2Client interface { + DRPCConn() drpc.Conn + + Upload(ctx context.Context, in *UploadRequest) (*UploadResponse, error) + RequestSign(ctx context.Context, in *RequestSignRequest) (*RequestSignResponse, error) + RequestDownload(ctx context.Context, in *RequestDownloadRequest) (*RequestDownloadResponse, error) + SpaceInfo(ctx context.Context, in *SpaceInfoRequest) (*SpaceInfoResponse, error) + Info(ctx context.Context, in *InfoRequest) (*InfoResponse, error) +} + +type drpcFileV2Client struct { + cc drpc.Conn +} + +func NewDRPCFileV2Client(cc drpc.Conn) DRPCFileV2Client { + return &drpcFileV2Client{cc} +} + +func (c *drpcFileV2Client) DRPCConn() drpc.Conn { return c.cc } + +func (c *drpcFileV2Client) Upload(ctx context.Context, in *UploadRequest) (*UploadResponse, error) { + out := new(UploadResponse) + err := c.cc.Invoke(ctx, "/filesyncv2.FileV2/Upload", drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}, in, out) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *drpcFileV2Client) RequestSign(ctx context.Context, in *RequestSignRequest) (*RequestSignResponse, error) { + out := new(RequestSignResponse) + err := c.cc.Invoke(ctx, "/filesyncv2.FileV2/RequestSign", drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}, in, out) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *drpcFileV2Client) RequestDownload(ctx context.Context, in *RequestDownloadRequest) (*RequestDownloadResponse, error) { + out := new(RequestDownloadResponse) + err := c.cc.Invoke(ctx, "/filesyncv2.FileV2/RequestDownload", drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}, in, out) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *drpcFileV2Client) SpaceInfo(ctx context.Context, in *SpaceInfoRequest) (*SpaceInfoResponse, error) { + out := new(SpaceInfoResponse) + err := c.cc.Invoke(ctx, "/filesyncv2.FileV2/SpaceInfo", drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}, in, out) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *drpcFileV2Client) Info(ctx context.Context, in *InfoRequest) (*InfoResponse, error) { + out := new(InfoResponse) + err := c.cc.Invoke(ctx, "/filesyncv2.FileV2/Info", drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}, in, out) + if err != nil { + return nil, err + } + return out, nil +} + +type DRPCFileV2Server interface { + Upload(context.Context, *UploadRequest) (*UploadResponse, error) + RequestSign(context.Context, *RequestSignRequest) (*RequestSignResponse, error) + RequestDownload(context.Context, *RequestDownloadRequest) (*RequestDownloadResponse, error) + SpaceInfo(context.Context, *SpaceInfoRequest) (*SpaceInfoResponse, error) + Info(context.Context, *InfoRequest) (*InfoResponse, error) +} + +type DRPCFileV2UnimplementedServer struct{} + +func (s *DRPCFileV2UnimplementedServer) Upload(context.Context, *UploadRequest) (*UploadResponse, error) { + return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) +} + +func (s *DRPCFileV2UnimplementedServer) RequestSign(context.Context, *RequestSignRequest) (*RequestSignResponse, error) { + return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) +} + +func (s *DRPCFileV2UnimplementedServer) RequestDownload(context.Context, *RequestDownloadRequest) (*RequestDownloadResponse, error) { + return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) +} + +func (s *DRPCFileV2UnimplementedServer) SpaceInfo(context.Context, *SpaceInfoRequest) (*SpaceInfoResponse, error) { + return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) +} + +func (s *DRPCFileV2UnimplementedServer) Info(context.Context, *InfoRequest) (*InfoResponse, error) { + return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) +} + +type DRPCFileV2Description struct{} + +func (DRPCFileV2Description) NumMethods() int { return 5 } + +func (DRPCFileV2Description) Method(n int) (string, drpc.Encoding, drpc.Receiver, interface{}, bool) { + switch n { + case 0: + return "/filesyncv2.FileV2/Upload", drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}, + func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) { + return srv.(DRPCFileV2Server). + Upload( + ctx, + in1.(*UploadRequest), + ) + }, DRPCFileV2Server.Upload, true + case 1: + return "/filesyncv2.FileV2/RequestSign", drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}, + func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) { + return srv.(DRPCFileV2Server). + RequestSign( + ctx, + in1.(*RequestSignRequest), + ) + }, DRPCFileV2Server.RequestSign, true + case 2: + return "/filesyncv2.FileV2/RequestDownload", drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}, + func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) { + return srv.(DRPCFileV2Server). + RequestDownload( + ctx, + in1.(*RequestDownloadRequest), + ) + }, DRPCFileV2Server.RequestDownload, true + case 3: + return "/filesyncv2.FileV2/SpaceInfo", drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}, + func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) { + return srv.(DRPCFileV2Server). + SpaceInfo( + ctx, + in1.(*SpaceInfoRequest), + ) + }, DRPCFileV2Server.SpaceInfo, true + case 4: + return "/filesyncv2.FileV2/Info", drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}, + func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) { + return srv.(DRPCFileV2Server). + Info( + ctx, + in1.(*InfoRequest), + ) + }, DRPCFileV2Server.Info, true + default: + return "", nil, nil, nil, false + } +} + +func DRPCRegisterFileV2(mux drpc.Mux, impl DRPCFileV2Server) error { + return mux.Register(impl, DRPCFileV2Description{}) +} + +type DRPCFileV2_UploadStream interface { + drpc.Stream + SendAndClose(*UploadResponse) error +} + +type drpcFileV2_UploadStream struct { + drpc.Stream +} + +func (x *drpcFileV2_UploadStream) SendAndClose(m *UploadResponse) error { + if err := x.MsgSend(m, drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}); err != nil { + return err + } + return x.CloseSend() +} + +type DRPCFileV2_RequestSignStream interface { + drpc.Stream + SendAndClose(*RequestSignResponse) error +} + +type drpcFileV2_RequestSignStream struct { + drpc.Stream +} + +func (x *drpcFileV2_RequestSignStream) SendAndClose(m *RequestSignResponse) error { + if err := x.MsgSend(m, drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}); err != nil { + return err + } + return x.CloseSend() +} + +type DRPCFileV2_RequestDownloadStream interface { + drpc.Stream + SendAndClose(*RequestDownloadResponse) error +} + +type drpcFileV2_RequestDownloadStream struct { + drpc.Stream +} + +func (x *drpcFileV2_RequestDownloadStream) SendAndClose(m *RequestDownloadResponse) error { + if err := x.MsgSend(m, drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}); err != nil { + return err + } + return x.CloseSend() +} + +type DRPCFileV2_SpaceInfoStream interface { + drpc.Stream + SendAndClose(*SpaceInfoResponse) error +} + +type drpcFileV2_SpaceInfoStream struct { + drpc.Stream +} + +func (x *drpcFileV2_SpaceInfoStream) SendAndClose(m *SpaceInfoResponse) error { + if err := x.MsgSend(m, drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}); err != nil { + return err + } + return x.CloseSend() +} + +type DRPCFileV2_InfoStream interface { + drpc.Stream + SendAndClose(*InfoResponse) error +} + +type drpcFileV2_InfoStream struct { + drpc.Stream +} + +func (x *drpcFileV2_InfoStream) SendAndClose(m *InfoResponse) error { + if err := x.MsgSend(m, drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}); err != nil { + return err + } + return x.CloseSend() +} diff --git a/commonfile/fileproto/fileprotov2/filev2_vtproto.pb.go b/commonfile/fileproto/fileprotov2/filev2_vtproto.pb.go new file mode 100644 index 000000000..ad91628c1 --- /dev/null +++ b/commonfile/fileproto/fileprotov2/filev2_vtproto.pb.go @@ -0,0 +1,3615 @@ +// Code generated by protoc-gen-go-vtproto. DO NOT EDIT. +// protoc-gen-go-vtproto version: v0.6.0 +// source: commonfile/fileproto/fileprotov2/protos/filev2.proto + +package fileprotov2 + +import ( + fmt "fmt" + protohelpers "github.com/planetscale/vtprotobuf/protohelpers" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + io "io" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +func (m *UploadRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *UploadRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *UploadRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Items) > 0 { + for iNdEx := len(m.Items) - 1; iNdEx >= 0; iNdEx-- { + size, err := m.Items[iNdEx].MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x12 + } + } + if len(m.SpaceId) > 0 { + i -= len(m.SpaceId) + copy(dAtA[i:], m.SpaceId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *UploadRequestItem) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *UploadRequestItem) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *UploadRequestItem) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.Size != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Size)) + i-- + dAtA[i] = 0x10 + } + if len(m.RootCid) > 0 { + i -= len(m.RootCid) + copy(dAtA[i:], m.RootCid) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.RootCid))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *UploadResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *UploadResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *UploadResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Results) > 0 { + for iNdEx := len(m.Results) - 1; iNdEx >= 0; iNdEx-- { + size, err := m.Results[iNdEx].MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *UploadResult) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *UploadResult) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *UploadResult) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.Upload != nil { + size, err := m.Upload.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x1a + } + if m.Code != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Code)) + i-- + dAtA[i] = 0x10 + } + if len(m.RootCid) > 0 { + i -= len(m.RootCid) + copy(dAtA[i:], m.RootCid) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.RootCid))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *PresignedUpload) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PresignedUpload) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *PresignedUpload) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.Expiry != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Expiry)) + i-- + dAtA[i] = 0x20 + } + if m.MaxContentLength != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.MaxContentLength)) + i-- + dAtA[i] = 0x18 + } + if len(m.Fields) > 0 { + for iNdEx := len(m.Fields) - 1; iNdEx >= 0; iNdEx-- { + size, err := m.Fields[iNdEx].MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x12 + } + } + if len(m.Url) > 0 { + i -= len(m.Url) + copy(dAtA[i:], m.Url) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Url))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *PresignField) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PresignField) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *PresignField) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Value) > 0 { + i -= len(m.Value) + copy(dAtA[i:], m.Value) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Value))) + i-- + dAtA[i] = 0x12 + } + if len(m.Key) > 0 { + i -= len(m.Key) + copy(dAtA[i:], m.Key) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Key))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *RequestSignRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RequestSignRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *RequestSignRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.RootCids) > 0 { + for iNdEx := len(m.RootCids) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.RootCids[iNdEx]) + copy(dAtA[i:], m.RootCids[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.RootCids[iNdEx]))) + i-- + dAtA[i] = 0x12 + } + } + if len(m.SpaceId) > 0 { + i -= len(m.SpaceId) + copy(dAtA[i:], m.SpaceId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *RequestSignResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RequestSignResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *RequestSignResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Results) > 0 { + for iNdEx := len(m.Results) - 1; iNdEx >= 0; iNdEx-- { + size, err := m.Results[iNdEx].MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *RequestSignResult) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RequestSignResult) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *RequestSignResult) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.Receipt != nil { + size, err := m.Receipt.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x1a + } + if m.Code != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Code)) + i-- + dAtA[i] = 0x10 + } + if len(m.RootCid) > 0 { + i -= len(m.RootCid) + copy(dAtA[i:], m.RootCid) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.RootCid))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *NetworkSignReceipt) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *NetworkSignReceipt) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *NetworkSignReceipt) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Signature) > 0 { + i -= len(m.Signature) + copy(dAtA[i:], m.Signature) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Signature))) + i-- + dAtA[i] = 0x12 + } + if len(m.ReceiptPayload) > 0 { + i -= len(m.ReceiptPayload) + copy(dAtA[i:], m.ReceiptPayload) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.ReceiptPayload))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *RequestDownloadRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RequestDownloadRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *RequestDownloadRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.RootCids) > 0 { + for iNdEx := len(m.RootCids) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.RootCids[iNdEx]) + copy(dAtA[i:], m.RootCids[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.RootCids[iNdEx]))) + i-- + dAtA[i] = 0x12 + } + } + if len(m.SpaceId) > 0 { + i -= len(m.SpaceId) + copy(dAtA[i:], m.SpaceId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *RequestDownloadResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RequestDownloadResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *RequestDownloadResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Results) > 0 { + for iNdEx := len(m.Results) - 1; iNdEx >= 0; iNdEx-- { + size, err := m.Results[iNdEx].MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *RequestDownloadResult) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RequestDownloadResult) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *RequestDownloadResult) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.Download != nil { + size, err := m.Download.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x1a + } + if m.Code != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Code)) + i-- + dAtA[i] = 0x10 + } + if len(m.RootCid) > 0 { + i -= len(m.RootCid) + copy(dAtA[i:], m.RootCid) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.RootCid))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *SignedDownload) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SignedDownload) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *SignedDownload) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.Expiry != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Expiry)) + i-- + dAtA[i] = 0x10 + } + if len(m.Url) > 0 { + i -= len(m.Url) + copy(dAtA[i:], m.Url) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Url))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *SpaceInfoRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SpaceInfoRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *SpaceInfoRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.SpaceIds) > 0 { + for iNdEx := len(m.SpaceIds) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.SpaceIds[iNdEx]) + copy(dAtA[i:], m.SpaceIds[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceIds[iNdEx]))) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *SpaceInfoResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SpaceInfoResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *SpaceInfoResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Results) > 0 { + for iNdEx := len(m.Results) - 1; iNdEx >= 0; iNdEx-- { + size, err := m.Results[iNdEx].MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *SpaceInfoResult) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SpaceInfoResult) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *SpaceInfoResult) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.Quota != nil { + size, err := m.Quota.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x1a + } + if m.Code != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Code)) + i-- + dAtA[i] = 0x10 + } + if len(m.SpaceId) > 0 { + i -= len(m.SpaceId) + copy(dAtA[i:], m.SpaceId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *SpaceQuota) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SpaceQuota) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *SpaceQuota) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.AccountUsageBytes != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.AccountUsageBytes)) + i-- + dAtA[i] = 0x38 + } + if m.AccountLimitBytes != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.AccountLimitBytes)) + i-- + dAtA[i] = 0x30 + } + if m.FilesCount != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.FilesCount)) + i-- + dAtA[i] = 0x28 + } + if m.InflightUsageBytes != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.InflightUsageBytes)) + i-- + dAtA[i] = 0x20 + } + if m.DurableUsageBytes != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.DurableUsageBytes)) + i-- + dAtA[i] = 0x18 + } + if m.TotalUsageBytes != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.TotalUsageBytes)) + i-- + dAtA[i] = 0x10 + } + if m.LimitBytes != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.LimitBytes)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *InfoRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *InfoRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *InfoRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + return len(dAtA) - i, nil +} + +func (m *InfoResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *InfoResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *InfoResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.PublicReadBaseUrl) > 0 { + i -= len(m.PublicReadBaseUrl) + copy(dAtA[i:], m.PublicReadBaseUrl) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.PublicReadBaseUrl))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *UploadRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.SpaceId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if len(m.Items) > 0 { + for _, e := range m.Items { + l = e.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *UploadRequestItem) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.RootCid) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.Size != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Size)) + } + n += len(m.unknownFields) + return n +} + +func (m *UploadResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Results) > 0 { + for _, e := range m.Results { + l = e.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *UploadResult) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.RootCid) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.Code != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Code)) + } + if m.Upload != nil { + l = m.Upload.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + +func (m *PresignedUpload) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Url) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if len(m.Fields) > 0 { + for _, e := range m.Fields { + l = e.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + if m.MaxContentLength != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.MaxContentLength)) + } + if m.Expiry != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Expiry)) + } + n += len(m.unknownFields) + return n +} + +func (m *PresignField) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Key) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Value) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + +func (m *RequestSignRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.SpaceId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if len(m.RootCids) > 0 { + for _, b := range m.RootCids { + l = len(b) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *RequestSignResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Results) > 0 { + for _, e := range m.Results { + l = e.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *RequestSignResult) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.RootCid) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.Code != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Code)) + } + if m.Receipt != nil { + l = m.Receipt.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + +func (m *NetworkSignReceipt) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.ReceiptPayload) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Signature) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + +func (m *RequestDownloadRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.SpaceId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if len(m.RootCids) > 0 { + for _, b := range m.RootCids { + l = len(b) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *RequestDownloadResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Results) > 0 { + for _, e := range m.Results { + l = e.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *RequestDownloadResult) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.RootCid) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.Code != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Code)) + } + if m.Download != nil { + l = m.Download.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + +func (m *SignedDownload) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Url) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.Expiry != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Expiry)) + } + n += len(m.unknownFields) + return n +} + +func (m *SpaceInfoRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.SpaceIds) > 0 { + for _, s := range m.SpaceIds { + l = len(s) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *SpaceInfoResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Results) > 0 { + for _, e := range m.Results { + l = e.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *SpaceInfoResult) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.SpaceId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.Code != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Code)) + } + if m.Quota != nil { + l = m.Quota.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + +func (m *SpaceQuota) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.LimitBytes != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.LimitBytes)) + } + if m.TotalUsageBytes != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.TotalUsageBytes)) + } + if m.DurableUsageBytes != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.DurableUsageBytes)) + } + if m.InflightUsageBytes != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.InflightUsageBytes)) + } + if m.FilesCount != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.FilesCount)) + } + if m.AccountLimitBytes != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.AccountLimitBytes)) + } + if m.AccountUsageBytes != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.AccountUsageBytes)) + } + n += len(m.unknownFields) + return n +} + +func (m *InfoRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + n += len(m.unknownFields) + return n +} + +func (m *InfoResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.PublicReadBaseUrl) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + +func (m *UploadRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: UploadRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: UploadRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Items", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Items = append(m.Items, &UploadRequestItem{}) + if err := m.Items[len(m.Items)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *UploadRequestItem) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: UploadRequestItem: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: UploadRequestItem: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RootCid", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RootCid = append(m.RootCid[:0], dAtA[iNdEx:postIndex]...) + if m.RootCid == nil { + m.RootCid = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Size", wireType) + } + m.Size = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Size |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *UploadResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: UploadResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: UploadResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Results", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Results = append(m.Results, &UploadResult{}) + if err := m.Results[len(m.Results)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *UploadResult) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: UploadResult: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: UploadResult: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RootCid", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RootCid = append(m.RootCid[:0], dAtA[iNdEx:postIndex]...) + if m.RootCid == nil { + m.RootCid = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Code", wireType) + } + m.Code = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Code |= ErrCode(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Upload", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Upload == nil { + m.Upload = &PresignedUpload{} + } + if err := m.Upload.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *PresignedUpload) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PresignedUpload: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PresignedUpload: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Url", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Url = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Fields", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Fields = append(m.Fields, &PresignField{}) + if err := m.Fields[len(m.Fields)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field MaxContentLength", wireType) + } + m.MaxContentLength = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.MaxContentLength |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Expiry", wireType) + } + m.Expiry = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Expiry |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *PresignField) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PresignField: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PresignField: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Key = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Value", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Value = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *RequestSignRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RequestSignRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RequestSignRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RootCids", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RootCids = append(m.RootCids, make([]byte, postIndex-iNdEx)) + copy(m.RootCids[len(m.RootCids)-1], dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *RequestSignResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RequestSignResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RequestSignResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Results", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Results = append(m.Results, &RequestSignResult{}) + if err := m.Results[len(m.Results)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *RequestSignResult) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RequestSignResult: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RequestSignResult: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RootCid", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RootCid = append(m.RootCid[:0], dAtA[iNdEx:postIndex]...) + if m.RootCid == nil { + m.RootCid = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Code", wireType) + } + m.Code = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Code |= ErrCode(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Receipt", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Receipt == nil { + m.Receipt = &NetworkSignReceipt{} + } + if err := m.Receipt.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *NetworkSignReceipt) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: NetworkSignReceipt: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: NetworkSignReceipt: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ReceiptPayload", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ReceiptPayload = append(m.ReceiptPayload[:0], dAtA[iNdEx:postIndex]...) + if m.ReceiptPayload == nil { + m.ReceiptPayload = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Signature", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Signature = append(m.Signature[:0], dAtA[iNdEx:postIndex]...) + if m.Signature == nil { + m.Signature = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *RequestDownloadRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RequestDownloadRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RequestDownloadRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RootCids", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RootCids = append(m.RootCids, make([]byte, postIndex-iNdEx)) + copy(m.RootCids[len(m.RootCids)-1], dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *RequestDownloadResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RequestDownloadResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RequestDownloadResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Results", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Results = append(m.Results, &RequestDownloadResult{}) + if err := m.Results[len(m.Results)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *RequestDownloadResult) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RequestDownloadResult: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RequestDownloadResult: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RootCid", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RootCid = append(m.RootCid[:0], dAtA[iNdEx:postIndex]...) + if m.RootCid == nil { + m.RootCid = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Code", wireType) + } + m.Code = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Code |= ErrCode(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Download", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Download == nil { + m.Download = &SignedDownload{} + } + if err := m.Download.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SignedDownload) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SignedDownload: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SignedDownload: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Url", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Url = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Expiry", wireType) + } + m.Expiry = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Expiry |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SpaceInfoRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SpaceInfoRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SpaceInfoRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceIds", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceIds = append(m.SpaceIds, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SpaceInfoResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SpaceInfoResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SpaceInfoResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Results", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Results = append(m.Results, &SpaceInfoResult{}) + if err := m.Results[len(m.Results)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SpaceInfoResult) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SpaceInfoResult: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SpaceInfoResult: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Code", wireType) + } + m.Code = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Code |= ErrCode(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Quota", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Quota == nil { + m.Quota = &SpaceQuota{} + } + if err := m.Quota.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SpaceQuota) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SpaceQuota: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SpaceQuota: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LimitBytes", wireType) + } + m.LimitBytes = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.LimitBytes |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field TotalUsageBytes", wireType) + } + m.TotalUsageBytes = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.TotalUsageBytes |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field DurableUsageBytes", wireType) + } + m.DurableUsageBytes = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.DurableUsageBytes |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field InflightUsageBytes", wireType) + } + m.InflightUsageBytes = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.InflightUsageBytes |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FilesCount", wireType) + } + m.FilesCount = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.FilesCount |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AccountLimitBytes", wireType) + } + m.AccountLimitBytes = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.AccountLimitBytes |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 7: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AccountUsageBytes", wireType) + } + m.AccountUsageBytes = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.AccountUsageBytes |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *InfoRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: InfoRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: InfoRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *InfoResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: InfoResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: InfoResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PublicReadBaseUrl", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PublicReadBaseUrl = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} diff --git a/commonfile/fileproto/fileprotov2/protos/filev2.proto b/commonfile/fileproto/fileprotov2/protos/filev2.proto new file mode 100644 index 000000000..67dcf3d1a --- /dev/null +++ b/commonfile/fileproto/fileprotov2/protos/filev2.proto @@ -0,0 +1,212 @@ +syntax = "proto3"; +package filesyncv2; + +option go_package = "commonfile/fileproto/fileprotov2"; + +// FileV2 is the v2 filenode broker (S3-direct uploads/downloads). +// +// drpc route prefix is "/filesyncv2.FileV2/..." — DISTINCT from v1 +// "/filesync.File/..." so both services can be mounted on the same drpc mux +// without a Register() panic. The package/service names are load-bearing: +// method paths are derived from `package filesyncv2` + `service FileV2`. +// +// Conventions (all four SPACE-SCOPED RPCs are BATCH; Info is node-level): +// * request carries a single shared spaceId (or repeated spaceIds for +// SpaceInfo) + a repeated list of items; +// * identity is NEVER a request field — it is taken from the authenticated +// connection, and all per-space/per-cid authorization derives from it; +// * rootCid is bytes (raw self-describing CID, as in v1's `bytes cid`); +// * every result echoes its own key (rootCid bytes, or spaceId string) so +// clients build map[string(key)]->result in one pass — results are matched +// by key, NEVER by positional index; the server may reorder/dedupe; +// * per-item result = flat status enum (ErrCode, Ok=0) + exactly one typed +// payload message that is valid ONLY when code == Ok; +// * one bad/forbidden item never fails the whole batch. +// +// Two error layers: +// 1. whole-RPC drpc error (err != nil), carrying an ErrCodes value from the +// offset-800 rpcerr registry, returned ONLY for cross-cutting failures: +// failed/absent auth (Forbidden), malformed input (InvalidRequest), an +// over-sized batch (QuerySizeExceeded), or a misrouted single-space +// request (NotResponsible — retry against the space's responsible pair); +// 2. per-item business outcome lives in the in-body ErrCode `code` field +// (err == nil for the RPC as a whole). +service FileV2 { + // Upload issues a presigned upload target per requested file root. + rpc Upload(UploadRequest) returns (UploadResponse); + // RequestSign returns a network-signed durable-custody receipt per rootCid. + rpc RequestSign(RequestSignRequest) returns (RequestSignResponse); + // RequestDownload returns a signed GET target per rootCid. + rpc RequestDownload(RequestDownloadRequest) returns (RequestDownloadResponse); + // SpaceInfo returns quota/usage per requested space (batch quota-info). + rpc SpaceInfo(SpaceInfoRequest) returns (SpaceInfoResponse); + // Info returns static node-level metadata (no spaceId, no responsibility + // gate, no auth beyond the connection); clients cache it. + rpc Info(InfoRequest) returns (InfoResponse); +} + +// ErrCodes are WHOLE-RPC (drpc-level) errors, mirroring v1 filesync.ErrCodes. +// Wired through rpcerr.ErrGroup(ErrCodes_ErrorOffset) in the fileprotov2err +// package. Returned as a NON-nil drpc error ONLY for auth / malformed / +// batch-too-large / misrouted. Never used to report the outcome of an +// individual item (that is the per-item ErrCode below). Zero value +// Unexpected=0 registers at offset+0 (=800), so an uncoded server fault reads +// as Unexpected — exactly as v1 does at 200. Offset 800 is the only free +// rpcerr block (v1=200, 100/300/400/500/600/700 are taken). +enum ErrCodes { + Unexpected = 0; // unclassified server fault (registry fallback) + Forbidden = 1; // connection identity is not permitted at all + InvalidRequest = 2; // malformed: empty spaceId, empty items, bad rootCid bytes + QuerySizeExceeded = 3; // batch too large (too many items in one request) + NotResponsible = 4; // retryable routing signal: this node is not in the space's chash pair + ErrorOffset = 800; // rpcerr.ErrGroup base for v2 (constant, never an error) +} + +// ErrCode is the PER-ITEM, in-body business outcome (err == nil at the RPC +// layer). Flat status enum with an explicit success sentinel Ok=0. The typed +// payload on a *Result is valid ONLY when code == Ok. Deliberately a SEPARATE +// enum from ErrCodes: its zero value means success (not error), it never flows +// through rpcerr, and its value short-names are kept disjoint from ErrCodes so +// proto3's package-level enum-value scoping stays clean. Servers MUST set +// `code` explicitly on every result (Ok is the proto3 default, so an unset +// code would be misread as a success with an empty payload). +enum ErrCode { + Ok = 0; // item succeeded; the typed payload is populated + ErrUnexpected = 1; // internal error handling this single item + ErrCidNotFound = 2; // sign/download: no such stored file for this rootCid + ErrForbidden = 3; // identity may not access this item's space/file + ErrLimitExceeded = 4; // upload rejected: space/account storage limit reached + ErrSpaceNotFound = 5; // quota-info: unknown/deleted space id + ErrNotResponsible = 6; // retryable routing signal: this node is not in this item's space chash pair +} + +// ---- Upload --------------------------------------------------------------- + +message UploadRequest { + string spaceId = 1; // shared space for every item (auth scope) + repeated UploadRequestItem items = 2; // files to obtain an upload target for +} + +message UploadRequestItem { + bytes rootCid = 1; // CID of the file root (result match key) + uint64 size = 2; // declared total byte size of the file +} + +message UploadResponse { + repeated UploadResult results = 1; // one per requested item, keyed by rootCid +} + +message UploadResult { + bytes rootCid = 1; // echoes UploadRequestItem.rootCid (match key) + ErrCode code = 2; // per-item outcome; `upload` valid only when Ok + PresignedUpload upload = 3; // set ONLY when code == Ok +} + +// PresignedUpload is an OPAQUE presigned upload target. No S3 vendor structs +// leak into the shared proto: the client POSTs `fields` + the object body to +// `url` verbatim. Modeled as a presigned POST form (fields are ordered +// key/value pairs rather than a map) so the signing order is deterministic and +// reproducible. +message PresignedUpload { + string url = 1; // presigned POST endpoint + repeated PresignField fields = 2; // multipart form fields to send verbatim, in order + uint64 maxContentLength = 3; // hard upper bound the presign enforces (bytes) + int64 expiry = 4; // unix seconds; POST must complete before this +} + +message PresignField { + string key = 1; + string value = 2; +} + +// ---- RequestSign ---------------------------------------------------------- + +message RequestSignRequest { + string spaceId = 1; // shared space for every rootCid (auth scope) + repeated bytes rootCids = 2; // files to obtain a durable-custody receipt for +} + +message RequestSignResponse { + repeated RequestSignResult results = 1; // one per rootCid, keyed by rootCid +} + +message RequestSignResult { + bytes rootCid = 1; // echoes the requested rootCid (match key) + ErrCode code = 2; // per-item outcome; `receipt` valid only when Ok + NetworkSignReceipt receipt = 3; // set ONLY when code == Ok +} + +// NetworkSignReceipt mirrors coordinator SpaceReceiptWithSignature: an opaque +// protobuf-encoded payload plus the network-key signature over it. The payload +// schema is intentionally not defined here (opaque blob forwarded verbatim). +message NetworkSignReceipt { + bytes receiptPayload = 1; // protobuf-encoded signed receipt payload (opaque) + bytes signature = 2; // network key signature over receiptPayload +} + +// ---- RequestDownload ------------------------------------------------------ + +message RequestDownloadRequest { + string spaceId = 1; // shared space for every rootCid (auth scope) + repeated bytes rootCids = 2; // files to obtain a signed GET target for +} + +message RequestDownloadResponse { + repeated RequestDownloadResult results = 1; // one per rootCid, keyed by rootCid +} + +message RequestDownloadResult { + bytes rootCid = 1; // echoes the requested rootCid (match key) + ErrCode code = 2; // per-item outcome; `download` valid only when Ok + SignedDownload download = 3; // set ONLY when code == Ok +} + +// SignedDownload is an OPAQUE signed GET target (kept as a message, symmetric +// with PresignedUpload, so download can grow response headers etc. without a +// wire break on the result envelope). +message SignedDownload { + string url = 1; // opaque signed GET url + int64 expiry = 2; // unix seconds the url is valid until +} + +// ---- SpaceInfo (batch quota-info) ----------------------------------------- + +message SpaceInfoRequest { + repeated string spaceIds = 1; // batch of spaces to report on (result match key) +} + +message SpaceInfoResponse { + repeated SpaceInfoResult results = 1; // one per requested space, keyed by spaceId +} + +message SpaceInfoResult { + string spaceId = 1; // echoes the requested spaceId (match key) + ErrCode code = 2; // per-item outcome; `quota` valid only when Ok + SpaceQuota quota = 3; // set ONLY when code == Ok +} + +message SpaceQuota { + uint64 limitBytes = 1; // effective storage limit for the requester in this space + uint64 totalUsageBytes = 2; // counted usage == durableUsageBytes + inflightUsageBytes + uint64 durableUsageBytes = 3; // bytes fully committed to durable storage + uint64 inflightUsageBytes = 4; // bytes reserved by presigned/not-yet-committed uploads + uint64 filesCount = 5; // number of file roots stored in the space + // account-pool view for the requesting identity (files are charged to + // their uploader; all the identity's spaces share one pool). Zero when + // the node has no coordinator-backed limits (self-hosted fallback). + uint64 accountLimitBytes = 6; // the identity's total file-storage cap + uint64 accountUsageBytes = 7; // aggregated durable usage across the identity's spaces +} + +// ---- Info (node-level metadata) --------------------------------------------- + +// InfoRequest is deliberately empty: Info is static node metadata, +// extensible later (capabilities, limits) without a wire break. +message InfoRequest {} + +message InfoResponse { + // publicReadBaseUrl, when non-empty, lets clients construct durable object + // URLs as {publicReadBaseUrl}/blob/{spaceId}/{rootCid} with zero broker + // round-trips; clients cache it. Empty = no public read: use RequestDownload. + string publicReadBaseUrl = 1; +} diff --git a/commonspace/clientspaceproto/clientspace.pb.go b/commonspace/clientspaceproto/clientspace.pb.go index e38e3206d..2dfd4a434 100644 --- a/commonspace/clientspaceproto/clientspace.pb.go +++ b/commonspace/clientspaceproto/clientspace.pb.go @@ -117,6 +117,115 @@ func (x *SpaceExchangeResponse) GetSpaceIds() []string { return nil } +type SpaceExchangeV2Request struct { + state protoimpl.MessageState `protogen:"open.v1"` + // nonce is 32 random bytes chosen by the caller; it challenges both request and response tokens + Nonce []byte `protobuf:"bytes,1,opt,name=nonce,proto3" json:"nonce,omitempty"` + // spaceTokens holds one 32-byte request token per caller space, + // padded with random tokens up to a bucket size and sorted to hide the true count + SpaceTokens [][]byte `protobuf:"bytes,2,rep,name=spaceTokens,proto3" json:"spaceTokens,omitempty"` + LocalServer *LocalServer `protobuf:"bytes,3,opt,name=localServer,proto3" json:"localServer,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SpaceExchangeV2Request) Reset() { + *x = SpaceExchangeV2Request{} + mi := &file_commonspace_clientspaceproto_protos_clientspace_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SpaceExchangeV2Request) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SpaceExchangeV2Request) ProtoMessage() {} + +func (x *SpaceExchangeV2Request) ProtoReflect() protoreflect.Message { + mi := &file_commonspace_clientspaceproto_protos_clientspace_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SpaceExchangeV2Request.ProtoReflect.Descriptor instead. +func (*SpaceExchangeV2Request) Descriptor() ([]byte, []int) { + return file_commonspace_clientspaceproto_protos_clientspace_proto_rawDescGZIP(), []int{2} +} + +func (x *SpaceExchangeV2Request) GetNonce() []byte { + if x != nil { + return x.Nonce + } + return nil +} + +func (x *SpaceExchangeV2Request) GetSpaceTokens() [][]byte { + if x != nil { + return x.SpaceTokens + } + return nil +} + +func (x *SpaceExchangeV2Request) GetLocalServer() *LocalServer { + if x != nil { + return x.LocalServer + } + return nil +} + +type SpaceExchangeV2Response struct { + state protoimpl.MessageState `protogen:"open.v1"` + // spaceTokens holds one 32-byte response token per intersecting space — + // the responder's proof of membership, keyed by the caller's nonce + SpaceTokens [][]byte `protobuf:"bytes,1,rep,name=spaceTokens,proto3" json:"spaceTokens,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SpaceExchangeV2Response) Reset() { + *x = SpaceExchangeV2Response{} + mi := &file_commonspace_clientspaceproto_protos_clientspace_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SpaceExchangeV2Response) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SpaceExchangeV2Response) ProtoMessage() {} + +func (x *SpaceExchangeV2Response) ProtoReflect() protoreflect.Message { + mi := &file_commonspace_clientspaceproto_protos_clientspace_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SpaceExchangeV2Response.ProtoReflect.Descriptor instead. +func (*SpaceExchangeV2Response) Descriptor() ([]byte, []int) { + return file_commonspace_clientspaceproto_protos_clientspace_proto_rawDescGZIP(), []int{3} +} + +func (x *SpaceExchangeV2Response) GetSpaceTokens() [][]byte { + if x != nil { + return x.SpaceTokens + } + return nil +} + type LocalServer struct { state protoimpl.MessageState `protogen:"open.v1"` Ips []string `protobuf:"bytes,1,rep,name=Ips,proto3" json:"Ips,omitempty"` @@ -127,7 +236,7 @@ type LocalServer struct { func (x *LocalServer) Reset() { *x = LocalServer{} - mi := &file_commonspace_clientspaceproto_protos_clientspace_proto_msgTypes[2] + mi := &file_commonspace_clientspaceproto_protos_clientspace_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -139,7 +248,7 @@ func (x *LocalServer) String() string { func (*LocalServer) ProtoMessage() {} func (x *LocalServer) ProtoReflect() protoreflect.Message { - mi := &file_commonspace_clientspaceproto_protos_clientspace_proto_msgTypes[2] + mi := &file_commonspace_clientspaceproto_protos_clientspace_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -152,7 +261,7 @@ func (x *LocalServer) ProtoReflect() protoreflect.Message { // Deprecated: Use LocalServer.ProtoReflect.Descriptor instead. func (*LocalServer) Descriptor() ([]byte, []int) { - return file_commonspace_clientspaceproto_protos_clientspace_proto_rawDescGZIP(), []int{2} + return file_commonspace_clientspaceproto_protos_clientspace_proto_rawDescGZIP(), []int{4} } func (x *LocalServer) GetIps() []string { @@ -178,12 +287,19 @@ const file_commonspace_clientspaceproto_protos_clientspace_proto_rawDesc = "" + "\bspaceIds\x18\x01 \x03(\tR\bspaceIds\x12:\n" + "\vlocalServer\x18\x02 \x01(\v2\x18.clientspace.LocalServerR\vlocalServer\"3\n" + "\x15SpaceExchangeResponse\x12\x1a\n" + - "\bspaceIds\x18\x01 \x03(\tR\bspaceIds\"3\n" + + "\bspaceIds\x18\x01 \x03(\tR\bspaceIds\"\x8c\x01\n" + + "\x16SpaceExchangeV2Request\x12\x14\n" + + "\x05nonce\x18\x01 \x01(\fR\x05nonce\x12 \n" + + "\vspaceTokens\x18\x02 \x03(\fR\vspaceTokens\x12:\n" + + "\vlocalServer\x18\x03 \x01(\v2\x18.clientspace.LocalServerR\vlocalServer\";\n" + + "\x17SpaceExchangeV2Response\x12 \n" + + "\vspaceTokens\x18\x01 \x03(\fR\vspaceTokens\"3\n" + "\vLocalServer\x12\x10\n" + "\x03Ips\x18\x01 \x03(\tR\x03Ips\x12\x12\n" + - "\x04port\x18\x02 \x01(\x05R\x04port2e\n" + + "\x04port\x18\x02 \x01(\x05R\x04port2\xc3\x01\n" + "\vClientSpace\x12V\n" + - "\rSpaceExchange\x12!.clientspace.SpaceExchangeRequest\x1a\".clientspace.SpaceExchangeResponseB\x1eZ\x1ccommonspace/clientspaceprotob\x06proto3" + "\rSpaceExchange\x12!.clientspace.SpaceExchangeRequest\x1a\".clientspace.SpaceExchangeResponse\x12\\\n" + + "\x0fSpaceExchangeV2\x12#.clientspace.SpaceExchangeV2Request\x1a$.clientspace.SpaceExchangeV2ResponseB\x1eZ\x1ccommonspace/clientspaceprotob\x06proto3" var ( file_commonspace_clientspaceproto_protos_clientspace_proto_rawDescOnce sync.Once @@ -197,21 +313,26 @@ func file_commonspace_clientspaceproto_protos_clientspace_proto_rawDescGZIP() [] return file_commonspace_clientspaceproto_protos_clientspace_proto_rawDescData } -var file_commonspace_clientspaceproto_protos_clientspace_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_commonspace_clientspaceproto_protos_clientspace_proto_msgTypes = make([]protoimpl.MessageInfo, 5) var file_commonspace_clientspaceproto_protos_clientspace_proto_goTypes = []any{ - (*SpaceExchangeRequest)(nil), // 0: clientspace.SpaceExchangeRequest - (*SpaceExchangeResponse)(nil), // 1: clientspace.SpaceExchangeResponse - (*LocalServer)(nil), // 2: clientspace.LocalServer + (*SpaceExchangeRequest)(nil), // 0: clientspace.SpaceExchangeRequest + (*SpaceExchangeResponse)(nil), // 1: clientspace.SpaceExchangeResponse + (*SpaceExchangeV2Request)(nil), // 2: clientspace.SpaceExchangeV2Request + (*SpaceExchangeV2Response)(nil), // 3: clientspace.SpaceExchangeV2Response + (*LocalServer)(nil), // 4: clientspace.LocalServer } var file_commonspace_clientspaceproto_protos_clientspace_proto_depIdxs = []int32{ - 2, // 0: clientspace.SpaceExchangeRequest.localServer:type_name -> clientspace.LocalServer - 0, // 1: clientspace.ClientSpace.SpaceExchange:input_type -> clientspace.SpaceExchangeRequest - 1, // 2: clientspace.ClientSpace.SpaceExchange:output_type -> clientspace.SpaceExchangeResponse - 2, // [2:3] is the sub-list for method output_type - 1, // [1:2] is the sub-list for method input_type - 1, // [1:1] is the sub-list for extension type_name - 1, // [1:1] is the sub-list for extension extendee - 0, // [0:1] is the sub-list for field type_name + 4, // 0: clientspace.SpaceExchangeRequest.localServer:type_name -> clientspace.LocalServer + 4, // 1: clientspace.SpaceExchangeV2Request.localServer:type_name -> clientspace.LocalServer + 0, // 2: clientspace.ClientSpace.SpaceExchange:input_type -> clientspace.SpaceExchangeRequest + 2, // 3: clientspace.ClientSpace.SpaceExchangeV2:input_type -> clientspace.SpaceExchangeV2Request + 1, // 4: clientspace.ClientSpace.SpaceExchange:output_type -> clientspace.SpaceExchangeResponse + 3, // 5: clientspace.ClientSpace.SpaceExchangeV2:output_type -> clientspace.SpaceExchangeV2Response + 4, // [4:6] is the sub-list for method output_type + 2, // [2:4] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name } func init() { file_commonspace_clientspaceproto_protos_clientspace_proto_init() } @@ -225,7 +346,7 @@ func file_commonspace_clientspaceproto_protos_clientspace_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_commonspace_clientspaceproto_protos_clientspace_proto_rawDesc), len(file_commonspace_clientspaceproto_protos_clientspace_proto_rawDesc)), NumEnums: 0, - NumMessages: 3, + NumMessages: 5, NumExtensions: 0, NumServices: 1, }, diff --git a/commonspace/clientspaceproto/clientspace_drpc.pb.go b/commonspace/clientspaceproto/clientspace_drpc.pb.go index eea12aefa..1a707ff40 100644 --- a/commonspace/clientspaceproto/clientspace_drpc.pb.go +++ b/commonspace/clientspaceproto/clientspace_drpc.pb.go @@ -34,6 +34,7 @@ type DRPCClientSpaceClient interface { DRPCConn() drpc.Conn SpaceExchange(ctx context.Context, in *SpaceExchangeRequest) (*SpaceExchangeResponse, error) + SpaceExchangeV2(ctx context.Context, in *SpaceExchangeV2Request) (*SpaceExchangeV2Response, error) } type drpcClientSpaceClient struct { @@ -55,8 +56,18 @@ func (c *drpcClientSpaceClient) SpaceExchange(ctx context.Context, in *SpaceExch return out, nil } +func (c *drpcClientSpaceClient) SpaceExchangeV2(ctx context.Context, in *SpaceExchangeV2Request) (*SpaceExchangeV2Response, error) { + out := new(SpaceExchangeV2Response) + err := c.cc.Invoke(ctx, "/clientspace.ClientSpace/SpaceExchangeV2", drpcEncoding_File_commonspace_clientspaceproto_protos_clientspace_proto{}, in, out) + if err != nil { + return nil, err + } + return out, nil +} + type DRPCClientSpaceServer interface { SpaceExchange(context.Context, *SpaceExchangeRequest) (*SpaceExchangeResponse, error) + SpaceExchangeV2(context.Context, *SpaceExchangeV2Request) (*SpaceExchangeV2Response, error) } type DRPCClientSpaceUnimplementedServer struct{} @@ -65,9 +76,13 @@ func (s *DRPCClientSpaceUnimplementedServer) SpaceExchange(context.Context, *Spa return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) } +func (s *DRPCClientSpaceUnimplementedServer) SpaceExchangeV2(context.Context, *SpaceExchangeV2Request) (*SpaceExchangeV2Response, error) { + return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) +} + type DRPCClientSpaceDescription struct{} -func (DRPCClientSpaceDescription) NumMethods() int { return 1 } +func (DRPCClientSpaceDescription) NumMethods() int { return 2 } func (DRPCClientSpaceDescription) Method(n int) (string, drpc.Encoding, drpc.Receiver, interface{}, bool) { switch n { @@ -80,6 +95,15 @@ func (DRPCClientSpaceDescription) Method(n int) (string, drpc.Encoding, drpc.Rec in1.(*SpaceExchangeRequest), ) }, DRPCClientSpaceServer.SpaceExchange, true + case 1: + return "/clientspace.ClientSpace/SpaceExchangeV2", drpcEncoding_File_commonspace_clientspaceproto_protos_clientspace_proto{}, + func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) { + return srv.(DRPCClientSpaceServer). + SpaceExchangeV2( + ctx, + in1.(*SpaceExchangeV2Request), + ) + }, DRPCClientSpaceServer.SpaceExchangeV2, true default: return "", nil, nil, nil, false } @@ -104,3 +128,19 @@ func (x *drpcClientSpace_SpaceExchangeStream) SendAndClose(m *SpaceExchangeRespo } return x.CloseSend() } + +type DRPCClientSpace_SpaceExchangeV2Stream interface { + drpc.Stream + SendAndClose(*SpaceExchangeV2Response) error +} + +type drpcClientSpace_SpaceExchangeV2Stream struct { + drpc.Stream +} + +func (x *drpcClientSpace_SpaceExchangeV2Stream) SendAndClose(m *SpaceExchangeV2Response) error { + if err := x.MsgSend(m, drpcEncoding_File_commonspace_clientspaceproto_protos_clientspace_proto{}); err != nil { + return err + } + return x.CloseSend() +} diff --git a/commonspace/clientspaceproto/clientspace_vtproto.pb.go b/commonspace/clientspaceproto/clientspace_vtproto.pb.go index dc6f59246..02e0c9b31 100644 --- a/commonspace/clientspaceproto/clientspace_vtproto.pb.go +++ b/commonspace/clientspaceproto/clientspace_vtproto.pb.go @@ -112,6 +112,107 @@ func (m *SpaceExchangeResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) return len(dAtA) - i, nil } +func (m *SpaceExchangeV2Request) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SpaceExchangeV2Request) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *SpaceExchangeV2Request) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.LocalServer != nil { + size, err := m.LocalServer.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x1a + } + if len(m.SpaceTokens) > 0 { + for iNdEx := len(m.SpaceTokens) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.SpaceTokens[iNdEx]) + copy(dAtA[i:], m.SpaceTokens[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceTokens[iNdEx]))) + i-- + dAtA[i] = 0x12 + } + } + if len(m.Nonce) > 0 { + i -= len(m.Nonce) + copy(dAtA[i:], m.Nonce) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Nonce))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *SpaceExchangeV2Response) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SpaceExchangeV2Response) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *SpaceExchangeV2Response) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.SpaceTokens) > 0 { + for iNdEx := len(m.SpaceTokens) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.SpaceTokens[iNdEx]) + copy(dAtA[i:], m.SpaceTokens[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceTokens[iNdEx]))) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + func (m *LocalServer) MarshalVT() (dAtA []byte, err error) { if m == nil { return nil, nil @@ -195,6 +296,46 @@ func (m *SpaceExchangeResponse) SizeVT() (n int) { return n } +func (m *SpaceExchangeV2Request) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Nonce) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if len(m.SpaceTokens) > 0 { + for _, b := range m.SpaceTokens { + l = len(b) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + if m.LocalServer != nil { + l = m.LocalServer.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + +func (m *SpaceExchangeV2Response) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.SpaceTokens) > 0 { + for _, b := range m.SpaceTokens { + l = len(b) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + func (m *LocalServer) SizeVT() (n int) { if m == nil { return 0 @@ -416,6 +557,242 @@ func (m *SpaceExchangeResponse) UnmarshalVT(dAtA []byte) error { } return nil } +func (m *SpaceExchangeV2Request) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SpaceExchangeV2Request: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SpaceExchangeV2Request: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Nonce", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Nonce = append(m.Nonce[:0], dAtA[iNdEx:postIndex]...) + if m.Nonce == nil { + m.Nonce = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceTokens", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceTokens = append(m.SpaceTokens, make([]byte, postIndex-iNdEx)) + copy(m.SpaceTokens[len(m.SpaceTokens)-1], dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field LocalServer", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.LocalServer == nil { + m.LocalServer = &LocalServer{} + } + if err := m.LocalServer.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SpaceExchangeV2Response) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SpaceExchangeV2Response: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SpaceExchangeV2Response: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceTokens", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceTokens = append(m.SpaceTokens, make([]byte, postIndex-iNdEx)) + copy(m.SpaceTokens[len(m.SpaceTokens)-1], dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *LocalServer) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/commonspace/clientspaceproto/exchange2.go b/commonspace/clientspaceproto/exchange2.go new file mode 100644 index 000000000..0f8d6e314 --- /dev/null +++ b/commonspace/clientspaceproto/exchange2.go @@ -0,0 +1,144 @@ +package clientspaceproto + +import ( + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "errors" + "io" + "sort" + + "golang.org/x/crypto/hkdf" + + "github.com/anyproto/any-sync/util/crypto" +) + +// SpaceExchangeV2 token scheme. +// +// Peers never send space ids. Instead, for every space a peer holds it sends +// HMAC-SHA256(discoveryKey, nonce || label || callerPeerId || responderPeerId), +// where discoveryKey is derived from the space's first ACL read key — a stable +// secret available to every member. A matching token therefore simultaneously +// identifies a shared space and proves the sender's membership in it: strangers +// on the LAN see values indistinguishable from random, cannot correlate them +// across sessions (fresh nonce each run), and cannot claim spaces they are not +// members of. Spaces whose ACL state is not available locally are skipped. +// +// The responder answers only for the intersection, keying its tokens by the +// caller's nonce (challenge-response), so a response cannot be precomputed or +// replayed. Both request and response tokens are bound to the ordered pair of +// peer ids, so tokens observed on one connection are useless on another. + +const ( + // NonceSizeV2 is the required size of SpaceExchangeV2Request.nonce + NonceSizeV2 = 32 + // TokenSizeV2 is the size of every element of spaceTokens + TokenSizeV2 = sha256.Size + // MaxTokensV2 bounds spaceTokens in a request; handlers must reject bigger lists + MaxTokensV2 = 4096 +) + +// tokenBucketsV2 are the sizes request token lists are padded to, hiding the real space count +var tokenBucketsV2 = []int{16, 32, 64, 128, 256} + +var ( + labelRequestV2 = []byte("any-sync:space-exchange:v2:req") + labelResponseV2 = []byte("any-sync:space-exchange:v2:resp") + discoveryInfoV2 = []byte("any-sync:space-exchange:v2:discovery-key") +) + +var ErrInvalidNonce = errors.New("space exchange v2: invalid nonce size") + +// DeriveDiscoveryKey derives the per-space LAN discovery key from the first ACL read key. +// The first read key never rotates, so discovery keeps working for members whose local +// ACL copy is behind; former members retain it, but downstream sync still enforces the +// current ACL, so they can at most learn that a peer holds the space. +func DeriveDiscoveryKey(firstReadKey crypto.SymKey, spaceId string) ([]byte, error) { + ikm, err := firstReadKey.Raw() + if err != nil { + return nil, err + } + key := make([]byte, 32) + if _, err = io.ReadFull(hkdf.New(sha256.New, ikm, []byte(spaceId), discoveryInfoV2), key); err != nil { + return nil, err + } + return key, nil +} + +// NewNonceV2 returns a fresh random nonce for a SpaceExchangeV2Request +func NewNonceV2() ([]byte, error) { + nonce := make([]byte, NonceSizeV2) + if _, err := rand.Read(nonce); err != nil { + return nil, err + } + return nonce, nil +} + +// RequestTokenV2 computes the caller's token for one space +func RequestTokenV2(discoveryKey, nonce []byte, callerPeerId, responderPeerId string) []byte { + return tokenV2(discoveryKey, labelRequestV2, nonce, callerPeerId, responderPeerId) +} + +// ResponseTokenV2 computes the responder's proof token for one intersecting space, +// keyed by the caller's nonce +func ResponseTokenV2(discoveryKey, nonce []byte, callerPeerId, responderPeerId string) []byte { + return tokenV2(discoveryKey, labelResponseV2, nonce, callerPeerId, responderPeerId) +} + +func tokenV2(discoveryKey, label, nonce []byte, callerPeerId, responderPeerId string) []byte { + mac := hmac.New(sha256.New, discoveryKey) + writeField(mac, label) + writeField(mac, nonce) + writeField(mac, []byte(callerPeerId)) + writeField(mac, []byte(responderPeerId)) + return mac.Sum(nil) +} + +// writeField length-prefixes every field so variable-length peer ids cannot +// produce colliding concatenations +func writeField(w io.Writer, field []byte) { + var lenBuf [4]byte + binary.BigEndian.PutUint32(lenBuf[:], uint32(len(field))) + _, _ = w.Write(lenBuf[:]) + _, _ = w.Write(field) +} + +// PadTokensV2 pads tokens with random fillers up to the next bucket size (beyond the +// largest bucket — up to the next multiple of it) and sorts the result. Sorting is +// required: without it the position of real tokens would reveal the true space count +// the padding is meant to hide. +func PadTokensV2(tokens [][]byte) ([][]byte, error) { + maxBucket := tokenBucketsV2[len(tokenBucketsV2)-1] + target := (len(tokens) + maxBucket - 1) / maxBucket * maxBucket + for _, bucket := range tokenBucketsV2 { + if len(tokens) <= bucket { + target = bucket + break + } + } + padded := make([][]byte, 0, target) + padded = append(padded, tokens...) + for len(padded) < target { + filler := make([]byte, TokenSizeV2) + if _, err := rand.Read(filler); err != nil { + return nil, err + } + padded = append(padded, filler) + } + sort.Slice(padded, func(i, j int) bool { + return string(padded[i]) < string(padded[j]) + }) + return padded, nil +} + +// ContainsTokenV2 reports whether token is present in tokens, comparing in constant time +func ContainsTokenV2(tokens [][]byte, token []byte) bool { + var found bool + for _, t := range tokens { + if hmac.Equal(t, token) { + found = true + } + } + return found +} diff --git a/commonspace/clientspaceproto/exchange2_bench_test.go b/commonspace/clientspaceproto/exchange2_bench_test.go new file mode 100644 index 000000000..5bf86760e --- /dev/null +++ b/commonspace/clientspaceproto/exchange2_bench_test.go @@ -0,0 +1,135 @@ +package clientspaceproto + +import ( + "testing" + + "github.com/anyproto/any-sync/util/crypto" +) + +// BenchmarkExchangeV2 measures a full exchange where both peers hold 1000 spaces, +// 500 of them shared. Discovery keys are precomputed once outside the loops — +// they are stable per space and meant to be cached; DeriveKeys measures that +// one-time cost separately. Matching uses a hash set, the right approach at this +// scale (ContainsTokenV2's linear scan is for small sets). +func BenchmarkExchangeV2(b *testing.B) { + const ( + spacesPerPeer = 1000 + shared = 500 + ) + caller, responder := "peerA", "peerB" + + newKeys := func(n int) []crypto.SymKey { + keys := make([]crypto.SymKey, n) + for i := range keys { + key, err := crypto.NewRandomAES() + if err != nil { + b.Fatal(err) + } + keys[i] = key + } + return keys + } + deriveAll := func(keys []crypto.SymKey, ids []string) [][]byte { + dks := make([][]byte, len(keys)) + for i := range keys { + dk, err := DeriveDiscoveryKey(keys[i], ids[i]) + if err != nil { + b.Fatal(err) + } + dks[i] = dk + } + return dks + } + + sharedKeys := newKeys(shared) + callerKeys := append(append([]crypto.SymKey{}, sharedKeys...), newKeys(spacesPerPeer-shared)...) + responderKeys := append(append([]crypto.SymKey{}, sharedKeys...), newKeys(spacesPerPeer-shared)...) + callerIds := make([]string, spacesPerPeer) + responderIds := make([]string, spacesPerPeer) + for i := range spacesPerPeer { + if i < shared { + callerIds[i] = "shared" + string(rune('0'+i%10)) + string(rune('a'+i/10%26)) + string(rune('a'+i/260)) + responderIds[i] = callerIds[i] + } else { + callerIds[i] = "caller-only-" + string(rune('a'+i%26)) + string(rune('a'+i/26%26)) + string(rune('a'+i/676)) + responderIds[i] = "responder-only-" + string(rune('a'+i%26)) + string(rune('a'+i/26%26)) + string(rune('a'+i/676)) + } + } + callerDks := deriveAll(callerKeys, callerIds) + responderDks := deriveAll(responderKeys, responderIds) + + nonce, err := NewNonceV2() + if err != nil { + b.Fatal(err) + } + + buildRequest := func() [][]byte { + tokens := make([][]byte, len(callerDks)) + for i, dk := range callerDks { + tokens[i] = RequestTokenV2(dk, nonce, caller, responder) + } + padded, err := PadTokensV2(tokens) + if err != nil { + b.Fatal(err) + } + return padded + } + handleRequest := func(reqTokens [][]byte) [][]byte { + received := make(map[string]struct{}, len(reqTokens)) + for _, t := range reqTokens { + received[string(t)] = struct{}{} + } + var respTokens [][]byte + for _, dk := range responderDks { + if _, ok := received[string(RequestTokenV2(dk, nonce, caller, responder))]; ok { + respTokens = append(respTokens, ResponseTokenV2(dk, nonce, caller, responder)) + } + } + return respTokens + } + verifyResponse := func(respTokens [][]byte) int { + received := make(map[string]struct{}, len(respTokens)) + for _, t := range respTokens { + received[string(t)] = struct{}{} + } + var matched int + for _, dk := range callerDks { + if _, ok := received[string(ResponseTokenV2(dk, nonce, caller, responder))]; ok { + matched++ + } + } + return matched + } + + reqTokens := buildRequest() + respTokens := handleRequest(reqTokens) + if got := verifyResponse(respTokens); got != shared { + b.Fatalf("expected %d shared spaces, got %d", shared, got) + } + + b.Run("DeriveKeys", func(b *testing.B) { + for b.Loop() { + deriveAll(callerKeys, callerIds) + } + }) + b.Run("CallerBuildRequest", func(b *testing.B) { + for b.Loop() { + buildRequest() + } + }) + b.Run("ResponderHandle", func(b *testing.B) { + for b.Loop() { + handleRequest(reqTokens) + } + }) + b.Run("CallerVerifyResponse", func(b *testing.B) { + for b.Loop() { + verifyResponse(respTokens) + } + }) + b.Run("FullRoundTrip", func(b *testing.B) { + for b.Loop() { + verifyResponse(handleRequest(buildRequest())) + } + }) +} diff --git a/commonspace/clientspaceproto/exchange2_test.go b/commonspace/clientspaceproto/exchange2_test.go new file mode 100644 index 000000000..38caec71e --- /dev/null +++ b/commonspace/clientspaceproto/exchange2_test.go @@ -0,0 +1,165 @@ +package clientspaceproto + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/anyproto/any-sync/util/crypto" +) + +func TestDeriveDiscoveryKey(t *testing.T) { + readKey, err := crypto.NewRandomAES() + require.NoError(t, err) + + key1, err := DeriveDiscoveryKey(readKey, "space1") + require.NoError(t, err) + require.Len(t, key1, 32) + + // deterministic for the same inputs + key1Again, err := DeriveDiscoveryKey(readKey, "space1") + require.NoError(t, err) + require.Equal(t, key1, key1Again) + + // different per space even with the same read key + key2, err := DeriveDiscoveryKey(readKey, "space2") + require.NoError(t, err) + require.NotEqual(t, key1, key2) + + // raw read key never appears as the discovery key + raw, err := readKey.Raw() + require.NoError(t, err) + require.NotEqual(t, raw, key1) +} + +func TestTokensV2(t *testing.T) { + readKey, err := crypto.NewRandomAES() + require.NoError(t, err) + dk, err := DeriveDiscoveryKey(readKey, "space1") + require.NoError(t, err) + nonce, err := NewNonceV2() + require.NoError(t, err) + + req := RequestTokenV2(dk, nonce, "peerA", "peerB") + require.Len(t, req, TokenSizeV2) + + // request and response tokens must differ (domain separation) + resp := ResponseTokenV2(dk, nonce, "peerA", "peerB") + require.NotEqual(t, req, resp) + + // bound to the peer pair + require.NotEqual(t, req, RequestTokenV2(dk, nonce, "peerA", "peerC")) + require.NotEqual(t, req, RequestTokenV2(dk, nonce, "peerB", "peerA")) + + // bound to the nonce + nonce2, err := NewNonceV2() + require.NoError(t, err) + require.NotEqual(t, req, RequestTokenV2(dk, nonce2, "peerA", "peerB")) + + // length prefixing: shifting a boundary between peer ids must change the token + require.NotEqual(t, RequestTokenV2(dk, nonce, "peerAp", "eerB"), RequestTokenV2(dk, nonce, "peerA", "peerB")) +} + +func TestPadTokensV2(t *testing.T) { + tokens := make([][]byte, 0, 20) + for range 20 { + tok, err := NewNonceV2() + require.NoError(t, err) + tokens = append(tokens, tok) + } + + padded, err := PadTokensV2(tokens[:3]) + require.NoError(t, err) + require.Len(t, padded, 16) + + padded, err = PadTokensV2(tokens[:16]) + require.NoError(t, err) + require.Len(t, padded, 16) + + padded, err = PadTokensV2(tokens[:17]) + require.NoError(t, err) + require.Len(t, padded, 32) + + // all real tokens survive padding + for _, tok := range tokens[:17] { + require.True(t, ContainsTokenV2(padded, tok)) + } + + // sorted, so real token positions don't leak the true count + for i := 1; i < len(padded); i++ { + require.LessOrEqual(t, string(padded[i-1]), string(padded[i])) + } + + // beyond the largest bucket: next multiple of it + big := make([][]byte, 300) + for i := range big { + big[i], err = NewNonceV2() + require.NoError(t, err) + } + padded, err = PadTokensV2(big) + require.NoError(t, err) + require.Len(t, padded, 512) +} + +// TestExchangeV2RoundTrip simulates a full exchange between two peers +func TestExchangeV2RoundTrip(t *testing.T) { + shared1, err := crypto.NewRandomAES() + require.NoError(t, err) + shared2, err := crypto.NewRandomAES() + require.NoError(t, err) + callerOnly, err := crypto.NewRandomAES() + require.NoError(t, err) + responderOnly, err := crypto.NewRandomAES() + require.NoError(t, err) + + type space struct { + id string + readKey crypto.SymKey + } + callerSpaces := []space{{"shared1", shared1}, {"shared2", shared2}, {"callerOnly", callerOnly}} + responderSpaces := []space{{"shared1", shared1}, {"shared2", shared2}, {"responderOnly", responderOnly}} + const caller, responder = "peerA", "peerB" + + // caller builds the request + nonce, err := NewNonceV2() + require.NoError(t, err) + var reqTokens [][]byte + for _, sp := range callerSpaces { + dk, err := DeriveDiscoveryKey(sp.readKey, sp.id) + require.NoError(t, err) + reqTokens = append(reqTokens, RequestTokenV2(dk, nonce, caller, responder)) + } + reqTokens, err = PadTokensV2(reqTokens) + require.NoError(t, err) + + // responder matches against its own spaces and answers for the intersection + var respTokens [][]byte + var responderIntersection []string + for _, sp := range responderSpaces { + dk, err := DeriveDiscoveryKey(sp.readKey, sp.id) + require.NoError(t, err) + if ContainsTokenV2(reqTokens, RequestTokenV2(dk, nonce, caller, responder)) { + responderIntersection = append(responderIntersection, sp.id) + respTokens = append(respTokens, ResponseTokenV2(dk, nonce, caller, responder)) + } + } + require.Equal(t, []string{"shared1", "shared2"}, responderIntersection) + + // caller verifies the response + var callerIntersection []string + for _, sp := range callerSpaces { + dk, err := DeriveDiscoveryKey(sp.readKey, sp.id) + require.NoError(t, err) + if ContainsTokenV2(respTokens, ResponseTokenV2(dk, nonce, caller, responder)) { + callerIntersection = append(callerIntersection, sp.id) + } + } + require.Equal(t, []string{"shared1", "shared2"}, callerIntersection) + + // a peer that knows only the space id (no read key) gets no match + wrongKey, err := crypto.NewRandomAES() + require.NoError(t, err) + dk, err := DeriveDiscoveryKey(wrongKey, "shared1") + require.NoError(t, err) + require.False(t, ContainsTokenV2(reqTokens, RequestTokenV2(dk, nonce, caller, responder))) +} diff --git a/commonspace/clientspaceproto/protos/clientspace.proto b/commonspace/clientspaceproto/protos/clientspace.proto index 046c3b19c..d4c0aea7d 100644 --- a/commonspace/clientspaceproto/protos/clientspace.proto +++ b/commonspace/clientspaceproto/protos/clientspace.proto @@ -4,7 +4,15 @@ package clientspace; option go_package = "commonspace/clientspaceproto"; service ClientSpace { + // SpaceExchange sends plaintext space id lists to any LAN peer. + // Deprecated: leaks the full space set to unauthenticated strangers; use SpaceExchangeV2. rpc SpaceExchange(SpaceExchangeRequest) returns (SpaceExchangeResponse); + // SpaceExchangeV2 lets two LAN peers learn only the intersection of their space sets. + // Instead of space ids, peers exchange per-space HMAC tokens keyed by a discovery key + // derived from the space's first ACL read key — a secret only members hold. A valid + // token both hides the space from non-members and proves the sender's membership. + // See clientspaceproto/exchange2.go for token derivation. + rpc SpaceExchangeV2(SpaceExchangeV2Request) returns (SpaceExchangeV2Response); } message SpaceExchangeRequest { @@ -16,6 +24,21 @@ message SpaceExchangeResponse { repeated string spaceIds = 1; } +message SpaceExchangeV2Request { + // nonce is 32 random bytes chosen by the caller; it challenges both request and response tokens + bytes nonce = 1; + // spaceTokens holds one 32-byte request token per caller space, + // padded with random tokens up to a bucket size and sorted to hide the true count + repeated bytes spaceTokens = 2; + LocalServer localServer = 3; +} + +message SpaceExchangeV2Response { + // spaceTokens holds one 32-byte response token per intersecting space — + // the responder's proof of membership, keyed by the caller's nonce + repeated bytes spaceTokens = 1; +} + message LocalServer { repeated string Ips = 1; int32 port = 2; diff --git a/commonspace/deletionmanager/deleteloop.go b/commonspace/deletionmanager/deleteloop.go index e7e1e1802..eafcde87e 100644 --- a/commonspace/deletionmanager/deleteloop.go +++ b/commonspace/deletionmanager/deleteloop.go @@ -2,10 +2,16 @@ package deletionmanager import ( "context" + "sync/atomic" "time" + + "go.uber.org/zap" ) -const deleteLoopInterval = time.Second * 20 +const ( + deleteLoopInterval = time.Second * 20 + deleteCloseTimeout = time.Second * 10 +) type deleteLoop struct { deleteCtx context.Context @@ -13,6 +19,8 @@ type deleteLoop struct { deleteChan chan struct{} deleteFunc func(ctx context.Context) loopDone chan struct{} + closeTimeout time.Duration + running atomic.Bool } func newDeleteLoop(deleteFunc func(ctx context.Context)) *deleteLoop { @@ -23,10 +31,12 @@ func newDeleteLoop(deleteFunc func(ctx context.Context)) *deleteLoop { deleteChan: make(chan struct{}, 1), deleteFunc: deleteFunc, loopDone: make(chan struct{}), + closeTimeout: deleteCloseTimeout, } } func (dl *deleteLoop) Run() { + dl.running.Store(true) go dl.loop() } @@ -55,7 +65,23 @@ func (dl *deleteLoop) notify() { } } -func (dl *deleteLoop) Close() { +// Close cancels the delete context and waits for the loop to exit. The wait is bounded: +// deleteFunc calls into treemanager implementations that may not honour ctx, and blocking +// here forever wedges the whole app.Close. +func (dl *deleteLoop) Close(ctx context.Context) { dl.deleteCancel() - <-dl.loopDone + // loopDone is closed by loop, which only Run starts: app.Start closes + // components it never ran + if !dl.running.Load() { + return + } + timer := time.NewTimer(dl.closeTimeout) + defer timer.Stop() + select { + case <-dl.loopDone: + case <-ctx.Done(): + log.WarnCtx(ctx, "delete loop close interrupted, delete is still in flight", zap.Error(ctx.Err())) + case <-timer.C: + log.WarnCtx(ctx, "delete loop close timed out, delete is still in flight") + } } diff --git a/commonspace/deletionmanager/deleteloop_test.go b/commonspace/deletionmanager/deleteloop_test.go new file mode 100644 index 000000000..0018812b2 --- /dev/null +++ b/commonspace/deletionmanager/deleteloop_test.go @@ -0,0 +1,95 @@ +package deletionmanager + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestDeleteLoop_CloseCancelsDeleteCtx(t *testing.T) { + var ( + started = make(chan struct{}) + done = make(chan struct{}) + ) + dl := newDeleteLoop(func(ctx context.Context) { + close(started) + <-ctx.Done() + close(done) + }) + dl.Run() + <-started + dl.Close(context.Background()) + select { + case <-done: + default: + require.Fail(t, "deleteFunc was not cancelled") + } +} + +func TestDeleteLoop_CloseBoundedWhenDeleteFuncIgnoresCtx(t *testing.T) { + var ( + started = make(chan struct{}) + release = make(chan struct{}) + ) + dl := newDeleteLoop(func(ctx context.Context) { + close(started) + <-release + }) + dl.closeTimeout = 10 * time.Millisecond + dl.Run() + <-started + closed := make(chan struct{}) + go func() { + dl.Close(context.Background()) + close(closed) + }() + select { + case <-closed: + case <-time.After(time.Second): + require.Fail(t, "Close blocked on a deleteFunc that ignores ctx") + } + close(release) +} + +func TestDeleteLoop_CloseHonoursCtx(t *testing.T) { + var ( + started = make(chan struct{}) + release = make(chan struct{}) + ) + dl := newDeleteLoop(func(ctx context.Context) { + close(started) + <-release + }) + dl.Run() + <-started + ctx, cancel := context.WithCancel(context.Background()) + closed := make(chan struct{}) + go func() { + dl.Close(ctx) + close(closed) + }() + cancel() + select { + case <-closed: + case <-time.After(time.Second): + require.Fail(t, "Close ignored the cancelled ctx") + } + close(release) +} + +// app.Start closes components it never ran: loopDone is only closed by the loop. +func TestDeleteLoop_CloseWithoutRun(t *testing.T) { + dl := newDeleteLoop(func(ctx context.Context) {}) + closed := make(chan struct{}) + go func() { + dl.Close(context.Background()) + close(closed) + }() + select { + case <-closed: + case <-time.After(time.Second): + require.Fail(t, "Close waited for a loop that was never started") + } +} diff --git a/commonspace/deletionmanager/deleter.go b/commonspace/deletionmanager/deleter.go index ada11d988..a33490e68 100644 --- a/commonspace/deletionmanager/deleter.go +++ b/commonspace/deletionmanager/deleter.go @@ -36,6 +36,9 @@ func (d *deleter) Delete(ctx context.Context) { spaceId = d.st.Id() ) for _, id := range allQueued { + if ctx.Err() != nil { + return + } log := d.log.With(zap.String("treeId", id)) shouldDelete, err := d.tryMarkDeleted(ctx, spaceId, id) if !shouldDelete { @@ -67,6 +70,9 @@ func (d *deleter) deleteBoundChildren(ctx context.Context, spaceId, parentId str return } for _, child := range children { + if ctx.Err() != nil { + return + } if child.DeletedStatus >= headstorage.DeletedStatusDeleted { continue } @@ -100,5 +106,5 @@ func (d *deleter) tryMarkDeleted(ctx context.Context, spaceId, treeId string) (b if !errors.Is(err, treestorage.ErrUnknownTreeId) { return false, err } - return false, d.getter.MarkTreeDeleted(context.Background(), spaceId, treeId) + return false, d.getter.MarkTreeDeleted(ctx, spaceId, treeId) } diff --git a/commonspace/deletionmanager/deletionmanager.go b/commonspace/deletionmanager/deletionmanager.go index c490a35fa..8981c1cbd 100644 --- a/commonspace/deletionmanager/deletionmanager.go +++ b/commonspace/deletionmanager/deletionmanager.go @@ -61,7 +61,7 @@ func (d *deletionManager) Run(ctx context.Context) (err error) { } func (d *deletionManager) Close(ctx context.Context) (err error) { - d.loop.Close() + d.loop.Close(ctx) return } diff --git a/commonspace/headsync/diffmanager.go b/commonspace/headsync/diffmanager.go index 9c534b264..8c85539a2 100644 --- a/commonspace/headsync/diffmanager.go +++ b/commonspace/headsync/diffmanager.go @@ -10,122 +10,91 @@ import ( "github.com/anyproto/any-sync/commonspace/deletionstate" "github.com/anyproto/any-sync/commonspace/headsync/headstorage" "github.com/anyproto/any-sync/commonspace/object/acl/syncacl" - "github.com/anyproto/any-sync/commonspace/object/keyvalue/kvinterfaces" "github.com/anyproto/any-sync/commonspace/spacestorage" "github.com/anyproto/any-sync/commonspace/spacesyncproto" "github.com/anyproto/any-sync/net/rpc/rpcerr" ) +// DiffManager holds the current diff (DiffType_V3). +// +// Upgrading the diff algorithm to a next DiffType requires a coexistence +// period, see how V2/V3 lived together before V2 removal (git history up to +// this commit): a DiffContainer with old/new diffs filled by DiffManager, +// dual hashes in statestorage, a switch by req.DiffType in HandleRangeRequest +// and by resp.DiffType in remote.DiffTypeCheck, so that the newest common +// type wins. The old type can be dropped once the secureservice +// compatibleVersions exclude all peers that don't support the new one. +// +// Note for that future upgrade: V3-only responders (this code and the node's +// HeadSync handler) reject unknown diff types with an error instead of +// answering with their best supported type, so a V(n+1) requester must treat +// such an error as "type unsupported" and fall back to requesting V(n). type DiffManager struct { - diffContainer ldiff.DiffContainer + diff ldiff.Diff storage spacestorage.SpaceStorage syncAcl syncacl.SyncAcl log logger.CtxLogger ctx context.Context deletionState deletionstate.ObjectDeletionState - keyValue kvinterfaces.KeyValueService } func NewDiffManager( - diffContainer ldiff.DiffContainer, + diff ldiff.Diff, storage spacestorage.SpaceStorage, syncAcl syncacl.SyncAcl, log logger.CtxLogger, ctx context.Context, deletionState deletionstate.ObjectDeletionState, - keyValue kvinterfaces.KeyValueService, ) *DiffManager { return &DiffManager{ - diffContainer: diffContainer, + diff: diff, storage: storage, syncAcl: syncAcl, log: log, ctx: ctx, deletionState: deletionState, - keyValue: keyValue, } } func (dm *DiffManager) FillDiff(ctx context.Context) error { - var ( - commonEls = make([]ldiff.Element, 0, 100) - onlyOldEls = make([]ldiff.Element, 0, 100) - noCommonSnapshotEls = make([]ldiff.Element, 0, 2) - ) + els := make([]ldiff.Element, 0, 100) + hasher := ldiff.NewHasher() + defer ldiff.ReleaseHasher(hasher) err := dm.storage.HeadStorage().IterateEntries(ctx, headstorage.IterOpts{}, func(entry headstorage.HeadsEntry) (bool, error) { - // empty derived roots shouldn't be set in all hashes - if entry.IsDerived && entry.Heads[0] == entry.Id { + // skip empty roots, except non-derived ones without a common snapshot: + // those have historically been part of the diff and must stay to keep + // the space hash stable (UpdateHeads deliberately differs, see there) + if len(entry.Heads) > 0 && entry.Heads[0] == entry.Id && (entry.IsDerived || entry.CommonSnapshot != "") { return true, nil } - if entry.CommonSnapshot != "" { - // empty roots shouldn't be set in new hashe - if entry.Heads[0] == entry.Id { - onlyOldEls = append(onlyOldEls, ldiff.Element{ - Id: entry.Id, - Head: concatStrings(entry.Heads), - }) - return true, nil - } - commonEls = append(commonEls, ldiff.Element{ - Id: entry.Id, - Head: concatStrings(entry.Heads), - }) - } else { - noCommonSnapshotEls = append(noCommonSnapshotEls, ldiff.Element{ - Id: entry.Id, - Head: concatStrings(entry.Heads), - }) - } + els = append(els, ldiff.Element{ + Id: entry.Id, + Head: hasher.HashId(concatStrings(entry.Heads)), + }) return true, nil }) if err != nil { return err } log.Debug("setting acl", zap.String("aclId", dm.syncAcl.Id()), zap.String("headId", dm.syncAcl.Head().Id)) - hasher := ldiff.NewHasher() - dm.setHeadsForNewDiff(commonEls, noCommonSnapshotEls, hasher) - dm.setHeadsForOldDiff(commonEls, onlyOldEls, noCommonSnapshotEls, hasher) - ldiff.ReleaseHasher(hasher) - oldHash := dm.diffContainer.OldDiff().Hash() - newHash := dm.diffContainer.NewDiff().Hash() - if err := dm.storage.StateStorage().SetHash(ctx, oldHash, newHash); err != nil { + if len(els) > 0 { + dm.diff.Set(els...) + } + if err := dm.storage.StateStorage().SetHash(ctx, dm.diff.Hash()); err != nil { dm.log.Error("can't write space hash", zap.Error(err)) return err } return nil } -func (dm *DiffManager) setHeadsForNewDiff(commonEls, noCommonSnapshotEls []ldiff.Element, hasher *ldiff.Hasher) { - for _, el := range append(commonEls, noCommonSnapshotEls...) { - hash := hasher.HashId(el.Head) - dm.diffContainer.NewDiff().Set(ldiff.Element{ - Id: el.Id, - Head: hash, - }) - } -} - -func (dm *DiffManager) setHeadsForOldDiff(commonEls, onlyOldEls, noCommonSnapshotEls []ldiff.Element, hasher *ldiff.Hasher) { - for _, el := range append(commonEls, onlyOldEls...) { - hash := hasher.HashId(el.Head) - dm.diffContainer.OldDiff().Set(ldiff.Element{ - Id: el.Id, - Head: hash, - }) - } - for _, el := range noCommonSnapshotEls { - dm.diffContainer.OldDiff().Set(el) - } -} - func (dm *DiffManager) TryDiff(ctx context.Context, rdiff RemoteDiff) (newIds, changedIds, removedIds []string, err error) { - needsSync, diff, err := dm.diffContainer.DiffTypeCheck(ctx, rdiff) + needsSync, err := rdiff.DiffTypeCheck(ctx, dm.diff) err = rpcerr.Unwrap(err) if err != nil { return nil, nil, nil, err } if needsSync { - newIds, changedIds, removedIds, err = diff.Diff(ctx, rdiff) + newIds, changedIds, removedIds, err = dm.diff.Diff(ctx, rdiff) err = rpcerr.Unwrap(err) if err != nil { return nil, nil, nil, err @@ -136,64 +105,38 @@ func (dm *DiffManager) TryDiff(ctx context.Context, rdiff RemoteDiff) (newIds, c func (dm *DiffManager) UpdateHeads(update headstorage.HeadsEntry) { if update.DeletedStatus != headstorage.DeletedStatusNotDeleted { - _ = dm.diffContainer.RemoveId(update.Id) + _ = dm.diff.RemoveId(update.Id) } else { if dm.deletionState.Exists(update.Id) { return } - // don't update for derived in both cases - if update.IsDerived && len(update.Heads) == 1 && update.Heads[0] == update.Id { + // live updates never add empty roots; unlike FillDiff there is no + // common-snapshot exception here — this asymmetry predates the V2 + // removal and both conditions must stay as is, or every existing + // space hash changes + if len(update.Heads) == 1 && update.Heads[0] == update.Id { return } hasher := ldiff.NewHasher() defer ldiff.ReleaseHasher(hasher) - if len(update.Heads) == 1 && update.Heads[0] == update.Id { - // empty roots should be updated only for old - dm.diffContainer.OldDiff().Set(ldiff.Element{ - Id: update.Id, - Head: hasher.HashId(update.Heads[0]), - }) - return - } - concatHeads := concatStrings(update.Heads) - if update.Id == dm.keyValue.DefaultStore().Id() { - dm.diffContainer.NewDiff().Set(ldiff.Element{ - Id: update.Id, - Head: hasher.HashId(concatHeads), - }) - // this happens due to old bug, so we should update only the heads as it is - dm.diffContainer.OldDiff().Set(ldiff.Element{ - Id: update.Id, - Head: concatHeads, - }) - } else { - for _, diff := range []ldiff.Diff{dm.diffContainer.OldDiff(), dm.diffContainer.NewDiff()} { - diff.Set(ldiff.Element{ - Id: update.Id, - Head: hasher.HashId(concatHeads), - }) - } - } + dm.diff.Set(ldiff.Element{ + Id: update.Id, + Head: hasher.HashId(concatStrings(update.Heads)), + }) } - oldHash := dm.diffContainer.OldDiff().Hash() - newHash := dm.diffContainer.NewDiff().Hash() - err := dm.storage.StateStorage().SetHash(dm.ctx, oldHash, newHash) + err := dm.storage.StateStorage().SetHash(dm.ctx, dm.diff.Hash()) if err != nil { dm.log.Warn("can't write space hash", zap.Error(err)) } } func (dm *DiffManager) HandleRangeRequest(ctx context.Context, req *spacesyncproto.HeadSyncRequest) (resp *spacesyncproto.HeadSyncResponse, err error) { - switch req.DiffType { - case spacesyncproto.DiffType_V3: - return HandleRangeRequest(ctx, dm.diffContainer.NewDiff(), req) - case spacesyncproto.DiffType_V2: - return HandleRangeRequest(ctx, dm.diffContainer.OldDiff(), req) - default: + if req.DiffType != spacesyncproto.DiffType_V3 { return nil, spacesyncproto.ErrUnexpected } + return HandleRangeRequest(ctx, dm.diff, req) } func (dm *DiffManager) AllIds() []string { - return dm.diffContainer.NewDiff().Ids() + return dm.diff.Ids() } diff --git a/commonspace/headsync/diffmanager_test.go b/commonspace/headsync/diffmanager_test.go index 334d6c1a4..94e1984ec 100644 --- a/commonspace/headsync/diffmanager_test.go +++ b/commonspace/headsync/diffmanager_test.go @@ -17,20 +17,15 @@ import ( "github.com/anyproto/any-sync/commonspace/headsync/statestorage/mock_statestorage" "github.com/anyproto/any-sync/commonspace/object/acl/list" "github.com/anyproto/any-sync/commonspace/object/acl/syncacl/mock_syncacl" - "github.com/anyproto/any-sync/commonspace/object/keyvalue/keyvaluestorage/mock_keyvaluestorage" - "github.com/anyproto/any-sync/commonspace/object/keyvalue/kvinterfaces/mock_kvinterfaces" "github.com/anyproto/any-sync/commonspace/spacestorage/mock_spacestorage" "github.com/anyproto/any-sync/commonspace/spacesyncproto" ) type diffManagerFixture struct { ctrl *gomock.Controller - diffContainerMock *mock_ldiff.MockDiffContainer storageMock *mock_spacestorage.MockSpaceStorage aclMock *mock_syncacl.MockSyncAcl deletionStateMock *mock_deletionstate.MockObjectDeletionState - kvMock *mock_kvinterfaces.MockKeyValueService - defStoreMock *mock_keyvaluestorage.MockStorage headStorage *mock_headstorage.MockHeadStorage stateStorage *mock_statestorage.MockStateStorage diffMock *mock_ldiff.MockDiff @@ -39,40 +34,31 @@ type diffManagerFixture struct { func newDiffManagerFixture(t *testing.T) *diffManagerFixture { ctrl := gomock.NewController(t) - diffContainerMock := mock_ldiff.NewMockDiffContainer(ctrl) storageMock := mock_spacestorage.NewMockSpaceStorage(ctrl) aclMock := mock_syncacl.NewMockSyncAcl(ctrl) deletionStateMock := mock_deletionstate.NewMockObjectDeletionState(ctrl) - kvMock := mock_kvinterfaces.NewMockKeyValueService(ctrl) - defStoreMock := mock_keyvaluestorage.NewMockStorage(ctrl) headStorage := mock_headstorage.NewMockHeadStorage(ctrl) stateStorage := mock_statestorage.NewMockStateStorage(ctrl) diffMock := mock_ldiff.NewMockDiff(ctrl) - kvMock.EXPECT().DefaultStore().Return(defStoreMock).AnyTimes() - defStoreMock.EXPECT().Id().Return("store").AnyTimes() storageMock.EXPECT().HeadStorage().Return(headStorage).AnyTimes() storageMock.EXPECT().StateStorage().Return(stateStorage).AnyTimes() log := logger.NewNamed("test") diffManager := NewDiffManager( - diffContainerMock, + diffMock, storageMock, aclMock, log, context.Background(), deletionStateMock, - kvMock, ) return &diffManagerFixture{ ctrl: ctrl, - diffContainerMock: diffContainerMock, storageMock: storageMock, aclMock: aclMock, deletionStateMock: deletionStateMock, - kvMock: kvMock, - defStoreMock: defStoreMock, headStorage: headStorage, stateStorage: stateStorage, diffMock: diffMock, @@ -84,6 +70,18 @@ func (fx *diffManagerFixture) stop() { fx.ctrl.Finish() } +func (fx *diffManagerFixture) expectIterateEntries(headEntries []headstorage.HeadsEntry) { + fx.headStorage.EXPECT().IterateEntries(gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx context.Context, opts headstorage.IterOpts, entryIter headstorage.EntryIterator) error { + for _, entry := range headEntries { + if res, err := entryIter(entry); err != nil || !res { + return err + } + } + return nil + }) +} + func TestDiffManager_FillDiff(t *testing.T) { ctx := context.Background() @@ -91,7 +89,7 @@ func TestDiffManager_FillDiff(t *testing.T) { fx := newDiffManagerFixture(t) defer fx.stop() - headEntries := []headstorage.HeadsEntry{ + fx.expectIterateEntries([]headstorage.HeadsEntry{ { Id: "id1", Heads: []string{"h1", "h2"}, @@ -104,54 +102,26 @@ func TestDiffManager_FillDiff(t *testing.T) { CommonSnapshot: "", IsDerived: false, }, - } - - fx.headStorage.EXPECT().IterateEntries(gomock.Any(), gomock.Any(), gomock.Any()). - DoAndReturn(func(ctx context.Context, opts headstorage.IterOpts, entryIter headstorage.EntryIterator) error { - for _, entry := range headEntries { - if res, err := entryIter(entry); err != nil || !res { - return err - } - } - return nil - }) + }) fx.aclMock.EXPECT().Id().Return("aclId").Times(1) fx.aclMock.EXPECT().Head().Return(&list.AclRecord{Id: "headId"}).Times(1) - // Test expects hashed values now hasher := ldiff.NewHasher() hash1 := hasher.HashId("h1h2") hash2 := hasher.HashId("h3") ldiff.ReleaseHasher(hasher) - // NewDiff gets both common and no common snapshot elements - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock).Times(2) fx.diffMock.EXPECT().Set(ldiff.Element{ Id: "id1", Head: hash1, - }) - fx.diffMock.EXPECT().Set(ldiff.Element{ + }, ldiff.Element{ Id: "id2", Head: hash2, }) - // OldDiff gets common elements with hash and no common snapshot elements without hash - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock).Times(2) - fx.diffMock.EXPECT().Set(ldiff.Element{ - Id: "id1", - Head: hash1, - }) - fx.diffMock.EXPECT().Set(ldiff.Element{ - Id: "id2", - Head: "h3", - }) - - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Hash().Return("oldHash") - fx.diffMock.EXPECT().Hash().Return("newHash") - fx.stateStorage.EXPECT().SetHash(ctx, "oldHash", "newHash").Return(nil) + fx.diffMock.EXPECT().Hash().Return("hash") + fx.stateStorage.EXPECT().SetHash(ctx, "hash").Return(nil) err := fx.diffManager.FillDiff(ctx) require.NoError(t, err) @@ -161,44 +131,31 @@ func TestDiffManager_FillDiff(t *testing.T) { fx := newDiffManagerFixture(t) defer fx.stop() - headEntries := []headstorage.HeadsEntry{ + fx.expectIterateEntries([]headstorage.HeadsEntry{ { Id: "id1", Heads: []string{"id1"}, CommonSnapshot: "snapshot1", IsDerived: true, }, - } - - fx.headStorage.EXPECT().IterateEntries(gomock.Any(), gomock.Any(), gomock.Any()). - DoAndReturn(func(ctx context.Context, opts headstorage.IterOpts, entryIter headstorage.EntryIterator) error { - for _, entry := range headEntries { - if res, err := entryIter(entry); err != nil || !res { - return err - } - } - return nil - }) + }) fx.aclMock.EXPECT().Id().Return("aclId").Times(1) fx.aclMock.EXPECT().Head().Return(&list.AclRecord{Id: "headId"}).Times(1) - // No elements should be set for derived entries - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Hash().Return("oldHash") - fx.diffMock.EXPECT().Hash().Return("newHash") - fx.stateStorage.EXPECT().SetHash(ctx, "oldHash", "newHash").Return(nil) + // no elements should be set for derived entries + fx.diffMock.EXPECT().Hash().Return("hash") + fx.stateStorage.EXPECT().SetHash(ctx, "hash").Return(nil) err := fx.diffManager.FillDiff(ctx) require.NoError(t, err) }) - t.Run("handle singular roots", func(t *testing.T) { + t.Run("skip singular roots", func(t *testing.T) { fx := newDiffManagerFixture(t) defer fx.stop() - headEntries := []headstorage.HeadsEntry{ + fx.expectIterateEntries([]headstorage.HeadsEntry{ { Id: "id1", Heads: []string{"id1"}, // Singular root @@ -211,49 +168,55 @@ func TestDiffManager_FillDiff(t *testing.T) { CommonSnapshot: "snapshot2", IsDerived: false, }, - } - - fx.headStorage.EXPECT().IterateEntries(gomock.Any(), gomock.Any(), gomock.Any()). - DoAndReturn(func(ctx context.Context, opts headstorage.IterOpts, entryIter headstorage.EntryIterator) error { - for _, entry := range headEntries { - if res, err := entryIter(entry); err != nil || !res { - return err - } - } - return nil - }) + }) fx.aclMock.EXPECT().Id().Return("aclId").Times(1) fx.aclMock.EXPECT().Head().Return(&list.AclRecord{Id: "headId"}).Times(1) hasher := ldiff.NewHasher() - hash1 := hasher.HashId("id1") hash2 := hasher.HashId("h1h2") ldiff.ReleaseHasher(hasher) - // NewDiff should only get id2 - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) + // only id2 should be set fx.diffMock.EXPECT().Set(ldiff.Element{ Id: "id2", Head: hash2, }) - // OldDiff should get both id1 and id2 - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock).Times(2) - fx.diffMock.EXPECT().Set(ldiff.Element{ - Id: "id2", - Head: hash2, + fx.diffMock.EXPECT().Hash().Return("hash") + fx.stateStorage.EXPECT().SetHash(ctx, "hash").Return(nil) + + err := fx.diffManager.FillDiff(ctx) + require.NoError(t, err) + }) + + t.Run("include singular root without common snapshot", func(t *testing.T) { + fx := newDiffManagerFixture(t) + defer fx.stop() + + fx.expectIterateEntries([]headstorage.HeadsEntry{ + { + Id: "id1", + Heads: []string{"id1"}, + CommonSnapshot: "", + IsDerived: false, + }, }) + + fx.aclMock.EXPECT().Id().Return("aclId").Times(1) + fx.aclMock.EXPECT().Head().Return(&list.AclRecord{Id: "headId"}).Times(1) + + hasher := ldiff.NewHasher() + hash1 := hasher.HashId("id1") + ldiff.ReleaseHasher(hasher) + fx.diffMock.EXPECT().Set(ldiff.Element{ Id: "id1", Head: hash1, }) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Hash().Return("oldHash") - fx.diffMock.EXPECT().Hash().Return("newHash") - fx.stateStorage.EXPECT().SetHash(ctx, "oldHash", "newHash").Return(nil) + fx.diffMock.EXPECT().Hash().Return("hash") + fx.stateStorage.EXPECT().SetHash(ctx, "hash").Return(nil) err := fx.diffManager.FillDiff(ctx) require.NoError(t, err) @@ -278,19 +241,29 @@ func TestDiffManager_FillDiff(t *testing.T) { fx.aclMock.EXPECT().Id().Return("aclId").Times(1) fx.aclMock.EXPECT().Head().Return(&list.AclRecord{Id: "headId"}).Times(1) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Hash().Return("oldHash") - fx.diffMock.EXPECT().Hash().Return("newHash") + fx.diffMock.EXPECT().Hash().Return("hash") expectedErr := fmt.Errorf("hash error") - fx.stateStorage.EXPECT().SetHash(ctx, "oldHash", "newHash").Return(expectedErr) + fx.stateStorage.EXPECT().SetHash(ctx, "hash").Return(expectedErr) err := fx.diffManager.FillDiff(ctx) require.ErrorIs(t, err, expectedErr) }) } +type fakeRemoteDiff struct { + needsSync bool + err error +} + +func (f *fakeRemoteDiff) Ranges(ctx context.Context, ranges []ldiff.Range, resBuf []ldiff.RangeResult) ([]ldiff.RangeResult, error) { + return nil, nil +} + +func (f *fakeRemoteDiff) DiffTypeCheck(ctx context.Context, diff ldiff.Diff) (bool, error) { + return f.needsSync, f.err +} + func TestDiffManager_TryDiff(t *testing.T) { ctx := context.Background() @@ -298,12 +271,11 @@ func TestDiffManager_TryDiff(t *testing.T) { fx := newDiffManagerFixture(t) defer fx.stop() - remoteDiff := &remote{spaceId: "space1"} + remoteDiff := &fakeRemoteDiff{needsSync: true} expectedNewIds := []string{"new1", "new2"} expectedChangedIds := []string{"changed1"} expectedRemovedIds := []string{"removed1"} - fx.diffContainerMock.EXPECT().DiffTypeCheck(ctx, remoteDiff).Return(true, fx.diffMock, nil) fx.diffMock.EXPECT().Diff(ctx, remoteDiff).Return(expectedNewIds, expectedChangedIds, expectedRemovedIds, nil) newIds, changedIds, removedIds, err := fx.diffManager.TryDiff(ctx, remoteDiff) @@ -317,9 +289,7 @@ func TestDiffManager_TryDiff(t *testing.T) { fx := newDiffManagerFixture(t) defer fx.stop() - remoteDiff := &remote{spaceId: "space1"} - - fx.diffContainerMock.EXPECT().DiffTypeCheck(ctx, remoteDiff).Return(false, nil, nil) + remoteDiff := &fakeRemoteDiff{needsSync: false} newIds, changedIds, removedIds, err := fx.diffManager.TryDiff(ctx, remoteDiff) require.NoError(t, err) @@ -327,57 +297,33 @@ func TestDiffManager_TryDiff(t *testing.T) { require.Nil(t, changedIds) require.Nil(t, removedIds) }) -} -func TestDiffManager_UpdateHeads(t *testing.T) { - t.Run("delete head", func(t *testing.T) { + t.Run("diff type check fails", func(t *testing.T) { fx := newDiffManagerFixture(t) defer fx.stop() - deleteStatus := headstorage.DeletedStatusDeleted - update := headstorage.HeadsEntry{ - Id: "id1", - DeletedStatus: deleteStatus, - } + expectedErr := fmt.Errorf("check error") + remoteDiff := &fakeRemoteDiff{err: expectedErr} - fx.diffContainerMock.EXPECT().RemoveId("id1").Return(nil) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Hash().Return("hash").Times(2) - fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash", "hash").Return(nil) - - fx.diffManager.UpdateHeads(update) + _, _, _, err := fx.diffManager.TryDiff(ctx, remoteDiff) + require.ErrorIs(t, err, expectedErr) }) +} - t.Run("update head for key-value store", func(t *testing.T) { +func TestDiffManager_UpdateHeads(t *testing.T) { + t.Run("delete head", func(t *testing.T) { fx := newDiffManagerFixture(t) defer fx.stop() + deleteStatus := headstorage.DeletedStatusDeleted update := headstorage.HeadsEntry{ - Id: "store", - Heads: []string{"head1"}, + Id: "id1", + DeletedStatus: deleteStatus, } - fx.deletionStateMock.EXPECT().Exists("store").Return(false) - - hasher := ldiff.NewHasher() - hash := hasher.HashId("head1") - ldiff.ReleaseHasher(hasher) - - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Set(ldiff.Element{ - Id: "store", - Head: hash, - }) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Set(ldiff.Element{ - Id: "store", - Head: "head1", - }) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Hash().Return("hash").Times(2) - fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash", "hash").Return(nil) + fx.diffMock.EXPECT().RemoveId("id1").Return(nil) + fx.diffMock.EXPECT().Hash().Return("hash") + fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash").Return(nil) fx.diffManager.UpdateHeads(update) }) @@ -397,20 +343,12 @@ func TestDiffManager_UpdateHeads(t *testing.T) { hash := hasher.HashId("head1head2") ldiff.ReleaseHasher(hasher) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) fx.diffMock.EXPECT().Set(ldiff.Element{ Id: "id1", Head: hash, }) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Set(ldiff.Element{ - Id: "id1", - Head: hash, - }) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Hash().Return("hash").Times(2) - fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash", "hash").Return(nil) + fx.diffMock.EXPECT().Hash().Return("hash") + fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash").Return(nil) fx.diffManager.UpdateHeads(update) }) @@ -444,7 +382,7 @@ func TestDiffManager_UpdateHeads(t *testing.T) { fx.diffManager.UpdateHeads(update) }) - t.Run("update singular root object", func(t *testing.T) { + t.Run("skip singular root object", func(t *testing.T) { fx := newDiffManagerFixture(t) defer fx.stop() @@ -455,18 +393,7 @@ func TestDiffManager_UpdateHeads(t *testing.T) { fx.deletionStateMock.EXPECT().Exists("id1").Return(false) - hasher := ldiff.NewHasher() - hash := hasher.HashId("id1") - ldiff.ReleaseHasher(hasher) - - // Singular roots should only be added to OldDiff and then return early - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Set(ldiff.Element{ - Id: "id1", - Head: hash, - }) - // No hash update since the function returns early for singular roots - + // no diff update and no hash update for singular roots fx.diffManager.UpdateHeads(update) }) @@ -485,20 +412,12 @@ func TestDiffManager_UpdateHeads(t *testing.T) { hash := hasher.HashId("") ldiff.ReleaseHasher(hasher) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) fx.diffMock.EXPECT().Set(ldiff.Element{ Id: "id1", Head: hash, }) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Set(ldiff.Element{ - Id: "id1", - Head: hash, - }) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Hash().Return("hash").Times(2) - fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash", "hash").Return(nil) + fx.diffMock.EXPECT().Hash().Return("hash") + fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash").Return(nil) fx.diffManager.UpdateHeads(update) }) @@ -513,26 +432,18 @@ func TestDiffManager_UpdateHeads(t *testing.T) { } fx.deletionStateMock.EXPECT().Exists("id1").Return(false) - + hasher := ldiff.NewHasher() hash := hasher.HashId("head1") ldiff.ReleaseHasher(hasher) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) fx.diffMock.EXPECT().Set(ldiff.Element{ Id: "id1", Head: hash, }) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Set(ldiff.Element{ - Id: "id1", - Head: hash, - }) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Hash().Return("hash").Times(2) + fx.diffMock.EXPECT().Hash().Return("hash") // UpdateHeads logs warning but doesn't fail - fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash", "hash").Return(fmt.Errorf("hash error")) + fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash").Return(fmt.Errorf("hash error")) // Should not panic or fail fx.diffManager.UpdateHeads(update) @@ -550,7 +461,6 @@ func TestDiffManager_HandleRangeRequest(t *testing.T) { DiffType: spacesyncproto.DiffType_V3, } - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) fx.diffMock.EXPECT().DiffType().Return(spacesyncproto.DiffType_V3) fx.diffMock.EXPECT().Ranges(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) @@ -566,12 +476,9 @@ func TestDiffManager_HandleRangeRequest(t *testing.T) { DiffType: spacesyncproto.DiffType_V2, } - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().DiffType().Return(spacesyncproto.DiffType_V2) - fx.diffMock.EXPECT().Ranges(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) - _, err := fx.diffManager.HandleRangeRequest(ctx, req) - require.NoError(t, err) + require.Error(t, err) + require.Equal(t, spacesyncproto.ErrUnexpected, err) }) t.Run("handle range request with unsupported diff type", func(t *testing.T) { @@ -594,10 +501,45 @@ func TestDiffManager_AllIds(t *testing.T) { defer fx.stop() expectedIds := []string{"id1", "id2", "id3"} - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) fx.diffMock.EXPECT().Ids().Return(expectedIds) ids := fx.diffManager.AllIds() require.Equal(t, expectedIds, ids) }) } + +// TestDiffManager_HashStability pins the V3 space hash for a scenario covering +// every head-entry class. The golden value was produced by the pre-V2-removal +// DiffManager (dual old/new diffs) on the same input: if this test fails, the +// change breaks hash compatibility with deployed peers and will force a +// network-wide resync — see the skip conditions in FillDiff and UpdateHeads. +func TestDiffManager_HashStability(t *testing.T) { + fx := newDiffManagerFixture(t) + defer fx.stop() + + fx.stateStorage.EXPECT().SetHash(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + fx.aclMock.EXPECT().Id().Return("aclId").AnyTimes() + fx.aclMock.EXPECT().Head().Return(&list.AclRecord{Id: "headId"}).AnyTimes() + fx.deletionStateMock.EXPECT().Exists(gomock.Any()).Return(false).AnyTimes() + + fx.expectIterateEntries([]headstorage.HeadsEntry{ + {Id: "A", Heads: []string{"h1", "h2"}, CommonSnapshot: "s"}, + {Id: "B", Heads: []string{"h3"}, CommonSnapshot: ""}, + {Id: "C", Heads: []string{"C"}, CommonSnapshot: "s"}, + {Id: "D", Heads: []string{"D"}, CommonSnapshot: "s", IsDerived: true}, + {Id: "E", Heads: []string{"E"}, CommonSnapshot: ""}, + }) + + diff := ldiff.New(32, 256) + dm := NewDiffManager(diff, fx.storageMock, fx.aclMock, logger.NewNamed("test"), context.Background(), fx.deletionStateMock) + require.NoError(t, dm.FillDiff(context.Background())) + for _, u := range []headstorage.HeadsEntry{ + {Id: "A", Heads: []string{"h1", "h2", "h5"}}, + {Id: "G", Heads: []string{"G"}}, + {Id: "store", Heads: []string{"k1"}}, + {Id: "B", DeletedStatus: headstorage.DeletedStatusDeleted}, + } { + dm.UpdateHeads(u) + } + require.Equal(t, "9d2101bbe6775d7cd4f1d322d0f227e48ccc27ae3bc745dbda5d05bd4ba61fc1", diff.Hash()) +} diff --git a/commonspace/headsync/diffsyncer_test.go b/commonspace/headsync/diffsyncer_test.go index 7eb62bd63..54fea011c 100644 --- a/commonspace/headsync/diffsyncer_test.go +++ b/commonspace/headsync/diffsyncer_test.go @@ -81,7 +81,8 @@ func TestDiffSyncer(t *testing.T) { fx.peerManagerMock.EXPECT(). GetResponsiblePeers(gomock.Any()). Return([]peer.Peer{mPeer}, nil) - fx.diffContainerMock.EXPECT().DiffTypeCheck(gomock.Any(), gomock.Any()).Return(true, fx.diffMock, nil) + fx.diffMock.EXPECT().Hash().Return("aabbcc") + fx.clientMock.EXPECT().HeadSync(gomock.Any(), gomock.Any()).Return(&spacesyncproto.HeadSyncResponse{DiffType: spacesyncproto.DiffType_V3}, nil) fx.diffMock.EXPECT(). Diff(gomock.Any(), gomock.Eq(NewRemoteDiff(fx.spaceState.SpaceId, fx.clientMock))). Return([]string{"new"}, []string{"changed"}, nil, nil) @@ -105,7 +106,8 @@ func TestDiffSyncer(t *testing.T) { fx.peerManagerMock.EXPECT(). GetResponsiblePeers(gomock.Any()). Return([]peer.Peer{mPeer}, nil) - fx.diffContainerMock.EXPECT().DiffTypeCheck(gomock.Any(), gomock.Any()).Return(true, fx.diffMock, nil) + fx.diffMock.EXPECT().Hash().Return("aabbcc") + fx.clientMock.EXPECT().HeadSync(gomock.Any(), gomock.Any()).Return(&spacesyncproto.HeadSyncResponse{DiffType: spacesyncproto.DiffType_V3}, nil) fx.diffMock.EXPECT(). Diff(gomock.Any(), gomock.Eq(remDiff)). Return([]string{"new"}, []string{"changed"}, nil, nil) @@ -130,7 +132,8 @@ func TestDiffSyncer(t *testing.T) { fx.peerManagerMock.EXPECT(). GetResponsiblePeers(gomock.Any()). Return([]peer.Peer{mPeer}, nil) - fx.diffContainerMock.EXPECT().DiffTypeCheck(gomock.Any(), gomock.Any()).Return(true, fx.diffMock, nil) + fx.diffMock.EXPECT().Hash().Return("aabbcc") + fx.clientMock.EXPECT().HeadSync(gomock.Any(), gomock.Any()).Return(&spacesyncproto.HeadSyncResponse{DiffType: spacesyncproto.DiffType_V3}, nil) fx.diffMock.EXPECT(). Diff(gomock.Any(), gomock.Eq(remDiff)). Return([]string{"new"}, []string{"changed"}, nil, nil) @@ -161,11 +164,9 @@ func TestDiffSyncer(t *testing.T) { fx.initDiffSyncer(t) defer fx.stop() deletedId := "id" - fx.diffContainerMock.EXPECT().RemoveId(deletedId).Return(nil) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) + fx.diffMock.EXPECT().RemoveId(deletedId).Return(nil) fx.diffMock.EXPECT().Hash().AnyTimes().Return("hash") - fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash", "hash").Return(nil) + fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash").Return(nil) upd := headstorage.DeletedStatusDeleted fx.diffSyncer.updateHeads(headstorage.HeadsEntry{ @@ -184,21 +185,13 @@ func TestDiffSyncer(t *testing.T) { hasher := ldiff.NewHasher() hash := hasher.HashId("head") ldiff.ReleaseHasher(hasher) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Set(ldiff.Element{ - Id: updatedId, - Head: hash, - }) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) fx.diffMock.EXPECT().Set(ldiff.Element{ Id: updatedId, Head: hash, }) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Hash().Return("hash").Times(2) - fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash", "hash").Return(nil) + fx.diffMock.EXPECT().Hash().Return("hash") + fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash").Return(nil) fx.diffSyncer.updateHeads(headstorage.HeadsEntry{ Id: "id", Heads: []string{"head"}, @@ -228,7 +221,8 @@ func TestDiffSyncer(t *testing.T) { // in the same flow. Here the peer has not yet committed the space, so the // second TryDiff also returns ErrSpaceMissing and we fall back to the // periodic round (no second push, no tree sync). - fx.diffContainerMock.EXPECT().DiffTypeCheck(gomock.Any(), gomock.Any()).Return(true, fx.diffMock, nil).Times(2) + fx.diffMock.EXPECT().Hash().Return("aabbcc").Times(2) + fx.clientMock.EXPECT().HeadSync(gomock.Any(), gomock.Any()).Return(&spacesyncproto.HeadSyncResponse{DiffType: spacesyncproto.DiffType_V3}, nil).Times(2) fx.diffMock.EXPECT(). Diff(gomock.Any(), gomock.Eq(remDiff)). Return(nil, nil, nil, spacesyncproto.ErrSpaceMissing). @@ -279,7 +273,8 @@ func TestDiffSyncer(t *testing.T) { fx.peerManagerMock.EXPECT(). GetResponsiblePeers(gomock.Any()). Return([]peer.Peer{mPeer}, nil) - fx.diffContainerMock.EXPECT().DiffTypeCheck(gomock.Any(), gomock.Any()).Return(true, fx.diffMock, nil).Times(2) + fx.diffMock.EXPECT().Hash().Return("aabbcc").Times(2) + fx.clientMock.EXPECT().HeadSync(gomock.Any(), gomock.Any()).Return(&spacesyncproto.HeadSyncResponse{DiffType: spacesyncproto.DiffType_V3}, nil).Times(2) // First diff: space missing -> triggers push. Second diff (after push, same // flow): the peer now holds the (still empty) space, so the owner's tree // surfaces as a REMOVED id (owner has it, peer lacks it -> ldiff @@ -339,7 +334,8 @@ func TestDiffSyncer(t *testing.T) { fx.peerManagerMock.EXPECT(). GetResponsiblePeers(gomock.Any()). Return([]peer.Peer{rpctest.MockPeer{}}, nil) - fx.diffContainerMock.EXPECT().DiffTypeCheck(gomock.Any(), gomock.Any()).Return(true, fx.diffMock, nil).Times(2) + fx.diffMock.EXPECT().Hash().Return("aabbcc").Times(2) + fx.clientMock.EXPECT().HeadSync(gomock.Any(), gomock.Any()).Return(&spacesyncproto.HeadSyncResponse{DiffType: spacesyncproto.DiffType_V3}, nil).Times(2) // First diff: space missing -> push. Second diff (after push) fails with a // transient error that is NOT ErrSpaceMissing: we must not push again and // must not sync trees; the round falls back to the periodic loop and Sync @@ -385,7 +381,8 @@ func TestDiffSyncer(t *testing.T) { GetResponsiblePeers(gomock.Any()). Return([]peer.Peer{rpctest.MockPeer{}}, nil) - fx.diffContainerMock.EXPECT().DiffTypeCheck(gomock.Any(), gomock.Any()).Return(true, fx.diffMock, nil) + fx.diffMock.EXPECT().Hash().Return("aabbcc") + fx.clientMock.EXPECT().HeadSync(gomock.Any(), gomock.Any()).Return(&spacesyncproto.HeadSyncResponse{DiffType: spacesyncproto.DiffType_V3}, nil) fx.diffMock.EXPECT(). Diff(gomock.Any(), gomock.Eq(remDiff)). Return(nil, nil, nil, spacesyncproto.ErrUnexpected) @@ -404,7 +401,7 @@ func TestDiffSyncer(t *testing.T) { fx.peerManagerMock.EXPECT(). GetResponsiblePeers(gomock.Any()). Return([]peer.Peer{mPeer}, nil) - fx.diffContainerMock.EXPECT().DiffTypeCheck(gomock.Any(), gomock.Any()).Return(true, fx.diffMock, spacesyncproto.ErrSpaceIsDeleted) + fx.clientMock.EXPECT().HeadSync(gomock.Any(), gomock.Any()).Return(nil, spacesyncproto.ErrSpaceIsDeleted) fx.peerManagerMock.EXPECT().KeepAlive(gomock.Any()) require.NoError(t, fx.diffSyncer.Sync(ctx)) diff --git a/commonspace/headsync/headsync.go b/commonspace/headsync/headsync.go index 50d1220e6..f07738ca6 100644 --- a/commonspace/headsync/headsync.go +++ b/commonspace/headsync/headsync.go @@ -10,7 +10,6 @@ import ( "github.com/anyproto/any-sync/app" "github.com/anyproto/any-sync/app/ldiff" "github.com/anyproto/any-sync/app/logger" - "github.com/anyproto/any-sync/app/olddiff" "github.com/anyproto/any-sync/commonspace/config" "github.com/anyproto/any-sync/commonspace/credentialprovider" "github.com/anyproto/any-sync/commonspace/deletionstate" @@ -77,13 +76,13 @@ func (h *headSync) Init(a *app.App) (err error) { h.configuration = a.MustComponent(nodeconf.CName).(nodeconf.NodeConf) h.log = log.With(zap.String("spaceId", h.spaceId)) h.storage = a.MustComponent(spacestorage.CName).(spacestorage.SpaceStorage) - diffContainer := ldiff.NewDiffContainer(ldiff.New(32, 256), olddiff.New(32, 256)) + diff := ldiff.New(32, 256) h.peerManager = a.MustComponent(peermanager.CName).(peermanager.PeerManager) h.credentialProvider = a.MustComponent(credentialprovider.CName).(credentialprovider.CredentialProvider) h.treeSyncer = a.MustComponent(treesyncer.CName).(treesyncer.TreeSyncer) h.deletionState = a.MustComponent(deletionstate.CName).(deletionstate.ObjectDeletionState) h.keyValue = a.MustComponent(kvinterfaces.CName).(kvinterfaces.KeyValueService) - h.diffManager = NewDiffManager(diffContainer, h.storage, h.syncAcl, h.log, context.Background(), h.deletionState, h.keyValue) + h.diffManager = NewDiffManager(diff, h.storage, h.syncAcl, h.log, context.Background(), h.deletionState) h.syncer = createDiffSyncer(h) sync := func(ctx context.Context) (err error) { return h.syncer.Sync(ctx) diff --git a/commonspace/headsync/headsync_test.go b/commonspace/headsync/headsync_test.go index eaf8741fe..4314e9023 100644 --- a/commonspace/headsync/headsync_test.go +++ b/commonspace/headsync/headsync_test.go @@ -71,7 +71,6 @@ type headSyncFixture struct { diffSyncerMock *mock_headsync.MockDiffSyncer treeSyncerMock *mock_treesyncer.MockTreeSyncer diffMock *mock_ldiff.MockDiff - diffContainerMock *mock_ldiff.MockDiffContainer clientMock *mock_spacesyncproto.MockDRPCSpaceSyncClient aclMock *mock_syncacl.MockSyncAcl headStorage *mock_headstorage.MockHeadStorage @@ -98,7 +97,6 @@ func newHeadSyncFixture(t *testing.T) *headSyncFixture { deletionStateMock := mock_deletionstate.NewMockObjectDeletionState(ctrl) deletionStateMock.EXPECT().Name().AnyTimes().Return(deletionstate.CName) diffSyncerMock := mock_headsync.NewMockDiffSyncer(ctrl) - diffContainerMock := mock_ldiff.NewMockDiffContainer(ctrl) treeSyncerMock := mock_treesyncer.NewMockTreeSyncer(ctrl) headStorage := mock_headstorage.NewMockHeadStorage(ctrl) stateStorage := mock_statestorage.NewMockStateStorage(ctrl) @@ -137,7 +135,6 @@ func newHeadSyncFixture(t *testing.T) *headSyncFixture { defStoreMock: defStore, configurationMock: configurationMock, storageMock: storageMock, - diffContainerMock: diffContainerMock, peerManagerMock: peerManagerMock, credentialProviderMock: credentialProviderMock, treeManagerMock: treeManagerMock, @@ -161,7 +158,7 @@ func (fx *headSyncFixture) init(t *testing.T) { fx.headStorage.EXPECT().AddObserver(gomock.Any()) err := fx.headSync.Init(fx.app) require.NoError(t, err) - fx.headSync.diffManager = NewDiffManager(fx.diffContainerMock, fx.storageMock, fx.aclMock, fx.headSync.log, context.Background(), fx.deletionStateMock, fx.kvMock) + fx.headSync.diffManager = NewDiffManager(fx.diffMock, fx.storageMock, fx.aclMock, fx.headSync.log, context.Background(), fx.deletionStateMock) } func (fx *headSyncFixture) stop() { @@ -206,30 +203,16 @@ func TestHeadSync(t *testing.T) { hash1 := hasher.HashId("h1h2") hash2 := hasher.HashId("h3h4") ldiff.ReleaseHasher(hasher) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock).Times(2) fx.diffMock.EXPECT().Set(ldiff.Element{ Id: "id1", Head: hash1, - }) - fx.diffMock.EXPECT().Set(ldiff.Element{ - Id: "id2", - Head: hash2, - }) - - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock).Times(2) - fx.diffMock.EXPECT().Set(ldiff.Element{ - Id: "id1", - Head: hash1, - }) - fx.diffMock.EXPECT().Set(ldiff.Element{ + }, ldiff.Element{ Id: "id2", Head: hash2, }) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Hash().Return("hash").Times(2) - fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash", "hash").Return(nil) + fx.diffMock.EXPECT().Hash().Return("hash") + fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash").Return(nil) fx.diffSyncerMock.EXPECT().Run() fx.diffSyncerMock.EXPECT().Sync(gomock.Any()).Return(nil) fx.diffSyncerMock.EXPECT().Close() @@ -273,22 +256,13 @@ func TestHeadSync(t *testing.T) { hasher := ldiff.NewHasher() hash2 := hasher.HashId("h3h4") ldiff.ReleaseHasher(hasher) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Set(ldiff.Element{ - Id: "id2", - Head: hash2, - }) - - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) fx.diffMock.EXPECT().Set(ldiff.Element{ Id: "id2", Head: hash2, }) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Hash().Return("hash").Times(2) - fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash", "hash").Return(nil) + fx.diffMock.EXPECT().Hash().Return("hash") + fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash").Return(nil) fx.diffSyncerMock.EXPECT().Run() fx.diffSyncerMock.EXPECT().Sync(gomock.Any()).Return(nil) fx.diffSyncerMock.EXPECT().Close() diff --git a/commonspace/headsync/remotediff.go b/commonspace/headsync/remotediff.go index 7608a6a35..1da2748f1 100644 --- a/commonspace/headsync/remotediff.go +++ b/commonspace/headsync/remotediff.go @@ -15,8 +15,9 @@ type Client interface { } type RemoteDiff interface { - ldiff.RemoteTypeChecker ldiff.Remote + // DiffTypeCheck asks the remote for its total hash and compares it with the local diff + DiffTypeCheck(ctx context.Context, diff ldiff.Diff) (needsSync bool, err error) } func NewRemoteDiff(spaceId string, client Client) RemoteDiff { @@ -27,9 +28,8 @@ func NewRemoteDiff(spaceId string, client Client) RemoteDiff { } type remote struct { - spaceId string - client Client - diffType spacesyncproto.DiffType + spaceId string + client Client } func (r *remote) Ranges(ctx context.Context, ranges []ldiff.Range, resBuf []ldiff.RangeResult) (results []ldiff.RangeResult, err error) { @@ -46,7 +46,7 @@ func (r *remote) Ranges(ctx context.Context, ranges []ldiff.Range, resBuf []ldif req := &spacesyncproto.HeadSyncRequest{ SpaceId: r.spaceId, Ranges: pbRanges, - DiffType: r.diffType, + DiffType: spacesyncproto.DiffType_V3, } resp, err := r.client.HeadSync(ctx, req) if err != nil { @@ -72,7 +72,7 @@ func (r *remote) Ranges(ctx context.Context, ranges []ldiff.Range, resBuf []ldif return } -func (r *remote) DiffTypeCheck(ctx context.Context, diffContainer ldiff.DiffContainer) (needsSync bool, diff ldiff.Diff, err error) { +func (r *remote) DiffTypeCheck(ctx context.Context, diff ldiff.Diff) (needsSync bool, err error) { req := &spacesyncproto.HeadSyncRequest{ SpaceId: r.spaceId, DiffType: spacesyncproto.DiffType_V3, @@ -82,34 +82,21 @@ func (r *remote) DiffTypeCheck(ctx context.Context, diffContainer ldiff.DiffCont if err != nil { return } - needsSync = true - checkHash := func(diff ldiff.Diff) (bool, error) { - hashB, err := hex.DecodeString(diff.Hash()) - if err != nil { - return false, err - } - if len(resp.Results) != 0 && bytes.Equal(hashB, resp.Results[0].Hash) { - return false, nil - } - return true, nil + if resp.DiffType != spacesyncproto.DiffType_V3 { + return false, spacesyncproto.ErrUnexpected } - r.diffType = resp.DiffType - switch resp.DiffType { - case spacesyncproto.DiffType_V3: - diff = diffContainer.NewDiff() - needsSync, err = checkHash(diff) - case spacesyncproto.DiffType_V2: - diff = diffContainer.OldDiff() - needsSync, err = checkHash(diff) - default: - err = spacesyncproto.ErrUnexpected + hashB, err := hex.DecodeString(diff.Hash()) + if err != nil { + return false, err } - return + if len(resp.Results) != 0 && bytes.Equal(hashB, resp.Results[0].Hash) { + return false, nil + } + return true, nil } func HandleRangeRequest(ctx context.Context, d ldiff.Diff, req *spacesyncproto.HeadSyncRequest) (resp *spacesyncproto.HeadSyncResponse, err error) { ranges := make([]ldiff.Range, 0, len(req.Ranges)) - // basically we gather data applicable for both diffs for _, reqRange := range req.Ranges { ranges = append(ranges, ldiff.Range{ From: reqRange.From, diff --git a/commonspace/headsync/remotediff_test.go b/commonspace/headsync/remotediff_test.go index 2a9e5dda1..7c0f74982 100644 --- a/commonspace/headsync/remotediff_test.go +++ b/commonspace/headsync/remotediff_test.go @@ -10,7 +10,6 @@ import ( "github.com/stretchr/testify/require" "github.com/anyproto/any-sync/app/ldiff" - "github.com/anyproto/any-sync/app/olddiff" "github.com/anyproto/any-sync/commonspace/spacesyncproto" ) @@ -67,12 +66,6 @@ func TestBenchRemoteWithDifferentCounts(t *testing.T) { return ldiff.New(32, 256) }, 32) }) - //old has higher head lengths because of hashes - t.Run("OldLdiff", func(t *testing.T) { - benchmarkDifferentDiffs(t, func() ldiff.Diff { - return olddiff.New(32, 256) - }, 100) - }) } type mockClient struct { @@ -108,31 +101,24 @@ func TestRemoteDiffTypeCheck(t *testing.T) { mockClient := &mockClientWithDiffType{t: t, l: contRemote, diffType: spacesyncproto.DiffType_V3} rd := NewRemoteDiff("space1", mockClient) - container := ldiff.NewDiffContainer(contLocal, ldiff.New(32, 256)) - needsSync, diff, err := rd.DiffTypeCheck(ctx, container) + needsSync, err := rd.DiffTypeCheck(ctx, contLocal) require.NoError(t, err) require.False(t, needsSync) - require.Equal(t, contLocal, diff) }) t.Run("diff type check with V2", func(t *testing.T) { ctx := context.Background() + contLocal := ldiff.New(32, 256) contRemote := ldiff.New(32, 256) - oldDiff := ldiff.New(32, 256) - - oldDiff.Set(ldiff.Element{Id: "1", Head: "head1"}) - contRemote.Set(ldiff.Element{Id: "1", Head: "head1"}) mockClient := &mockClientWithDiffType{t: t, l: contRemote, diffType: spacesyncproto.DiffType_V2} rd := NewRemoteDiff("space1", mockClient) - container := ldiff.NewDiffContainer(ldiff.New(32, 256), oldDiff) - needsSync, diff, err := rd.DiffTypeCheck(ctx, container) + _, err := rd.DiffTypeCheck(ctx, contLocal) - require.NoError(t, err) - require.False(t, needsSync) - require.Equal(t, oldDiff, diff) + require.Error(t, err) + require.Equal(t, spacesyncproto.ErrUnexpected, err) }) t.Run("diff type check with unsupported type", func(t *testing.T) { @@ -143,8 +129,7 @@ func TestRemoteDiffTypeCheck(t *testing.T) { mockClient := &mockClientWithDiffType{t: t, l: contRemote, diffType: spacesyncproto.DiffType_V1} rd := NewRemoteDiff("space1", mockClient) - container := ldiff.NewDiffContainer(contLocal, ldiff.New(32, 256)) - _, _, err := rd.DiffTypeCheck(ctx, container) + _, err := rd.DiffTypeCheck(ctx, contLocal) require.Error(t, err) require.Equal(t, spacesyncproto.ErrUnexpected, err) @@ -161,12 +146,10 @@ func TestRemoteDiffTypeCheck(t *testing.T) { mockClient := &mockClientWithDiffType{t: t, l: contRemote, diffType: spacesyncproto.DiffType_V3} rd := NewRemoteDiff("space1", mockClient) - container := ldiff.NewDiffContainer(contLocal, ldiff.New(32, 256)) - needsSync, diff, err := rd.DiffTypeCheck(ctx, container) + needsSync, err := rd.DiffTypeCheck(ctx, contLocal) require.NoError(t, err) require.True(t, needsSync) - require.Equal(t, contLocal, diff) }) t.Run("head sync request fails", func(t *testing.T) { @@ -176,12 +159,10 @@ func TestRemoteDiffTypeCheck(t *testing.T) { mockClient := &mockClientWithError{err: fmt.Errorf("network error")} rd := NewRemoteDiff("space1", mockClient) - container := ldiff.NewDiffContainer(contLocal, ldiff.New(32, 256)) - needsSync, diff, err := rd.DiffTypeCheck(ctx, container) + needsSync, err := rd.DiffTypeCheck(ctx, contLocal) require.Error(t, err) require.False(t, needsSync) - require.Nil(t, diff) }) } diff --git a/commonspace/headsync/statestorage/mock_statestorage/mock_statestorage.go b/commonspace/headsync/statestorage/mock_statestorage/mock_statestorage.go index 0b13c6e81..b6b21c97e 100644 --- a/commonspace/headsync/statestorage/mock_statestorage/mock_statestorage.go +++ b/commonspace/headsync/statestorage/mock_statestorage/mock_statestorage.go @@ -57,17 +57,17 @@ func (mr *MockStateStorageMockRecorder) GetState(ctx any) *gomock.Call { } // SetHash mocks base method. -func (m *MockStateStorage) SetHash(ctx context.Context, oldHash, newHash string) error { +func (m *MockStateStorage) SetHash(ctx context.Context, hash string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SetHash", ctx, oldHash, newHash) + ret := m.ctrl.Call(m, "SetHash", ctx, hash) ret0, _ := ret[0].(error) return ret0 } // SetHash indicates an expected call of SetHash. -func (mr *MockStateStorageMockRecorder) SetHash(ctx, oldHash, newHash any) *gomock.Call { +func (mr *MockStateStorageMockRecorder) SetHash(ctx, hash any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetHash", reflect.TypeOf((*MockStateStorage)(nil).SetHash), ctx, oldHash, newHash) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetHash", reflect.TypeOf((*MockStateStorage)(nil).SetHash), ctx, hash) } // SetObserver mocks base method. diff --git a/commonspace/headsync/statestorage/statestorage.go b/commonspace/headsync/statestorage/statestorage.go index 14d73e025..286dfba84 100644 --- a/commonspace/headsync/statestorage/statestorage.go +++ b/commonspace/headsync/statestorage/statestorage.go @@ -10,7 +10,6 @@ import ( ) type State struct { - OldHash string NewHash string AclId string SettingsId string @@ -19,25 +18,29 @@ type State struct { } type Observer interface { - OnHashChange(oldHash, newHash string) + OnHashChange(hash string) } type StateStorage interface { GetState(ctx context.Context) (State, error) SettingsId() string - SetHash(ctx context.Context, oldHash, newHash string) error + SetHash(ctx context.Context, hash string) error SetObserver(observer Observer) } const ( stateCollectionKey = "state" idKey = "id" - oldHashKey = "oh" newHashKey = "nh" - legacyHashKey = "h" - headerKey = "e" - aclIdKey = "a" - settingsIdKey = "s" + // oldHashKey held the removed V2-diff hash; it is still mirrored on write because + // releases before the V2 removal treat a doc with an empty "oh" as legacy format + // and misread NewHash as empty (see their stateFromDoc). Drop the mirror once + // downgrades/coldsync to those releases are out of support. + oldHashKey = "oh" + legacyHashKey = "h" + headerKey = "e" + aclIdKey = "a" + settingsIdKey = "s" ) type stateStorage struct { @@ -62,20 +65,20 @@ func (s *stateStorage) SetObserver(observer Observer) { s.observer = observer } -func (s *stateStorage) SetHash(ctx context.Context, oldHash, newHash string) (err error) { +func (s *stateStorage) SetHash(ctx context.Context, hash string) (err error) { var modifyResult anystore.ModifyResult defer func() { if s.observer != nil && err == nil && modifyResult.Modified > 0 { - s.observer.OnHashChange(oldHash, newHash) + s.observer.OnHashChange(hash) } }() mod := query.ModifyFunc(func(a *anyenc.Arena, v *anyenc.Value) (result *anyenc.Value, modified bool, err error) { - if storeutil.ModifyKey(v, oldHashKey, a.NewString(oldHash)) { + if storeutil.ModifyKey(v, newHashKey, a.NewString(hash)) { modified = true } - if storeutil.ModifyKey(v, newHashKey, a.NewString(newHash)) { + if storeutil.ModifyKey(v, oldHashKey, a.NewString(hash)) { modified = true } @@ -148,20 +151,15 @@ func (s *stateStorage) SettingsId() string { } func (s *stateStorage) stateFromDoc(doc anystore.Doc) State { - var ( - oldHash = doc.Value().GetString(oldHashKey) - newHash = doc.Value().GetString(newHashKey) - ) + newHash := doc.Value().GetString(newHashKey) // legacy hash is used for backward compatibility, which was due to a mistake in key names - if oldHash == "" || newHash == "" { - oldHash = doc.Value().GetString(legacyHashKey) - newHash = oldHash + if newHash == "" { + newHash = doc.Value().GetString(legacyHashKey) } return State{ SpaceId: doc.Value().GetString(idKey), SettingsId: doc.Value().GetString(settingsIdKey), AclId: doc.Value().GetString(aclIdKey), - OldHash: oldHash, NewHash: newHash, SpaceHeader: doc.Value().GetBytes(headerKey), } diff --git a/commonspace/object/acl/syncacl/syncacl.go b/commonspace/object/acl/syncacl/syncacl.go index 738991890..e9de38d69 100644 --- a/commonspace/object/acl/syncacl/syncacl.go +++ b/commonspace/object/acl/syncacl/syncacl.go @@ -48,6 +48,10 @@ type syncAcl struct { verifier recordverifier.RecordVerifier isClosed bool aclUpdater headupdater.AclUpdater + // broadcasts happen under the acl lock, so they get their own cancellable ctx: + // Close cancels it before taking the lock + broadcastCtx context.Context + broadcastCancel context.CancelFunc } func (s *syncAcl) SetAclUpdater(updater headupdater.AclUpdater) { @@ -63,6 +67,7 @@ func (s *syncAcl) Run(ctx context.Context) (err error) { } func (s *syncAcl) Init(a *app.App) (err error) { + s.broadcastCtx, s.broadcastCancel = context.WithCancel(context.Background()) storage := a.MustComponent(spacestorage.CName).(spacestorage.SpaceStorage) aclStorage, err := storage.AclStorage() if err != nil { @@ -103,7 +108,7 @@ func (s *syncAcl) AddRawRecord(rawRec *consensusproto.RawRecordWithId) (err erro } func (s *syncAcl) broadcast(headUpdate *objectmessages.HeadUpdate) { - err := s.syncClient.Broadcast(context.Background(), headUpdate) + err := s.syncClient.Broadcast(s.broadcastCtx, headUpdate) if err != nil { log.Error("broadcast acl message error", zap.Error(err)) } @@ -139,6 +144,10 @@ func (s *syncAcl) SyncWithPeer(ctx context.Context, p peer.Peer) (err error) { } func (s *syncAcl) Close(ctx context.Context) (err error) { + // unblocks a broadcast holding the acl lock, otherwise the Lock below never returns + if s.broadcastCancel != nil { + s.broadcastCancel() + } if s.AclList == nil { return } diff --git a/commonspace/object/keyvalue/keyvalue.go b/commonspace/object/keyvalue/keyvalue.go index c634d2a36..01ca32594 100644 --- a/commonspace/object/keyvalue/keyvalue.go +++ b/commonspace/object/keyvalue/keyvalue.go @@ -274,7 +274,7 @@ func (k *keyValueService) Run(ctx context.Context) (err error) { func (k *keyValueService) Close(ctx context.Context) (err error) { k.cancel() - k.limiter.Close() + k.limiter.Close(ctx) return nil } diff --git a/commonspace/object/keyvalue/keyvalue_ordering_test.go b/commonspace/object/keyvalue/keyvalue_ordering_test.go index 4b8ac0620..ccceeeee1 100644 --- a/commonspace/object/keyvalue/keyvalue_ordering_test.go +++ b/commonspace/object/keyvalue/keyvalue_ordering_test.go @@ -47,7 +47,7 @@ func TestStoreElementsNewestFirst(t *testing.T) { } require.NoError(t, fxClient.SyncWithPeer(serverPeer)) - fxClient.limiter.Close() + fxClient.limiter.Close(ctx) var sentKeys []string for _, id := range fxServer.ts.sentIds() { @@ -74,7 +74,7 @@ func TestPushedValuesPersistDespiteSendFailure(t *testing.T) { fxServer.ts.setFailTerminator(true) require.NoError(t, fxClient.SyncWithPeer(serverPeer)) - fxClient.limiter.Close() + fxClient.limiter.Close(ctx) require.True(t, fxServer.check(t, "pushed", []byte("pushed-value")), "server must persist pushed values even when its response send fails") @@ -91,7 +91,7 @@ func TestIncrementalApplyConvergence(t *testing.T) { } require.NoError(t, fxClient.SyncWithPeer(serverPeer)) - fxClient.limiter.Close() + fxClient.limiter.Close(ctx) // The client broadcasts once per applied SetRaw, so the broadcast count // proves the pull was applied incrementally rather than in one shot. diff --git a/commonspace/object/keyvalue/keyvalue_test.go b/commonspace/object/keyvalue/keyvalue_test.go index 41b654012..23ebf0f1c 100644 --- a/commonspace/object/keyvalue/keyvalue_test.go +++ b/commonspace/object/keyvalue/keyvalue_test.go @@ -40,7 +40,7 @@ func TestKeyValueService(t *testing.T) { fxServer.add(t, "key4", []byte("value4")) err := fxClient.SyncWithPeer(serverPeer) require.NoError(t, err) - fxClient.limiter.Close() + fxClient.limiter.Close(ctx) fxClient.check(t, "key3", []byte("value3")) fxClient.check(t, "key4", []byte("value4")) fxServer.check(t, "key1", []byte("value1")) @@ -53,7 +53,7 @@ func TestKeyValueService(t *testing.T) { fxServer.add(t, "key1", []byte("value2")) err := fxClient.SyncWithPeer(serverPeer) require.NoError(t, err) - fxClient.limiter.Close() + fxClient.limiter.Close(ctx) fxClient.check(t, "key1", []byte("value1")) fxClient.check(t, "key1", []byte("value2")) fxServer.check(t, "key1", []byte("value1")) @@ -62,7 +62,7 @@ func TestKeyValueService(t *testing.T) { fxServer.add(t, "key1", []byte("value2-2")) err = fxClient.SyncWithPeer(serverPeer) require.NoError(t, err) - fxClient.limiter.Close() + fxClient.limiter.Close(ctx) fxClient.check(t, "key1", []byte("value1-2")) fxClient.check(t, "key1", []byte("value2-2")) fxServer.check(t, "key1", []byte("value1-2")) @@ -100,7 +100,7 @@ func TestKeyValueService(t *testing.T) { } err := fxClient.SyncWithPeer(serverPeer) require.NoError(t, err) - fxClient.limiter.Close() + fxClient.limiter.Close(ctx) for key := range allKeys { if strings.HasPrefix(key, "client-key-") { diff --git a/commonspace/object/keyvalue/keyvaluestorage/innerstorage/keyvaluestorage.go b/commonspace/object/keyvalue/keyvaluestorage/innerstorage/keyvaluestorage.go index 8f7655aab..b60d7bfa0 100644 --- a/commonspace/object/keyvalue/keyvaluestorage/innerstorage/keyvaluestorage.go +++ b/commonspace/object/keyvalue/keyvaluestorage/innerstorage/keyvaluestorage.go @@ -98,7 +98,11 @@ func (s *storage) GetKeyPeerId(ctx context.Context, keyPeerId string) (value Key } func (s *storage) IterateValues(ctx context.Context, iterFunc func(kv KeyValue) (bool, error)) (err error) { - iter, err := s.collection.Find(nil).Iter(ctx) + // Sorted by id (key+"-"+peerId) so all rows of one key come out + // adjacent. Storage.Iterate builds its per-key callback groups from + // consecutive runs; an unsorted scan is insertion-ordered and can + // split a key across groups whenever peers' row batches interleave. + iter, err := s.collection.Find(nil).Sort("id").Iter(ctx) if err != nil { return } diff --git a/commonspace/object/keyvalue/keyvaluestorage/innerstorage/keyvaluestorage_test.go b/commonspace/object/keyvalue/keyvaluestorage/innerstorage/keyvaluestorage_test.go index f1acf5c33..5cd5fff14 100644 --- a/commonspace/object/keyvalue/keyvaluestorage/innerstorage/keyvaluestorage_test.go +++ b/commonspace/object/keyvalue/keyvaluestorage/innerstorage/keyvaluestorage_test.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "path/filepath" + "sort" "testing" anystore "github.com/anyproto/any-store" @@ -94,6 +95,47 @@ func testKeyValue(i int) innerstorage.KeyValue { } } +// TestIterateValuesKeyRowsAdjacent asserts the scan is id-ordered no matter +// the insertion order. Storage.Iterate builds its per-key callback groups +// from consecutive runs, so all rows of one key must come out adjacent — +// an insertion-ordered scan splits a key across groups whenever peers' row +// batches interleave (per-peer batch inserts put one key's rows far apart). +func TestIterateValuesKeyRowsAdjacent(t *testing.T) { + storage := newTestStorage(t) + const keys, peers = 16, 3 + for p := 0; p < peers; p++ { + batch := make([]innerstorage.KeyValue, 0, keys) + for k := 0; k < keys; k++ { + kv := testKeyValue(k) + kv.Key = fmt.Sprintf("key%02d", k) + kv.PeerId = fmt.Sprintf("peer%d", p) + kv.KeyPeerId = kv.Key + "-" + kv.PeerId + batch = append(batch, kv) + } + require.NoError(t, storage.Set(ctx, batch...)) + } + + var ids, keysSeen []string + require.NoError(t, storage.IterateValues(ctx, func(kv innerstorage.KeyValue) (bool, error) { + ids = append(ids, kv.KeyPeerId) + keysSeen = append(keysSeen, kv.Key) + return true, nil + })) + require.Len(t, ids, keys*peers) + require.True(t, sort.StringsAreSorted(ids), "scan must be id-ordered, got %v", ids) + + finished := map[string]bool{} + var current string + for _, key := range keysSeen { + if key == current { + continue + } + require.False(t, finished[key], "rows of %s split across non-adjacent runs", key) + finished[current] = true + current = key + } +} + // TestIterateValuesReturnsOwnedMemory asserts the KeyValues handed to the // iterator callback own their byte slices: retaining one across iterations must // not let the next document's parse overwrite its contents. diff --git a/commonspace/object/keyvalue/limiter.go b/commonspace/object/keyvalue/limiter.go index 7a36bffdc..8b4ee9733 100644 --- a/commonspace/object/keyvalue/limiter.go +++ b/commonspace/object/keyvalue/limiter.go @@ -3,23 +3,34 @@ package keyvalue import ( "context" "sync" + "time" + + "go.uber.org/zap" ) +const limiterCloseTimeout = time.Second * 10 + type concurrentLimiter struct { - mu sync.Mutex - inProgress map[string]bool - wg sync.WaitGroup + mu sync.Mutex + inProgress map[string]bool + wg sync.WaitGroup + closed bool + closeTimeout time.Duration } func newConcurrentLimiter() *concurrentLimiter { return &concurrentLimiter{ - inProgress: make(map[string]bool), + inProgress: make(map[string]bool), + closeTimeout: limiterCloseTimeout, } } func (cl *concurrentLimiter) ScheduleRequest(ctx context.Context, id string, action func()) bool { cl.mu.Lock() - if cl.inProgress[id] { + // a bounded Close can return while wg still has waiters parked: an Add after + // that panics with "WaitGroup is reused before previous Wait has returned", + // and SyncWithPeer stays callable after Close + if cl.closed || cl.inProgress[id] { cl.mu.Unlock() return false } @@ -47,6 +58,26 @@ func (cl *concurrentLimiter) ScheduleRequest(ctx context.Context, id string, act return true } -func (cl *concurrentLimiter) Close() { - cl.wg.Wait() +// Close rejects further requests and waits for the scheduled ones to finish. The +// wait is bounded: a request already past the ctx check is doing peer rpc that may +// not return while nodes are unreachable, and blocking here forever wedges the +// whole app.Close. On timeout the request is abandoned, not stopped. +func (cl *concurrentLimiter) Close(ctx context.Context) { + cl.mu.Lock() + cl.closed = true + cl.mu.Unlock() + done := make(chan struct{}) + go func() { + cl.wg.Wait() + close(done) + }() + timer := time.NewTimer(cl.closeTimeout) + defer timer.Stop() + select { + case <-done: + case <-ctx.Done(): + log.WarnCtx(ctx, "key value close interrupted, peer sync is still in flight", zap.Error(ctx.Err())) + case <-timer.C: + log.WarnCtx(ctx, "key value close timed out, peer sync is still in flight") + } } diff --git a/commonspace/object/keyvalue/limiter_test.go b/commonspace/object/keyvalue/limiter_test.go new file mode 100644 index 000000000..592a428c3 --- /dev/null +++ b/commonspace/object/keyvalue/limiter_test.go @@ -0,0 +1,88 @@ +package keyvalue + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestConcurrentLimiter_CloseBoundedWhenActionIgnoresCtx(t *testing.T) { + var ( + started = make(chan struct{}) + release = make(chan struct{}) + ) + cl := newConcurrentLimiter() + cl.closeTimeout = 10 * time.Millisecond + require.True(t, cl.ScheduleRequest(context.Background(), "peer", func() { + close(started) + <-release + })) + <-started + closed := make(chan struct{}) + go func() { + cl.Close(context.Background()) + close(closed) + }() + select { + case <-closed: + case <-time.After(time.Second): + require.Fail(t, "Close blocked on a request that ignores ctx") + } + close(release) +} + +func TestConcurrentLimiter_CloseHonoursCtx(t *testing.T) { + var ( + started = make(chan struct{}) + release = make(chan struct{}) + ) + cl := newConcurrentLimiter() + require.True(t, cl.ScheduleRequest(context.Background(), "peer", func() { + close(started) + <-release + })) + <-started + ctx, cancel := context.WithCancel(context.Background()) + closed := make(chan struct{}) + go func() { + cl.Close(ctx) + close(closed) + }() + cancel() + select { + case <-closed: + case <-time.After(time.Second): + require.Fail(t, "Close ignored the cancelled ctx") + } + close(release) +} + +// A timed-out Close leaves a waiter parked on the WaitGroup: a later Add would +// panic with "WaitGroup is reused before previous Wait has returned". +func TestConcurrentLimiter_NoScheduleAfterClose(t *testing.T) { + for i := 0; i < 500; i++ { + cl := newConcurrentLimiter() + cl.closeTimeout = time.Microsecond + release := make(chan struct{}) + require.True(t, cl.ScheduleRequest(context.Background(), "peer", func() { + <-release + })) + cl.Close(context.Background()) + + var start sync.WaitGroup + start.Add(2) + go func() { + start.Done() + start.Wait() + close(release) + }() + go func() { + start.Done() + start.Wait() + require.False(t, cl.ScheduleRequest(context.Background(), "peer2", func() {})) + }() + } +} diff --git a/commonspace/object/tree/synctree/request.go b/commonspace/object/tree/synctree/request.go index 3f092915d..a978e1590 100644 --- a/commonspace/object/tree/synctree/request.go +++ b/commonspace/object/tree/synctree/request.go @@ -9,6 +9,7 @@ type InnerRequest struct { heads []string snapshotPath []string root *treechangeproto.RawTreeChangeWithId + probe bool } func (r *InnerRequest) MsgSize() uint64 { @@ -29,10 +30,21 @@ func NewRequest(peerId, spaceId, objectId string, heads []string, snapshotPath [ }) } +// NewProbeRequest creates a request for the tree's root and current heads +// only — the responder sends no change bodies. Old responders ignore the +// probe flag and stream the full tree, so the caller's response collector +// must handle both shapes. +func NewProbeRequest(peerId, spaceId, objectId string) *objectmessages.Request { + return objectmessages.NewRequest(peerId, spaceId, objectId, &InnerRequest{ + probe: true, + }) +} + func (r *InnerRequest) Marshall() ([]byte, error) { msg := &treechangeproto.TreeFullSyncRequest{ Heads: r.heads, SnapshotPath: r.snapshotPath, + Probe: r.probe, } req := treechangeproto.WrapFullRequest(msg, r.root) return req.MarshalVT() diff --git a/commonspace/object/tree/synctree/synchandler.go b/commonspace/object/tree/synctree/synchandler.go index 0edeb7d91..7c62954c6 100644 --- a/commonspace/object/tree/synctree/synchandler.go +++ b/commonspace/object/tree/synctree/synchandler.go @@ -113,6 +113,31 @@ func (s *syncHandler) HandleStreamRequest(ctx context.Context, rq syncdeps.Reque if request == nil { return nil, ErrUnexpectedRequestType } + if request.Probe { + // A probe declares nothing about the requester's state, so no + // counter-request — just root + current heads, no change bodies. + s.tree.Lock() + heads := make([]string, len(s.tree.Heads())) + copy(heads, s.tree.Heads()) + snapshotPath, err := s.tree.SnapshotPath() + root := s.tree.Header() + s.tree.Unlock() + if err != nil { + return nil, err + } + resp := &response.Response{ + SpaceId: s.spaceId, + ObjectId: req.ObjectId(), + Heads: heads, + SnapshotPath: snapshotPath, + Root: root, + } + protoResp, err := resp.ProtoMessage() + if err != nil { + return nil, err + } + return nil, send(protoResp) + } s.tree.Lock() curHeads := s.tree.Heads() log.Debug("got stream request", diff --git a/commonspace/object/tree/synctree/synchandler_test.go b/commonspace/object/tree/synctree/synchandler_test.go index 1bbbcbb06..caa3d039e 100644 --- a/commonspace/object/tree/synctree/synchandler_test.go +++ b/commonspace/object/tree/synctree/synchandler_test.go @@ -12,6 +12,7 @@ import ( "github.com/anyproto/any-sync/commonspace/object/tree/synctree/response" "github.com/anyproto/any-sync/commonspace/object/tree/synctree/response/mock_response" "github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto" + "github.com/anyproto/any-sync/commonspace/spacesyncproto" "github.com/anyproto/any-sync/commonspace/sync/objectsync/objectmessages" "github.com/anyproto/any-sync/commonspace/syncstatus/mock_syncstatus" "github.com/anyproto/any-sync/net/peer" @@ -337,6 +338,61 @@ func TestSyncHandler_HandleStreamRequest(t *testing.T) { }) } +func TestSyncHandler_HandleStreamRequest_Probe(t *testing.T) { + t.Run("probe request, we send root and heads without changes, no request", func(t *testing.T) { + fx := newSyncHandlerFixture(t) + defer fx.finish() + fullRequest := &treechangeproto.TreeFullSyncRequest{ + Probe: true, + } + wrapped := treechangeproto.WrapFullRequest(fullRequest, nil) + marshaled, err := wrapped.MarshalVT() + require.NoError(t, err) + request := objectmessages.NewByteRequest("peerId", "spaceId", "objectId", marshaled) + root := &treechangeproto.RawTreeChangeWithId{ + RawChange: []byte("rootChange"), + Id: "objectId", + } + fx.tree.EXPECT().Heads().AnyTimes().Return([]string{"curHead"}) + fx.tree.EXPECT().SnapshotPath().Return([]string{"objectId"}, nil) + fx.tree.EXPECT().Header().Return(root) + ctx = peer.CtxWithPeerId(ctx, "peerId") + callCount := 0 + req, err := fx.syncHandler.HandleStreamRequest(ctx, request, testUpdater{}, func(resp proto.Message) error { + callCount++ + msg, ok := resp.(*spacesyncproto.ObjectSyncMessage) + require.True(t, ok) + treeMsg := &treechangeproto.TreeSyncMessage{} + require.NoError(t, treeMsg.UnmarshalVT(msg.Payload)) + fullResp := treeMsg.GetContent().GetFullSyncResponse() + require.NotNil(t, fullResp) + require.Equal(t, []string{"curHead"}, fullResp.Heads) + require.Empty(t, fullResp.Changes) + require.Equal(t, root.Id, treeMsg.RootChange.Id) + require.Equal(t, root.RawChange, treeMsg.RootChange.RawChange) + return nil + }) + require.NoError(t, err) + require.Nil(t, req) + require.Equal(t, 1, callCount) + }) +} + +func TestNewProbeRequest(t *testing.T) { + req := NewProbeRequest("peerId", "spaceId", "objectId") + protoMsg, err := req.Proto() + require.NoError(t, err) + msg, ok := protoMsg.(*spacesyncproto.ObjectSyncMessage) + require.True(t, ok) + treeMsg := &treechangeproto.TreeSyncMessage{} + require.NoError(t, treeMsg.UnmarshalVT(msg.Payload)) + fullReq := treeMsg.GetContent().GetFullSyncRequest() + require.NotNil(t, fullReq) + require.True(t, fullReq.Probe) + require.Empty(t, fullReq.Heads) + require.Empty(t, fullReq.Changes) +} + func TestSyncHandler_HandleResponse(t *testing.T) { t.Run("handle response with changes", func(t *testing.T) { fx := newSyncHandlerFixture(t) diff --git a/commonspace/object/tree/synctree/synctree.go b/commonspace/object/tree/synctree/synctree.go index dd597d502..33816a902 100644 --- a/commonspace/object/tree/synctree/synctree.go +++ b/commonspace/object/tree/synctree/synctree.go @@ -85,6 +85,12 @@ type BuildDeps struct { BuildObjectTree objecttree.BuildObjectTreeFunc ValidateObjectTree objecttree.ValidatorFunc StatsCollector *TreeStatsCollector + // Probe makes the remote fetch request the tree's root and current + // heads only (no change bodies). The response never yields a usable + // tree by itself, so Probe callers must supply a ValidateObjectTree + // that inspects the payload and returns an error to abort the build. + // Old responders ignore the flag and stream the full tree. + Probe bool } var newTreeGetter = func(deps BuildDeps, treeId string) treeGetter { diff --git a/commonspace/object/tree/synctree/treeremotegetter.go b/commonspace/object/tree/synctree/treeremotegetter.go index 0de5399a9..60d11825b 100644 --- a/commonspace/object/tree/synctree/treeremotegetter.go +++ b/commonspace/object/tree/synctree/treeremotegetter.go @@ -6,6 +6,7 @@ import ( "github.com/anyproto/any-sync/commonspace/object/tree/objecttree" "github.com/anyproto/any-sync/commonspace/object/tree/treestorage" + "github.com/anyproto/any-sync/commonspace/sync/objectsync/objectmessages" "github.com/anyproto/any-sync/net/peer" ) @@ -51,7 +52,12 @@ func (t treeRemoteGetter) getPeers(ctx context.Context) (peerIds []string, err e func (t treeRemoteGetter) treeRequest(ctx context.Context, peerId string) (collector *fullResponseCollector, err error) { collector = createCollector(t.deps) - req := t.deps.SyncClient.CreateNewTreeRequest(peerId, t.treeId) + var req *objectmessages.Request + if t.deps.Probe { + req = NewProbeRequest(peerId, t.deps.SpaceId, t.treeId) + } else { + req = t.deps.SyncClient.CreateNewTreeRequest(peerId, t.treeId) + } err = t.deps.SyncClient.SendTreeRequest(ctx, req, collector) if err != nil { return nil, err diff --git a/commonspace/object/tree/synctree/treeremotegetter_test.go b/commonspace/object/tree/synctree/treeremotegetter_test.go index d622df5ac..650fe3552 100644 --- a/commonspace/object/tree/synctree/treeremotegetter_test.go +++ b/commonspace/object/tree/synctree/treeremotegetter_test.go @@ -9,7 +9,9 @@ import ( "go.uber.org/mock/gomock" "github.com/anyproto/any-sync/commonspace/object/tree/synctree/mock_synctree" + "github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto" "github.com/anyproto/any-sync/commonspace/peermanager/mock_peermanager" + "github.com/anyproto/any-sync/commonspace/spacesyncproto" "github.com/anyproto/any-sync/commonspace/sync/objectsync/objectmessages" "github.com/anyproto/any-sync/net/peer" "github.com/anyproto/any-sync/net/peer/mock_peer" @@ -103,4 +105,32 @@ func TestTreeRemoteGetter(t *testing.T) { _, _, err := fx.treeGetter.treeRequestLoop(tCtx) require.Error(t, err) }) + + t.Run("probe deps send a probe request instead of a full one", func(t *testing.T) { + fx := newTreeRemoteGetterFixture(t) + defer fx.stop() + fx.treeGetter.deps.Probe = true + coll := newFullResponseCollector(BuildDeps{}) + createCollector = func(deps BuildDeps) *fullResponseCollector { + return coll + } + tCtx := peer.CtxWithPeerId(ctx, peerId) + fx.syncClientMock.EXPECT().SendTreeRequest(tCtx, gomock.Any(), coll).DoAndReturn( + func(_ context.Context, rq any, _ any) error { + req, ok := rq.(*objectmessages.Request) + require.True(t, ok) + protoMsg, err := req.Proto() + require.NoError(t, err) + treeMsg := &treechangeproto.TreeSyncMessage{} + require.NoError(t, treeMsg.UnmarshalVT(protoMsg.(*spacesyncproto.ObjectSyncMessage).Payload)) + fullReq := treeMsg.GetContent().GetFullSyncRequest() + require.NotNil(t, fullReq) + require.True(t, fullReq.Probe) + return nil + }) + retColl, pId, err := fx.treeGetter.treeRequestLoop(tCtx) + require.NoError(t, err) + require.Equal(t, peerId, pId) + require.Equal(t, coll, retColl) + }) } diff --git a/commonspace/object/tree/treechangeproto/protos/treechange.proto b/commonspace/object/tree/treechangeproto/protos/treechange.proto index c52ede9a7..f8e57becb 100644 --- a/commonspace/object/tree/treechangeproto/protos/treechange.proto +++ b/commonspace/object/tree/treechangeproto/protos/treechange.proto @@ -121,6 +121,10 @@ message TreeFullSyncRequest { repeated string heads = 1; repeated RawTreeChangeWithId changes = 2; repeated string snapshotPath = 3; + // probe asks the responder for the tree's root and current heads only, + // with no change bodies. Old responders ignore it and stream the full + // tree; a probing requester must be ready for both response shapes. + bool probe = 4; } // TreeFullSyncResponse is a message sent as a response for a specific full sync diff --git a/commonspace/object/tree/treechangeproto/treechange.pb.go b/commonspace/object/tree/treechangeproto/treechange.pb.go index a59611635..fd7057bca 100644 --- a/commonspace/object/tree/treechangeproto/treechange.pb.go +++ b/commonspace/object/tree/treechangeproto/treechange.pb.go @@ -803,10 +803,14 @@ func (x *TreeHeadUpdate) GetSnapshotPath() []string { // TreeHeadUpdate is a message sent when document needs full sync type TreeFullSyncRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` - Heads []string `protobuf:"bytes,1,rep,name=heads,proto3" json:"heads,omitempty"` - Changes []*RawTreeChangeWithId `protobuf:"bytes,2,rep,name=changes,proto3" json:"changes,omitempty"` - SnapshotPath []string `protobuf:"bytes,3,rep,name=snapshotPath,proto3" json:"snapshotPath,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + Heads []string `protobuf:"bytes,1,rep,name=heads,proto3" json:"heads,omitempty"` + Changes []*RawTreeChangeWithId `protobuf:"bytes,2,rep,name=changes,proto3" json:"changes,omitempty"` + SnapshotPath []string `protobuf:"bytes,3,rep,name=snapshotPath,proto3" json:"snapshotPath,omitempty"` + // probe asks the responder for the tree's root and current heads only, + // with no change bodies. Old responders ignore it and stream the full + // tree; a probing requester must be ready for both response shapes. + Probe bool `protobuf:"varint,4,opt,name=probe,proto3" json:"probe,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -862,6 +866,13 @@ func (x *TreeFullSyncRequest) GetSnapshotPath() []string { return nil } +func (x *TreeFullSyncRequest) GetProbe() bool { + if x != nil { + return x.Probe + } + return false +} + // TreeFullSyncResponse is a message sent as a response for a specific full sync type TreeFullSyncResponse struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -1096,11 +1107,12 @@ const file_treechange_proto_rawDesc = "" + "\x0eTreeHeadUpdate\x12\x14\n" + "\x05heads\x18\x01 \x03(\tR\x05heads\x129\n" + "\achanges\x18\x02 \x03(\v2\x1f.treechange.RawTreeChangeWithIdR\achanges\x12\"\n" + - "\fsnapshotPath\x18\x03 \x03(\tR\fsnapshotPath\"\x8a\x01\n" + + "\fsnapshotPath\x18\x03 \x03(\tR\fsnapshotPath\"\xa0\x01\n" + "\x13TreeFullSyncRequest\x12\x14\n" + "\x05heads\x18\x01 \x03(\tR\x05heads\x129\n" + "\achanges\x18\x02 \x03(\v2\x1f.treechange.RawTreeChangeWithIdR\achanges\x12\"\n" + - "\fsnapshotPath\x18\x03 \x03(\tR\fsnapshotPath\"\x8b\x01\n" + + "\fsnapshotPath\x18\x03 \x03(\tR\fsnapshotPath\x12\x14\n" + + "\x05probe\x18\x04 \x01(\bR\x05probe\"\x8b\x01\n" + "\x14TreeFullSyncResponse\x12\x14\n" + "\x05heads\x18\x01 \x03(\tR\x05heads\x129\n" + "\achanges\x18\x02 \x03(\v2\x1f.treechange.RawTreeChangeWithIdR\achanges\x12\"\n" + diff --git a/commonspace/object/tree/treechangeproto/treechange_vtproto.pb.go b/commonspace/object/tree/treechangeproto/treechange_vtproto.pb.go index 467552b47..2725bb80c 100644 --- a/commonspace/object/tree/treechangeproto/treechange_vtproto.pb.go +++ b/commonspace/object/tree/treechangeproto/treechange_vtproto.pb.go @@ -706,6 +706,16 @@ func (m *TreeFullSyncRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if m.Probe { + i-- + if m.Probe { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x20 + } if len(m.SnapshotPath) > 0 { for iNdEx := len(m.SnapshotPath) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.SnapshotPath[iNdEx]) @@ -1209,6 +1219,9 @@ func (m *TreeFullSyncRequest) SizeVT() (n int) { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } } + if m.Probe { + n += 2 + } n += len(m.unknownFields) return n } @@ -3131,6 +3144,26 @@ func (m *TreeFullSyncRequest) UnmarshalVT(dAtA []byte) error { } m.SnapshotPath = append(m.SnapshotPath, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Probe", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Probe = bool(v != 0) default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) diff --git a/commonspace/object/treesyncer/treesyncer.go b/commonspace/object/treesyncer/treesyncer.go index 1ebe8267b..463c42568 100644 --- a/commonspace/object/treesyncer/treesyncer.go +++ b/commonspace/object/treesyncer/treesyncer.go @@ -5,6 +5,7 @@ import ( "context" "github.com/anyproto/any-sync/app" + "github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto" "github.com/anyproto/any-sync/net/peer" ) @@ -17,3 +18,14 @@ type TreeSyncer interface { ShouldSync(peerId string) bool SyncAll(ctx context.Context, p peer.Peer, existing, missing []string) error } + +// PullFilter is an optional extension of the registered TreeSyncer. When a +// head update arrives for a tree that does not exist locally, objectsync +// consults it before queueing the fetch: returning false swallows the update +// and the tree is not pulled. rootChange is the tree's raw root (carried by +// every head update; its header — including changeType — is cleartext) and +// heads are the sender's current heads. TreeSyncers that do not implement +// the interface keep the default always-pull behavior. +type PullFilter interface { + ShouldPull(ctx context.Context, objectId string, rootChange *treechangeproto.RawTreeChangeWithId, heads []string) bool +} diff --git a/commonspace/objecttreebuilder/treebuilder.go b/commonspace/objecttreebuilder/treebuilder.go index a414f25a3..10723522f 100644 --- a/commonspace/objecttreebuilder/treebuilder.go +++ b/commonspace/objecttreebuilder/treebuilder.go @@ -30,6 +30,10 @@ type BuildTreeOpts struct { Listener updatelistener.UpdateListener TreeBuilder objecttree.BuildObjectTreeFunc TreeValidator objecttree.ValidatorFunc + // Probe requests the tree's root and current heads only (no change + // bodies) when the tree is fetched remotely. Requires a TreeValidator + // that inspects the payload and aborts — see synctree.BuildDeps.Probe. + Probe bool } const CName = "common.commonspace.objecttreebuilder" @@ -155,6 +159,7 @@ func (t *treeBuilder) BuildTree(ctx context.Context, id string, opts BuildTreeOp BuildObjectTree: treeBuilder, ValidateObjectTree: opts.TreeValidator, StatsCollector: t.treeStats, + Probe: opts.Probe, } t.treesUsed.Add(1) t.log.Debug("incrementing counter", zap.String("id", id), zap.Int32("trees", t.treesUsed.Load())) diff --git a/commonspace/pubsub/dedup.go b/commonspace/pubsub/dedup.go new file mode 100644 index 000000000..42b582816 --- /dev/null +++ b/commonspace/pubsub/dedup.go @@ -0,0 +1,48 @@ +package pubsub + +import "sync" + +const msgIdLen = 16 + +// msgIdDedup is a fixed-size duplicate-suppression cache keyed by message id. +// It is a FIFO ring: adding a new id evicts the oldest one, keeping memory flat. +type msgIdDedup struct { + mu sync.Mutex + set map[[msgIdLen]byte]struct{} + ring [][msgIdLen]byte + pos int + full bool +} + +func newMsgIdDedup(size int) *msgIdDedup { + return &msgIdDedup{ + set: make(map[[msgIdLen]byte]struct{}, size), + ring: make([][msgIdLen]byte, size), + } +} + +// seen reports whether id was already recorded; if not, it records it, +// evicting the oldest entry when the cache is full. +func (d *msgIdDedup) seen(id []byte) bool { + if len(id) != msgIdLen { + return false + } + var key [msgIdLen]byte + copy(key[:], id) + d.mu.Lock() + defer d.mu.Unlock() + if _, ok := d.set[key]; ok { + return true + } + if d.full { + delete(d.set, d.ring[d.pos]) + } + d.ring[d.pos] = key + d.set[key] = struct{}{} + d.pos++ + if d.pos == len(d.ring) { + d.pos = 0 + d.full = true + } + return false +} diff --git a/commonspace/pubsub/dedup_test.go b/commonspace/pubsub/dedup_test.go new file mode 100644 index 000000000..b0f02835f --- /dev/null +++ b/commonspace/pubsub/dedup_test.go @@ -0,0 +1,41 @@ +package pubsub + +import ( + "encoding/binary" + "testing" + + "github.com/stretchr/testify/require" +) + +func testMsgId(n int) []byte { + id := make([]byte, msgIdLen) + binary.LittleEndian.PutUint64(id, uint64(n)) + return id +} + +func TestDedupSeen(t *testing.T) { + d := newMsgIdDedup(4) + require.False(t, d.seen(testMsgId(1))) + require.True(t, d.seen(testMsgId(1))) + require.False(t, d.seen(testMsgId(2))) + require.True(t, d.seen(testMsgId(2))) +} + +func TestDedupEviction(t *testing.T) { + d := newMsgIdDedup(4) + for i := 1; i <= 4; i++ { + require.False(t, d.seen(testMsgId(i))) + } + // adding a fifth evicts the oldest (1) + require.False(t, d.seen(testMsgId(5))) + require.False(t, d.seen(testMsgId(1)), "oldest id should have been evicted") + // 1 was re-recorded above, evicting 2 + require.True(t, d.seen(testMsgId(5))) + require.False(t, d.seen(testMsgId(2))) +} + +func TestDedupInvalidLength(t *testing.T) { + d := newMsgIdDedup(4) + require.False(t, d.seen([]byte("short"))) + require.False(t, d.seen([]byte("short")), "invalid ids are never recorded") +} diff --git a/commonspace/pubsub/deps.go b/commonspace/pubsub/deps.go new file mode 100644 index 000000000..c85375f76 --- /dev/null +++ b/commonspace/pubsub/deps.go @@ -0,0 +1,141 @@ +package pubsub + +import ( + "context" + "time" + + "github.com/anyproto/any-sync/commonspace/pubsub/pubsubproto" + "github.com/anyproto/any-sync/metric" + "github.com/anyproto/any-sync/net/peer" + "github.com/anyproto/any-sync/net/streampool" + "github.com/anyproto/any-sync/util/crypto" +) + +// Handler receives a decrypted, signature-verified message on a subscribed topic. +// Handlers run on a bounded dispatch queue and must not block. +type Handler func(spaceId, topic string, identity crypto.PubKey, payload []byte) + +// MembershipChecker gates subscribe and publish on space membership. +// Implementations resolve the space's ACL state; any non-nil error rejects. +type MembershipChecker interface { + CheckMember(ctx context.Context, spaceId string, identity crypto.PubKey) error +} + +// Crypto encrypts and decrypts payloads with the space read key. +// A nil Crypto means plaintext payloads (keyless spaces). +type Crypto interface { + // Encrypt returns the key id used and the ciphertext. + Encrypt(spaceId string, payload []byte) (keyId string, encrypted []byte, err error) + // Decrypt resolves keyId to a historical read key and decrypts. + Decrypt(spaceId, keyId string, encrypted []byte) ([]byte, error) +} + +// PeerProvider resolves the peers a client sends publishes and interest to for a +// space: the responsible sync node plus any directly connected LAN peers. +type PeerProvider interface { + SpacePeers(ctx context.Context, spaceId string) ([]peer.Peer, error) +} + +// Relay is implemented only on responsible nodes; nil on clients. +type Relay interface { + // IsResponsible reports whether this node is responsible for the space. + IsResponsible(spaceId string) bool + // IsResponsibleNode reports whether peerId is a responsible node for the space + // (used to authorize inbound relayed publishes). + IsResponsibleNode(spaceId, peerId string) bool + // OtherResponsiblePeers returns the other responsible nodes, excluding self. + OtherResponsiblePeers(ctx context.Context, spaceId string) ([]peer.Peer, error) +} + +// StatusHandler observes Status frames received from serving peers (rejections). +type StatusHandler func(peerId string, status *pubsubproto.Status) + +// Deps carries the pluggable pieces wired by the node or client host. +type Deps struct { + Membership MembershipChecker + Crypto Crypto + Peers PeerProvider // client side; may be nil on nodes + Relay Relay // node side; nil on clients + OnStatus StatusHandler + // Metric, if set, registers the private pool's prometheus metrics so a node + // relaying at scale is observable. Optional. + Metric metric.Metric + Config Config +} + +// Config bounds the engine per DESIGN.md §9; zero values take defaults. +type Config struct { + MaxPayloadSize int + MaxPatternsPerStream int + MaxPatternsPerSpace int + PublishRps float64 + PublishBurst int + WriteQueueSize int + DispatchQueueSize int + DedupSize int + // MaxTimestampSkew bounds how stale (or future) a received message's timestamp + // may be before it is dropped, raising the replay bar even after dedup eviction. + MaxTimestampSkew time.Duration + // ResyncInterval is how often a client re-pushes its interest to space peers, + // keeping server-side interest alive across reconnects. + ResyncInterval time.Duration + // PeerTTL keeps a pubsub stream's peer from being reaped by idle pool GC. + PeerTTL time.Duration + // DialQueueWorkers/DialQueueSize size the pool's outbound dial pool. + DialQueueWorkers int + DialQueueSize int +} + +func (c Config) withDefaults() Config { + if c.MaxPayloadSize <= 0 { + c.MaxPayloadSize = 64 * 1024 + } + if c.MaxPatternsPerStream <= 0 { + c.MaxPatternsPerStream = 1000 + } + if c.MaxPatternsPerSpace <= 0 { + c.MaxPatternsPerSpace = 100 + } + if c.PublishRps <= 0 { + c.PublishRps = 30 + } + if c.PublishBurst <= 0 { + c.PublishBurst = 60 + } + if c.WriteQueueSize <= 0 { + c.WriteQueueSize = 100 + } + if c.DispatchQueueSize <= 0 { + c.DispatchQueueSize = 100 + } + if c.DedupSize <= 0 { + c.DedupSize = 4096 + } + if c.MaxTimestampSkew <= 0 { + c.MaxTimestampSkew = 5 * time.Minute + } + if c.ResyncInterval <= 0 { + c.ResyncInterval = 20 * time.Second + } + if c.PeerTTL <= 0 { + c.PeerTTL = time.Hour + } + if c.DialQueueWorkers <= 0 { + c.DialQueueWorkers = 4 + } + if c.DialQueueSize <= 0 { + c.DialQueueSize = 100 + } + return c +} + +// streamPoolConfig maps the pubsub config onto the streampool's own config. Only +// the dial knobs are consumed by the pool; per-stream queue size is passed +// explicitly to AddStream/ReadStream via WriteQueueSize. +func (c Config) streamPoolConfig() streampool.StreamConfig { + return streampool.StreamConfig{ + SendQueueSize: c.WriteQueueSize, + DialQueueWorkers: c.DialQueueWorkers, + DialQueueSize: c.DialQueueSize, + } +} diff --git a/commonspace/pubsub/pubsubproto/errors.go b/commonspace/pubsub/pubsubproto/errors.go new file mode 100644 index 000000000..3390ff7a0 --- /dev/null +++ b/commonspace/pubsub/pubsubproto/errors.go @@ -0,0 +1,20 @@ +package pubsubproto + +import ( + "errors" + + "github.com/anyproto/any-sync/net/rpc/rpcerr" +) + +var ( + errGroup = rpcerr.ErrGroup(ErrCodes_ErrorOffset) + + ErrUnexpected = errGroup.Register(errors.New("unexpected error"), uint64(ErrCodes_Unexpected)) + ErrNotAMember = errGroup.Register(errors.New("identity is not a space member"), uint64(ErrCodes_NotAMember)) + ErrNotResponsible = errGroup.Register(errors.New("peer is not responsible for space"), uint64(ErrCodes_NotResponsible)) + ErrRateLimited = errGroup.Register(errors.New("publish rate limit exceeded"), uint64(ErrCodes_RateLimited)) + ErrTooManyTopics = errGroup.Register(errors.New("too many topic patterns"), uint64(ErrCodes_TooManyTopics)) + ErrInvalidMessage = errGroup.Register(errors.New("invalid message"), uint64(ErrCodes_InvalidMessage)) + ErrTopicNotOwned = errGroup.Register(errors.New("topic is owned by another account"), uint64(ErrCodes_TopicNotOwned)) + ErrInvalidTopic = errGroup.Register(errors.New("invalid topic or pattern"), uint64(ErrCodes_InvalidTopic)) +) diff --git a/commonspace/pubsub/pubsubproto/protos/pubsub.proto b/commonspace/pubsub/pubsubproto/protos/pubsub.proto new file mode 100644 index 000000000..ba563feb4 --- /dev/null +++ b/commonspace/pubsub/pubsubproto/protos/pubsub.proto @@ -0,0 +1,86 @@ +syntax = "proto3"; +package pubsub; + +option go_package = "commonspace/pubsub/pubsubproto"; + +service PubSub { + // PubSubStream is a long-lived bidirectional stream multiplexing all spaces and topics between two peers + rpc PubSubStream(stream PubSubMessage) returns (stream PubSubMessage); +} + +enum ErrCodes { + Unexpected = 0; + // NotAMember - identity has no permissions in the space's ACL + NotAMember = 1; + // NotResponsible - the serving node is not responsible for the space + NotResponsible = 2; + // RateLimited - the per-peer publish rate limit was exceeded + RateLimited = 3; + // TooManyTopics - the per-stream or per-space pattern cap was exceeded + TooManyTopics = 4; + // InvalidMessage - malformed frame, oversized payload, identity mismatch or bad signature + InvalidMessage = 5; + // TopicNotOwned - publish into the acc/ namespace by an identity other than the topic owner + TopicNotOwned = 6; + // InvalidTopic - malformed topic or pattern: bad wildcard placement, reserved chars, non-canonical form + InvalidTopic = 7; + // ErrorOffset - error-code band for this proto package; see docs/rpc-error-offsets.md + // (100..900 are the any-sync core bands; pubsub uses 1100) + ErrorOffset = 1100; +} + +// PubSubMessage is the single frame type carried by PubSubStream +message PubSubMessage { + oneof content { + Subscribe subscribe = 1; + Unsubscribe unsubscribe = 2; + Publish publish = 3; + Status status = 4; + } +} + +// Subscribe adds topic patterns to the stream's interest set for the space. +// Patterns may contain wildcards: '*' matches exactly one segment, '>' matches one-or-more trailing segments (tail-only) +message Subscribe { + string spaceId = 1; + repeated string topics = 2; +} + +// Unsubscribe removes patterns from the stream's interest set, matched verbatim (not expanded); +// empty topics means remove all patterns of the space +message Unsubscribe { + string spaceId = 1; + repeated string topics = 2; +} + +// Publish carries one ephemeral fire-and-forget message to a fully-qualified topic +message Publish { + string spaceId = 1; + // topic is a fully-qualified '/'-separated topic; wildcards are not allowed + string topic = 2; + // msgId is 16 random bytes generated by the publisher, used for duplicate suppression + bytes msgId = 3; + // keyId is the space ReadKey id used to encrypt the payload; empty means plaintext (keyless spaces) + string keyId = 4; + // payload is ciphertext, or plaintext iff keyId is empty + bytes payload = 5; + // identity is the sender's marshalled account public key + bytes identity = 6; + // signature is the account-key signature over "anysync:pubsub:v1" | spaceId | topic | msgId | keyId | le64(timestampMilli) | payload + bytes signature = 7; + // timestampMilli is the sender's wall clock, informational + int64 timestampMilli = 8; + // relayed is set by a node when forwarding node-to-node; a relayed message is never forwarded again; excluded from the signature + bool relayed = 9; +} + +// Status is sent by the serving peer on a rejected subscribe or publish; success is silent +message Status { + string spaceId = 1; + // topics echoes the offending patterns (or the topic of a rejected publish) + repeated string topics = 2; + ErrCodes code = 3; + // msgId echoes a rejected publish's id so the caller can correlate the + // rejection to a specific Publish; empty for subscribe rejections + bytes msgId = 4; +} diff --git a/commonspace/pubsub/pubsubproto/pubsub.pb.go b/commonspace/pubsub/pubsubproto/pubsub.pb.go new file mode 100644 index 000000000..37141f5c8 --- /dev/null +++ b/commonspace/pubsub/pubsubproto/pubsub.pb.go @@ -0,0 +1,623 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v7.35.1 +// source: commonspace/pubsub/pubsubproto/protos/pubsub.proto + +package pubsubproto + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ErrCodes int32 + +const ( + ErrCodes_Unexpected ErrCodes = 0 + // NotAMember - identity has no permissions in the space's ACL + ErrCodes_NotAMember ErrCodes = 1 + // NotResponsible - the serving node is not responsible for the space + ErrCodes_NotResponsible ErrCodes = 2 + // RateLimited - the per-peer publish rate limit was exceeded + ErrCodes_RateLimited ErrCodes = 3 + // TooManyTopics - the per-stream or per-space pattern cap was exceeded + ErrCodes_TooManyTopics ErrCodes = 4 + // InvalidMessage - malformed frame, oversized payload, identity mismatch or bad signature + ErrCodes_InvalidMessage ErrCodes = 5 + // TopicNotOwned - publish into the acc/ namespace by an identity other than the topic owner + ErrCodes_TopicNotOwned ErrCodes = 6 + // InvalidTopic - malformed topic or pattern: bad wildcard placement, reserved chars, non-canonical form + ErrCodes_InvalidTopic ErrCodes = 7 + ErrCodes_ErrorOffset ErrCodes = 1100 +) + +// Enum value maps for ErrCodes. +var ( + ErrCodes_name = map[int32]string{ + 0: "Unexpected", + 1: "NotAMember", + 2: "NotResponsible", + 3: "RateLimited", + 4: "TooManyTopics", + 5: "InvalidMessage", + 6: "TopicNotOwned", + 7: "InvalidTopic", + 1100: "ErrorOffset", + } + ErrCodes_value = map[string]int32{ + "Unexpected": 0, + "NotAMember": 1, + "NotResponsible": 2, + "RateLimited": 3, + "TooManyTopics": 4, + "InvalidMessage": 5, + "TopicNotOwned": 6, + "InvalidTopic": 7, + "ErrorOffset": 1100, + } +) + +func (x ErrCodes) Enum() *ErrCodes { + p := new(ErrCodes) + *p = x + return p +} + +func (x ErrCodes) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ErrCodes) Descriptor() protoreflect.EnumDescriptor { + return file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_enumTypes[0].Descriptor() +} + +func (ErrCodes) Type() protoreflect.EnumType { + return &file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_enumTypes[0] +} + +func (x ErrCodes) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ErrCodes.Descriptor instead. +func (ErrCodes) EnumDescriptor() ([]byte, []int) { + return file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDescGZIP(), []int{0} +} + +// PubSubMessage is the single frame type carried by PubSubStream +type PubSubMessage struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Content: + // + // *PubSubMessage_Subscribe + // *PubSubMessage_Unsubscribe + // *PubSubMessage_Publish + // *PubSubMessage_Status + Content isPubSubMessage_Content `protobuf_oneof:"content"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PubSubMessage) Reset() { + *x = PubSubMessage{} + mi := &file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PubSubMessage) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PubSubMessage) ProtoMessage() {} + +func (x *PubSubMessage) ProtoReflect() protoreflect.Message { + mi := &file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PubSubMessage.ProtoReflect.Descriptor instead. +func (*PubSubMessage) Descriptor() ([]byte, []int) { + return file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDescGZIP(), []int{0} +} + +func (x *PubSubMessage) GetContent() isPubSubMessage_Content { + if x != nil { + return x.Content + } + return nil +} + +func (x *PubSubMessage) GetSubscribe() *Subscribe { + if x != nil { + if x, ok := x.Content.(*PubSubMessage_Subscribe); ok { + return x.Subscribe + } + } + return nil +} + +func (x *PubSubMessage) GetUnsubscribe() *Unsubscribe { + if x != nil { + if x, ok := x.Content.(*PubSubMessage_Unsubscribe); ok { + return x.Unsubscribe + } + } + return nil +} + +func (x *PubSubMessage) GetPublish() *Publish { + if x != nil { + if x, ok := x.Content.(*PubSubMessage_Publish); ok { + return x.Publish + } + } + return nil +} + +func (x *PubSubMessage) GetStatus() *Status { + if x != nil { + if x, ok := x.Content.(*PubSubMessage_Status); ok { + return x.Status + } + } + return nil +} + +type isPubSubMessage_Content interface { + isPubSubMessage_Content() +} + +type PubSubMessage_Subscribe struct { + Subscribe *Subscribe `protobuf:"bytes,1,opt,name=subscribe,proto3,oneof"` +} + +type PubSubMessage_Unsubscribe struct { + Unsubscribe *Unsubscribe `protobuf:"bytes,2,opt,name=unsubscribe,proto3,oneof"` +} + +type PubSubMessage_Publish struct { + Publish *Publish `protobuf:"bytes,3,opt,name=publish,proto3,oneof"` +} + +type PubSubMessage_Status struct { + Status *Status `protobuf:"bytes,4,opt,name=status,proto3,oneof"` +} + +func (*PubSubMessage_Subscribe) isPubSubMessage_Content() {} + +func (*PubSubMessage_Unsubscribe) isPubSubMessage_Content() {} + +func (*PubSubMessage_Publish) isPubSubMessage_Content() {} + +func (*PubSubMessage_Status) isPubSubMessage_Content() {} + +// Subscribe adds topic patterns to the stream's interest set for the space. +// Patterns may contain wildcards: '*' matches exactly one segment, '>' matches one-or-more trailing segments (tail-only) +type Subscribe struct { + state protoimpl.MessageState `protogen:"open.v1"` + SpaceId string `protobuf:"bytes,1,opt,name=spaceId,proto3" json:"spaceId,omitempty"` + Topics []string `protobuf:"bytes,2,rep,name=topics,proto3" json:"topics,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Subscribe) Reset() { + *x = Subscribe{} + mi := &file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Subscribe) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Subscribe) ProtoMessage() {} + +func (x *Subscribe) ProtoReflect() protoreflect.Message { + mi := &file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Subscribe.ProtoReflect.Descriptor instead. +func (*Subscribe) Descriptor() ([]byte, []int) { + return file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDescGZIP(), []int{1} +} + +func (x *Subscribe) GetSpaceId() string { + if x != nil { + return x.SpaceId + } + return "" +} + +func (x *Subscribe) GetTopics() []string { + if x != nil { + return x.Topics + } + return nil +} + +// Unsubscribe removes patterns from the stream's interest set, matched verbatim (not expanded); +// empty topics means remove all patterns of the space +type Unsubscribe struct { + state protoimpl.MessageState `protogen:"open.v1"` + SpaceId string `protobuf:"bytes,1,opt,name=spaceId,proto3" json:"spaceId,omitempty"` + Topics []string `protobuf:"bytes,2,rep,name=topics,proto3" json:"topics,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Unsubscribe) Reset() { + *x = Unsubscribe{} + mi := &file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Unsubscribe) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Unsubscribe) ProtoMessage() {} + +func (x *Unsubscribe) ProtoReflect() protoreflect.Message { + mi := &file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Unsubscribe.ProtoReflect.Descriptor instead. +func (*Unsubscribe) Descriptor() ([]byte, []int) { + return file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDescGZIP(), []int{2} +} + +func (x *Unsubscribe) GetSpaceId() string { + if x != nil { + return x.SpaceId + } + return "" +} + +func (x *Unsubscribe) GetTopics() []string { + if x != nil { + return x.Topics + } + return nil +} + +// Publish carries one ephemeral fire-and-forget message to a fully-qualified topic +type Publish struct { + state protoimpl.MessageState `protogen:"open.v1"` + SpaceId string `protobuf:"bytes,1,opt,name=spaceId,proto3" json:"spaceId,omitempty"` + // topic is a fully-qualified '/'-separated topic; wildcards are not allowed + Topic string `protobuf:"bytes,2,opt,name=topic,proto3" json:"topic,omitempty"` + // msgId is 16 random bytes generated by the publisher, used for duplicate suppression + MsgId []byte `protobuf:"bytes,3,opt,name=msgId,proto3" json:"msgId,omitempty"` + // keyId is the space ReadKey id used to encrypt the payload; empty means plaintext (keyless spaces) + KeyId string `protobuf:"bytes,4,opt,name=keyId,proto3" json:"keyId,omitempty"` + // payload is ciphertext, or plaintext iff keyId is empty + Payload []byte `protobuf:"bytes,5,opt,name=payload,proto3" json:"payload,omitempty"` + // identity is the sender's marshalled account public key + Identity []byte `protobuf:"bytes,6,opt,name=identity,proto3" json:"identity,omitempty"` + // signature is the account-key signature over "anysync:pubsub:v1" | spaceId | topic | msgId | keyId | le64(timestampMilli) | payload + Signature []byte `protobuf:"bytes,7,opt,name=signature,proto3" json:"signature,omitempty"` + // timestampMilli is the sender's wall clock, informational + TimestampMilli int64 `protobuf:"varint,8,opt,name=timestampMilli,proto3" json:"timestampMilli,omitempty"` + // relayed is set by a node when forwarding node-to-node; a relayed message is never forwarded again; excluded from the signature + Relayed bool `protobuf:"varint,9,opt,name=relayed,proto3" json:"relayed,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Publish) Reset() { + *x = Publish{} + mi := &file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Publish) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Publish) ProtoMessage() {} + +func (x *Publish) ProtoReflect() protoreflect.Message { + mi := &file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Publish.ProtoReflect.Descriptor instead. +func (*Publish) Descriptor() ([]byte, []int) { + return file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDescGZIP(), []int{3} +} + +func (x *Publish) GetSpaceId() string { + if x != nil { + return x.SpaceId + } + return "" +} + +func (x *Publish) GetTopic() string { + if x != nil { + return x.Topic + } + return "" +} + +func (x *Publish) GetMsgId() []byte { + if x != nil { + return x.MsgId + } + return nil +} + +func (x *Publish) GetKeyId() string { + if x != nil { + return x.KeyId + } + return "" +} + +func (x *Publish) GetPayload() []byte { + if x != nil { + return x.Payload + } + return nil +} + +func (x *Publish) GetIdentity() []byte { + if x != nil { + return x.Identity + } + return nil +} + +func (x *Publish) GetSignature() []byte { + if x != nil { + return x.Signature + } + return nil +} + +func (x *Publish) GetTimestampMilli() int64 { + if x != nil { + return x.TimestampMilli + } + return 0 +} + +func (x *Publish) GetRelayed() bool { + if x != nil { + return x.Relayed + } + return false +} + +// Status is sent by the serving peer on a rejected subscribe or publish; success is silent +type Status struct { + state protoimpl.MessageState `protogen:"open.v1"` + SpaceId string `protobuf:"bytes,1,opt,name=spaceId,proto3" json:"spaceId,omitempty"` + // topics echoes the offending patterns (or the topic of a rejected publish) + Topics []string `protobuf:"bytes,2,rep,name=topics,proto3" json:"topics,omitempty"` + Code ErrCodes `protobuf:"varint,3,opt,name=code,proto3,enum=pubsub.ErrCodes" json:"code,omitempty"` + // msgId echoes a rejected publish's id so the caller can correlate the + // rejection to a specific Publish; empty for subscribe rejections + MsgId []byte `protobuf:"bytes,4,opt,name=msgId,proto3" json:"msgId,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Status) Reset() { + *x = Status{} + mi := &file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Status) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Status) ProtoMessage() {} + +func (x *Status) ProtoReflect() protoreflect.Message { + mi := &file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Status.ProtoReflect.Descriptor instead. +func (*Status) Descriptor() ([]byte, []int) { + return file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDescGZIP(), []int{4} +} + +func (x *Status) GetSpaceId() string { + if x != nil { + return x.SpaceId + } + return "" +} + +func (x *Status) GetTopics() []string { + if x != nil { + return x.Topics + } + return nil +} + +func (x *Status) GetCode() ErrCodes { + if x != nil { + return x.Code + } + return ErrCodes_Unexpected +} + +func (x *Status) GetMsgId() []byte { + if x != nil { + return x.MsgId + } + return nil +} + +var File_commonspace_pubsub_pubsubproto_protos_pubsub_proto protoreflect.FileDescriptor + +const file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDesc = "" + + "\n" + + "2commonspace/pubsub/pubsubproto/protos/pubsub.proto\x12\x06pubsub\"\xdd\x01\n" + + "\rPubSubMessage\x121\n" + + "\tsubscribe\x18\x01 \x01(\v2\x11.pubsub.SubscribeH\x00R\tsubscribe\x127\n" + + "\vunsubscribe\x18\x02 \x01(\v2\x13.pubsub.UnsubscribeH\x00R\vunsubscribe\x12+\n" + + "\apublish\x18\x03 \x01(\v2\x0f.pubsub.PublishH\x00R\apublish\x12(\n" + + "\x06status\x18\x04 \x01(\v2\x0e.pubsub.StatusH\x00R\x06statusB\t\n" + + "\acontent\"=\n" + + "\tSubscribe\x12\x18\n" + + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x12\x16\n" + + "\x06topics\x18\x02 \x03(\tR\x06topics\"?\n" + + "\vUnsubscribe\x12\x18\n" + + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x12\x16\n" + + "\x06topics\x18\x02 \x03(\tR\x06topics\"\xfb\x01\n" + + "\aPublish\x12\x18\n" + + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x12\x14\n" + + "\x05topic\x18\x02 \x01(\tR\x05topic\x12\x14\n" + + "\x05msgId\x18\x03 \x01(\fR\x05msgId\x12\x14\n" + + "\x05keyId\x18\x04 \x01(\tR\x05keyId\x12\x18\n" + + "\apayload\x18\x05 \x01(\fR\apayload\x12\x1a\n" + + "\bidentity\x18\x06 \x01(\fR\bidentity\x12\x1c\n" + + "\tsignature\x18\a \x01(\fR\tsignature\x12&\n" + + "\x0etimestampMilli\x18\b \x01(\x03R\x0etimestampMilli\x12\x18\n" + + "\arelayed\x18\t \x01(\bR\arelayed\"v\n" + + "\x06Status\x12\x18\n" + + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x12\x16\n" + + "\x06topics\x18\x02 \x03(\tR\x06topics\x12$\n" + + "\x04code\x18\x03 \x01(\x0e2\x10.pubsub.ErrCodesR\x04code\x12\x14\n" + + "\x05msgId\x18\x04 \x01(\fR\x05msgId*\xad\x01\n" + + "\bErrCodes\x12\x0e\n" + + "\n" + + "Unexpected\x10\x00\x12\x0e\n" + + "\n" + + "NotAMember\x10\x01\x12\x12\n" + + "\x0eNotResponsible\x10\x02\x12\x0f\n" + + "\vRateLimited\x10\x03\x12\x11\n" + + "\rTooManyTopics\x10\x04\x12\x12\n" + + "\x0eInvalidMessage\x10\x05\x12\x11\n" + + "\rTopicNotOwned\x10\x06\x12\x10\n" + + "\fInvalidTopic\x10\a\x12\x10\n" + + "\vErrorOffset\x10\xcc\x082J\n" + + "\x06PubSub\x12@\n" + + "\fPubSubStream\x12\x15.pubsub.PubSubMessage\x1a\x15.pubsub.PubSubMessage(\x010\x01B Z\x1ecommonspace/pubsub/pubsubprotob\x06proto3" + +var ( + file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDescOnce sync.Once + file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDescData []byte +) + +func file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDescGZIP() []byte { + file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDescOnce.Do(func() { + file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDesc), len(file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDesc))) + }) + return file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDescData +} + +var file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_goTypes = []any{ + (ErrCodes)(0), // 0: pubsub.ErrCodes + (*PubSubMessage)(nil), // 1: pubsub.PubSubMessage + (*Subscribe)(nil), // 2: pubsub.Subscribe + (*Unsubscribe)(nil), // 3: pubsub.Unsubscribe + (*Publish)(nil), // 4: pubsub.Publish + (*Status)(nil), // 5: pubsub.Status +} +var file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_depIdxs = []int32{ + 2, // 0: pubsub.PubSubMessage.subscribe:type_name -> pubsub.Subscribe + 3, // 1: pubsub.PubSubMessage.unsubscribe:type_name -> pubsub.Unsubscribe + 4, // 2: pubsub.PubSubMessage.publish:type_name -> pubsub.Publish + 5, // 3: pubsub.PubSubMessage.status:type_name -> pubsub.Status + 0, // 4: pubsub.Status.code:type_name -> pubsub.ErrCodes + 1, // 5: pubsub.PubSub.PubSubStream:input_type -> pubsub.PubSubMessage + 1, // 6: pubsub.PubSub.PubSubStream:output_type -> pubsub.PubSubMessage + 6, // [6:7] is the sub-list for method output_type + 5, // [5:6] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name +} + +func init() { file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_init() } +func file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_init() { + if File_commonspace_pubsub_pubsubproto_protos_pubsub_proto != nil { + return + } + file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_msgTypes[0].OneofWrappers = []any{ + (*PubSubMessage_Subscribe)(nil), + (*PubSubMessage_Unsubscribe)(nil), + (*PubSubMessage_Publish)(nil), + (*PubSubMessage_Status)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDesc), len(file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_rawDesc)), + NumEnums: 1, + NumMessages: 5, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_goTypes, + DependencyIndexes: file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_depIdxs, + EnumInfos: file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_enumTypes, + MessageInfos: file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_msgTypes, + }.Build() + File_commonspace_pubsub_pubsubproto_protos_pubsub_proto = out.File + file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_goTypes = nil + file_commonspace_pubsub_pubsubproto_protos_pubsub_proto_depIdxs = nil +} diff --git a/commonspace/pubsub/pubsubproto/pubsub_drpc.pb.go b/commonspace/pubsub/pubsubproto/pubsub_drpc.pb.go new file mode 100644 index 000000000..923f1aa20 --- /dev/null +++ b/commonspace/pubsub/pubsubproto/pubsub_drpc.pb.go @@ -0,0 +1,149 @@ +// Code generated by protoc-gen-go-drpc. DO NOT EDIT. +// protoc-gen-go-drpc version: v1.0.0 +// source: commonspace/pubsub/pubsubproto/protos/pubsub.proto + +package pubsubproto + +import ( + context "context" + errors "errors" + drpc1 "github.com/planetscale/vtprotobuf/codec/drpc" + drpc "storj.io/drpc" + drpcerr "storj.io/drpc/drpcerr" +) + +type drpcEncoding_File_commonspace_pubsub_pubsubproto_protos_pubsub_proto struct{} + +func (drpcEncoding_File_commonspace_pubsub_pubsubproto_protos_pubsub_proto) Marshal(msg drpc.Message) ([]byte, error) { + return drpc1.Marshal(msg) +} + +func (drpcEncoding_File_commonspace_pubsub_pubsubproto_protos_pubsub_proto) Unmarshal(buf []byte, msg drpc.Message) error { + return drpc1.Unmarshal(buf, msg) +} + +func (drpcEncoding_File_commonspace_pubsub_pubsubproto_protos_pubsub_proto) JSONMarshal(msg drpc.Message) ([]byte, error) { + return drpc1.JSONMarshal(msg) +} + +func (drpcEncoding_File_commonspace_pubsub_pubsubproto_protos_pubsub_proto) JSONUnmarshal(buf []byte, msg drpc.Message) error { + return drpc1.JSONUnmarshal(buf, msg) +} + +type DRPCPubSubClient interface { + DRPCConn() drpc.Conn + + PubSubStream(ctx context.Context) (DRPCPubSub_PubSubStreamClient, error) +} + +type drpcPubSubClient struct { + cc drpc.Conn +} + +func NewDRPCPubSubClient(cc drpc.Conn) DRPCPubSubClient { + return &drpcPubSubClient{cc} +} + +func (c *drpcPubSubClient) DRPCConn() drpc.Conn { return c.cc } + +func (c *drpcPubSubClient) PubSubStream(ctx context.Context) (DRPCPubSub_PubSubStreamClient, error) { + stream, err := c.cc.NewStream(ctx, "/pubsub.PubSub/PubSubStream", drpcEncoding_File_commonspace_pubsub_pubsubproto_protos_pubsub_proto{}) + if err != nil { + return nil, err + } + x := &drpcPubSub_PubSubStreamClient{stream} + return x, nil +} + +type DRPCPubSub_PubSubStreamClient interface { + drpc.Stream + Send(*PubSubMessage) error + Recv() (*PubSubMessage, error) +} + +type drpcPubSub_PubSubStreamClient struct { + drpc.Stream +} + +func (x *drpcPubSub_PubSubStreamClient) GetStream() drpc.Stream { + return x.Stream +} + +func (x *drpcPubSub_PubSubStreamClient) Send(m *PubSubMessage) error { + return x.MsgSend(m, drpcEncoding_File_commonspace_pubsub_pubsubproto_protos_pubsub_proto{}) +} + +func (x *drpcPubSub_PubSubStreamClient) Recv() (*PubSubMessage, error) { + m := new(PubSubMessage) + if err := x.MsgRecv(m, drpcEncoding_File_commonspace_pubsub_pubsubproto_protos_pubsub_proto{}); err != nil { + return nil, err + } + return m, nil +} + +func (x *drpcPubSub_PubSubStreamClient) RecvMsg(m *PubSubMessage) error { + return x.MsgRecv(m, drpcEncoding_File_commonspace_pubsub_pubsubproto_protos_pubsub_proto{}) +} + +type DRPCPubSubServer interface { + PubSubStream(DRPCPubSub_PubSubStreamStream) error +} + +type DRPCPubSubUnimplementedServer struct{} + +func (s *DRPCPubSubUnimplementedServer) PubSubStream(DRPCPubSub_PubSubStreamStream) error { + return drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) +} + +type DRPCPubSubDescription struct{} + +func (DRPCPubSubDescription) NumMethods() int { return 1 } + +func (DRPCPubSubDescription) Method(n int) (string, drpc.Encoding, drpc.Receiver, interface{}, bool) { + switch n { + case 0: + return "/pubsub.PubSub/PubSubStream", drpcEncoding_File_commonspace_pubsub_pubsubproto_protos_pubsub_proto{}, + func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) { + return nil, srv.(DRPCPubSubServer). + PubSubStream( + &drpcPubSub_PubSubStreamStream{in1.(drpc.Stream)}, + ) + }, DRPCPubSubServer.PubSubStream, true + default: + return "", nil, nil, nil, false + } +} + +func DRPCRegisterPubSub(mux drpc.Mux, impl DRPCPubSubServer) error { + return mux.Register(impl, DRPCPubSubDescription{}) +} + +type DRPCPubSub_PubSubStreamStream interface { + drpc.Stream + Send(*PubSubMessage) error + Recv() (*PubSubMessage, error) +} + +type drpcPubSub_PubSubStreamStream struct { + drpc.Stream +} + +func (x *drpcPubSub_PubSubStreamStream) GetStream() drpc.Stream { + return x.Stream +} + +func (x *drpcPubSub_PubSubStreamStream) Send(m *PubSubMessage) error { + return x.MsgSend(m, drpcEncoding_File_commonspace_pubsub_pubsubproto_protos_pubsub_proto{}) +} + +func (x *drpcPubSub_PubSubStreamStream) Recv() (*PubSubMessage, error) { + m := new(PubSubMessage) + if err := x.MsgRecv(m, drpcEncoding_File_commonspace_pubsub_pubsubproto_protos_pubsub_proto{}); err != nil { + return nil, err + } + return m, nil +} + +func (x *drpcPubSub_PubSubStreamStream) RecvMsg(m *PubSubMessage) error { + return x.MsgRecv(m, drpcEncoding_File_commonspace_pubsub_pubsubproto_protos_pubsub_proto{}) +} diff --git a/commonspace/pubsub/pubsubproto/pubsub_vtproto.pb.go b/commonspace/pubsub/pubsubproto/pubsub_vtproto.pb.go new file mode 100644 index 000000000..362a0e66a --- /dev/null +++ b/commonspace/pubsub/pubsubproto/pubsub_vtproto.pb.go @@ -0,0 +1,1501 @@ +// Code generated by protoc-gen-go-vtproto. DO NOT EDIT. +// protoc-gen-go-vtproto version: v0.6.0 +// source: commonspace/pubsub/pubsubproto/protos/pubsub.proto + +package pubsubproto + +import ( + fmt "fmt" + protohelpers "github.com/planetscale/vtprotobuf/protohelpers" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + io "io" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +func (m *PubSubMessage) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PubSubMessage) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *PubSubMessage) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if vtmsg, ok := m.Content.(interface { + MarshalToSizedBufferVT([]byte) (int, error) + }); ok { + size, err := vtmsg.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + } + return len(dAtA) - i, nil +} + +func (m *PubSubMessage_Subscribe) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *PubSubMessage_Subscribe) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + i := len(dAtA) + if m.Subscribe != nil { + size, err := m.Subscribe.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} +func (m *PubSubMessage_Unsubscribe) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *PubSubMessage_Unsubscribe) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + i := len(dAtA) + if m.Unsubscribe != nil { + size, err := m.Unsubscribe.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x12 + } + return len(dAtA) - i, nil +} +func (m *PubSubMessage_Publish) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *PubSubMessage_Publish) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + i := len(dAtA) + if m.Publish != nil { + size, err := m.Publish.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x1a + } + return len(dAtA) - i, nil +} +func (m *PubSubMessage_Status) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *PubSubMessage_Status) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + i := len(dAtA) + if m.Status != nil { + size, err := m.Status.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x22 + } + return len(dAtA) - i, nil +} +func (m *Subscribe) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Subscribe) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *Subscribe) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Topics) > 0 { + for iNdEx := len(m.Topics) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Topics[iNdEx]) + copy(dAtA[i:], m.Topics[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Topics[iNdEx]))) + i-- + dAtA[i] = 0x12 + } + } + if len(m.SpaceId) > 0 { + i -= len(m.SpaceId) + copy(dAtA[i:], m.SpaceId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *Unsubscribe) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Unsubscribe) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *Unsubscribe) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Topics) > 0 { + for iNdEx := len(m.Topics) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Topics[iNdEx]) + copy(dAtA[i:], m.Topics[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Topics[iNdEx]))) + i-- + dAtA[i] = 0x12 + } + } + if len(m.SpaceId) > 0 { + i -= len(m.SpaceId) + copy(dAtA[i:], m.SpaceId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *Publish) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Publish) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *Publish) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.Relayed { + i-- + if m.Relayed { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x48 + } + if m.TimestampMilli != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.TimestampMilli)) + i-- + dAtA[i] = 0x40 + } + if len(m.Signature) > 0 { + i -= len(m.Signature) + copy(dAtA[i:], m.Signature) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Signature))) + i-- + dAtA[i] = 0x3a + } + if len(m.Identity) > 0 { + i -= len(m.Identity) + copy(dAtA[i:], m.Identity) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Identity))) + i-- + dAtA[i] = 0x32 + } + if len(m.Payload) > 0 { + i -= len(m.Payload) + copy(dAtA[i:], m.Payload) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Payload))) + i-- + dAtA[i] = 0x2a + } + if len(m.KeyId) > 0 { + i -= len(m.KeyId) + copy(dAtA[i:], m.KeyId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.KeyId))) + i-- + dAtA[i] = 0x22 + } + if len(m.MsgId) > 0 { + i -= len(m.MsgId) + copy(dAtA[i:], m.MsgId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.MsgId))) + i-- + dAtA[i] = 0x1a + } + if len(m.Topic) > 0 { + i -= len(m.Topic) + copy(dAtA[i:], m.Topic) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Topic))) + i-- + dAtA[i] = 0x12 + } + if len(m.SpaceId) > 0 { + i -= len(m.SpaceId) + copy(dAtA[i:], m.SpaceId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *Status) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Status) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *Status) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.MsgId) > 0 { + i -= len(m.MsgId) + copy(dAtA[i:], m.MsgId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.MsgId))) + i-- + dAtA[i] = 0x22 + } + if m.Code != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Code)) + i-- + dAtA[i] = 0x18 + } + if len(m.Topics) > 0 { + for iNdEx := len(m.Topics) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.Topics[iNdEx]) + copy(dAtA[i:], m.Topics[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Topics[iNdEx]))) + i-- + dAtA[i] = 0x12 + } + } + if len(m.SpaceId) > 0 { + i -= len(m.SpaceId) + copy(dAtA[i:], m.SpaceId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *PubSubMessage) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if vtmsg, ok := m.Content.(interface{ SizeVT() int }); ok { + n += vtmsg.SizeVT() + } + n += len(m.unknownFields) + return n +} + +func (m *PubSubMessage_Subscribe) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Subscribe != nil { + l = m.Subscribe.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + return n +} +func (m *PubSubMessage_Unsubscribe) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Unsubscribe != nil { + l = m.Unsubscribe.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + return n +} +func (m *PubSubMessage_Publish) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Publish != nil { + l = m.Publish.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + return n +} +func (m *PubSubMessage_Status) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Status != nil { + l = m.Status.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + return n +} +func (m *Subscribe) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.SpaceId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if len(m.Topics) > 0 { + for _, s := range m.Topics { + l = len(s) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *Unsubscribe) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.SpaceId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if len(m.Topics) > 0 { + for _, s := range m.Topics { + l = len(s) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *Publish) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.SpaceId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Topic) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.MsgId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.KeyId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Payload) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Identity) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Signature) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.TimestampMilli != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.TimestampMilli)) + } + if m.Relayed { + n += 2 + } + n += len(m.unknownFields) + return n +} + +func (m *Status) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.SpaceId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if len(m.Topics) > 0 { + for _, s := range m.Topics { + l = len(s) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + if m.Code != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Code)) + } + l = len(m.MsgId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + +func (m *PubSubMessage) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PubSubMessage: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PubSubMessage: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Subscribe", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if oneof, ok := m.Content.(*PubSubMessage_Subscribe); ok { + if err := oneof.Subscribe.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + } else { + v := &Subscribe{} + if err := v.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Content = &PubSubMessage_Subscribe{Subscribe: v} + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Unsubscribe", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if oneof, ok := m.Content.(*PubSubMessage_Unsubscribe); ok { + if err := oneof.Unsubscribe.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + } else { + v := &Unsubscribe{} + if err := v.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Content = &PubSubMessage_Unsubscribe{Unsubscribe: v} + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Publish", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if oneof, ok := m.Content.(*PubSubMessage_Publish); ok { + if err := oneof.Publish.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + } else { + v := &Publish{} + if err := v.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Content = &PubSubMessage_Publish{Publish: v} + } + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Status", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if oneof, ok := m.Content.(*PubSubMessage_Status); ok { + if err := oneof.Status.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + } else { + v := &Status{} + if err := v.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Content = &PubSubMessage_Status{Status: v} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Subscribe) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Subscribe: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Subscribe: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Topics", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Topics = append(m.Topics, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Unsubscribe) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Unsubscribe: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Unsubscribe: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Topics", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Topics = append(m.Topics, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Publish) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Publish: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Publish: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Topic", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Topic = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field MsgId", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.MsgId = append(m.MsgId[:0], dAtA[iNdEx:postIndex]...) + if m.MsgId == nil { + m.MsgId = []byte{} + } + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field KeyId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.KeyId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Payload", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Payload = append(m.Payload[:0], dAtA[iNdEx:postIndex]...) + if m.Payload == nil { + m.Payload = []byte{} + } + iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Identity", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Identity = append(m.Identity[:0], dAtA[iNdEx:postIndex]...) + if m.Identity == nil { + m.Identity = []byte{} + } + iNdEx = postIndex + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Signature", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Signature = append(m.Signature[:0], dAtA[iNdEx:postIndex]...) + if m.Signature == nil { + m.Signature = []byte{} + } + iNdEx = postIndex + case 8: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field TimestampMilli", wireType) + } + m.TimestampMilli = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.TimestampMilli |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 9: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Relayed", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Relayed = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Status) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Status: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Status: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Topics", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Topics = append(m.Topics, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Code", wireType) + } + m.Code = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Code |= ErrCodes(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field MsgId", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.MsgId = append(m.MsgId[:0], dAtA[iNdEx:postIndex]...) + if m.MsgId == nil { + m.MsgId = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} diff --git a/commonspace/pubsub/ratelimit.go b/commonspace/pubsub/ratelimit.go new file mode 100644 index 000000000..c83d42d63 --- /dev/null +++ b/commonspace/pubsub/ratelimit.go @@ -0,0 +1,55 @@ +package pubsub + +import ( + "sync" + "time" + + "golang.org/x/time/rate" +) + +const rateLimiterIdleTimeout = time.Minute + +// peerRateLimiter is a per-peer publish token bucket with lazy garbage collection +// of idle entries, so memory stays O(recently active peers). +type peerRateLimiter struct { + mu sync.Mutex + peers map[string]*peerRate + rps rate.Limit + burst int + lastGC time.Time +} + +type peerRate struct { + *rate.Limiter + lastUsage time.Time +} + +func newPeerRateLimiter(rps float64, burst int) *peerRateLimiter { + return &peerRateLimiter{ + peers: make(map[string]*peerRate), + rps: rate.Limit(rps), + burst: burst, + lastGC: time.Now(), + } +} + +func (r *peerRateLimiter) allow(peerId string) bool { + now := time.Now() + r.mu.Lock() + defer r.mu.Unlock() + if now.Sub(r.lastGC) > rateLimiterIdleTimeout { + for id, pr := range r.peers { + if now.Sub(pr.lastUsage) > rateLimiterIdleTimeout { + delete(r.peers, id) + } + } + r.lastGC = now + } + pr, ok := r.peers[peerId] + if !ok { + pr = &peerRate{Limiter: rate.NewLimiter(r.rps, r.burst)} + r.peers[peerId] = pr + } + pr.lastUsage = now + return pr.Allow() +} diff --git a/commonspace/pubsub/reconnect_test.go b/commonspace/pubsub/reconnect_test.go new file mode 100644 index 000000000..c29674a57 --- /dev/null +++ b/commonspace/pubsub/reconnect_test.go @@ -0,0 +1,318 @@ +package pubsub + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/anyproto/any-sync/commonspace/pubsub/pubsubproto" +) + +// rawStream opens a direct PubSubStream from client fx to server target, bypassing +// the client engine's pool so a test can drive two independent streams from the +// same peerId (as a reconnect produces server-side). +func rawStream(t *testing.T, client, target *engineFx) pubsubproto.DRPCPubSub_PubSubStreamClient { + p := connect(t, client, target) + conn, err := p.AcquireDrpcConn(testCtx) + require.NoError(t, err) + stream, err := pubsubproto.NewDRPCPubSubClient(conn).PubSubStream(p.Context()) + require.NoError(t, err) + return stream +} + +func remoteMatchCount(fx *engineFx, spaceId, topic string) int { + fx.svc.remoteMu.Lock() + defer fx.svc.remoteMu.Unlock() + si := fx.svc.remote[spaceId] + if si == nil { + return 0 + } + return len(si.trie.Match(topic, nil)) +} + +func waitMatchCount(t *testing.T, fx *engineFx, spaceId, topic string, want int) { + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if remoteMatchCount(fx, spaceId, topic) == want { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("match count for %s never reached %d (got %d)", topic, want, remoteMatchCount(fx, spaceId, topic)) +} + +// TestReconnectKeepsInterest is the regression test for the streamId-keyed interest +// fix: two streams from the SAME peer each hold interest, and closing one must not +// wipe the other's — a stale reconnecting stream must not take the fresh one down. +func TestReconnectKeepsInterest(t *testing.T) { + membership := &fakeMembership{} + relay := &fakeRelay{} + node := newEngineFx(t, "node", membership, relay) + defer node.finish() + client := newEngineFx(t, "client", membership, nil) + defer client.finish() + membership.allow(client.identity()) + + subMsg := &pubsubproto.PubSubMessage{Content: &pubsubproto.PubSubMessage_Subscribe{ + Subscribe: &pubsubproto.Subscribe{SpaceId: testSpace, Topics: []string{"chat/>"}}, + }} + + // stream 1 subscribes + s1 := rawStream(t, client, node) + require.NoError(t, s1.Send(subMsg)) + waitMatchCount(t, node, testSpace, "chat/x", 1) + + // stream 2 (same peerId) subscribes to the same pattern — the node must track + // it independently, so the trie refcount is now 2 even though Match still + // returns the single pattern + s2 := rawStream(t, client, node) + require.NoError(t, s2.Send(subMsg)) + // give the node time to process s2's subscribe + time.Sleep(200 * time.Millisecond) + require.Equal(t, 1, remoteMatchCount(node, testSpace, "chat/x")) + + // close stream 1 — the stale stream must NOT take down stream 2's interest + require.NoError(t, s1.Close()) + // interest must remain after s1's close is processed + time.Sleep(300 * time.Millisecond) + require.Equal(t, 1, remoteMatchCount(node, testSpace, "chat/x"), + "closing the stale stream wiped the fresh stream's interest") + + // closing stream 2 finally clears it + require.NoError(t, s2.Close()) + waitMatchCount(t, node, testSpace, "chat/x", 0) + + // and the per-stream bookkeeping is fully drained + node.svc.remoteMu.Lock() + remainingStreams := len(node.svc.streams) + remainingSpaces := len(node.svc.remote) + node.svc.remoteMu.Unlock() + require.Equal(t, 0, remainingStreams, "stream records leaked") + require.Equal(t, 0, remainingSpaces, "space records leaked") +} + +// TestStreamCloseDrainsInterest verifies a single stream's interest and all +// bookkeeping is removed when it closes without an explicit unsubscribe. +func TestStreamCloseDrainsInterest(t *testing.T) { + membership := &fakeMembership{} + relay := &fakeRelay{} + node := newEngineFx(t, "node", membership, relay) + defer node.finish() + client := newEngineFx(t, "client", membership, nil) + defer client.finish() + membership.allow(client.identity()) + + s := rawStream(t, client, node) + require.NoError(t, s.Send(&pubsubproto.PubSubMessage{Content: &pubsubproto.PubSubMessage_Subscribe{ + Subscribe: &pubsubproto.Subscribe{SpaceId: testSpace, Topics: []string{"a/*", "b/>", "c"}}, + }})) + waitMatchCount(t, node, testSpace, "a/x", 1) + + require.NoError(t, s.Close()) + waitMatchCount(t, node, testSpace, "a/x", 0) + node.svc.remoteMu.Lock() + defer node.svc.remoteMu.Unlock() + require.Empty(t, node.svc.streams) + require.Empty(t, node.svc.remote) +} + +// TestCapExceededReportsOnlyRejected verifies that when a subscribe exceeds the +// per-space pattern cap mid-batch, the TooManyTopics status echoes only the +// rejected patterns, and the ones accepted before the cap stay live. +func TestCapExceededReportsOnlyRejected(t *testing.T) { + membership := &fakeMembership{} + relay := &fakeRelay{} + node := newEngineFx(t, "node", membership, relay) + defer node.finish() + // tiny per-space cap so a 3-topic subscribe trips it after 2 + node.svc.cfg.MaxPatternsPerSpace = 2 + client := newEngineFx(t, "client", membership, nil) + defer client.finish() + membership.allow(client.identity()) + + s := rawStream(t, client, node) + require.NoError(t, s.Send(&pubsubproto.PubSubMessage{Content: &pubsubproto.PubSubMessage_Subscribe{ + Subscribe: &pubsubproto.Subscribe{SpaceId: testSpace, Topics: []string{"a", "b", "c"}}, + }})) + + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + msg, err := s.Recv() + require.NoError(t, err) + if st := msg.GetStatus(); st != nil { + require.Equal(t, pubsubproto.ErrCodes_TooManyTopics, st.Code) + require.Equal(t, []string{"c"}, st.Topics, "only the rejected topic is reported") + // the two accepted before the cap are live + require.Equal(t, 1, remoteMatchCount(node, testSpace, "a")) + require.Equal(t, 1, remoteMatchCount(node, testSpace, "b")) + return + } + } + t.Fatal("no TooManyTopics status received") +} + +// TestInvalidSpaceIdRejected ensures a spaceId containing '/' (which would break +// tag parsing) is rejected at subscribe with InvalidTopic. +func TestInvalidSpaceIdRejected(t *testing.T) { + membership := &fakeMembership{} + node := newEngineFx(t, "node", membership, &fakeRelay{}) + defer node.finish() + client := newEngineFx(t, "client", membership, nil) + defer client.finish() + membership.allow(client.identity()) + + s := rawStream(t, client, node) + require.NoError(t, s.Send(&pubsubproto.PubSubMessage{Content: &pubsubproto.PubSubMessage_Subscribe{ + Subscribe: &pubsubproto.Subscribe{SpaceId: "bad/space", Topics: []string{"chat/>"}}, + }})) + // the node replies with a Status frame we can read directly off the raw stream + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + msg, err := s.Recv() + require.NoError(t, err) + if st := msg.GetStatus(); st != nil { + require.Equal(t, pubsubproto.ErrCodes_InvalidTopic, st.Code) + return + } + } + t.Fatal("no InvalidTopic status received") +} + +// TestEvictMemberStopsDelivery verifies §6.4 active eviction: after EvictMember, +// a removed member's stream stops receiving even though its stream stays open, and +// another member subscribed to the same pattern keeps receiving. +func TestEvictMemberStopsDelivery(t *testing.T) { + n := newNetFx(t) + defer n.finish() + + _, err := n.clientA.svc.Subscribe(testSpace, "chat/>", n.clientA.handler()) + require.NoError(t, err) + _, err = n.clientC.svc.Subscribe(testSpace, "chat/>", n.clientC.handler()) + require.NoError(t, err) + waitMatchCount(t, n.nodeB, testSpace, "chat/x", 1) + // wait until both members' interest is registered on the node + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + n.nodeB.svc.remoteMu.Lock() + nstreams := len(n.nodeB.svc.streams) + n.nodeB.svc.remoteMu.Unlock() + if nstreams == 2 { + break + } + time.Sleep(10 * time.Millisecond) + } + + // evict clientA at the node + n.nodeB.svc.EvictMember(testSpace, n.clientA.identity()) + + // clientC (still a member) publishes; clientC receives, clientA does not + require.NoError(t, n.clientC.svc.Publish(testCtx, testSpace, "chat/x", []byte("members-only"))) + require.Equal(t, "members-only", waitReceived(t, n.clientC).payload) + expectSilence(t, n.clientA, 400*time.Millisecond) +} + +// TestRevalidateMembersEvictsNonMembers verifies the node-side ACL-change hook: +// RevalidateMembers evicts subscribers whose account no longer passes the member +// predicate while keeping current members subscribed. +func TestRevalidateMembersEvictsNonMembers(t *testing.T) { + n := newNetFx(t) + defer n.finish() + + _, err := n.clientA.svc.Subscribe(testSpace, "chat/>", n.clientA.handler()) + require.NoError(t, err) + _, err = n.clientC.svc.Subscribe(testSpace, "chat/>", n.clientC.handler()) + require.NoError(t, err) + waitMatchCount(t, n.nodeB, testSpace, "chat/x", 1) + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + n.nodeB.svc.remoteMu.Lock() + nstreams := len(n.nodeB.svc.streams) + n.nodeB.svc.remoteMu.Unlock() + if nstreams == 2 { + break + } + time.Sleep(10 * time.Millisecond) + } + + // only clientC remains a member; clientA is evicted in one pass + keep := n.clientC.identity().Account() + n.nodeB.svc.RevalidateMembers(testSpace, func(account string) bool { + return account == keep + }) + + require.NoError(t, n.clientC.svc.Publish(testCtx, testSpace, "chat/x", []byte("survivors"))) + require.Equal(t, "survivors", waitReceived(t, n.clientC).payload) + expectSilence(t, n.clientA, 400*time.Millisecond) +} + +// TestCloseSpaceDropsInterest verifies CloseSpace tears down both local and +// serving-side interest so a global service retains nothing for a closed space. +func TestCloseSpaceDropsInterest(t *testing.T) { + n := newNetFx(t) + defer n.finish() + + _, err := n.clientA.svc.Subscribe(testSpace, "chat/>", n.clientA.handler()) + require.NoError(t, err) + waitMatchCount(t, n.nodeB, testSpace, "chat/x", 1) + + // node unloads the space + n.nodeB.svc.CloseSpace(testSpace) + waitMatchCount(t, n.nodeB, testSpace, "chat/x", 0) + + // client closes the space: local interest is gone + n.clientA.svc.CloseSpace(testSpace) + n.clientA.svc.localMu.Lock() + _, hasTrie := n.clientA.svc.localTrie[testSpace] + n.clientA.svc.localMu.Unlock() + require.False(t, hasTrie, "local interest retained after CloseSpace") +} + +// TestResyncRestoresDeliveryAfterDrop verifies the reconnect watcher: after a +// subscriber's stream drops and a fresh peer replaces it, the periodic resync +// re-pushes interest and delivery resumes without the app re-subscribing. +func TestResyncRestoresDeliveryAfterDrop(t *testing.T) { + membership := &fakeMembership{} + relay := &fakeRelay{} + node := newEngineFx(t, "node", membership, relay) + defer node.finish() + sub := newEngineFx(t, "sub", membership, nil) + defer sub.finish() + pub := newEngineFx(t, "pub", membership, nil) + defer pub.finish() + membership.allow(sub.identity(), pub.identity()) + + sub.peers.add(connect(t, sub, node)) + pub.peers.add(connect(t, pub, node)) + + _, err := sub.svc.Subscribe(testSpace, "chat/>", sub.handler()) + require.NoError(t, err) + waitInterest(t, node, "chat/x") + + require.NoError(t, pub.svc.Publish(testCtx, testSpace, "chat/x", []byte("before"))) + require.Equal(t, "before", waitReceived(t, sub).payload) + + // drop the subscriber's connection and replace it with a fresh peer, as a real + // reconnect would; the node sees the stale stream close and (eventually) a new one + for _, p := range sub.ownedPeers { + _ = p.Close() + } + sub.peers.replace(connect(t, sub, node)) + + // the periodic resync re-pushes interest over the new peer; publish is + // fire-and-forget so retry until a message lands (delivery resumes once resync + // has reopened the stream and re-registered interest on the node) + deadline := time.Now().Add(3 * time.Second) + for { + require.NoError(t, pub.svc.Publish(testCtx, testSpace, "chat/x", []byte("after"))) + select { + case r := <-sub.received: + require.Equal(t, "after", r.payload) + return + case <-time.After(200 * time.Millisecond): + } + if time.Now().After(deadline) { + t.Fatal("delivery did not resume after reconnect") + } + } +} diff --git a/commonspace/pubsub/rpchandler.go b/commonspace/pubsub/rpchandler.go new file mode 100644 index 000000000..c80771741 --- /dev/null +++ b/commonspace/pubsub/rpchandler.go @@ -0,0 +1,21 @@ +package pubsub + +import ( + "storj.io/drpc" + + "github.com/anyproto/any-sync/commonspace/pubsub/pubsubproto" +) + +// rpcHandler adapts Service to the generated DRPC server interface. +type rpcHandler struct { + s Service +} + +func (r rpcHandler) PubSubStream(stream pubsubproto.DRPCPubSub_PubSubStreamStream) error { + return r.s.HandleStream(stream) +} + +// RegisterRpc registers the pubsub DRPC service on the given mux. +func RegisterRpc(mux drpc.Mux, s Service) error { + return pubsubproto.DRPCRegisterPubSub(mux, rpcHandler{s: s}) +} diff --git a/commonspace/pubsub/service.go b/commonspace/pubsub/service.go new file mode 100644 index 000000000..822a91dc3 --- /dev/null +++ b/commonspace/pubsub/service.go @@ -0,0 +1,957 @@ +package pubsub + +import ( + "context" + "crypto/rand" + "errors" + "sync" + "time" + + "github.com/cheggaaa/mb/v3" + "go.uber.org/zap" + "storj.io/drpc" + + "github.com/anyproto/any-sync/accountservice" + "github.com/anyproto/any-sync/app" + "github.com/anyproto/any-sync/app/logger" + "github.com/anyproto/any-sync/commonspace/object/accountdata" + "github.com/anyproto/any-sync/commonspace/pubsub/pubsubproto" + "github.com/anyproto/any-sync/net/peer" + "github.com/anyproto/any-sync/net/streampool" + "github.com/anyproto/any-sync/util/crypto" +) + +const CName = "common.commonspace.pubsub" + +var log = logger.NewNamed(CName) + +var ( + ErrClosed = errors.New("pubsub service closed") +) + +// Service is the shared pubsub engine used by nodes (relay role, Deps.Relay set) +// and clients (Deps.Peers set) alike. One long-lived PubSubStream per peer pair +// multiplexes all spaces and topics; interest and routing state are in-memory only +// and die with the streams. +type Service interface { + app.ComponentRunnable + // Publish encrypts, signs and fire-and-forgets payload to the topic within the space. + Publish(ctx context.Context, spaceId, topic string, payload []byte) error + // Subscribe registers a local handler for a pattern and pushes the interest to + // the space's peers. The returned func unregisters and unsubscribes. + Subscribe(spaceId, pattern string, h Handler) (unsubscribe func(), err error) + // SyncInterest (re)sends all local interest for the space to its current peers; + // call after (re)connecting to a space's peers. + SyncInterest(ctx context.Context, spaceId string) error + // CloseSpace drops all local and serving-side interest for the space and + // withdraws the local interest from its peers. Hosts call it on space + // unload/close so a global service doesn't retain closed-space state. + CloseSpace(spaceId string) + // EvictMember drops all serving-side interest of an identity in a space, + // enforcing DESIGN §6.4 (active drop on ACL removal). Nodes wire it to their + // ACL-update hook. No-op on clients (they hold no serving interest). + EvictMember(spaceId string, identity crypto.PubKey) + // RevalidateMembers evicts every subscriber of the space whose account no + // longer passes isMember. Nodes call it on an ACL change to cut off all + // removed members in one pass. No-op on clients. + RevalidateMembers(spaceId string, isMember func(account string) bool) + // HandleStream serves an inbound PubSubStream; blocks for the stream lifetime. + HandleStream(stream drpc.Stream) error +} + +func New(deps Deps) Service { + return &service{deps: deps} +} + +type service struct { + deps Deps + cfg Config + account *accountdata.AccountKeys + pool streampool.StreamPool + + // remote interest: what peers subscribed on us (serving side). + // Interest is keyed by streamId (not peerId): two streams from the same peer + // are independent, so a reconnecting peer's fresh stream keeps its interest + // when the stale stream closes, and the trie refcount counts subscribing + // streams. streams[streamId] is the authoritative record cleaned on close. + remoteMu sync.Mutex + remote map[string]*spaceInterest // spaceId -> match trie (refcount = subscribing streams) + streams map[uint32]*streamInterest // streamId -> what that stream subscribed + + // local interest: what our handlers subscribed to (client side) + localMu sync.Mutex + localTrie map[string]*patternTrie // spaceId -> trie of local patterns + localSubs map[string]map[string][]localSub // spaceId -> pattern -> handlers + nextSubId uint64 + localTopic map[string]int // spaceId -> distinct local patterns (cap bookkeeping) + + dedup *msgIdDedup + rate *peerRateLimiter + dispatch *mb.MB[dispatchItem] + resyncNow chan struct{} + loops sync.WaitGroup + + ctx context.Context + ctxCancel context.CancelFunc +} + +type spaceInterest struct { + trie *patternTrie +} + +// streamInterest records the patterns one inbound stream subscribed, per space, +// so a stream close can withdraw exactly its own contribution regardless of what +// tags the pool recorded. +type streamInterest struct { + account string // subscriber account id, for member eviction + bySpace map[string]map[string]struct{} // spaceId -> patterns + total int // total patterns across spaces on this stream +} + +type localSub struct { + id uint64 + h Handler +} + +type dispatchItem struct { + patterns []string + spaceId string + topic string + identity crypto.PubKey + payload []byte +} + +func (s *service) Init(a *app.App) (err error) { + s.cfg = s.deps.Config.withDefaults() + s.account = a.MustComponent(accountservice.CName).(accountservice.Service).Account() + poolOpts := []streampool.Option{streampool.WithStreamCloseHook(s.onStreamClose)} + if s.deps.Metric != nil { + poolOpts = append(poolOpts, streampool.WithMetric(s.deps.Metric, "pubsub")) + } + s.pool = streampool.NewStreamPool(s, s.cfg.streamPoolConfig(), poolOpts...) + s.remote = make(map[string]*spaceInterest) + s.streams = make(map[uint32]*streamInterest) + s.localTrie = make(map[string]*patternTrie) + s.localSubs = make(map[string]map[string][]localSub) + s.localTopic = make(map[string]int) + s.dedup = newMsgIdDedup(s.cfg.DedupSize) + s.rate = newPeerRateLimiter(s.cfg.PublishRps, s.cfg.PublishBurst) + s.dispatch = mb.New[dispatchItem](s.cfg.DispatchQueueSize) + s.resyncNow = make(chan struct{}, 1) + s.ctx, s.ctxCancel = context.WithCancel(context.Background()) + return nil +} + +func (s *service) Name() string { return CName } + +func (s *service) Run(ctx context.Context) error { + if err := s.pool.Run(ctx); err != nil { + return err + } + s.loops.Add(1) + go func() { + defer s.loops.Done() + s.dispatchLoop() + }() + if s.deps.Peers != nil { + s.loops.Add(1) + go func() { + defer s.loops.Done() + s.resyncLoop() + }() + } + return nil +} + +// resyncLoop keeps server-side interest alive across reconnects. streampool opens +// streams lazily and never re-sends interest on a fresh stream, so without this a +// subscriber goes silent after its stream drops (a routine event: idle pool GC +// reaps quiet streams). The loop re-pushes all local interest periodically, and +// immediately when a client-side stream closes. Re-sends are idempotent +// (handleSubscribe dedups per stream) and bounded by the local subscription set. +func (s *service) resyncLoop() { + ticker := time.NewTicker(s.cfg.ResyncInterval) + defer ticker.Stop() + for { + select { + case <-s.ctx.Done(): + return + case <-ticker.C: + case <-s.resyncNow: + } + s.localMu.Lock() + spaces := make([]string, 0, len(s.localSubs)) + for spaceId := range s.localSubs { + spaces = append(spaces, spaceId) + } + s.localMu.Unlock() + for _, spaceId := range spaces { + if err := s.SyncInterest(s.ctx, spaceId); err != nil { + log.Debug("resync interest failed", zap.String("spaceId", spaceId), zap.Error(err)) + } + } + } +} + +// triggerResync asks the resync loop to re-push interest now (non-blocking). +func (s *service) triggerResync() { + select { + case s.resyncNow <- struct{}{}: + default: + } +} + +func (s *service) Close(ctx context.Context) error { + s.ctxCancel() + _ = s.dispatch.Close() + s.loops.Wait() + return s.pool.Close(ctx) +} + +// +// public API (client side) +// + +func (s *service) Publish(ctx context.Context, spaceId, topic string, payload []byte) error { + if err := ValidateTopic(topic); err != nil { + return err + } + if len(payload) > s.cfg.MaxPayloadSize { + return pubsubproto.ErrInvalidMessage + } + if owner := TopicOwner(topic); owner != "" && owner != s.account.SignKey.GetPublic().Account() { + return pubsubproto.ErrTopicNotOwned + } + p := &pubsubproto.Publish{ + SpaceId: spaceId, + Topic: topic, + MsgId: make([]byte, msgIdLen), + TimestampMilli: time.Now().UnixMilli(), + } + if _, err := rand.Read(p.MsgId); err != nil { + return err + } + if s.deps.Crypto != nil { + keyId, enc, err := s.deps.Crypto.Encrypt(spaceId, payload) + if err != nil { + return err + } + p.KeyId, p.Payload = keyId, enc + } else { + p.Payload = payload + } + if err := signPublish(s.account.SignKey, p); err != nil { + return err + } + // record our own msgId so network echoes are suppressed, + // and deliver to local handlers synchronously with the plaintext + s.dedup.seen(p.MsgId) + s.enqueueLocal(spaceId, topic, s.account.SignKey.GetPublic(), payload) + + if s.deps.Peers == nil { + return nil + } + msg := wrapPublish(p) + return s.pool.Send(ctx, msg, func(ctx context.Context) ([]peer.Peer, error) { + return s.deps.Peers.SpacePeers(ctx, spaceId) + }) +} + +func (s *service) Subscribe(spaceId, pattern string, h Handler) (func(), error) { + if err := ValidatePattern(pattern); err != nil { + return nil, err + } + s.localMu.Lock() + if s.localTopic[spaceId] >= s.cfg.MaxPatternsPerSpace { + s.localMu.Unlock() + return nil, pubsubproto.ErrTooManyTopics + } + trie, ok := s.localTrie[spaceId] + if !ok { + trie = newPatternTrie() + s.localTrie[spaceId] = trie + s.localSubs[spaceId] = make(map[string][]localSub) + } + s.nextSubId++ + id := s.nextSubId + firstForPattern := len(s.localSubs[spaceId][pattern]) == 0 + s.localSubs[spaceId][pattern] = append(s.localSubs[spaceId][pattern], localSub{id: id, h: h}) + if firstForPattern { + trie.Add(pattern) + s.localTopic[spaceId]++ + } + s.localMu.Unlock() + + if firstForPattern { + s.sendInterest(spaceId, []string{pattern}, true) + } + return func() { s.unsubscribe(spaceId, pattern, id) }, nil +} + +func (s *service) unsubscribe(spaceId, pattern string, id uint64) { + s.localMu.Lock() + subs := s.localSubs[spaceId][pattern] + for i, sub := range subs { + if sub.id == id { + subs = append(subs[:i], subs[i+1:]...) + break + } + } + lastForPattern := len(subs) == 0 + if lastForPattern { + delete(s.localSubs[spaceId], pattern) + if trie := s.localTrie[spaceId]; trie != nil { + trie.Remove(pattern) + s.localTopic[spaceId]-- + if trie.Len() == 0 { + delete(s.localTrie, spaceId) + delete(s.localSubs, spaceId) + delete(s.localTopic, spaceId) + } + } + } else { + s.localSubs[spaceId][pattern] = subs + } + s.localMu.Unlock() + + if lastForPattern { + s.sendInterest(spaceId, []string{pattern}, false) + } +} + +func (s *service) SyncInterest(ctx context.Context, spaceId string) error { + s.localMu.Lock() + var patterns []string + for pattern := range s.localSubs[spaceId] { + patterns = append(patterns, pattern) + } + s.localMu.Unlock() + if len(patterns) == 0 || s.deps.Peers == nil { + return nil + } + msg := &pubsubproto.PubSubMessage{Content: &pubsubproto.PubSubMessage_Subscribe{ + Subscribe: &pubsubproto.Subscribe{SpaceId: spaceId, Topics: patterns}, + }} + return s.pool.Send(ctx, msg, func(ctx context.Context) ([]peer.Peer, error) { + return s.deps.Peers.SpacePeers(ctx, spaceId) + }) +} + +func (s *service) CloseSpace(spaceId string) { + // client side: drop local subscriptions and withdraw interest from peers + s.localMu.Lock() + var patterns []string + for pattern := range s.localSubs[spaceId] { + patterns = append(patterns, pattern) + } + delete(s.localTrie, spaceId) + delete(s.localSubs, spaceId) + delete(s.localTopic, spaceId) + s.localMu.Unlock() + if len(patterns) > 0 { + s.sendInterest(spaceId, patterns, false) + } + + // serving side: drop the space trie and every stream's interest in it, + // stripping the routing tags so no lingering tag delivers after close + s.remoteMu.Lock() + delete(s.remote, spaceId) + for streamId, strm := range s.streams { + spacePatterns := strm.bySpace[spaceId] + if len(spacePatterns) == 0 { + continue + } + tags := make([]string, 0, len(spacePatterns)) + for pattern := range spacePatterns { + tags = append(tags, interestTag(spaceId, pattern)) + strm.total-- + } + delete(strm.bySpace, spaceId) + _ = s.pool.RemoveTagsById(streamId, tags...) + if strm.total == 0 { + delete(s.streams, streamId) + } + } + s.remoteMu.Unlock() +} + +func (s *service) EvictMember(spaceId string, identity crypto.PubKey) { + account := identity.Account() + s.evictSpaceStreams(spaceId, func(strm *streamInterest) bool { + return strm.account == account + }) +} + +// RevalidateMembers evicts every subscriber of the space whose account no longer +// passes isMember. A node calls it on an ACL change to actively cut off removed +// members (DESIGN §6.4) rather than waiting for their streams to close. +func (s *service) RevalidateMembers(spaceId string, isMember func(account string) bool) { + s.evictSpaceStreams(spaceId, func(strm *streamInterest) bool { + return !isMember(strm.account) + }) +} + +// evictSpaceStreams drops the space interest of every stream matching evict, +// stripping its routing tags so delivery stops even while the stream stays open. +func (s *service) evictSpaceStreams(spaceId string, evict func(*streamInterest) bool) { + s.remoteMu.Lock() + defer s.remoteMu.Unlock() + si := s.remote[spaceId] + for streamId, strm := range s.streams { + spacePatterns := strm.bySpace[spaceId] + if len(spacePatterns) == 0 || !evict(strm) { + continue + } + tags := make([]string, 0, len(spacePatterns)) + for pattern := range spacePatterns { + tags = append(tags, interestTag(spaceId, pattern)) + strm.total-- + if si != nil { + si.trie.Remove(pattern) + } + } + delete(strm.bySpace, spaceId) + _ = s.pool.RemoveTagsById(streamId, tags...) + if strm.total == 0 { + delete(s.streams, streamId) + } + } + if si != nil { + s.pruneSpace(spaceId, si) + } +} + +func (s *service) sendInterest(spaceId string, patterns []string, subscribe bool) { + if s.deps.Peers == nil { + return + } + var msg *pubsubproto.PubSubMessage + if subscribe { + msg = &pubsubproto.PubSubMessage{Content: &pubsubproto.PubSubMessage_Subscribe{ + Subscribe: &pubsubproto.Subscribe{SpaceId: spaceId, Topics: patterns}, + }} + } else { + msg = &pubsubproto.PubSubMessage{Content: &pubsubproto.PubSubMessage_Unsubscribe{ + Unsubscribe: &pubsubproto.Unsubscribe{SpaceId: spaceId, Topics: patterns}, + }} + } + if err := s.pool.Send(s.ctx, msg, func(ctx context.Context) ([]peer.Peer, error) { + return s.deps.Peers.SpacePeers(ctx, spaceId) + }); err != nil { + log.Info("send interest failed", zap.String("spaceId", spaceId), zap.Error(err)) + } +} + +// +// streamhandler.StreamHandler (the engine is its own pool handler) +// + +func (s *service) OpenStream(ctx context.Context, p peer.Peer) (stream drpc.Stream, tags []string, queueSize int, err error) { + // hold the peer so idle pool GC (default ~1m TTL) does not silently reap a + // quiet pubsub stream out from under a subscriber + p.SetTTL(s.cfg.PeerTTL) + conn, err := p.AcquireDrpcConn(ctx) + if err != nil { + return nil, nil, 0, err + } + objectStream, err := pubsubproto.NewDRPCPubSubClient(conn).PubSubStream(ctx) + if err != nil { + return nil, nil, 0, err + } + return objectStream, nil, s.cfg.WriteQueueSize, nil +} + +func (s *service) NewReadMessage() drpc.Message { + return &pubsubproto.PubSubMessage{} +} + +func (s *service) HandleMessage(ctx context.Context, peerId string, msg drpc.Message) error { + m, ok := msg.(*pubsubproto.PubSubMessage) + if !ok { + return pubsubproto.ErrUnexpected + } + switch { + case m.GetSubscribe() != nil: + s.handleSubscribe(ctx, peerId, m.GetSubscribe()) + case m.GetUnsubscribe() != nil: + s.handleUnsubscribe(ctx, peerId, m.GetUnsubscribe()) + case m.GetPublish() != nil: + s.handlePublish(ctx, peerId, m.GetPublish()) + case m.GetStatus() != nil: + st := m.GetStatus() + log.Debug("pubsub status from peer", + zap.String("peerId", peerId), zap.String("spaceId", st.SpaceId), + zap.String("code", st.Code.String()), zap.Strings("topics", st.Topics)) + if s.deps.OnStatus != nil { + s.deps.OnStatus(peerId, st) + } + } + // frame-level rejections answer with Status; never break the stream + return nil +} + +// HandleStream serves an inbound PubSubStream (DRPC server entry); blocks. +func (s *service) HandleStream(stream drpc.Stream) error { + return s.pool.ReadStream(stream, s.cfg.WriteQueueSize) +} + +// +// serving-side frame handling +// + +func (s *service) handleSubscribe(ctx context.Context, peerId string, sub *pubsubproto.Subscribe) { + streamId, ok := streampool.CtxStreamId(ctx) + if !ok { + s.sendStatus(ctx, peerId, sub.SpaceId, sub.Topics, pubsubproto.ErrCodes_InvalidMessage) + return + } + identity, err := peer.CtxPubKey(ctx) + if err != nil { + s.sendStatus(ctx, peerId, sub.SpaceId, sub.Topics, pubsubproto.ErrCodes_InvalidMessage) + return + } + if err = validateSpaceId(sub.SpaceId); err != nil { + s.sendStatus(ctx, peerId, sub.SpaceId, sub.Topics, pubsubproto.ErrCodes_InvalidTopic) + return + } + if s.deps.Relay != nil && !s.deps.Relay.IsResponsible(sub.SpaceId) { + s.sendStatus(ctx, peerId, sub.SpaceId, sub.Topics, pubsubproto.ErrCodes_NotResponsible) + return + } + for _, pattern := range sub.Topics { + if err = ValidatePattern(pattern); err != nil { + s.sendStatus(ctx, peerId, sub.SpaceId, sub.Topics, pubsubproto.ErrCodes_InvalidTopic) + return + } + } + if s.deps.Membership != nil { + if err = s.deps.Membership.CheckMember(ctx, sub.SpaceId, identity); err != nil { + s.sendStatus(ctx, peerId, sub.SpaceId, sub.Topics, pubsubproto.ErrCodes_NotAMember) + return + } + } + + // Record interest and register the routing tags under a single remoteMu hold. + // remoteMu -> pool.mu is a consistent lock order (removeStream releases pool.mu + // before onStreamClose takes remoteMu), so holding across AddTagsCtx is safe and + // makes interest+tag atomic w.r.t. a concurrent close: if the stream was already + // removed, AddTagsCtx fails and we roll the interest back, so nothing leaks. + s.remoteMu.Lock() + si := s.remote[sub.SpaceId] + if si == nil { + si = &spaceInterest{trie: newPatternTrie()} + s.remote[sub.SpaceId] = si + } + strm := s.streams[streamId] + if strm == nil { + strm = &streamInterest{account: identity.Account(), bySpace: make(map[string]map[string]struct{})} + s.streams[streamId] = strm + } + spacePatterns := strm.bySpace[sub.SpaceId] + if spacePatterns == nil { + spacePatterns = make(map[string]struct{}) + strm.bySpace[sub.SpaceId] = spacePatterns + } + var accepted, rejected []string + for i, pattern := range sub.Topics { + if _, exists := spacePatterns[pattern]; exists { + continue + } + if len(spacePatterns) >= s.cfg.MaxPatternsPerSpace || strm.total >= s.cfg.MaxPatternsPerStream { + // cap hit: this pattern and every remaining one are rejected + rejected = sub.Topics[i:] + break + } + spacePatterns[pattern] = struct{}{} + strm.total++ + si.trie.Add(pattern) + accepted = append(accepted, pattern) + } + if len(accepted) > 0 { + tags := make([]string, len(accepted)) + for i, pattern := range accepted { + tags[i] = interestTag(sub.SpaceId, pattern) + } + if err = s.pool.AddTagsCtx(ctx, tags...); err != nil { + // the stream was removed between accept and tagging: undo the interest + // so onStreamClose (which found nothing) leaves no orphan behind + for _, pattern := range accepted { + s.removeStreamPattern(strm, si, sub.SpaceId, pattern) + } + s.pruneStream(streamId, strm) + s.pruneSpace(sub.SpaceId, si) + } + } + s.remoteMu.Unlock() + + if len(rejected) > 0 { + s.sendStatus(ctx, peerId, sub.SpaceId, rejected, pubsubproto.ErrCodes_TooManyTopics) + } +} + +func (s *service) handleUnsubscribe(ctx context.Context, peerId string, unsub *pubsubproto.Unsubscribe) { + streamId, ok := streampool.CtxStreamId(ctx) + if !ok { + return + } + s.remoteMu.Lock() + strm := s.streams[streamId] + si := s.remote[unsub.SpaceId] + if strm == nil || si == nil { + s.remoteMu.Unlock() + return + } + patterns := unsub.Topics + if len(patterns) == 0 { // empty means all patterns of the space + for pattern := range strm.bySpace[unsub.SpaceId] { + patterns = append(patterns, pattern) + } + } + var removed []string + for _, pattern := range patterns { + if s.removeStreamPattern(strm, si, unsub.SpaceId, pattern) { + removed = append(removed, pattern) + } + } + s.pruneStream(streamId, strm) + s.pruneSpace(unsub.SpaceId, si) + s.remoteMu.Unlock() + + if len(removed) > 0 { + tags := make([]string, len(removed)) + for i, pattern := range removed { + tags[i] = interestTag(unsub.SpaceId, pattern) + } + if err := s.pool.RemoveTagsCtx(ctx, tags...); err != nil { + log.Warn("remove tags failed", zap.Error(err)) + } + } +} + +func (s *service) handlePublish(ctx context.Context, peerId string, p *pubsubproto.Publish) { + if len(p.MsgId) != msgIdLen || len(p.Payload) > s.cfg.MaxPayloadSize { + s.sendPubStatus(ctx, peerId, p, pubsubproto.ErrCodes_InvalidMessage) + return + } + if ValidateTopic(p.Topic) != nil { + s.sendPubStatus(ctx, peerId, p, pubsubproto.ErrCodes_InvalidTopic) + return + } + if s.deps.Relay != nil { + s.relayPublish(ctx, peerId, p) + return + } + // client role: deliver locally only, never forward (relay rule 1) + s.receivePublish(ctx, p) +} + +// relayPublish is the node ingress path: authorize, fan out to subscribed streams, +// and forward client-originated messages once to the other responsible nodes. +func (s *service) relayPublish(ctx context.Context, peerId string, p *pubsubproto.Publish) { + if !s.deps.Relay.IsResponsible(p.SpaceId) { + s.sendPubStatus(ctx, peerId, p, pubsubproto.ErrCodes_NotResponsible) + return + } + if p.Relayed { + // only responsible peer nodes may relay; relayed messages are never re-forwarded + if !s.deps.Relay.IsResponsibleNode(p.SpaceId, peerId) { + return + } + s.fanout(ctx, p) + return + } + // client-originated: bind attribution to the handshake-proven identity. + // Reject an empty identity explicitly: CtxIdentity returns (nil,nil) for an + // unverified inbound, and bytesEqual(nil,nil) would otherwise pass the bind. + ctxIdentity, err := peer.CtxIdentity(ctx) + if err != nil || len(ctxIdentity) == 0 || len(p.Identity) == 0 || !bytesEqual(ctxIdentity, p.Identity) { + s.sendPubStatus(ctx, peerId, p, pubsubproto.ErrCodes_InvalidMessage) + return + } + if s.deps.Membership != nil { + identity, err := peer.CtxPubKey(ctx) + if err != nil || s.deps.Membership.CheckMember(ctx, p.SpaceId, identity) != nil { + s.sendPubStatus(ctx, peerId, p, pubsubproto.ErrCodes_NotAMember) + return + } + } + if owner := TopicOwner(p.Topic); owner != "" { + identity, err := peer.CtxPubKey(ctx) + if err != nil || identity.Account() != owner { + s.sendPubStatus(ctx, peerId, p, pubsubproto.ErrCodes_TopicNotOwned) + return + } + } + if !s.rate.allow(peerId) { + s.sendPubStatus(ctx, peerId, p, pubsubproto.ErrCodes_RateLimited) + return + } + s.fanout(ctx, p) + // forward exactly once to the other responsible nodes; + // byte-slice fields are shared with the original but never mutated + relayed := &pubsubproto.Publish{ + SpaceId: p.SpaceId, + Topic: p.Topic, + MsgId: p.MsgId, + KeyId: p.KeyId, + Payload: p.Payload, + Identity: p.Identity, + Signature: p.Signature, + TimestampMilli: p.TimestampMilli, + Relayed: true, + } + if err := s.pool.Send(ctx, wrapPublish(relayed), func(ctx context.Context) ([]peer.Peer, error) { + return s.deps.Relay.OtherResponsiblePeers(ctx, p.SpaceId) + }); err != nil { + log.Info("forward to responsible nodes failed", zap.Error(err)) + } +} + +// fanout writes the message to every stream whose interest matches the topic. +func (s *service) fanout(ctx context.Context, p *pubsubproto.Publish) { + s.remoteMu.Lock() + si := s.remote[p.SpaceId] + var patterns []string + if si != nil { + patterns = si.trie.Match(p.Topic, nil) + } + s.remoteMu.Unlock() + if len(patterns) == 0 { + return + } + tags := make([]string, len(patterns)) + for i, pattern := range patterns { + tags[i] = interestTag(p.SpaceId, pattern) + } + // Broadcast dedups streams subscribed to several matching patterns + if err := s.pool.Broadcast(ctx, wrapPublish(p), tags...); err != nil { + log.Info("fanout failed", zap.Error(err)) + } +} + +// receivePublish is the client receive path. Cheap filters (local interest, +// membership, ownership, timestamp) run before the expensive Ed25519 verify so a +// relay/LAN-peer flood of forged messages is shed cheaply; the dedup ring is +// recorded only after verify so junk msgIds can't evict legitimate ones. +func (s *service) receivePublish(ctx context.Context, p *pubsubproto.Publish) { + s.localMu.Lock() + var patterns []string + if trie := s.localTrie[p.SpaceId]; trie != nil { + patterns = trie.Match(p.Topic, nil) + } + s.localMu.Unlock() + if len(patterns) == 0 { + return + } + // cheap: unmarshal the claimed identity (not yet trusted) to run filters + identity, err := identityOf(p) + if err != nil { + log.Debug("dropping publish with bad identity", zap.String("topic", p.Topic), zap.Error(err)) + return + } + if s.deps.Membership != nil { + if err = s.deps.Membership.CheckMember(ctx, p.SpaceId, identity); err != nil { + log.Debug("dropping publish from non-member", zap.String("topic", p.Topic)) + return + } + } + // end-to-end acc/ ownership: the signature covers the topic, so not even a + // malicious relay can inject into someone else's self-owned topic + if owner := TopicOwner(p.Topic); owner != "" && identity.Account() != owner { + log.Debug("dropping publish into unowned topic", zap.String("topic", p.Topic)) + return + } + if s.isStale(p.TimestampMilli) { + log.Debug("dropping stale publish", zap.String("topic", p.Topic)) + return + } + // expensive: verify only after the cheap filters pass + if err = verifySignature(identity, p); err != nil { + log.Debug("dropping publish with bad signature", zap.String("topic", p.Topic), zap.Error(err)) + return + } + // record dedup only for messages we would deliver, so forged floods can't + // flush the ring and re-open a replay window + if s.dedup.seen(p.MsgId) { + return + } + payload := p.Payload + if p.KeyId != "" { + if s.deps.Crypto == nil { + return + } + if payload, err = s.deps.Crypto.Decrypt(p.SpaceId, p.KeyId, p.Payload); err != nil { + log.Debug("dropping undecryptable publish", zap.String("topic", p.Topic), zap.Error(err)) + return + } + } + s.enqueueLocalMatched(patterns, p.SpaceId, p.Topic, identity, payload) +} + +// isStale reports whether a signed timestamp is outside the accepted skew window, +// raising the replay bar even after dedup eviction. Zero is treated as absent +// (never stale) to stay compatible with senders that omit it. +func (s *service) isStale(timestampMilli int64) bool { + if timestampMilli == 0 { + return false + } + skew := s.cfg.MaxTimestampSkew.Milliseconds() + now := time.Now().UnixMilli() + delta := now - timestampMilli + return delta > skew || delta < -skew +} + +// +// local dispatch +// + +func (s *service) enqueueLocal(spaceId, topic string, identity crypto.PubKey, payload []byte) { + s.localMu.Lock() + var patterns []string + if trie := s.localTrie[spaceId]; trie != nil { + patterns = trie.Match(topic, nil) + } + s.localMu.Unlock() + if len(patterns) == 0 { + return + } + s.enqueueLocalMatched(patterns, spaceId, topic, identity, payload) +} + +func (s *service) enqueueLocalMatched(patterns []string, spaceId, topic string, identity crypto.PubKey, payload []byte) { + err := s.dispatch.TryAdd(dispatchItem{ + patterns: patterns, + spaceId: spaceId, + topic: topic, + identity: identity, + payload: payload, + }) + if err != nil && !errors.Is(err, mb.ErrClosed) { + log.Debug("dispatch queue overflow, message dropped", zap.String("topic", topic)) + } +} + +func (s *service) dispatchLoop() { + for { + item, err := s.dispatch.WaitOne(s.ctx) + if err != nil { + return + } + s.localMu.Lock() + var handlers []Handler + for _, pattern := range item.patterns { + for _, sub := range s.localSubs[item.spaceId][pattern] { + handlers = append(handlers, sub.h) + } + } + s.localMu.Unlock() + for _, h := range handlers { + h(item.spaceId, item.topic, item.identity, item.payload) + } + } +} + +// +// housekeeping +// + +// onStreamClose withdraws exactly the closed stream's interest, keyed by streamId. +// It uses the engine's own per-stream record (streams[streamId]) rather than the +// pool's tag snapshot, so a stream that closed before its tags were registered is +// still cleaned, and a sibling stream of the same peer is never touched. +func (s *service) onStreamClose(streamId uint32, _ string, _ []string) { + // A client-side outbound stream dropping means our pushed interest is gone on + // the peer; re-push it promptly rather than waiting for the periodic tick. + if s.deps.Peers != nil { + s.triggerResync() + } + s.remoteMu.Lock() + defer s.remoteMu.Unlock() + strm := s.streams[streamId] + if strm == nil { + return + } + for spaceId, patterns := range strm.bySpace { + si := s.remote[spaceId] + if si == nil { + continue + } + for pattern := range patterns { + si.trie.Remove(pattern) + } + s.pruneSpace(spaceId, si) + } + delete(s.streams, streamId) +} + +// removeStreamPattern withdraws one pattern of one space from a stream's record and +// the space trie. Returns true if the pattern was present. Caller holds remoteMu. +func (s *service) removeStreamPattern(strm *streamInterest, si *spaceInterest, spaceId, pattern string) bool { + patterns := strm.bySpace[spaceId] + if _, ok := patterns[pattern]; !ok { + return false + } + delete(patterns, pattern) + strm.total-- + if len(patterns) == 0 { + delete(strm.bySpace, spaceId) + } + si.trie.Remove(pattern) + return true +} + +// pruneStream drops an empty stream record. Caller holds remoteMu. +func (s *service) pruneStream(streamId uint32, strm *streamInterest) { + if strm.total == 0 { + delete(s.streams, streamId) + } +} + +// pruneSpace drops an empty space trie. Caller holds remoteMu. +func (s *service) pruneSpace(spaceId string, si *spaceInterest) { + if si.trie.Len() == 0 { + delete(s.remote, spaceId) + } +} + +func (s *service) sendStatus(ctx context.Context, peerId, spaceId string, topics []string, code pubsubproto.ErrCodes) { + s.sendStatusMsg(ctx, peerId, &pubsubproto.Status{SpaceId: spaceId, Topics: topics, Code: code}) +} + +// sendPubStatus reports a rejected publish, echoing its msgId so the caller can +// correlate the rejection to the originating Publish. +func (s *service) sendPubStatus(ctx context.Context, peerId string, p *pubsubproto.Publish, code pubsubproto.ErrCodes) { + s.sendStatusMsg(ctx, peerId, &pubsubproto.Status{ + SpaceId: p.SpaceId, + Topics: []string{p.Topic}, + Code: code, + MsgId: p.MsgId, + }) +} + +func (s *service) sendStatusMsg(ctx context.Context, peerId string, st *pubsubproto.Status) { + msg := &pubsubproto.PubSubMessage{Content: &pubsubproto.PubSubMessage_Status{Status: st}} + if err := s.pool.SendById(ctx, msg, peerId); err != nil { + log.Debug("send status failed", zap.String("peerId", peerId), zap.Error(err)) + } +} + +func interestTag(spaceId, pattern string) string { + return spaceId + "/" + pattern +} + +func wrapPublish(p *pubsubproto.Publish) *pubsubproto.PubSubMessage { + return &pubsubproto.PubSubMessage{Content: &pubsubproto.PubSubMessage_Publish{Publish: p}} +} + +func bytesEqual(a, b []byte) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} diff --git a/commonspace/pubsub/service_test.go b/commonspace/pubsub/service_test.go new file mode 100644 index 000000000..1dd416897 --- /dev/null +++ b/commonspace/pubsub/service_test.go @@ -0,0 +1,493 @@ +package pubsub + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/anyproto/any-sync/app" + "github.com/anyproto/any-sync/commonspace/object/accountdata" + "github.com/anyproto/any-sync/commonspace/pubsub/pubsubproto" + "github.com/anyproto/any-sync/net/peer" + "github.com/anyproto/any-sync/net/rpc/rpctest" + "github.com/anyproto/any-sync/testutil/accounttest" + "github.com/anyproto/any-sync/util/crypto" +) + +var testCtx = context.Background() + +const testSpace = "space1" + +// +// fakes +// + +type fakeMembership struct { + mu sync.Mutex + allowed map[string]bool // account id -> member +} + +func (f *fakeMembership) allow(accounts ...crypto.PubKey) { + f.mu.Lock() + defer f.mu.Unlock() + if f.allowed == nil { + f.allowed = make(map[string]bool) + } + for _, a := range accounts { + f.allowed[a.Account()] = true + } +} + +func (f *fakeMembership) CheckMember(_ context.Context, _ string, identity crypto.PubKey) error { + f.mu.Lock() + defer f.mu.Unlock() + if f.allowed[identity.Account()] { + return nil + } + return fmt.Errorf("not a member") +} + +type staticPeers struct { + mu sync.Mutex + peers []peer.Peer +} + +func (s *staticPeers) add(p peer.Peer) { + s.mu.Lock() + defer s.mu.Unlock() + s.peers = append(s.peers, p) +} + +func (s *staticPeers) replace(p peer.Peer) { + s.mu.Lock() + defer s.mu.Unlock() + s.peers = []peer.Peer{p} +} + +func (s *staticPeers) SpacePeers(_ context.Context, _ string) ([]peer.Peer, error) { + s.mu.Lock() + defer s.mu.Unlock() + return append([]peer.Peer(nil), s.peers...), nil +} + +type fakeRelay struct { + mu sync.Mutex + nodePeerIds map[string]bool + others []peer.Peer + forwardCalls atomic.Int32 +} + +func (f *fakeRelay) addNodePeer(peerId string) { + f.mu.Lock() + defer f.mu.Unlock() + if f.nodePeerIds == nil { + f.nodePeerIds = make(map[string]bool) + } + f.nodePeerIds[peerId] = true +} + +func (f *fakeRelay) addOther(p peer.Peer) { + f.mu.Lock() + defer f.mu.Unlock() + f.others = append(f.others, p) +} + +func (f *fakeRelay) IsResponsible(string) bool { return true } + +func (f *fakeRelay) IsResponsibleNode(_, peerId string) bool { + f.mu.Lock() + defer f.mu.Unlock() + return f.nodePeerIds[peerId] +} + +func (f *fakeRelay) OtherResponsiblePeers(_ context.Context, _ string) ([]peer.Peer, error) { + f.forwardCalls.Add(1) + f.mu.Lock() + defer f.mu.Unlock() + return append([]peer.Peer(nil), f.others...), nil +} + +// +// fixture +// + +type received struct { + topic string + account string + payload string +} + +type engineFx struct { + t *testing.T + name string + acc *accountdata.AccountKeys + svc *service + app *app.App + ts *rpctest.TestServer + membership *fakeMembership + peers *staticPeers + relay *fakeRelay + statuses chan *pubsubproto.Status + received chan received + ownedPeers []peer.Peer +} + +func (fx *engineFx) identity() crypto.PubKey { return fx.acc.SignKey.GetPublic() } + +func (fx *engineFx) finish() { + require.NoError(fx.t, fx.app.Close(testCtx)) + for _, p := range fx.ownedPeers { + _ = p.Close() + } +} + +func newEngineFx(t *testing.T, name string, membership *fakeMembership, relay *fakeRelay) *engineFx { + acc, err := accountdata.NewRandom() + require.NoError(t, err) + fx := &engineFx{ + t: t, + name: name, + acc: acc, + membership: membership, + relay: relay, + statuses: make(chan *pubsubproto.Status, 16), + received: make(chan received, 64), + } + deps := Deps{ + Membership: membership, + OnStatus: func(_ string, st *pubsubproto.Status) { fx.statuses <- st }, + // fast resync so reconnect tests don't wait the 20s default + Config: Config{ResyncInterval: 150 * time.Millisecond}, + } + if relay != nil { + deps.Relay = relay + } else { + fx.peers = &staticPeers{} + deps.Peers = fx.peers + } + fx.svc = New(deps).(*service) + + fx.app = new(app.App) + fx.app.Register(accounttest.NewWithAcc(acc)).Register(fx.svc) + require.NoError(t, fx.app.Start(testCtx)) + + fx.ts = rpctest.NewTestServer() + require.NoError(t, RegisterRpc(fx.ts.Mux, fx.svc)) + return fx +} + +func (fx *engineFx) handler() Handler { + return func(_, topic string, identity crypto.PubKey, payload []byte) { + fx.received <- received{topic: topic, account: identity.Account(), payload: string(payload)} + } +} + +// connect wires from -> to and returns from's peer handle for to. +// Both directions carry the respective remote's peerId and identity in ctx, +// mirroring what secureservice puts there in production. +func connect(t *testing.T, from, to *engineFx) peer.Peer { + fromIdentity, err := from.identity().Marshall() + require.NoError(t, err) + toIdentity, err := to.identity().Marshall() + require.NoError(t, err) + mcAtTo, mcAtFrom := rpctest.MultiConnPairWithClientServerIdentity( + from.acc.PeerId, to.acc.PeerId, fromIdentity, toIdentity) + pAtTo, err := peer.NewPeer(mcAtTo, to.ts) + require.NoError(t, err) + to.ownedPeers = append(to.ownedPeers, pAtTo) + pAtFrom, err := peer.NewPeer(mcAtFrom, from.ts) + require.NoError(t, err) + from.ownedPeers = append(from.ownedPeers, pAtFrom) + return pAtFrom +} + +// waitInterest polls the serving engine until topic matches remote interest, +// removing the subscribe/publish race inherent to fire-and-forget semantics. +func waitInterest(t *testing.T, serving *engineFx, topic string) { + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + serving.svc.remoteMu.Lock() + si := serving.svc.remote[testSpace] + var n int + if si != nil { + n = len(si.trie.Match(topic, nil)) + } + serving.svc.remoteMu.Unlock() + if n > 0 { + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("interest for %s never registered on %s", topic, serving.name) +} + +func waitReceived(t *testing.T, fx *engineFx) received { + select { + case r := <-fx.received: + return r + case <-time.After(2 * time.Second): + t.Fatalf("%s: timeout waiting for message", fx.name) + return received{} + } +} + +func expectSilence(t *testing.T, fx *engineFx, d time.Duration) { + select { + case r := <-fx.received: + t.Fatalf("%s: unexpected message %+v", fx.name, r) + case <-time.After(d): + } +} + +func waitStatus(t *testing.T, fx *engineFx, code pubsubproto.ErrCodes) { + deadline := time.After(2 * time.Second) + for { + select { + case st := <-fx.statuses: + if st.Code == code { + return + } + case <-deadline: + t.Fatalf("%s: timeout waiting for status %s", fx.name, code) + } + } +} + +// +// tests +// + +// topology: clientA, clientC -> nodeB <-> nodeB2 <- clientA2 +type netFx struct { + nodeB, nodeB2, clientA, clientC, clientA2 *engineFx +} + +func newNetFx(t *testing.T) *netFx { + membership := &fakeMembership{} + relayB := &fakeRelay{} + relayB2 := &fakeRelay{} + + n := &netFx{ + nodeB: newEngineFx(t, "nodeB", membership, relayB), + nodeB2: newEngineFx(t, "nodeB2", membership, relayB2), + clientA: newEngineFx(t, "clientA", membership, nil), + clientC: newEngineFx(t, "clientC", membership, nil), + clientA2: newEngineFx(t, "clientA2", membership, nil), + } + membership.allow(n.clientA.identity(), n.clientC.identity(), n.clientA2.identity()) + + n.clientA.peers.add(connect(t, n.clientA, n.nodeB)) + n.clientC.peers.add(connect(t, n.clientC, n.nodeB)) + n.clientA2.peers.add(connect(t, n.clientA2, n.nodeB2)) + relayB.addOther(connect(t, n.nodeB, n.nodeB2)) + relayB2.addNodePeer(n.nodeB.acc.PeerId) + relayB.addNodePeer(n.nodeB2.acc.PeerId) + return n +} + +func (n *netFx) finish() { + for _, fx := range []*engineFx{n.clientA, n.clientC, n.clientA2, n.nodeB, n.nodeB2} { + fx.finish() + } +} + +func TestPubSubFanoutAndWildcards(t *testing.T) { + n := newNetFx(t) + defer n.finish() + + _, err := n.clientA.svc.Subscribe(testSpace, "chat/>", n.clientA.handler()) + require.NoError(t, err) + _, err = n.clientA.svc.Subscribe(testSpace, "acc/online/*", n.clientA.handler()) + require.NoError(t, err) + waitInterest(t, n.nodeB, "chat/room1/typing") + waitInterest(t, n.nodeB, "acc/online/"+n.clientC.identity().Account()) + + require.NoError(t, n.clientC.svc.Publish(testCtx, testSpace, "chat/room1/typing", []byte("tick"))) + r := waitReceived(t, n.clientA) + require.Equal(t, "chat/room1/typing", r.topic) + require.Equal(t, n.clientC.identity().Account(), r.account) + require.Equal(t, "tick", r.payload) + + ownTopic := "acc/online/" + n.clientC.identity().Account() + require.NoError(t, n.clientC.svc.Publish(testCtx, testSpace, ownTopic, []byte("on"))) + r = waitReceived(t, n.clientA) + require.Equal(t, ownTopic, r.topic) + + // no matching interest: fire-and-forget discards silently + require.NoError(t, n.clientC.svc.Publish(testCtx, testSpace, "other/topic", []byte("x"))) + expectSilence(t, n.clientA, 300*time.Millisecond) +} + +func TestPubSubNodeRelay(t *testing.T) { + n := newNetFx(t) + defer n.finish() + + _, err := n.clientA2.svc.Subscribe(testSpace, "chat/>", n.clientA2.handler()) + require.NoError(t, err) + waitInterest(t, n.nodeB2, "chat/x") + + require.NoError(t, n.clientC.svc.Publish(testCtx, testSpace, "chat/x", []byte("cross-node"))) + r := waitReceived(t, n.clientA2) + require.Equal(t, "chat/x", r.topic) + require.Equal(t, "cross-node", r.payload) + + // hop limit: B2 must not re-forward the relayed message + require.Equal(t, int32(0), n.nodeB2.relay.forwardCalls.Load()) + // B forwarded exactly once + require.Equal(t, int32(1), n.nodeB.relay.forwardCalls.Load()) +} + +func TestPubSubMembershipRejection(t *testing.T) { + n := newNetFx(t) + defer n.finish() + + outsider := newEngineFx(t, "outsider", n.nodeB.membership, nil) + defer outsider.finish() + outsider.peers.add(connect(t, outsider, n.nodeB)) + + _, err := outsider.svc.Subscribe(testSpace, "chat/>", outsider.handler()) + require.NoError(t, err) // local registration succeeds, the node rejects async + waitStatus(t, outsider, pubsubproto.ErrCodes_NotAMember) + + require.NoError(t, outsider.svc.Publish(testCtx, testSpace, "chat/x", []byte("spam"))) + waitStatus(t, outsider, pubsubproto.ErrCodes_NotAMember) +} + +func TestPubSubTopicOwnership(t *testing.T) { + n := newNetFx(t) + defer n.finish() + + victimTopic := "acc/online/" + n.clientA.identity().Account() + _, err := n.clientA.svc.Subscribe(testSpace, "acc/online/*", n.clientA.handler()) + require.NoError(t, err) + waitInterest(t, n.nodeB, victimTopic) + + // the client fails fast when publishing into someone else's self-owned topic + require.ErrorIs(t, n.clientC.svc.Publish(testCtx, testSpace, victimTopic, []byte("spoof")), + pubsubproto.ErrTopicNotOwned) + + // relay-side enforcement: a malicious client bypassing the local check is + // rejected at the node ingress and never fanned out + spoofed := &pubsubproto.Publish{ + SpaceId: testSpace, + Topic: victimTopic, + MsgId: testMsgId(776), + Payload: []byte("spoof"), + TimestampMilli: time.Now().UnixMilli(), + } + require.NoError(t, signPublish(n.clientC.acc.SignKey, spoofed)) + spoofCtx := peer.CtxWithPeerId(peer.CtxWithIdentity(testCtx, spoofed.Identity), n.clientC.acc.PeerId) + n.nodeB.svc.handlePublish(spoofCtx, n.clientC.acc.PeerId, spoofed) + expectSilence(t, n.clientA, 300*time.Millisecond) + + // receive-side enforcement: a forged message injected past the relay is dropped + forged := &pubsubproto.Publish{ + SpaceId: testSpace, + Topic: victimTopic, + MsgId: testMsgId(777), + Payload: []byte("forged"), + TimestampMilli: time.Now().UnixMilli(), + } + require.NoError(t, signPublish(n.clientC.acc.SignKey, forged)) + n.clientA.svc.receivePublish(testCtx, forged) + expectSilence(t, n.clientA, 300*time.Millisecond) +} + +func TestPubSubEchoSuppression(t *testing.T) { + n := newNetFx(t) + defer n.finish() + + _, err := n.clientA.svc.Subscribe(testSpace, "chat/>", n.clientA.handler()) + require.NoError(t, err) + waitInterest(t, n.nodeB, "chat/self") + + require.NoError(t, n.clientA.svc.Publish(testCtx, testSpace, "chat/self", []byte("echo?"))) + r := waitReceived(t, n.clientA) + require.Equal(t, "echo?", r.payload) + // the node echoes the message back to A's subscribed stream; dedup must drop it + expectSilence(t, n.clientA, 500*time.Millisecond) +} + +func TestPubSubStaleMessageDropped(t *testing.T) { + n := newNetFx(t) + defer n.finish() + + _, err := n.clientA.svc.Subscribe(testSpace, "chat/>", n.clientA.handler()) + require.NoError(t, err) + + // a validly-signed message with a timestamp far in the past is dropped even + // though its signature verifies — raising the replay bar after dedup eviction + stale := &pubsubproto.Publish{ + SpaceId: testSpace, + Topic: "chat/old", + MsgId: testMsgId(999), + Payload: []byte("replayed"), + TimestampMilli: time.Now().Add(-time.Hour).UnixMilli(), + } + require.NoError(t, signPublish(n.clientC.acc.SignKey, stale)) + n.clientA.svc.receivePublish(testCtx, stale) + expectSilence(t, n.clientA, 300*time.Millisecond) + + // a fresh message from the same sender is delivered + fresh := &pubsubproto.Publish{ + SpaceId: testSpace, + Topic: "chat/new", + MsgId: testMsgId(1000), + Payload: []byte("fresh"), + TimestampMilli: time.Now().UnixMilli(), + } + require.NoError(t, signPublish(n.clientC.acc.SignKey, fresh)) + n.clientA.svc.receivePublish(testCtx, fresh) + require.Equal(t, "fresh", waitReceived(t, n.clientA).payload) +} + +func TestPubSubDuplicatePathSuppression(t *testing.T) { + n := newNetFx(t) + defer n.finish() + + _, err := n.clientA.svc.Subscribe(testSpace, "chat/>", n.clientA.handler()) + require.NoError(t, err) + + p := &pubsubproto.Publish{ + SpaceId: testSpace, + Topic: "chat/dup", + MsgId: testMsgId(555), + Payload: []byte("once"), + TimestampMilli: time.Now().UnixMilli(), + } + require.NoError(t, signPublish(n.clientC.acc.SignKey, p)) + // the same message arrives twice (LAN path + node path) + n.clientA.svc.receivePublish(testCtx, p) + n.clientA.svc.receivePublish(testCtx, p) + r := waitReceived(t, n.clientA) + require.Equal(t, "once", r.payload) + expectSilence(t, n.clientA, 300*time.Millisecond) +} + +func TestPubSubUnsubscribe(t *testing.T) { + n := newNetFx(t) + defer n.finish() + + unsub, err := n.clientA.svc.Subscribe(testSpace, "chat/>", n.clientA.handler()) + require.NoError(t, err) + waitInterest(t, n.nodeB, "chat/x") + + unsub() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + n.nodeB.svc.remoteMu.Lock() + si := n.nodeB.svc.remote[testSpace] + n.nodeB.svc.remoteMu.Unlock() + if si == nil { + break + } + time.Sleep(10 * time.Millisecond) + } + require.NoError(t, n.clientC.svc.Publish(testCtx, testSpace, "chat/x", []byte("gone"))) + expectSilence(t, n.clientA, 300*time.Millisecond) +} diff --git a/commonspace/pubsub/sign.go b/commonspace/pubsub/sign.go new file mode 100644 index 000000000..9517dfccc --- /dev/null +++ b/commonspace/pubsub/sign.go @@ -0,0 +1,69 @@ +package pubsub + +import ( + "encoding/binary" + + "github.com/anyproto/any-sync/commonspace/pubsub/pubsubproto" + "github.com/anyproto/any-sync/util/crypto" +) + +const signPrefix = "anysync:pubsub:v1" + +// publishSignData builds the byte string covered by a Publish signature. +// Every variable-length field is length-prefixed so the encoding is unambiguous +// (plain concatenation would let field boundaries shift). The relayed flag is +// excluded because nodes mutate it in transit. +func publishSignData(p *pubsubproto.Publish) []byte { + size := len(signPrefix) + 4*4 + 8 + + len(p.SpaceId) + len(p.Topic) + len(p.MsgId) + len(p.KeyId) + len(p.Payload) + buf := make([]byte, 0, size) + buf = append(buf, signPrefix...) + for _, f := range [][]byte{[]byte(p.SpaceId), []byte(p.Topic), p.MsgId, []byte(p.KeyId)} { + buf = binary.LittleEndian.AppendUint32(buf, uint32(len(f))) + buf = append(buf, f...) + } + buf = binary.LittleEndian.AppendUint64(buf, uint64(p.TimestampMilli)) + buf = append(buf, p.Payload...) + return buf +} + +// signPublish stamps identity and signature on the message using the account key. +func signPublish(key crypto.PrivKey, p *pubsubproto.Publish) (err error) { + p.Identity, err = key.GetPublic().Marshall() + if err != nil { + return + } + p.Signature, err = key.Sign(publishSignData(p)) + return +} + +// identityOf unmarshals the sender's public key from the message without verifying +// the signature. Cheap: lets the receiver run membership/ownership filters before +// the expensive Ed25519 verify, so a forged-signature flood is shed cheaply. +func identityOf(p *pubsubproto.Publish) (crypto.PubKey, error) { + return crypto.UnmarshalEd25519PublicKeyProto(p.Identity) +} + +// verifySignature checks the message signature against an already-unmarshalled key. +func verifySignature(pubKey crypto.PubKey, p *pubsubproto.Publish) error { + ok, err := pubKey.Verify(publishSignData(p), p.Signature) + if err != nil { + return err + } + if !ok { + return pubsubproto.ErrInvalidMessage + } + return nil +} + +// verifyPublish unmarshals the identity and verifies the signature in one step. +func verifyPublish(p *pubsubproto.Publish) (crypto.PubKey, error) { + pubKey, err := identityOf(p) + if err != nil { + return nil, err + } + if err = verifySignature(pubKey, p); err != nil { + return nil, err + } + return pubKey, nil +} diff --git a/commonspace/pubsub/sign_test.go b/commonspace/pubsub/sign_test.go new file mode 100644 index 000000000..1019a91ba --- /dev/null +++ b/commonspace/pubsub/sign_test.go @@ -0,0 +1,82 @@ +package pubsub + +import ( + "crypto/rand" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/anyproto/any-sync/commonspace/pubsub/pubsubproto" + "github.com/anyproto/any-sync/util/crypto" +) + +func newTestPublish() *pubsubproto.Publish { + return &pubsubproto.Publish{ + SpaceId: "space1", + Topic: "chat/abc/typing", + MsgId: testMsgId(42), + KeyId: "key1", + Payload: []byte("hello"), + TimestampMilli: 1751800000000, + } +} + +func TestSignVerifyPublish(t *testing.T) { + priv, _, err := crypto.GenerateEd25519Key(rand.Reader) + require.NoError(t, err) + + p := newTestPublish() + require.NoError(t, signPublish(priv, p)) + + identity, err := verifyPublish(p) + require.NoError(t, err) + require.True(t, identity.Equals(priv.GetPublic())) + + // the relayed flag is excluded from the signature + p.Relayed = true + _, err = verifyPublish(p) + require.NoError(t, err) +} + +func TestVerifyPublishTampered(t *testing.T) { + priv, _, err := crypto.GenerateEd25519Key(rand.Reader) + require.NoError(t, err) + + fields := map[string]func(p *pubsubproto.Publish){ + "payload": func(p *pubsubproto.Publish) { p.Payload = []byte("evil") }, + "topic": func(p *pubsubproto.Publish) { p.Topic = "acc/online/victim" }, + "spaceId": func(p *pubsubproto.Publish) { p.SpaceId = "space2" }, + "msgId": func(p *pubsubproto.Publish) { p.MsgId = testMsgId(43) }, + "keyId": func(p *pubsubproto.Publish) { p.KeyId = "key2" }, + "ts": func(p *pubsubproto.Publish) { p.TimestampMilli++ }, + } + for name, tamper := range fields { + p := newTestPublish() + require.NoError(t, signPublish(priv, p)) + tamper(p) + _, err = verifyPublish(p) + require.Error(t, err, "tampered %s must fail verification", name) + } +} + +func TestVerifyPublishForeignIdentity(t *testing.T) { + priv1, _, err := crypto.GenerateEd25519Key(rand.Reader) + require.NoError(t, err) + _, pub2, err := crypto.GenerateEd25519Key(rand.Reader) + require.NoError(t, err) + + p := newTestPublish() + require.NoError(t, signPublish(priv1, p)) + // swap in another identity: signature no longer matches + p.Identity, err = pub2.Marshall() + require.NoError(t, err) + _, err = verifyPublish(p) + require.Error(t, err) +} + +func TestSignDataUnambiguous(t *testing.T) { + // shifting bytes between adjacent fields must change the signed data + p1 := &pubsubproto.Publish{SpaceId: "ab", Topic: "c"} + p2 := &pubsubproto.Publish{SpaceId: "a", Topic: "bc"} + require.NotEqual(t, publishSignData(p1), publishSignData(p2)) +} diff --git a/commonspace/pubsub/topic.go b/commonspace/pubsub/topic.go new file mode 100644 index 000000000..c1afa44a1 --- /dev/null +++ b/commonspace/pubsub/topic.go @@ -0,0 +1,112 @@ +package pubsub + +import ( + "strings" + + "github.com/anyproto/any-sync/commonspace/pubsub/pubsubproto" +) + +const ( + maxTopicLen = 256 + maxSegments = 16 + wildcardOne = "*" + wildcardTail = ">" + accNamespace = "acc" +) + +// splitTopic splits a topic into segments using a stack-allocated array for the +// common case. Leading, trailing and doubled separators produce empty segments, +// which validation rejects. +func splitTopic(topic string) []string { + var tsa [maxSegments]string + n := 0 + rest := topic + for n < maxSegments { + idx := strings.IndexByte(rest, '/') + if idx < 0 { + tsa[n] = rest + n++ + return tsa[:n:n] + } + tsa[n] = rest[:idx] + n++ + rest = rest[idx+1:] + } + // over maxSegments: return an over-length slice so validation rejects it + return append(tsa[:n:n], rest) +} + +// validateSegments applies the shared structural rules for topics and patterns. +func validateSegments(topic string, segs []string) error { + if len(topic) == 0 || len(topic) > maxTopicLen { + return pubsubproto.ErrInvalidTopic + } + if len(segs) > maxSegments { + return pubsubproto.ErrInvalidTopic + } + // canonical form: no leading/trailing separator, no empty segments + for _, s := range segs { + if s == "" { + return pubsubproto.ErrInvalidTopic + } + } + return nil +} + +// ValidateTopic checks a fully-qualified publish topic: canonical form, no wildcards. +func ValidateTopic(topic string) error { + segs := splitTopic(topic) + if err := validateSegments(topic, segs); err != nil { + return err + } + for _, s := range segs { + if strings.ContainsAny(s, "*>") { + return pubsubproto.ErrInvalidTopic + } + } + return nil +} + +// ValidatePattern checks a subscription pattern: canonical form, wildcards only as +// whole segments, '>' only in tail position. +func ValidatePattern(pattern string) error { + segs := splitTopic(pattern) + if err := validateSegments(pattern, segs); err != nil { + return err + } + for i, s := range segs { + switch s { + case wildcardOne: + continue + case wildcardTail: + if i != len(segs)-1 { + return pubsubproto.ErrInvalidTopic + } + default: + if strings.ContainsAny(s, "*>") { + return pubsubproto.ErrInvalidTopic + } + } + } + return nil +} + +// validateSpaceId rejects a spaceId that would break the "spaceId/pattern" tag +// encoding. spaceId must be non-empty and contain no '/'. +func validateSpaceId(spaceId string) error { + if spaceId == "" || strings.IndexByte(spaceId, '/') >= 0 { + return pubsubproto.ErrInvalidTopic + } + return nil +} + +// TopicOwner returns the account id that exclusively may publish to the topic, or "" +// if the topic is not in the self-owned acc/ namespace. The owner is the last segment. +// Assumes a validated fully-qualified topic. +func TopicOwner(topic string) string { + segs := splitTopic(topic) + if len(segs) < 2 || segs[0] != accNamespace { + return "" + } + return segs[len(segs)-1] +} diff --git a/commonspace/pubsub/topic_test.go b/commonspace/pubsub/topic_test.go new file mode 100644 index 000000000..45461cb19 --- /dev/null +++ b/commonspace/pubsub/topic_test.go @@ -0,0 +1,72 @@ +package pubsub + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestValidateTopic(t *testing.T) { + valid := []string{ + "presence", + "chat/abc/typing", + "acc/online/A5xyz", + strings.Repeat("a/", 15) + "a", // 16 segments + } + for _, topic := range valid { + require.NoError(t, ValidateTopic(topic), topic) + } + invalid := []string{ + "", + "/presence", + "presence/", + "a//b", + "chat/*/typing", + "chat/>", + "chat/ty*", + "chat/ty>pe", + strings.Repeat("a/", 16) + "a", // 17 segments + strings.Repeat("x", 257), + } + for _, topic := range invalid { + require.Error(t, ValidateTopic(topic), topic) + } +} + +func TestValidatePattern(t *testing.T) { + valid := []string{ + "presence", + "chat/*/typing", + "chat/>", + ">", + "*", + "acc/online/*", + "a/*/*/b", + } + for _, p := range valid { + require.NoError(t, ValidatePattern(p), p) + } + invalid := []string{ + "", + "/chat/*", + "chat/>/typing", // '>' not in tail position + "chat/ty*", // wildcard not a whole segment + "chat/*>", + "a//>", + } + for _, p := range invalid { + require.Error(t, ValidatePattern(p), p) + } +} + +func TestTopicOwner(t *testing.T) { + require.Equal(t, "A5xyz", TopicOwner("acc/online/A5xyz")) + require.Equal(t, "A5xyz", TopicOwner("acc/cursor/obj1/A5xyz")) + require.Equal(t, "", TopicOwner("presence")) + require.Equal(t, "", TopicOwner("chat/abc/typing")) + // bare "acc" has no owner segment + require.Equal(t, "", TopicOwner("acc")) + // non-acc first segment + require.Equal(t, "", TopicOwner("accounts/online/A5xyz")) +} diff --git a/commonspace/pubsub/trie.go b/commonspace/pubsub/trie.go new file mode 100644 index 000000000..ba91d28a3 --- /dev/null +++ b/commonspace/pubsub/trie.go @@ -0,0 +1,176 @@ +package pubsub + +// patternTrie is a per-space interest trie over '/'-separated topic segments, +// mirroring the NATS sublist shape: one level per segment, a literal-child map plus +// dedicated single-segment ('*') and tail ('>') wildcard slots per level. Terminals +// hold a refcount of subscribing streams so identical patterns from many streams +// share one node. Not goroutine-safe; the engine serializes access. +type patternTrie struct { + root *trieLevel + size int // number of live distinct patterns +} + +type trieLevel struct { + nodes map[string]*trieNode + pwc *trieNode // '*' + fwc *trieNode // '>' +} + +type trieNode struct { + next *trieLevel + pattern string // set iff this node terminates a pattern + refs int // subscriber refcount for the terminated pattern +} + +func newPatternTrie() *patternTrie { + return &patternTrie{root: &trieLevel{}} +} + +func (t *patternTrie) Len() int { + return t.size +} + +// Add registers one subscriber reference for the pattern (assumed validated). +// Returns true when the pattern is new to the trie (0 -> 1 transition). +func (t *patternTrie) Add(pattern string) bool { + segs := splitTopic(pattern) + level := t.root + var node *trieNode + for _, seg := range segs { + node = level.child(seg) + if node == nil { + node = &trieNode{} + level.setChild(seg, node) + } + if node.next == nil { + node.next = &trieLevel{} + } + level = node.next + } + node.pattern = pattern + node.refs++ + if node.refs == 1 { + t.size++ + return true + } + return false +} + +// Remove drops one subscriber reference; on the last reference the pattern is pruned. +// Returns true when the pattern was removed entirely (1 -> 0 transition). +func (t *patternTrie) Remove(pattern string) bool { + segs := splitTopic(pattern) + return t.remove(t.root, segs, pattern) +} + +func (t *patternTrie) remove(level *trieLevel, segs []string, pattern string) bool { + if len(segs) == 0 { + return false + } + node := level.child(segs[0]) + if node == nil { + return false + } + var removed bool + if len(segs) == 1 { + if node.refs == 0 { + return false + } + node.refs-- + if node.refs > 0 { + return false + } + node.pattern = "" + t.size-- + removed = true + } else { + if node.next == nil { + return false + } + removed = t.remove(node.next, segs[1:], pattern) + } + if node.refs == 0 && (node.next == nil || node.next.empty()) { + level.deleteChild(segs[0]) + } + return removed +} + +// Match walks the trie with the segments of a fully-qualified topic and appends +// every matching pattern to dst. Match order follows NATS matchLevel: the +// tail-wildcard terminal is collected at each level (it matches one-or-more +// remaining segments), then the '*' branch, then the literal branch. +func (t *patternTrie) Match(topic string, dst []string) []string { + segs := splitTopic(topic) + return matchLevel(t.root, segs, dst) +} + +func matchLevel(level *trieLevel, segs []string, dst []string) []string { + if level == nil || len(segs) == 0 { + return dst + } + if level.fwc != nil && level.fwc.refs > 0 { + dst = append(dst, level.fwc.pattern) + } + if level.pwc != nil { + dst = matchNode(level.pwc, segs[1:], dst) + } + if node := level.literal(segs[0]); node != nil { + dst = matchNode(node, segs[1:], dst) + } + return dst +} + +// matchNode resolves a node that consumed one segment against the remaining ones. +func matchNode(node *trieNode, rest []string, dst []string) []string { + if len(rest) == 0 { + if node.refs > 0 { + dst = append(dst, node.pattern) + } + return dst + } + return matchLevel(node.next, rest, dst) +} + +func (l *trieLevel) child(seg string) *trieNode { + switch seg { + case wildcardOne: + return l.pwc + case wildcardTail: + return l.fwc + default: + return l.nodes[seg] + } +} + +func (l *trieLevel) literal(seg string) *trieNode { + return l.nodes[seg] +} + +func (l *trieLevel) setChild(seg string, n *trieNode) { + switch seg { + case wildcardOne: + l.pwc = n + case wildcardTail: + l.fwc = n + default: + if l.nodes == nil { + l.nodes = make(map[string]*trieNode) + } + l.nodes[seg] = n + } +} + +func (l *trieLevel) deleteChild(seg string) { + switch seg { + case wildcardOne: + l.pwc = nil + case wildcardTail: + l.fwc = nil + default: + delete(l.nodes, seg) + } +} + +func (l *trieLevel) empty() bool { + return len(l.nodes) == 0 && l.pwc == nil && l.fwc == nil +} diff --git a/commonspace/pubsub/trie_test.go b/commonspace/pubsub/trie_test.go new file mode 100644 index 000000000..b721db666 --- /dev/null +++ b/commonspace/pubsub/trie_test.go @@ -0,0 +1,118 @@ +package pubsub + +import ( + "sort" + "testing" + + "github.com/stretchr/testify/require" +) + +func match(t *patternTrie, topic string) []string { + res := t.Match(topic, nil) + sort.Strings(res) + return res +} + +func TestTrieExactMatch(t *testing.T) { + tr := newPatternTrie() + require.True(t, tr.Add("presence")) + require.True(t, tr.Add("chat/abc/typing")) + + require.Equal(t, []string{"presence"}, match(tr, "presence")) + require.Equal(t, []string{"chat/abc/typing"}, match(tr, "chat/abc/typing")) + require.Empty(t, match(tr, "chat/abc")) + require.Empty(t, match(tr, "chat/abc/typing/extra")) + require.Empty(t, match(tr, "other")) +} + +func TestTrieSingleSegmentWildcard(t *testing.T) { + tr := newPatternTrie() + tr.Add("chat/*/typing") + + require.Equal(t, []string{"chat/*/typing"}, match(tr, "chat/abc/typing")) + require.Empty(t, match(tr, "chat/typing")) // '*' must consume one segment + require.Empty(t, match(tr, "chat/a/b/typing")) // '*' consumes exactly one + require.Empty(t, match(tr, "chat/abc/typing/late")) // trailing extra segment + + tr.Add("*") + require.Equal(t, []string{"*"}, match(tr, "presence")) + require.Empty(t, match(tr, "a/b")) +} + +func TestTrieTailWildcard(t *testing.T) { + tr := newPatternTrie() + tr.Add("chat/>") + + require.Equal(t, []string{"chat/>"}, match(tr, "chat/abc")) + require.Equal(t, []string{"chat/>"}, match(tr, "chat/a/b/c")) + require.Empty(t, match(tr, "chat")) // '>' requires at least one segment + + tr.Add(">") + require.Equal(t, []string{">"}, match(tr, "anything")) + require.Equal(t, []string{">", "chat/>"}, match(tr, "chat/x")) +} + +func TestTrieOverlappingPatterns(t *testing.T) { + tr := newPatternTrie() + tr.Add("chat/>") + tr.Add("chat/*/typing") + tr.Add("chat/abc/typing") + + require.Equal(t, + []string{"chat/*/typing", "chat/>", "chat/abc/typing"}, + match(tr, "chat/abc/typing")) + require.Equal(t, []string{"chat/>"}, match(tr, "chat/abc/presence")) +} + +func TestTrieAccWildcard(t *testing.T) { + tr := newPatternTrie() + tr.Add("acc/online/*") + require.Equal(t, []string{"acc/online/*"}, match(tr, "acc/online/A1")) + require.Equal(t, []string{"acc/online/*"}, match(tr, "acc/online/A2")) + require.Empty(t, match(tr, "acc/cursor/A1")) +} + +func TestTrieRefcounting(t *testing.T) { + tr := newPatternTrie() + require.True(t, tr.Add("chat/>")) + require.False(t, tr.Add("chat/>")) // second subscriber, same pattern + require.Equal(t, 1, tr.Len()) + + require.False(t, tr.Remove("chat/>")) // still one ref left + require.Equal(t, []string{"chat/>"}, match(tr, "chat/x")) + + require.True(t, tr.Remove("chat/>")) // last ref gone + require.Empty(t, match(tr, "chat/x")) + require.Equal(t, 0, tr.Len()) + + // removing a non-existent pattern is a no-op + require.False(t, tr.Remove("chat/>")) + require.False(t, tr.Remove("never/added")) +} + +func TestTriePruning(t *testing.T) { + tr := newPatternTrie() + tr.Add("a/b/c/d") + tr.Add("a/b/x") + require.True(t, tr.Remove("a/b/c/d")) + // sibling under the shared prefix still matches + require.Equal(t, []string{"a/b/x"}, match(tr, "a/b/x")) + require.Empty(t, match(tr, "a/b/c/d")) + + tr.Remove("a/b/x") + require.True(t, tr.root.empty(), "trie should be fully pruned") +} + +func TestTrieInteriorTerminal(t *testing.T) { + tr := newPatternTrie() + // a pattern that is a prefix of another + tr.Add("chat") + tr.Add("chat/abc") + require.Equal(t, []string{"chat"}, match(tr, "chat")) + require.Equal(t, []string{"chat/abc"}, match(tr, "chat/abc")) + + // removing the prefix pattern keeps the longer one intact + require.True(t, tr.Remove("chat")) + require.Empty(t, match(tr, "chat")) + require.Equal(t, []string{"chat/abc"}, match(tr, "chat/abc")) +} diff --git a/commonspace/settings/settingsobject.go b/commonspace/settings/settingsobject.go index 9288c207a..0cc4b9aea 100644 --- a/commonspace/settings/settingsobject.go +++ b/commonspace/settings/settingsobject.go @@ -230,12 +230,12 @@ func (s *settingsObject) DeleteObject(ctx context.Context, id string) (err error return } - return s.addContent(res, isSnapshot) + return s.addContent(ctx, res, isSnapshot) } -func (s *settingsObject) addContent(data []byte, isSnapshot bool) (err error) { +func (s *settingsObject) addContent(ctx context.Context, data []byte, isSnapshot bool) (err error) { accountData := s.account.Account() - res, err := s.AddContent(context.Background(), objecttree.SignableChangeContent{ + res, err := s.AddContent(ctx, objecttree.SignableChangeContent{ Data: data, Key: accountData.SignKey, IsSnapshot: isSnapshot, diff --git a/commonspace/spacepayloads/payloads.go b/commonspace/spacepayloads/payloads.go index 9a30f09ec..9517a70cb 100644 --- a/commonspace/spacepayloads/payloads.go +++ b/commonspace/spacepayloads/payloads.go @@ -42,6 +42,8 @@ type SpaceCreatePayload struct { Metadata []byte // Options is the ACL space options (e.g. delete restrictions) Options *aclrecordproto.AclSpaceOptions + // FileProtoVersion gates the file protocol the space uses (embedded in the signed header) + FileProtoVersion spacesyncproto.SpaceFileProtoVersion } type SpaceDerivePayload struct { @@ -49,6 +51,8 @@ type SpaceDerivePayload struct { MasterKey crypto.PrivKey SpaceType string SpacePayload []byte + // FileProtoVersion gates the file protocol the space uses (embedded in the signed header) + FileProtoVersion spacesyncproto.SpaceFileProtoVersion } const ( @@ -78,6 +82,7 @@ func StoragePayloadForSpaceCreate(payload SpaceCreatePayload) (storagePayload sp SpaceHeaderPayload: payload.SpacePayload, ReplicationKey: payload.ReplicationKey, Seed: spaceHeaderSeed, + FileprotoVersion: payload.FileProtoVersion, } marshalled, err := header.MarshalVT() if err != nil { @@ -165,6 +170,7 @@ func StoragePayloadForSpaceCreateV1(payload SpaceCreatePayload) (storagePayload SpaceHeaderPayload: payload.SpacePayload, ReplicationKey: payload.ReplicationKey, Seed: spaceHeaderSeed, + FileprotoVersion: payload.FileProtoVersion, Version: spacesyncproto.SpaceHeaderVersion_SpaceHeaderVersion1, } @@ -261,6 +267,7 @@ func StoragePayloadForSpaceDerive(payload SpaceDerivePayload) (storagePayload sp SpaceType: payload.SpaceType, SpaceHeaderPayload: payload.SpacePayload, ReplicationKey: repKey, + FileprotoVersion: payload.FileProtoVersion, } marshalled, err := header.MarshalVT() if err != nil { @@ -315,6 +322,90 @@ func StoragePayloadForSpaceDerive(payload SpaceDerivePayload) (storagePayload sp return } +func StoragePayloadForSpaceDeriveV1(payload SpaceDerivePayload) (storagePayload spacestorage.SpaceStorageCreatePayload, err error) { + // marshalling keys + identity, err := payload.SigningKey.GetPublic().Marshall() + if err != nil { + return + } + pubKey, err := payload.SigningKey.GetPublic().Raw() + if err != nil { + return + } + + // preparing replication key + hasher := fnv.New64() + _, err = hasher.Write(pubKey) + if err != nil { + return + } + repKey := hasher.Sum64() + + // preparing header (acl and settings payloads are embedded below, before signing) + header := &spacesyncproto.SpaceHeader{ + Identity: identity, + SpaceType: payload.SpaceType, + SpaceHeaderPayload: payload.SpacePayload, + ReplicationKey: repKey, + FileprotoVersion: payload.FileProtoVersion, + Version: spacesyncproto.SpaceHeaderVersion_SpaceHeaderVersion1, + } + + // building acl root (no SpaceId: for v1 the space id derives from the header that embeds this payload) + keyStorage := crypto.NewKeyStorage() + aclBuilder := list.NewAclRecordBuilder("", keyStorage, nil, recordverifier.NewValidateFull()) + aclRoot, err := aclBuilder.BuildRoot(list.RootContent{ + PrivKey: payload.SigningKey, + MasterKey: payload.MasterKey, + }) + if err != nil { + return + } + + // building settings (deterministic: no seed, no timestamp) + builder := objecttree.NewChangeBuilder(keyStorage, nil) + _, settingsRoot, err := builder.BuildRoot(objecttree.InitialContent{ + AclHeadId: aclRoot.Id, + PrivKey: payload.SigningKey, + ChangeType: SpaceReserved, + }) + if err != nil { + return + } + + // build header + header.AclPayload = aclRoot.Payload + header.SettingPayload = settingsRoot.RawChange + + marshalled, err := header.MarshalVT() + if err != nil { + return + } + signature, err := payload.SigningKey.Sign(marshalled) + if err != nil { + return + } + rawHeader := &spacesyncproto.RawSpaceHeader{SpaceHeader: marshalled, Signature: signature} + marshalled, err = rawHeader.MarshalVT() + if err != nil { + return + } + id, err := cidutil.NewCidFromBytes(marshalled) + spaceId := NewSpaceId(id, repKey) + rawHeaderWithId := &spacesyncproto.RawSpaceHeaderWithId{ + RawHeader: marshalled, + Id: spaceId, + } + + // creating storage + storagePayload = spacestorage.SpaceStorageCreatePayload{ + AclWithId: aclRoot, + SpaceHeaderWithId: rawHeaderWithId, + SpaceSettingsWithId: settingsRoot, + } + return +} + func makeOneToOneInfo(sharedSk crypto.PrivKey, aPk, bPk crypto.PubKey) (oneToOneInfo aclrecordproto.AclOneToOneInfo, err error) { writers := make([][]byte, 2) diff --git a/commonspace/spacepayloads/payloads_test.go b/commonspace/spacepayloads/payloads_test.go index d7f68cbd1..8508a869a 100644 --- a/commonspace/spacepayloads/payloads_test.go +++ b/commonspace/spacepayloads/payloads_test.go @@ -878,6 +878,146 @@ func TestStoragePayloadForSpaceDerive(t *testing.T) { }) } +func TestStoragePayloadForSpaceDeriveV1(t *testing.T) { + t.Run("success: builds valid v1 derived space payload", func(t *testing.T) { + acc, err := accountdata.NewRandom() + require.NoError(t, err) + + master, _, err := crypto.GenerateRandomEd25519KeyPair() + require.NoError(t, err) + + pl := SpaceDerivePayload{ + SigningKey: acc.SignKey, + MasterKey: master, + SpaceType: "derived.space", + SpacePayload: randBytes(10), + } + + out, err := StoragePayloadForSpaceDeriveV1(pl) + require.NoError(t, err) + + // v1 payload should validate as a set (acl + settings embedded in header). + require.NoError(t, ValidateSpaceStorageCreatePayload(out)) + }) + + t.Run("deterministic: same keys yield same space id", func(t *testing.T) { + acc, err := accountdata.NewRandom() + require.NoError(t, err) + + master, _, err := crypto.GenerateRandomEd25519KeyPair() + require.NoError(t, err) + + pl := SpaceDerivePayload{ + SigningKey: acc.SignKey, + MasterKey: master, + SpaceType: "derived.space", + SpacePayload: []byte("stable-payload"), + } + + out1, err := StoragePayloadForSpaceDeriveV1(pl) + require.NoError(t, err) + out2, err := StoragePayloadForSpaceDeriveV1(pl) + require.NoError(t, err) + + require.Equal(t, out1.SpaceHeaderWithId.Id, out2.SpaceHeaderWithId.Id) + require.Equal(t, out1.SpaceHeaderWithId.RawHeader, out2.SpaceHeaderWithId.RawHeader) + }) +} + +func decodeSpaceHeader(t *testing.T, out spacestorage.SpaceStorageCreatePayload) *spacesyncproto.SpaceHeader { + var raw spacesyncproto.RawSpaceHeader + require.NoError(t, raw.UnmarshalVT(out.SpaceHeaderWithId.RawHeader)) + var h spacesyncproto.SpaceHeader + require.NoError(t, h.UnmarshalVT(raw.SpaceHeader)) + return &h +} + +func TestSpaceHeaderFileProtoVersion(t *testing.T) { + newCreatePayload := func(t *testing.T, fpv spacesyncproto.SpaceFileProtoVersion) SpaceCreatePayload { + acc, err := accountdata.NewRandom() + require.NoError(t, err) + master, _, err := crypto.GenerateRandomEd25519KeyPair() + require.NoError(t, err) + metaKey, _, err := crypto.GenerateRandomEd25519KeyPair() + require.NoError(t, err) + readKey, _ := crypto.NewRandomAES() + return SpaceCreatePayload{ + SigningKey: acc.SignKey, + SpaceType: "test.space", + ReplicationKey: mrand.Uint64(), + SpacePayload: randBytes(12), + MasterKey: master, + ReadKey: readKey, + MetadataKey: metaKey, + Metadata: randBytes(6), + FileProtoVersion: fpv, + } + } + + t.Run("create v1 carries fileprotoVersion=V2 and survives round-trip", func(t *testing.T) { + out, err := StoragePayloadForSpaceCreateV1(newCreatePayload(t, spacesyncproto.SpaceFileProtoVersion_SpaceFileProtoVersionV2)) + require.NoError(t, err) + require.NoError(t, ValidateSpaceStorageCreatePayload(out)) + require.Equal(t, spacesyncproto.SpaceFileProtoVersion_SpaceFileProtoVersionV2, decodeSpaceHeader(t, out).GetFileprotoVersion()) + }) + + t.Run("create v0 carries fileprotoVersion=V2", func(t *testing.T) { + out, err := StoragePayloadForSpaceCreate(newCreatePayload(t, spacesyncproto.SpaceFileProtoVersion_SpaceFileProtoVersionV2)) + require.NoError(t, err) + require.NoError(t, ValidateSpaceStorageCreatePayload(out)) + require.Equal(t, spacesyncproto.SpaceFileProtoVersion_SpaceFileProtoVersionV2, decodeSpaceHeader(t, out).GetFileprotoVersion()) + }) + + t.Run("derive v1 carries fileprotoVersion=V2", func(t *testing.T) { + acc, err := accountdata.NewRandom() + require.NoError(t, err) + master, _, err := crypto.GenerateRandomEd25519KeyPair() + require.NoError(t, err) + out, err := StoragePayloadForSpaceDeriveV1(SpaceDerivePayload{ + SigningKey: acc.SignKey, + MasterKey: master, + SpaceType: "derived.space", + SpacePayload: randBytes(10), + FileProtoVersion: spacesyncproto.SpaceFileProtoVersion_SpaceFileProtoVersionV2, + }) + require.NoError(t, err) + require.NoError(t, ValidateSpaceStorageCreatePayload(out)) + require.Equal(t, spacesyncproto.SpaceFileProtoVersion_SpaceFileProtoVersionV2, decodeSpaceHeader(t, out).GetFileprotoVersion()) + }) + + t.Run("unset defaults to Unspecified", func(t *testing.T) { + out, err := StoragePayloadForSpaceCreateV1(newCreatePayload(t, spacesyncproto.SpaceFileProtoVersion_SpaceFileProtoVersionUnspecified)) + require.NoError(t, err) + require.Equal(t, spacesyncproto.SpaceFileProtoVersion_SpaceFileProtoVersionUnspecified, decodeSpaceHeader(t, out).GetFileprotoVersion()) + }) + + t.Run("fileprotoVersion is covered by the header signature (tamper rejected)", func(t *testing.T) { + out, err := StoragePayloadForSpaceCreateV1(newCreatePayload(t, spacesyncproto.SpaceFileProtoVersion_SpaceFileProtoVersionUnspecified)) + require.NoError(t, err) + // original validates + _, err = ValidateSpaceHeader(out.SpaceHeaderWithId, nil, out.AclWithId.Payload, out.SpaceSettingsWithId.RawChange) + require.NoError(t, err) + + // flip fileprotoVersion inside the signed header body, keep the original signature + var raw spacesyncproto.RawSpaceHeader + require.NoError(t, raw.UnmarshalVT(out.SpaceHeaderWithId.RawHeader)) + var h spacesyncproto.SpaceHeader + require.NoError(t, h.UnmarshalVT(raw.SpaceHeader)) + h.FileprotoVersion = spacesyncproto.SpaceFileProtoVersion_SpaceFileProtoVersionV2 + raw.SpaceHeader, err = h.MarshalVT() + require.NoError(t, err) + tamperedRaw, err := raw.MarshalVT() + require.NoError(t, err) + + // recompute the cid so we pass the cid check and reach the signature check + newCid, err := cidutil.NewCidFromBytes(tamperedRaw) + require.NoError(t, err) + tampered := &spacesyncproto.RawSpaceHeaderWithId{RawHeader: tamperedRaw, Id: NewSpaceId(newCid, h.ReplicationKey)} + _, err = ValidateSpaceHeader(tampered, nil, out.AclWithId.Payload, out.SpaceSettingsWithId.RawChange) + require.ErrorIs(t, err, spacestorage.ErrIncorrectSpaceHeader) + }) +} + func randBytes(n int) []byte { b := make([]byte, n) rand.Read(b) diff --git a/commonspace/spaceservice.go b/commonspace/spaceservice.go index d6384f2d9..8aa84d627 100644 --- a/commonspace/spaceservice.go +++ b/commonspace/spaceservice.go @@ -127,9 +127,7 @@ func (s *spaceService) buildRecordVerifier() (recordverifier.RecordVerifier, err } func (s *spaceService) CreateSpace(ctx context.Context, payload spacepayloads.SpaceCreatePayload) (id string, err error) { - // TODO: switch to SpaceCreatePayloadV1 after next proto update - // storageCreate, err := spacepayloads.StoragePayloadForSpaceCreateV1(payload) - storageCreate, err := spacepayloads.StoragePayloadForSpaceCreate(payload) + storageCreate, err := spacepayloads.StoragePayloadForSpaceCreateV1(payload) if err != nil { return } @@ -145,7 +143,7 @@ func (s *spaceService) CreateSpace(ctx context.Context, payload spacepayloads.Sp } func (s *spaceService) DeriveId(ctx context.Context, payload spacepayloads.SpaceDerivePayload) (id string, err error) { - storageCreate, err := spacepayloads.StoragePayloadForSpaceDerive(payload) + storageCreate, err := spacepayloads.StoragePayloadForSpaceDeriveV1(payload) if err != nil { return } @@ -154,7 +152,7 @@ func (s *spaceService) DeriveId(ctx context.Context, payload spacepayloads.Space } func (s *spaceService) DeriveSpace(ctx context.Context, payload spacepayloads.SpaceDerivePayload) (id string, err error) { - storageCreate, err := spacepayloads.StoragePayloadForSpaceDerive(payload) + storageCreate, err := spacepayloads.StoragePayloadForSpaceDeriveV1(payload) if err != nil { return } diff --git a/commonspace/spacestorage/migration/spacemigrator.go b/commonspace/spacestorage/migration/spacemigrator.go index 182b1e8b4..8fba4ce67 100644 --- a/commonspace/spacestorage/migration/spacemigrator.go +++ b/commonspace/spacestorage/migration/spacemigrator.go @@ -160,7 +160,7 @@ func (s *spaceMigrator) migrateHash(ctx context.Context, oldStorage oldstorage.S if err != nil { return err } - return newStorage.StateStorage().SetHash(ctx, spaceHash, spaceHash) + return newStorage.StateStorage().SetHash(ctx, spaceHash) } func (s *spaceMigrator) checkMigrated(ctx context.Context, id string) (bool, spacestorage.SpaceStorage) { diff --git a/commonspace/spacesyncproto/protos/spacesync.proto b/commonspace/spacesyncproto/protos/spacesync.proto index 92f1d8b70..21c9b7bd2 100644 --- a/commonspace/spacesyncproto/protos/spacesync.proto +++ b/commonspace/spacesyncproto/protos/spacesync.proto @@ -125,6 +125,13 @@ enum SpaceHeaderVersion { SpaceHeaderVersion1 = 1; } +// SpaceFileProtoVersion gates the file protocol a space uses. It lives in the signed, +// immutable header so it can't be stripped/downgraded without invalidating the owner signature. +enum SpaceFileProtoVersion { + SpaceFileProtoVersionUnspecified = 0; + SpaceFileProtoVersionV2 = 2; +} + // SpaceHeader is a header for a space message SpaceHeader { bytes identity = 1; @@ -138,6 +145,9 @@ message SpaceHeader { bytes aclPayload = 7; bytes settingPayload = 8; + // fileprotoVersion gates the file protocol (v2 = filenode v2 + S3-direct + networkSign) + SpaceFileProtoVersion fileprotoVersion = 9; + SpaceHeaderVersion version = 100; } @@ -262,8 +272,8 @@ message StorageHeader { // DiffType is a type of diff enum DiffType { Initial = 0; - V1 = 1; - V2 = 2; + V1 = 1; // deprecated, not supported anymore + V2 = 2; // deprecated, not supported anymore V3 = 3; } diff --git a/commonspace/spacesyncproto/spacesync.pb.go b/commonspace/spacesyncproto/spacesync.pb.go index 7907ba194..f18393ebd 100644 --- a/commonspace/spacesyncproto/spacesync.pb.go +++ b/commonspace/spacesyncproto/spacesync.pb.go @@ -140,6 +140,54 @@ func (SpaceHeaderVersion) EnumDescriptor() ([]byte, []int) { return file_commonspace_spacesyncproto_protos_spacesync_proto_rawDescGZIP(), []int{1} } +// SpaceFileProtoVersion gates the file protocol a space uses. It lives in the signed, +// immutable header so it can't be stripped/downgraded without invalidating the owner signature. +type SpaceFileProtoVersion int32 + +const ( + SpaceFileProtoVersion_SpaceFileProtoVersionUnspecified SpaceFileProtoVersion = 0 + SpaceFileProtoVersion_SpaceFileProtoVersionV2 SpaceFileProtoVersion = 2 +) + +// Enum value maps for SpaceFileProtoVersion. +var ( + SpaceFileProtoVersion_name = map[int32]string{ + 0: "SpaceFileProtoVersionUnspecified", + 2: "SpaceFileProtoVersionV2", + } + SpaceFileProtoVersion_value = map[string]int32{ + "SpaceFileProtoVersionUnspecified": 0, + "SpaceFileProtoVersionV2": 2, + } +) + +func (x SpaceFileProtoVersion) Enum() *SpaceFileProtoVersion { + p := new(SpaceFileProtoVersion) + *p = x + return p +} + +func (x SpaceFileProtoVersion) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (SpaceFileProtoVersion) Descriptor() protoreflect.EnumDescriptor { + return file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes[2].Descriptor() +} + +func (SpaceFileProtoVersion) Type() protoreflect.EnumType { + return &file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes[2] +} + +func (x SpaceFileProtoVersion) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use SpaceFileProtoVersion.Descriptor instead. +func (SpaceFileProtoVersion) EnumDescriptor() ([]byte, []int) { + return file_commonspace_spacesyncproto_protos_spacesync_proto_rawDescGZIP(), []int{2} +} + // SpaceSubscription contains in ObjectSyncMessage.Payload and indicates that we need to subscribe or unsubscribe the current stream to this space type SpaceSubscriptionAction int32 @@ -171,11 +219,11 @@ func (x SpaceSubscriptionAction) String() string { } func (SpaceSubscriptionAction) Descriptor() protoreflect.EnumDescriptor { - return file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes[2].Descriptor() + return file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes[3].Descriptor() } func (SpaceSubscriptionAction) Type() protoreflect.EnumType { - return &file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes[2] + return &file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes[3] } func (x SpaceSubscriptionAction) Number() protoreflect.EnumNumber { @@ -184,7 +232,7 @@ func (x SpaceSubscriptionAction) Number() protoreflect.EnumNumber { // Deprecated: Use SpaceSubscriptionAction.Descriptor instead. func (SpaceSubscriptionAction) EnumDescriptor() ([]byte, []int) { - return file_commonspace_spacesyncproto_protos_spacesync_proto_rawDescGZIP(), []int{2} + return file_commonspace_spacesyncproto_protos_spacesync_proto_rawDescGZIP(), []int{3} } // DiffType is a type of diff @@ -224,11 +272,11 @@ func (x DiffType) String() string { } func (DiffType) Descriptor() protoreflect.EnumDescriptor { - return file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes[3].Descriptor() + return file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes[4].Descriptor() } func (DiffType) Type() protoreflect.EnumType { - return &file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes[3] + return &file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes[4] } func (x DiffType) Number() protoreflect.EnumNumber { @@ -237,7 +285,7 @@ func (x DiffType) Number() protoreflect.EnumNumber { // Deprecated: Use DiffType.Descriptor instead. func (DiffType) EnumDescriptor() ([]byte, []int) { - return file_commonspace_spacesyncproto_protos_spacesync_proto_rawDescGZIP(), []int{3} + return file_commonspace_spacesyncproto_protos_spacesync_proto_rawDescGZIP(), []int{4} } // ObjectType is a type of object @@ -274,11 +322,11 @@ func (x ObjectType) String() string { } func (ObjectType) Descriptor() protoreflect.EnumDescriptor { - return file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes[4].Descriptor() + return file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes[5].Descriptor() } func (ObjectType) Type() protoreflect.EnumType { - return &file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes[4] + return &file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes[5] } func (x ObjectType) Number() protoreflect.EnumNumber { @@ -287,7 +335,7 @@ func (x ObjectType) Number() protoreflect.EnumNumber { // Deprecated: Use ObjectType.Descriptor instead. func (ObjectType) EnumDescriptor() ([]byte, []int) { - return file_commonspace_spacesyncproto_protos_spacesync_proto_rawDescGZIP(), []int{4} + return file_commonspace_spacesyncproto_protos_spacesync_proto_rawDescGZIP(), []int{5} } // HeadSyncRange presenting a request for one range @@ -999,11 +1047,13 @@ type SpaceHeader struct { Seed []byte `protobuf:"bytes,5,opt,name=seed,proto3" json:"seed,omitempty"` SpaceHeaderPayload []byte `protobuf:"bytes,6,opt,name=spaceHeaderPayload,proto3" json:"spaceHeaderPayload,omitempty"` // since v1 fields - AclPayload []byte `protobuf:"bytes,7,opt,name=aclPayload,proto3" json:"aclPayload,omitempty"` - SettingPayload []byte `protobuf:"bytes,8,opt,name=settingPayload,proto3" json:"settingPayload,omitempty"` - Version SpaceHeaderVersion `protobuf:"varint,100,opt,name=version,proto3,enum=spacesync.SpaceHeaderVersion" json:"version,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + AclPayload []byte `protobuf:"bytes,7,opt,name=aclPayload,proto3" json:"aclPayload,omitempty"` + SettingPayload []byte `protobuf:"bytes,8,opt,name=settingPayload,proto3" json:"settingPayload,omitempty"` + // fileprotoVersion gates the file protocol (v2 = filenode v2 + S3-direct + networkSign) + FileprotoVersion SpaceFileProtoVersion `protobuf:"varint,9,opt,name=fileprotoVersion,proto3,enum=spacesync.SpaceFileProtoVersion" json:"fileprotoVersion,omitempty"` + Version SpaceHeaderVersion `protobuf:"varint,100,opt,name=version,proto3,enum=spacesync.SpaceHeaderVersion" json:"version,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SpaceHeader) Reset() { @@ -1092,6 +1142,13 @@ func (x *SpaceHeader) GetSettingPayload() []byte { return nil } +func (x *SpaceHeader) GetFileprotoVersion() SpaceFileProtoVersion { + if x != nil { + return x.FileprotoVersion + } + return SpaceFileProtoVersion_SpaceFileProtoVersionUnspecified +} + func (x *SpaceHeader) GetVersion() SpaceHeaderVersion { if x != nil { return x.Version @@ -2204,7 +2261,7 @@ const file_commonspace_spacesyncproto_protos_spacesync_proto_rawDesc = "" + "aclPayload\x12\"\n" + "\faclPayloadId\x18\x03 \x01(\tR\faclPayloadId\x122\n" + "\x14spaceSettingsPayload\x18\x04 \x01(\fR\x14spaceSettingsPayload\x126\n" + - "\x16spaceSettingsPayloadId\x18\x05 \x01(\tR\x16spaceSettingsPayloadId\"\xd2\x02\n" + + "\x16spaceSettingsPayloadId\x18\x05 \x01(\tR\x16spaceSettingsPayloadId\"\xa0\x03\n" + "\vSpaceHeader\x12\x1a\n" + "\bidentity\x18\x01 \x01(\fR\bidentity\x12\x1c\n" + "\ttimestamp\x18\x02 \x01(\x03R\ttimestamp\x12\x1c\n" + @@ -2215,7 +2272,8 @@ const file_commonspace_spacesyncproto_protos_spacesync_proto_rawDesc = "" + "\n" + "aclPayload\x18\a \x01(\fR\n" + "aclPayload\x12&\n" + - "\x0esettingPayload\x18\b \x01(\fR\x0esettingPayload\x127\n" + + "\x0esettingPayload\x18\b \x01(\fR\x0esettingPayload\x12L\n" + + "\x10fileprotoVersion\x18\t \x01(\x0e2 .spacesync.SpaceFileProtoVersionR\x10fileprotoVersion\x127\n" + "\aversion\x18d \x01(\x0e2\x1d.spacesync.SpaceHeaderVersionR\aversion\"P\n" + "\x0eRawSpaceHeader\x12 \n" + "\vspaceHeader\x18\x01 \x01(\fR\vspaceHeader\x12\x1c\n" + @@ -2294,7 +2352,10 @@ const file_commonspace_spacesyncproto_protos_spacesync_proto_rawDesc = "" + "\vErrorOffset\x10d*F\n" + "\x12SpaceHeaderVersion\x12\x17\n" + "\x13SpaceHeaderVersion0\x10\x00\x12\x17\n" + - "\x13SpaceHeaderVersion1\x10\x01*9\n" + + "\x13SpaceHeaderVersion1\x10\x01*Z\n" + + "\x15SpaceFileProtoVersion\x12$\n" + + " SpaceFileProtoVersionUnspecified\x10\x00\x12\x1b\n" + + "\x17SpaceFileProtoVersionV2\x10\x02*9\n" + "\x17SpaceSubscriptionAction\x12\r\n" + "\tSubscribe\x10\x00\x12\x0f\n" + "\vUnsubscribe\x10\x01*/\n" + @@ -2333,92 +2394,94 @@ func file_commonspace_spacesyncproto_protos_spacesync_proto_rawDescGZIP() []byte return file_commonspace_spacesyncproto_protos_spacesync_proto_rawDescData } -var file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes = make([]protoimpl.EnumInfo, 5) +var file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes = make([]protoimpl.EnumInfo, 6) var file_commonspace_spacesyncproto_protos_spacesync_proto_msgTypes = make([]protoimpl.MessageInfo, 32) var file_commonspace_spacesyncproto_protos_spacesync_proto_goTypes = []any{ (ErrCodes)(0), // 0: spacesync.ErrCodes (SpaceHeaderVersion)(0), // 1: spacesync.SpaceHeaderVersion - (SpaceSubscriptionAction)(0), // 2: spacesync.SpaceSubscriptionAction - (DiffType)(0), // 3: spacesync.DiffType - (ObjectType)(0), // 4: spacesync.ObjectType - (*HeadSyncRange)(nil), // 5: spacesync.HeadSyncRange - (*HeadSyncResult)(nil), // 6: spacesync.HeadSyncResult - (*HeadSyncResultElement)(nil), // 7: spacesync.HeadSyncResultElement - (*HeadSyncRequest)(nil), // 8: spacesync.HeadSyncRequest - (*HeadSyncResponse)(nil), // 9: spacesync.HeadSyncResponse - (*ObjectSyncMessage)(nil), // 10: spacesync.ObjectSyncMessage - (*SpacePushRequest)(nil), // 11: spacesync.SpacePushRequest - (*SpacePushResponse)(nil), // 12: spacesync.SpacePushResponse - (*SpacePullRequest)(nil), // 13: spacesync.SpacePullRequest - (*SpacePullResponse)(nil), // 14: spacesync.SpacePullResponse - (*AclRecord)(nil), // 15: spacesync.AclRecord - (*SpacePayload)(nil), // 16: spacesync.SpacePayload - (*SpaceHeader)(nil), // 17: spacesync.SpaceHeader - (*RawSpaceHeader)(nil), // 18: spacesync.RawSpaceHeader - (*RawSpaceHeaderWithId)(nil), // 19: spacesync.RawSpaceHeaderWithId - (*SpaceSettingsContent)(nil), // 20: spacesync.SpaceSettingsContent - (*ObjectDelete)(nil), // 21: spacesync.ObjectDelete - (*StoreHeader)(nil), // 22: spacesync.StoreHeader - (*SpaceDelete)(nil), // 23: spacesync.SpaceDelete - (*SpaceSettingsSnapshot)(nil), // 24: spacesync.SpaceSettingsSnapshot - (*SettingsData)(nil), // 25: spacesync.SettingsData - (*SpaceSubscription)(nil), // 26: spacesync.SpaceSubscription - (*AclAddRecordRequest)(nil), // 27: spacesync.AclAddRecordRequest - (*AclAddRecordResponse)(nil), // 28: spacesync.AclAddRecordResponse - (*AclGetRecordsRequest)(nil), // 29: spacesync.AclGetRecordsRequest - (*AclGetRecordsResponse)(nil), // 30: spacesync.AclGetRecordsResponse - (*StoreDiffRequest)(nil), // 31: spacesync.StoreDiffRequest - (*StoreDiffResponse)(nil), // 32: spacesync.StoreDiffResponse - (*StoreKeyValue)(nil), // 33: spacesync.StoreKeyValue - (*StoreKeyValues)(nil), // 34: spacesync.StoreKeyValues - (*StoreKeyInner)(nil), // 35: spacesync.StoreKeyInner - (*StorageHeader)(nil), // 36: spacesync.StorageHeader + (SpaceFileProtoVersion)(0), // 2: spacesync.SpaceFileProtoVersion + (SpaceSubscriptionAction)(0), // 3: spacesync.SpaceSubscriptionAction + (DiffType)(0), // 4: spacesync.DiffType + (ObjectType)(0), // 5: spacesync.ObjectType + (*HeadSyncRange)(nil), // 6: spacesync.HeadSyncRange + (*HeadSyncResult)(nil), // 7: spacesync.HeadSyncResult + (*HeadSyncResultElement)(nil), // 8: spacesync.HeadSyncResultElement + (*HeadSyncRequest)(nil), // 9: spacesync.HeadSyncRequest + (*HeadSyncResponse)(nil), // 10: spacesync.HeadSyncResponse + (*ObjectSyncMessage)(nil), // 11: spacesync.ObjectSyncMessage + (*SpacePushRequest)(nil), // 12: spacesync.SpacePushRequest + (*SpacePushResponse)(nil), // 13: spacesync.SpacePushResponse + (*SpacePullRequest)(nil), // 14: spacesync.SpacePullRequest + (*SpacePullResponse)(nil), // 15: spacesync.SpacePullResponse + (*AclRecord)(nil), // 16: spacesync.AclRecord + (*SpacePayload)(nil), // 17: spacesync.SpacePayload + (*SpaceHeader)(nil), // 18: spacesync.SpaceHeader + (*RawSpaceHeader)(nil), // 19: spacesync.RawSpaceHeader + (*RawSpaceHeaderWithId)(nil), // 20: spacesync.RawSpaceHeaderWithId + (*SpaceSettingsContent)(nil), // 21: spacesync.SpaceSettingsContent + (*ObjectDelete)(nil), // 22: spacesync.ObjectDelete + (*StoreHeader)(nil), // 23: spacesync.StoreHeader + (*SpaceDelete)(nil), // 24: spacesync.SpaceDelete + (*SpaceSettingsSnapshot)(nil), // 25: spacesync.SpaceSettingsSnapshot + (*SettingsData)(nil), // 26: spacesync.SettingsData + (*SpaceSubscription)(nil), // 27: spacesync.SpaceSubscription + (*AclAddRecordRequest)(nil), // 28: spacesync.AclAddRecordRequest + (*AclAddRecordResponse)(nil), // 29: spacesync.AclAddRecordResponse + (*AclGetRecordsRequest)(nil), // 30: spacesync.AclGetRecordsRequest + (*AclGetRecordsResponse)(nil), // 31: spacesync.AclGetRecordsResponse + (*StoreDiffRequest)(nil), // 32: spacesync.StoreDiffRequest + (*StoreDiffResponse)(nil), // 33: spacesync.StoreDiffResponse + (*StoreKeyValue)(nil), // 34: spacesync.StoreKeyValue + (*StoreKeyValues)(nil), // 35: spacesync.StoreKeyValues + (*StoreKeyInner)(nil), // 36: spacesync.StoreKeyInner + (*StorageHeader)(nil), // 37: spacesync.StorageHeader } var file_commonspace_spacesyncproto_protos_spacesync_proto_depIdxs = []int32{ - 7, // 0: spacesync.HeadSyncResult.elements:type_name -> spacesync.HeadSyncResultElement - 5, // 1: spacesync.HeadSyncRequest.ranges:type_name -> spacesync.HeadSyncRange - 3, // 2: spacesync.HeadSyncRequest.diffType:type_name -> spacesync.DiffType - 6, // 3: spacesync.HeadSyncResponse.results:type_name -> spacesync.HeadSyncResult - 3, // 4: spacesync.HeadSyncResponse.diffType:type_name -> spacesync.DiffType - 4, // 5: spacesync.ObjectSyncMessage.objectType:type_name -> spacesync.ObjectType - 16, // 6: spacesync.SpacePushRequest.payload:type_name -> spacesync.SpacePayload - 16, // 7: spacesync.SpacePullResponse.payload:type_name -> spacesync.SpacePayload - 15, // 8: spacesync.SpacePullResponse.aclRecords:type_name -> spacesync.AclRecord - 19, // 9: spacesync.SpacePayload.spaceHeader:type_name -> spacesync.RawSpaceHeaderWithId - 1, // 10: spacesync.SpaceHeader.version:type_name -> spacesync.SpaceHeaderVersion - 21, // 11: spacesync.SpaceSettingsContent.objectDelete:type_name -> spacesync.ObjectDelete - 23, // 12: spacesync.SpaceSettingsContent.spaceDelete:type_name -> spacesync.SpaceDelete - 20, // 13: spacesync.SettingsData.content:type_name -> spacesync.SpaceSettingsContent - 24, // 14: spacesync.SettingsData.snapshot:type_name -> spacesync.SpaceSettingsSnapshot - 2, // 15: spacesync.SpaceSubscription.action:type_name -> spacesync.SpaceSubscriptionAction - 5, // 16: spacesync.StoreDiffRequest.ranges:type_name -> spacesync.HeadSyncRange - 6, // 17: spacesync.StoreDiffResponse.results:type_name -> spacesync.HeadSyncResult - 33, // 18: spacesync.StoreKeyValues.keyValues:type_name -> spacesync.StoreKeyValue - 8, // 19: spacesync.SpaceSync.HeadSync:input_type -> spacesync.HeadSyncRequest - 31, // 20: spacesync.SpaceSync.StoreDiff:input_type -> spacesync.StoreDiffRequest - 33, // 21: spacesync.SpaceSync.StoreElements:input_type -> spacesync.StoreKeyValue - 11, // 22: spacesync.SpaceSync.SpacePush:input_type -> spacesync.SpacePushRequest - 13, // 23: spacesync.SpaceSync.SpacePull:input_type -> spacesync.SpacePullRequest - 10, // 24: spacesync.SpaceSync.ObjectSyncStream:input_type -> spacesync.ObjectSyncMessage - 10, // 25: spacesync.SpaceSync.ObjectSync:input_type -> spacesync.ObjectSyncMessage - 10, // 26: spacesync.SpaceSync.ObjectSyncRequestStream:input_type -> spacesync.ObjectSyncMessage - 27, // 27: spacesync.SpaceSync.AclAddRecord:input_type -> spacesync.AclAddRecordRequest - 29, // 28: spacesync.SpaceSync.AclGetRecords:input_type -> spacesync.AclGetRecordsRequest - 9, // 29: spacesync.SpaceSync.HeadSync:output_type -> spacesync.HeadSyncResponse - 32, // 30: spacesync.SpaceSync.StoreDiff:output_type -> spacesync.StoreDiffResponse - 33, // 31: spacesync.SpaceSync.StoreElements:output_type -> spacesync.StoreKeyValue - 12, // 32: spacesync.SpaceSync.SpacePush:output_type -> spacesync.SpacePushResponse - 14, // 33: spacesync.SpaceSync.SpacePull:output_type -> spacesync.SpacePullResponse - 10, // 34: spacesync.SpaceSync.ObjectSyncStream:output_type -> spacesync.ObjectSyncMessage - 10, // 35: spacesync.SpaceSync.ObjectSync:output_type -> spacesync.ObjectSyncMessage - 10, // 36: spacesync.SpaceSync.ObjectSyncRequestStream:output_type -> spacesync.ObjectSyncMessage - 28, // 37: spacesync.SpaceSync.AclAddRecord:output_type -> spacesync.AclAddRecordResponse - 30, // 38: spacesync.SpaceSync.AclGetRecords:output_type -> spacesync.AclGetRecordsResponse - 29, // [29:39] is the sub-list for method output_type - 19, // [19:29] is the sub-list for method input_type - 19, // [19:19] is the sub-list for extension type_name - 19, // [19:19] is the sub-list for extension extendee - 0, // [0:19] is the sub-list for field type_name + 8, // 0: spacesync.HeadSyncResult.elements:type_name -> spacesync.HeadSyncResultElement + 6, // 1: spacesync.HeadSyncRequest.ranges:type_name -> spacesync.HeadSyncRange + 4, // 2: spacesync.HeadSyncRequest.diffType:type_name -> spacesync.DiffType + 7, // 3: spacesync.HeadSyncResponse.results:type_name -> spacesync.HeadSyncResult + 4, // 4: spacesync.HeadSyncResponse.diffType:type_name -> spacesync.DiffType + 5, // 5: spacesync.ObjectSyncMessage.objectType:type_name -> spacesync.ObjectType + 17, // 6: spacesync.SpacePushRequest.payload:type_name -> spacesync.SpacePayload + 17, // 7: spacesync.SpacePullResponse.payload:type_name -> spacesync.SpacePayload + 16, // 8: spacesync.SpacePullResponse.aclRecords:type_name -> spacesync.AclRecord + 20, // 9: spacesync.SpacePayload.spaceHeader:type_name -> spacesync.RawSpaceHeaderWithId + 2, // 10: spacesync.SpaceHeader.fileprotoVersion:type_name -> spacesync.SpaceFileProtoVersion + 1, // 11: spacesync.SpaceHeader.version:type_name -> spacesync.SpaceHeaderVersion + 22, // 12: spacesync.SpaceSettingsContent.objectDelete:type_name -> spacesync.ObjectDelete + 24, // 13: spacesync.SpaceSettingsContent.spaceDelete:type_name -> spacesync.SpaceDelete + 21, // 14: spacesync.SettingsData.content:type_name -> spacesync.SpaceSettingsContent + 25, // 15: spacesync.SettingsData.snapshot:type_name -> spacesync.SpaceSettingsSnapshot + 3, // 16: spacesync.SpaceSubscription.action:type_name -> spacesync.SpaceSubscriptionAction + 6, // 17: spacesync.StoreDiffRequest.ranges:type_name -> spacesync.HeadSyncRange + 7, // 18: spacesync.StoreDiffResponse.results:type_name -> spacesync.HeadSyncResult + 34, // 19: spacesync.StoreKeyValues.keyValues:type_name -> spacesync.StoreKeyValue + 9, // 20: spacesync.SpaceSync.HeadSync:input_type -> spacesync.HeadSyncRequest + 32, // 21: spacesync.SpaceSync.StoreDiff:input_type -> spacesync.StoreDiffRequest + 34, // 22: spacesync.SpaceSync.StoreElements:input_type -> spacesync.StoreKeyValue + 12, // 23: spacesync.SpaceSync.SpacePush:input_type -> spacesync.SpacePushRequest + 14, // 24: spacesync.SpaceSync.SpacePull:input_type -> spacesync.SpacePullRequest + 11, // 25: spacesync.SpaceSync.ObjectSyncStream:input_type -> spacesync.ObjectSyncMessage + 11, // 26: spacesync.SpaceSync.ObjectSync:input_type -> spacesync.ObjectSyncMessage + 11, // 27: spacesync.SpaceSync.ObjectSyncRequestStream:input_type -> spacesync.ObjectSyncMessage + 28, // 28: spacesync.SpaceSync.AclAddRecord:input_type -> spacesync.AclAddRecordRequest + 30, // 29: spacesync.SpaceSync.AclGetRecords:input_type -> spacesync.AclGetRecordsRequest + 10, // 30: spacesync.SpaceSync.HeadSync:output_type -> spacesync.HeadSyncResponse + 33, // 31: spacesync.SpaceSync.StoreDiff:output_type -> spacesync.StoreDiffResponse + 34, // 32: spacesync.SpaceSync.StoreElements:output_type -> spacesync.StoreKeyValue + 13, // 33: spacesync.SpaceSync.SpacePush:output_type -> spacesync.SpacePushResponse + 15, // 34: spacesync.SpaceSync.SpacePull:output_type -> spacesync.SpacePullResponse + 11, // 35: spacesync.SpaceSync.ObjectSyncStream:output_type -> spacesync.ObjectSyncMessage + 11, // 36: spacesync.SpaceSync.ObjectSync:output_type -> spacesync.ObjectSyncMessage + 11, // 37: spacesync.SpaceSync.ObjectSyncRequestStream:output_type -> spacesync.ObjectSyncMessage + 29, // 38: spacesync.SpaceSync.AclAddRecord:output_type -> spacesync.AclAddRecordResponse + 31, // 39: spacesync.SpaceSync.AclGetRecords:output_type -> spacesync.AclGetRecordsResponse + 30, // [30:40] is the sub-list for method output_type + 20, // [20:30] is the sub-list for method input_type + 20, // [20:20] is the sub-list for extension type_name + 20, // [20:20] is the sub-list for extension extendee + 0, // [0:20] is the sub-list for field type_name } func init() { file_commonspace_spacesyncproto_protos_spacesync_proto_init() } @@ -2435,7 +2498,7 @@ func file_commonspace_spacesyncproto_protos_spacesync_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_commonspace_spacesyncproto_protos_spacesync_proto_rawDesc), len(file_commonspace_spacesyncproto_protos_spacesync_proto_rawDesc)), - NumEnums: 5, + NumEnums: 6, NumMessages: 32, NumExtensions: 0, NumServices: 1, diff --git a/commonspace/spacesyncproto/spacesync_vtproto.pb.go b/commonspace/spacesyncproto/spacesync_vtproto.pb.go index 6ce78ed40..1f1ae6093 100644 --- a/commonspace/spacesyncproto/spacesync_vtproto.pb.go +++ b/commonspace/spacesyncproto/spacesync_vtproto.pb.go @@ -693,6 +693,11 @@ func (m *SpaceHeader) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i-- dAtA[i] = 0xa0 } + if m.FileprotoVersion != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.FileprotoVersion)) + i-- + dAtA[i] = 0x48 + } if len(m.SettingPayload) > 0 { i -= len(m.SettingPayload) copy(dAtA[i:], m.SettingPayload) @@ -1997,6 +2002,9 @@ func (m *SpaceHeader) SizeVT() (n int) { if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } + if m.FileprotoVersion != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.FileprotoVersion)) + } if m.Version != 0 { n += 2 + protohelpers.SizeOfVarint(uint64(m.Version)) } @@ -4219,6 +4227,25 @@ func (m *SpaceHeader) UnmarshalVT(dAtA []byte) error { m.SettingPayload = []byte{} } iNdEx = postIndex + case 9: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FileprotoVersion", wireType) + } + m.FileprotoVersion = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.FileprotoVersion |= SpaceFileProtoVersion(b&0x7F) << shift + if b < 0x80 { + break + } + } case 100: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType) diff --git a/commonspace/sync/objectsync/objectsync_test.go b/commonspace/sync/objectsync/objectsync_test.go index 1062538eb..3ac7a5e9a 100644 --- a/commonspace/sync/objectsync/objectsync_test.go +++ b/commonspace/sync/objectsync/objectsync_test.go @@ -15,6 +15,7 @@ import ( "github.com/anyproto/any-sync/commonspace/object/tree/synctree" "github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto" "github.com/anyproto/any-sync/commonspace/object/treemanager" + "github.com/anyproto/any-sync/commonspace/object/treesyncer" "github.com/anyproto/any-sync/commonspace/objectmanager/mock_objectmanager" "github.com/anyproto/any-sync/commonspace/spacestate" "github.com/anyproto/any-sync/commonspace/sync/objectsync/objectmessages" @@ -79,6 +80,85 @@ func TestObjectSync_HandleHeadUpdate(t *testing.T) { req := synctree.NewRequest(update.Meta.PeerId, update.Meta.SpaceId, update.Meta.ObjectId, nil, nil, nil) require.Equal(t, req, r) }) + t.Run("handle head update object missing, pull filter declines, update swallowed", func(t *testing.T) { + root := &treechangeproto.RawTreeChangeWithId{ + RawChange: []byte("rootChange"), + Id: "objectId", + } + var gotRoot *treechangeproto.RawTreeChangeWithId + var gotHeads []string + fx := newFixtureWithPullFilter(t, func(_ context.Context, objectId string, rootChange *treechangeproto.RawTreeChangeWithId, heads []string) bool { + require.Equal(t, "objectId", objectId) + gotRoot = rootChange + gotHeads = heads + return false + }) + defer fx.close(t) + update := headUpdateWithRoot(t, root, []string{"headId"}) + ctx = peer.CtxWithPeerId(ctx, "peerId") + fx.objectManager.EXPECT().GetObject(context.Background(), "objectId").Return(nil, fmt.Errorf("no object")) + r, err := fx.objectSync.HandleHeadUpdate(ctx, update) + require.NoError(t, err) + require.Nil(t, r) + require.Equal(t, root.Id, gotRoot.Id) + require.Equal(t, root.RawChange, gotRoot.RawChange) + require.Equal(t, []string{"headId"}, gotHeads) + }) + t.Run("handle head update object missing, pull filter accepts, return request", func(t *testing.T) { + root := &treechangeproto.RawTreeChangeWithId{ + RawChange: []byte("rootChange"), + Id: "objectId", + } + fx := newFixtureWithPullFilter(t, func(_ context.Context, _ string, _ *treechangeproto.RawTreeChangeWithId, _ []string) bool { + return true + }) + defer fx.close(t) + update := headUpdateWithRoot(t, root, []string{"headId"}) + ctx = peer.CtxWithPeerId(ctx, "peerId") + fx.objectManager.EXPECT().GetObject(context.Background(), "objectId").Return(nil, fmt.Errorf("no object")) + r, err := fx.objectSync.HandleHeadUpdate(ctx, update) + require.NoError(t, err) + req := synctree.NewRequest("peerId", "spaceId", "objectId", nil, nil, nil) + require.Equal(t, req, r) + }) + t.Run("handle head update object missing, no root in update, pull filter skipped, return request", func(t *testing.T) { + fx := newFixtureWithPullFilter(t, func(_ context.Context, _ string, _ *treechangeproto.RawTreeChangeWithId, _ []string) bool { + require.Fail(t, "filter must not be consulted without a root change") + return false + }) + defer fx.close(t) + update := &objectmessages.HeadUpdate{ + Meta: objectmessages.ObjectMeta{ + PeerId: "peerId", + ObjectId: "objectId", + SpaceId: "spaceId", + }, + } + ctx = peer.CtxWithPeerId(ctx, "peerId") + fx.objectManager.EXPECT().GetObject(context.Background(), "objectId").Return(nil, fmt.Errorf("no object")) + r, err := fx.objectSync.HandleHeadUpdate(ctx, update) + require.NoError(t, err) + req := synctree.NewRequest("peerId", "spaceId", "objectId", nil, nil, nil) + require.Equal(t, req, r) + }) +} + +func headUpdateWithRoot(t *testing.T, root *treechangeproto.RawTreeChangeWithId, heads []string) *objectmessages.HeadUpdate { + treeHeadUpdate := &treechangeproto.TreeHeadUpdate{ + Heads: heads, + SnapshotPath: []string{root.Id}, + } + wrapped := treechangeproto.WrapHeadUpdate(treeHeadUpdate, root) + marshaled, err := wrapped.MarshalVT() + require.NoError(t, err) + return &objectmessages.HeadUpdate{ + Bytes: marshaled, + Meta: objectmessages.ObjectMeta{ + PeerId: "peerId", + ObjectId: "objectId", + SpaceId: "spaceId", + }, + } } func TestObjectSync_HandleStreamRequest(t *testing.T) { @@ -168,6 +248,31 @@ type fixture struct { } func newFixture(t *testing.T) *fixture { + return newFixtureWithPullFilter(t, nil) +} + +// filteringTreeSyncer is a minimal TreeSyncer that also implements the +// optional treesyncer.PullFilter extension. +type filteringTreeSyncer struct { + shouldPull func(ctx context.Context, objectId string, rootChange *treechangeproto.RawTreeChangeWithId, heads []string) bool +} + +func (f *filteringTreeSyncer) Init(a *app.App) error { return nil } +func (f *filteringTreeSyncer) Name() string { return treesyncer.CName } +func (f *filteringTreeSyncer) Run(ctx context.Context) error { return nil } +func (f *filteringTreeSyncer) Close(ctx context.Context) error { return nil } +func (f *filteringTreeSyncer) StartSync() {} +func (f *filteringTreeSyncer) StopSync() {} +func (f *filteringTreeSyncer) ShouldSync(string) bool { return true } +func (f *filteringTreeSyncer) SyncAll(ctx context.Context, p peer.Peer, existing, missing []string) error { + return nil +} + +func (f *filteringTreeSyncer) ShouldPull(ctx context.Context, objectId string, rootChange *treechangeproto.RawTreeChangeWithId, heads []string) bool { + return f.shouldPull(ctx, objectId, rootChange, heads) +} + +func newFixtureWithPullFilter(t *testing.T, shouldPull func(ctx context.Context, objectId string, rootChange *treechangeproto.RawTreeChangeWithId, heads []string) bool) *fixture { fx := &fixture{ a: &app.App{}, } @@ -186,6 +291,9 @@ func newFixture(t *testing.T) *fixture { Register(fx.keyValue). Register(syncstatus.NewNoOpSyncStatus()). Register(fx.objectSync) + if shouldPull != nil { + fx.a.Register(&filteringTreeSyncer{shouldPull: shouldPull}) + } require.NoError(t, fx.a.Start(context.Background())) return fx } diff --git a/commonspace/sync/objectsync/synchandler.go b/commonspace/sync/objectsync/synchandler.go index 89509bf69..73c1aebee 100644 --- a/commonspace/sync/objectsync/synchandler.go +++ b/commonspace/sync/objectsync/synchandler.go @@ -15,6 +15,7 @@ import ( "github.com/anyproto/any-sync/commonspace/object/tree/synctree" "github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto" "github.com/anyproto/any-sync/commonspace/object/treemanager" + "github.com/anyproto/any-sync/commonspace/object/treesyncer" "github.com/anyproto/any-sync/commonspace/objectmanager" "github.com/anyproto/any-sync/commonspace/spacestate" "github.com/anyproto/any-sync/commonspace/spacesyncproto" @@ -31,11 +32,12 @@ var ErrUnexpectedHeadUpdateType = errors.New("unexpected head update type") var log = logger.NewNamed(syncdeps.CName) type objectSync struct { - spaceId string - pool pool.Service - manager objectmanager.ObjectManager - status syncstatus.StatusUpdater - keyValue kvinterfaces.KeyValueService + spaceId string + pool pool.Service + manager objectmanager.ObjectManager + status syncstatus.StatusUpdater + keyValue kvinterfaces.KeyValueService + pullFilter treesyncer.PullFilter } func New() syncdeps.SyncHandler { @@ -48,6 +50,8 @@ func (o *objectSync) Init(a *app.App) (err error) { o.keyValue = a.MustComponent(kvinterfaces.CName).(kvinterfaces.KeyValueService) o.status = a.MustComponent(syncstatus.CName).(syncstatus.StatusUpdater) o.spaceId = a.MustComponent(spacestate.CName).(*spacestate.SpaceState).SpaceId + // optional extension of the registered tree syncer + o.pullFilter, _ = a.Component(treesyncer.CName).(treesyncer.PullFilter) return } @@ -69,6 +73,9 @@ func (o *objectSync) HandleHeadUpdate(ctx context.Context, headUpdate drpc.Messa } obj, err := o.manager.GetObject(context.Background(), update.Meta.ObjectId) if err != nil { + if o.pullFilter != nil && !o.shouldPull(ctx, update) { + return nil, nil + } return synctree.NewRequest(peerId, update.Meta.SpaceId, update.Meta.ObjectId, nil, nil, nil), nil } objHandler, ok := obj.(syncdeps.ObjectSyncHandler) @@ -78,6 +85,21 @@ func (o *objectSync) HandleHeadUpdate(ctx context.Context, headUpdate drpc.Messa return objHandler.HandleHeadUpdate(ctx, o.status, update) } +// shouldPull consults the pull filter for a locally-missing tree. Head +// updates carry the tree's raw root change, so the filter can classify the +// tree without fetching anything. Malformed updates default to pull. +func (o *objectSync) shouldPull(ctx context.Context, update *objectmessages.HeadUpdate) bool { + treeSyncMsg := &treechangeproto.TreeSyncMessage{} + if err := treeSyncMsg.UnmarshalVT(update.Bytes); err != nil { + return true + } + contentUpdate := treeSyncMsg.GetContent().GetHeadUpdate() + if contentUpdate == nil || treeSyncMsg.RootChange == nil { + return true + } + return o.pullFilter.ShouldPull(ctx, update.Meta.ObjectId, treeSyncMsg.RootChange, contentUpdate.Heads) +} + func (o *objectSync) HandleStreamRequest(ctx context.Context, rq syncdeps.Request, updater syncdeps.QueueSizeUpdater, sendResponse func(resp proto.Message) error) (syncdeps.Request, error) { obj, err := o.manager.GetObject(context.Background(), rq.ObjectId()) if err != nil { diff --git a/coordinator/coordinatorclient/coordinatorclient.go b/coordinator/coordinatorclient/coordinatorclient.go index ac8952fe0..c9fcbd3ba 100644 --- a/coordinator/coordinatorclient/coordinatorclient.go +++ b/coordinator/coordinatorclient/coordinatorclient.go @@ -59,6 +59,9 @@ type CoordinatorClient interface { AclUploadInvite(ctx context.Context, spaceId string, inviteKey crypto.PubKey, block blocks.Block) (err error) AclDeleteInvite(ctx context.Context, spaceId string, inviteCid cid.Cid) (err error) + FileLimitsGet(ctx context.Context, spaceId, identity string) (resp *coordinatorproto.FileLimitsGetResponse, err error) + FileUsageReport(ctx context.Context, rows []*coordinatorproto.FileUsageRow) (err error) + app.Component } @@ -299,6 +302,29 @@ func (c *coordinatorClient) AccountLimitsSet(ctx context.Context, req *coordinat }) } +func (c *coordinatorClient) FileLimitsGet(ctx context.Context, spaceId, identity string) (resp *coordinatorproto.FileLimitsGetResponse, err error) { + err = c.doClient(ctx, func(cl coordinatorproto.DRPCCoordinatorClient) error { + resp, err = cl.FileLimitsGet(ctx, &coordinatorproto.FileLimitsGetRequest{ + SpaceId: spaceId, + Identity: identity, + }) + if err != nil { + return rpcerr.Unwrap(err) + } + return nil + }) + return +} + +func (c *coordinatorClient) FileUsageReport(ctx context.Context, rows []*coordinatorproto.FileUsageRow) (err error) { + return c.doClient(ctx, func(cl coordinatorproto.DRPCCoordinatorClient) error { + if _, err := cl.FileUsageReport(ctx, &coordinatorproto.FileUsageReportRequest{Rows: rows}); err != nil { + return rpcerr.Unwrap(err) + } + return nil + }) +} + func (c *coordinatorClient) SpaceMakeShareable(ctx context.Context, spaceId string) (err error) { return c.doClient(ctx, func(cl coordinatorproto.DRPCCoordinatorClient) error { if _, err := cl.SpaceMakeShareable(ctx, &coordinatorproto.SpaceMakeShareableRequest{ diff --git a/coordinator/coordinatorclient/mock_coordinatorclient/mock_coordinatorclient.go b/coordinator/coordinatorclient/mock_coordinatorclient/mock_coordinatorclient.go index 916402717..7214e47a0 100644 --- a/coordinator/coordinatorclient/mock_coordinatorclient/mock_coordinatorclient.go +++ b/coordinator/coordinatorclient/mock_coordinatorclient/mock_coordinatorclient.go @@ -179,6 +179,35 @@ func (mr *MockCoordinatorClientMockRecorder) DeletionLog(ctx, lastRecordId, limi return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeletionLog", reflect.TypeOf((*MockCoordinatorClient)(nil).DeletionLog), ctx, lastRecordId, limit) } +// FileLimitsGet mocks base method. +func (m *MockCoordinatorClient) FileLimitsGet(ctx context.Context, spaceId, identity string) (*coordinatorproto.FileLimitsGetResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FileLimitsGet", ctx, spaceId, identity) + ret0, _ := ret[0].(*coordinatorproto.FileLimitsGetResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FileLimitsGet indicates an expected call of FileLimitsGet. +func (mr *MockCoordinatorClientMockRecorder) FileLimitsGet(ctx, spaceId, identity any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FileLimitsGet", reflect.TypeOf((*MockCoordinatorClient)(nil).FileLimitsGet), ctx, spaceId, identity) +} + +// FileUsageReport mocks base method. +func (m *MockCoordinatorClient) FileUsageReport(ctx context.Context, rows []*coordinatorproto.FileUsageRow) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FileUsageReport", ctx, rows) + ret0, _ := ret[0].(error) + return ret0 +} + +// FileUsageReport indicates an expected call of FileUsageReport. +func (mr *MockCoordinatorClientMockRecorder) FileUsageReport(ctx, rows any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FileUsageReport", reflect.TypeOf((*MockCoordinatorClient)(nil).FileUsageReport), ctx, rows) +} + // IdentityRepoGet mocks base method. func (m *MockCoordinatorClient) IdentityRepoGet(ctx context.Context, identities, kinds []string) ([]*identityrepoproto.DataWithIdentity, error) { m.ctrl.T.Helper() diff --git a/coordinator/coordinatorproto/coordinator.pb.go b/coordinator/coordinatorproto/coordinator.pb.go index 42bdb3676..e26f56f45 100644 --- a/coordinator/coordinatorproto/coordinator.pb.go +++ b/coordinator/coordinatorproto/coordinator.pb.go @@ -225,6 +225,8 @@ const ( // PaymentProcessingAPI supports payment processing api // (see any-pp-node repository) NodeType_PaymentProcessingAPI NodeType = 5 + // FileV2API supports the v2 file api (filenode v2 broker pool) + NodeType_FileV2API NodeType = 6 ) // Enum value maps for NodeType. @@ -236,6 +238,7 @@ var ( 3: "ConsensusAPI", 4: "NamingNodeAPI", 5: "PaymentProcessingAPI", + 6: "FileV2API", } NodeType_value = map[string]int32{ "TreeAPI": 0, @@ -244,6 +247,7 @@ var ( "ConsensusAPI": 3, "NamingNodeAPI": 4, "PaymentProcessingAPI": 5, + "FileV2API": 6, } ) @@ -1582,8 +1586,13 @@ type NetworkConfigurationResponse struct { Nodes []*Node `protobuf:"bytes,3,rep,name=nodes,proto3" json:"nodes,omitempty"` // unix timestamp of the creation time of configuration CreationTimeUnix uint64 `protobuf:"varint,4,opt,name=creationTimeUnix,proto3" json:"creationTimeUnix,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // monotonically increasing version of the configuration, 0 if the coordinator predates epoch support + Epoch uint64 `protobuf:"varint,5,opt,name=epoch,proto3" json:"epoch,omitempty"` + // identity of the fileV2 fleet's shared receipt-signing key (network + // string encoding); empty if the network has no fileV2 fleet + FileNetworkId string `protobuf:"bytes,6,opt,name=fileNetworkId,proto3" json:"fileNetworkId,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NetworkConfigurationResponse) Reset() { @@ -1644,6 +1653,20 @@ func (x *NetworkConfigurationResponse) GetCreationTimeUnix() uint64 { return 0 } +func (x *NetworkConfigurationResponse) GetEpoch() uint64 { + if x != nil { + return x.Epoch + } + return 0 +} + +func (x *NetworkConfigurationResponse) GetFileNetworkId() string { + if x != nil { + return x.FileNetworkId + } + return "" +} + // Node describes one node in the network type Node struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -3590,6 +3613,272 @@ func (*AclDeleteInviteResponse) Descriptor() ([]byte, []int) { return file_coordinator_coordinatorproto_protos_coordinator_proto_rawDescGZIP(), []int{53} } +type FileLimitsGetRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + SpaceId string `protobuf:"bytes,1,opt,name=spaceId,proto3" json:"spaceId,omitempty"` + // Identity is the uploader account identity whose pool bounds are requested + Identity string `protobuf:"bytes,2,opt,name=identity,proto3" json:"identity,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileLimitsGetRequest) Reset() { + *x = FileLimitsGetRequest{} + mi := &file_coordinator_coordinatorproto_protos_coordinator_proto_msgTypes[54] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileLimitsGetRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileLimitsGetRequest) ProtoMessage() {} + +func (x *FileLimitsGetRequest) ProtoReflect() protoreflect.Message { + mi := &file_coordinator_coordinatorproto_protos_coordinator_proto_msgTypes[54] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileLimitsGetRequest.ProtoReflect.Descriptor instead. +func (*FileLimitsGetRequest) Descriptor() ([]byte, []int) { + return file_coordinator_coordinatorproto_protos_coordinator_proto_rawDescGZIP(), []int{54} +} + +func (x *FileLimitsGetRequest) GetSpaceId() string { + if x != nil { + return x.SpaceId + } + return "" +} + +func (x *FileLimitsGetRequest) GetIdentity() string { + if x != nil { + return x.Identity + } + return "" +} + +type FileLimitsGetResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // AccountLimitBytes is the identity's total file-storage cap (the common account pool) + AccountLimitBytes uint64 `protobuf:"varint,1,opt,name=accountLimitBytes,proto3" json:"accountLimitBytes,omitempty"` + // AccountTotalUsageBytes is the coordinator-aggregated durable usage across all the identity's spaces + AccountTotalUsageBytes uint64 `protobuf:"varint,2,opt,name=accountTotalUsageBytes,proto3" json:"accountTotalUsageBytes,omitempty"` + // SpaceLimitBytes is an optional space-scoped cap; 0 = unset + SpaceLimitBytes uint64 `protobuf:"varint,3,opt,name=spaceLimitBytes,proto3" json:"spaceLimitBytes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileLimitsGetResponse) Reset() { + *x = FileLimitsGetResponse{} + mi := &file_coordinator_coordinatorproto_protos_coordinator_proto_msgTypes[55] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileLimitsGetResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileLimitsGetResponse) ProtoMessage() {} + +func (x *FileLimitsGetResponse) ProtoReflect() protoreflect.Message { + mi := &file_coordinator_coordinatorproto_protos_coordinator_proto_msgTypes[55] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileLimitsGetResponse.ProtoReflect.Descriptor instead. +func (*FileLimitsGetResponse) Descriptor() ([]byte, []int) { + return file_coordinator_coordinatorproto_protos_coordinator_proto_rawDescGZIP(), []int{55} +} + +func (x *FileLimitsGetResponse) GetAccountLimitBytes() uint64 { + if x != nil { + return x.AccountLimitBytes + } + return 0 +} + +func (x *FileLimitsGetResponse) GetAccountTotalUsageBytes() uint64 { + if x != nil { + return x.AccountTotalUsageBytes + } + return 0 +} + +func (x *FileLimitsGetResponse) GetSpaceLimitBytes() uint64 { + if x != nil { + return x.SpaceLimitBytes + } + return 0 +} + +type FileUsageReportRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Rows []*FileUsageRow `protobuf:"bytes,1,rep,name=rows,proto3" json:"rows,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileUsageReportRequest) Reset() { + *x = FileUsageReportRequest{} + mi := &file_coordinator_coordinatorproto_protos_coordinator_proto_msgTypes[56] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileUsageReportRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileUsageReportRequest) ProtoMessage() {} + +func (x *FileUsageReportRequest) ProtoReflect() protoreflect.Message { + mi := &file_coordinator_coordinatorproto_protos_coordinator_proto_msgTypes[56] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileUsageReportRequest.ProtoReflect.Descriptor instead. +func (*FileUsageReportRequest) Descriptor() ([]byte, []int) { + return file_coordinator_coordinatorproto_protos_coordinator_proto_rawDescGZIP(), []int{56} +} + +func (x *FileUsageReportRequest) GetRows() []*FileUsageRow { + if x != nil { + return x.Rows + } + return nil +} + +type FileUsageRow struct { + state protoimpl.MessageState `protogen:"open.v1"` + SpaceId string `protobuf:"bytes,1,opt,name=spaceId,proto3" json:"spaceId,omitempty"` + // Identity is the charged uploader identity (the author of the root's earliest bind row) + Identity string `protobuf:"bytes,2,opt,name=identity,proto3" json:"identity,omitempty"` + // DurableBytes is the HEAD-measured durable usage of this (space, identity) slice; in-flight is never reported + DurableBytes uint64 `protobuf:"varint,3,opt,name=durableBytes,proto3" json:"durableBytes,omitempty"` + DurableFilesCount uint64 `protobuf:"varint,4,opt,name=durableFilesCount,proto3" json:"durableFilesCount,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileUsageRow) Reset() { + *x = FileUsageRow{} + mi := &file_coordinator_coordinatorproto_protos_coordinator_proto_msgTypes[57] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileUsageRow) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileUsageRow) ProtoMessage() {} + +func (x *FileUsageRow) ProtoReflect() protoreflect.Message { + mi := &file_coordinator_coordinatorproto_protos_coordinator_proto_msgTypes[57] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileUsageRow.ProtoReflect.Descriptor instead. +func (*FileUsageRow) Descriptor() ([]byte, []int) { + return file_coordinator_coordinatorproto_protos_coordinator_proto_rawDescGZIP(), []int{57} +} + +func (x *FileUsageRow) GetSpaceId() string { + if x != nil { + return x.SpaceId + } + return "" +} + +func (x *FileUsageRow) GetIdentity() string { + if x != nil { + return x.Identity + } + return "" +} + +func (x *FileUsageRow) GetDurableBytes() uint64 { + if x != nil { + return x.DurableBytes + } + return 0 +} + +func (x *FileUsageRow) GetDurableFilesCount() uint64 { + if x != nil { + return x.DurableFilesCount + } + return 0 +} + +type FileUsageReportResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileUsageReportResponse) Reset() { + *x = FileUsageReportResponse{} + mi := &file_coordinator_coordinatorproto_protos_coordinator_proto_msgTypes[58] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileUsageReportResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileUsageReportResponse) ProtoMessage() {} + +func (x *FileUsageReportResponse) ProtoReflect() protoreflect.Message { + mi := &file_coordinator_coordinatorproto_protos_coordinator_proto_msgTypes[58] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileUsageReportResponse.ProtoReflect.Descriptor instead. +func (*FileUsageReportResponse) Descriptor() ([]byte, []int) { + return file_coordinator_coordinatorproto_protos_coordinator_proto_rawDescGZIP(), []int{58} +} + var File_coordinator_coordinatorproto_protos_coordinator_proto protoreflect.FileDescriptor const file_coordinator_coordinatorproto_protos_coordinator_proto_rawDesc = "" + @@ -3649,12 +3938,14 @@ const file_coordinator_coordinatorproto_protos_coordinator_proto_rawDesc = "" + "\aaclHead\x18\x02 \x01(\tR\aaclHead\"\x1e\n" + "\x1cSpaceMakeUnshareableResponse\";\n" + "\x1bNetworkConfigurationRequest\x12\x1c\n" + - "\tcurrentId\x18\x01 \x01(\tR\tcurrentId\"\xbb\x01\n" + + "\tcurrentId\x18\x01 \x01(\tR\tcurrentId\"\xf7\x01\n" + "\x1cNetworkConfigurationResponse\x12(\n" + "\x0fconfigurationId\x18\x01 \x01(\tR\x0fconfigurationId\x12\x1c\n" + "\tnetworkId\x18\x02 \x01(\tR\tnetworkId\x12'\n" + "\x05nodes\x18\x03 \x03(\v2\x11.coordinator.NodeR\x05nodes\x12*\n" + - "\x10creationTimeUnix\x18\x04 \x01(\x04R\x10creationTimeUnix\"i\n" + + "\x10creationTimeUnix\x18\x04 \x01(\x04R\x10creationTimeUnix\x12\x14\n" + + "\x05epoch\x18\x05 \x01(\x04R\x05epoch\x12$\n" + + "\rfileNetworkId\x18\x06 \x01(\tR\rfileNetworkId\"i\n" + "\x04Node\x12\x16\n" + "\x06peerId\x18\x01 \x01(\tR\x06peerId\x12\x1c\n" + "\taddresses\x18\x02 \x03(\tR\taddresses\x12+\n" + @@ -3770,7 +4061,22 @@ const file_coordinator_coordinatorproto_protos_coordinator_proto_rawDesc = "" + "\x16AclDeleteInviteRequest\x12\x18\n" + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x12\x1c\n" + "\tinviteCid\x18\x02 \x01(\fR\tinviteCid\"\x19\n" + - "\x17AclDeleteInviteResponse*\xd1\x02\n" + + "\x17AclDeleteInviteResponse\"L\n" + + "\x14FileLimitsGetRequest\x12\x18\n" + + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x12\x1a\n" + + "\bidentity\x18\x02 \x01(\tR\bidentity\"\xa7\x01\n" + + "\x15FileLimitsGetResponse\x12,\n" + + "\x11accountLimitBytes\x18\x01 \x01(\x04R\x11accountLimitBytes\x126\n" + + "\x16accountTotalUsageBytes\x18\x02 \x01(\x04R\x16accountTotalUsageBytes\x12(\n" + + "\x0fspaceLimitBytes\x18\x03 \x01(\x04R\x0fspaceLimitBytes\"G\n" + + "\x16FileUsageReportRequest\x12-\n" + + "\x04rows\x18\x01 \x03(\v2\x19.coordinator.FileUsageRowR\x04rows\"\x96\x01\n" + + "\fFileUsageRow\x12\x18\n" + + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x12\x1a\n" + + "\bidentity\x18\x02 \x01(\tR\bidentity\x12\"\n" + + "\fdurableBytes\x18\x03 \x01(\x04R\fdurableBytes\x12,\n" + + "\x11durableFilesCount\x18\x04 \x01(\x04R\x11durableFilesCount\"\x19\n" + + "\x17FileUsageReportResponse*\xd1\x02\n" + "\n" + "ErrorCodes\x12\x0e\n" + "\n" + @@ -3798,14 +4104,15 @@ const file_coordinator_coordinatorproto_protos_coordinator_proto_rawDesc = "" + "\x14SpaceStatusNotExists\x10\x04*J\n" + "\x10SpacePermissions\x12\x1b\n" + "\x17SpacePermissionsUnknown\x10\x00\x12\x19\n" + - "\x15SpacePermissionsOwner\x10\x01*w\n" + + "\x15SpacePermissionsOwner\x10\x01*\x86\x01\n" + "\bNodeType\x12\v\n" + "\aTreeAPI\x10\x00\x12\v\n" + "\aFileAPI\x10\x01\x12\x12\n" + "\x0eCoordinatorAPI\x10\x02\x12\x10\n" + "\fConsensusAPI\x10\x03\x12\x11\n" + "\rNamingNodeAPI\x10\x04\x12\x18\n" + - "\x14PaymentProcessingAPI\x10\x05*9\n" + + "\x14PaymentProcessingAPI\x10\x05\x12\r\n" + + "\tFileV2API\x10\x06*9\n" + "\x13DeletionPayloadType\x12\b\n" + "\x04Tree\x10\x00\x12\v\n" + "\aConfirm\x10\x01\x12\v\n" + @@ -3831,7 +4138,7 @@ const file_coordinator_coordinatorproto_protos_coordinator_proto_rawDesc = "" + "\x0fNotifyEventType\x12\x14\n" + "\x10UnspecifiedEvent\x10\x00\x12\x18\n" + "\x14InboxNewMessageEvent\x10\x01\x12\x1d\n" + - "\x19NetworkConfigChangedEvent\x10\x022\xde\x0e\n" + + "\x19NetworkConfigChangedEvent\x10\x022\x94\x10\n" + "\vCoordinator\x12J\n" + "\tSpaceSign\x12\x1d.coordinator.SpaceSignRequest\x1a\x1e.coordinator.SpaceSignResponse\x12_\n" + "\x10SpaceStatusCheck\x12$.coordinator.SpaceStatusCheckRequest\x1a%.coordinator.SpaceStatusCheckResponse\x12k\n" + @@ -3853,7 +4160,9 @@ const file_coordinator_coordinatorproto_protos_coordinator_proto_rawDesc = "" + "\x0fInboxAddMessage\x12#.coordinator.InboxAddMessageRequest\x1a$.coordinator.InboxAddMessageResponse\x12[\n" + "\x0fNotifySubscribe\x12#.coordinator.NotifySubscribeRequest\x1a!.coordinator.NotifySubscribeEvent0\x01\x12\\\n" + "\x0fAclUploadInvite\x12#.coordinator.AclUploadInviteRequest\x1a$.coordinator.AclUploadInviteResponse\x12\\\n" + - "\x0fAclDeleteInvite\x12#.coordinator.AclDeleteInviteRequest\x1a$.coordinator.AclDeleteInviteResponseB\x1eZ\x1ccoordinator/coordinatorprotob\x06proto3" + "\x0fAclDeleteInvite\x12#.coordinator.AclDeleteInviteRequest\x1a$.coordinator.AclDeleteInviteResponse\x12V\n" + + "\rFileLimitsGet\x12!.coordinator.FileLimitsGetRequest\x1a\".coordinator.FileLimitsGetResponse\x12\\\n" + + "\x0fFileUsageReport\x12#.coordinator.FileUsageReportRequest\x1a$.coordinator.FileUsageReportResponseB\x1eZ\x1ccoordinator/coordinatorprotob\x06proto3" var ( file_coordinator_coordinatorproto_protos_coordinator_proto_rawDescOnce sync.Once @@ -3868,7 +4177,7 @@ func file_coordinator_coordinatorproto_protos_coordinator_proto_rawDescGZIP() [] } var file_coordinator_coordinatorproto_protos_coordinator_proto_enumTypes = make([]protoimpl.EnumInfo, 11) -var file_coordinator_coordinatorproto_protos_coordinator_proto_msgTypes = make([]protoimpl.MessageInfo, 54) +var file_coordinator_coordinatorproto_protos_coordinator_proto_msgTypes = make([]protoimpl.MessageInfo, 59) var file_coordinator_coordinatorproto_protos_coordinator_proto_goTypes = []any{ (ErrorCodes)(0), // 0: coordinator.ErrorCodes (SpaceStatus)(0), // 1: coordinator.SpaceStatus @@ -3935,6 +4244,11 @@ var file_coordinator_coordinatorproto_protos_coordinator_proto_goTypes = []any{ (*AclUploadInviteResponse)(nil), // 62: coordinator.AclUploadInviteResponse (*AclDeleteInviteRequest)(nil), // 63: coordinator.AclDeleteInviteRequest (*AclDeleteInviteResponse)(nil), // 64: coordinator.AclDeleteInviteResponse + (*FileLimitsGetRequest)(nil), // 65: coordinator.FileLimitsGetRequest + (*FileLimitsGetResponse)(nil), // 66: coordinator.FileLimitsGetResponse + (*FileUsageReportRequest)(nil), // 67: coordinator.FileUsageReportRequest + (*FileUsageRow)(nil), // 68: coordinator.FileUsageRow + (*FileUsageReportResponse)(nil), // 69: coordinator.FileUsageReportResponse } var file_coordinator_coordinatorproto_protos_coordinator_proto_depIdxs = []int32{ 1, // 0: coordinator.SpaceStatusPayload.status:type_name -> coordinator.SpaceStatus @@ -3961,51 +4275,56 @@ var file_coordinator_coordinatorproto_protos_coordinator_proto_depIdxs = []int32 52, // 21: coordinator.InboxAddMessageRequest.message:type_name -> coordinator.InboxMessage 10, // 22: coordinator.NotifySubscribeRequest.eventType:type_name -> coordinator.NotifyEventType 10, // 23: coordinator.NotifySubscribeEvent.eventType:type_name -> coordinator.NotifyEventType - 11, // 24: coordinator.Coordinator.SpaceSign:input_type -> coordinator.SpaceSignRequest - 17, // 25: coordinator.Coordinator.SpaceStatusCheck:input_type -> coordinator.SpaceStatusCheckRequest - 19, // 26: coordinator.Coordinator.SpaceStatusCheckMany:input_type -> coordinator.SpaceStatusCheckManyRequest - 22, // 27: coordinator.Coordinator.SpaceStatusChange:input_type -> coordinator.SpaceStatusChangeRequest - 24, // 28: coordinator.Coordinator.SpaceMakeShareable:input_type -> coordinator.SpaceMakeShareableRequest - 26, // 29: coordinator.Coordinator.SpaceMakeUnshareable:input_type -> coordinator.SpaceMakeUnshareableRequest - 28, // 30: coordinator.Coordinator.NetworkConfiguration:input_type -> coordinator.NetworkConfigurationRequest - 33, // 31: coordinator.Coordinator.DeletionLog:input_type -> coordinator.DeletionLogRequest - 36, // 32: coordinator.Coordinator.SpaceDelete:input_type -> coordinator.SpaceDeleteRequest - 38, // 33: coordinator.Coordinator.AccountDelete:input_type -> coordinator.AccountDeleteRequest - 41, // 34: coordinator.Coordinator.AccountRevertDeletion:input_type -> coordinator.AccountRevertDeletionRequest - 43, // 35: coordinator.Coordinator.AclAddRecord:input_type -> coordinator.AclAddRecordRequest - 45, // 36: coordinator.Coordinator.AclGetRecords:input_type -> coordinator.AclGetRecordsRequest - 47, // 37: coordinator.Coordinator.AccountLimitsSet:input_type -> coordinator.AccountLimitsSetRequest - 49, // 38: coordinator.Coordinator.AclEventLog:input_type -> coordinator.AclEventLogRequest - 55, // 39: coordinator.Coordinator.InboxFetch:input_type -> coordinator.InboxFetchRequest - 57, // 40: coordinator.Coordinator.InboxAddMessage:input_type -> coordinator.InboxAddMessageRequest - 59, // 41: coordinator.Coordinator.NotifySubscribe:input_type -> coordinator.NotifySubscribeRequest - 61, // 42: coordinator.Coordinator.AclUploadInvite:input_type -> coordinator.AclUploadInviteRequest - 63, // 43: coordinator.Coordinator.AclDeleteInvite:input_type -> coordinator.AclDeleteInviteRequest - 14, // 44: coordinator.Coordinator.SpaceSign:output_type -> coordinator.SpaceSignResponse - 18, // 45: coordinator.Coordinator.SpaceStatusCheck:output_type -> coordinator.SpaceStatusCheckResponse - 20, // 46: coordinator.Coordinator.SpaceStatusCheckMany:output_type -> coordinator.SpaceStatusCheckManyResponse - 23, // 47: coordinator.Coordinator.SpaceStatusChange:output_type -> coordinator.SpaceStatusChangeResponse - 25, // 48: coordinator.Coordinator.SpaceMakeShareable:output_type -> coordinator.SpaceMakeShareableResponse - 27, // 49: coordinator.Coordinator.SpaceMakeUnshareable:output_type -> coordinator.SpaceMakeUnshareableResponse - 29, // 50: coordinator.Coordinator.NetworkConfiguration:output_type -> coordinator.NetworkConfigurationResponse - 34, // 51: coordinator.Coordinator.DeletionLog:output_type -> coordinator.DeletionLogResponse - 37, // 52: coordinator.Coordinator.SpaceDelete:output_type -> coordinator.SpaceDeleteResponse - 40, // 53: coordinator.Coordinator.AccountDelete:output_type -> coordinator.AccountDeleteResponse - 42, // 54: coordinator.Coordinator.AccountRevertDeletion:output_type -> coordinator.AccountRevertDeletionResponse - 44, // 55: coordinator.Coordinator.AclAddRecord:output_type -> coordinator.AclAddRecordResponse - 46, // 56: coordinator.Coordinator.AclGetRecords:output_type -> coordinator.AclGetRecordsResponse - 48, // 57: coordinator.Coordinator.AccountLimitsSet:output_type -> coordinator.AccountLimitsSetResponse - 50, // 58: coordinator.Coordinator.AclEventLog:output_type -> coordinator.AclEventLogResponse - 56, // 59: coordinator.Coordinator.InboxFetch:output_type -> coordinator.InboxFetchResponse - 58, // 60: coordinator.Coordinator.InboxAddMessage:output_type -> coordinator.InboxAddMessageResponse - 60, // 61: coordinator.Coordinator.NotifySubscribe:output_type -> coordinator.NotifySubscribeEvent - 62, // 62: coordinator.Coordinator.AclUploadInvite:output_type -> coordinator.AclUploadInviteResponse - 64, // 63: coordinator.Coordinator.AclDeleteInvite:output_type -> coordinator.AclDeleteInviteResponse - 44, // [44:64] is the sub-list for method output_type - 24, // [24:44] is the sub-list for method input_type - 24, // [24:24] is the sub-list for extension type_name - 24, // [24:24] is the sub-list for extension extendee - 0, // [0:24] is the sub-list for field type_name + 68, // 24: coordinator.FileUsageReportRequest.rows:type_name -> coordinator.FileUsageRow + 11, // 25: coordinator.Coordinator.SpaceSign:input_type -> coordinator.SpaceSignRequest + 17, // 26: coordinator.Coordinator.SpaceStatusCheck:input_type -> coordinator.SpaceStatusCheckRequest + 19, // 27: coordinator.Coordinator.SpaceStatusCheckMany:input_type -> coordinator.SpaceStatusCheckManyRequest + 22, // 28: coordinator.Coordinator.SpaceStatusChange:input_type -> coordinator.SpaceStatusChangeRequest + 24, // 29: coordinator.Coordinator.SpaceMakeShareable:input_type -> coordinator.SpaceMakeShareableRequest + 26, // 30: coordinator.Coordinator.SpaceMakeUnshareable:input_type -> coordinator.SpaceMakeUnshareableRequest + 28, // 31: coordinator.Coordinator.NetworkConfiguration:input_type -> coordinator.NetworkConfigurationRequest + 33, // 32: coordinator.Coordinator.DeletionLog:input_type -> coordinator.DeletionLogRequest + 36, // 33: coordinator.Coordinator.SpaceDelete:input_type -> coordinator.SpaceDeleteRequest + 38, // 34: coordinator.Coordinator.AccountDelete:input_type -> coordinator.AccountDeleteRequest + 41, // 35: coordinator.Coordinator.AccountRevertDeletion:input_type -> coordinator.AccountRevertDeletionRequest + 43, // 36: coordinator.Coordinator.AclAddRecord:input_type -> coordinator.AclAddRecordRequest + 45, // 37: coordinator.Coordinator.AclGetRecords:input_type -> coordinator.AclGetRecordsRequest + 47, // 38: coordinator.Coordinator.AccountLimitsSet:input_type -> coordinator.AccountLimitsSetRequest + 49, // 39: coordinator.Coordinator.AclEventLog:input_type -> coordinator.AclEventLogRequest + 55, // 40: coordinator.Coordinator.InboxFetch:input_type -> coordinator.InboxFetchRequest + 57, // 41: coordinator.Coordinator.InboxAddMessage:input_type -> coordinator.InboxAddMessageRequest + 59, // 42: coordinator.Coordinator.NotifySubscribe:input_type -> coordinator.NotifySubscribeRequest + 61, // 43: coordinator.Coordinator.AclUploadInvite:input_type -> coordinator.AclUploadInviteRequest + 63, // 44: coordinator.Coordinator.AclDeleteInvite:input_type -> coordinator.AclDeleteInviteRequest + 65, // 45: coordinator.Coordinator.FileLimitsGet:input_type -> coordinator.FileLimitsGetRequest + 67, // 46: coordinator.Coordinator.FileUsageReport:input_type -> coordinator.FileUsageReportRequest + 14, // 47: coordinator.Coordinator.SpaceSign:output_type -> coordinator.SpaceSignResponse + 18, // 48: coordinator.Coordinator.SpaceStatusCheck:output_type -> coordinator.SpaceStatusCheckResponse + 20, // 49: coordinator.Coordinator.SpaceStatusCheckMany:output_type -> coordinator.SpaceStatusCheckManyResponse + 23, // 50: coordinator.Coordinator.SpaceStatusChange:output_type -> coordinator.SpaceStatusChangeResponse + 25, // 51: coordinator.Coordinator.SpaceMakeShareable:output_type -> coordinator.SpaceMakeShareableResponse + 27, // 52: coordinator.Coordinator.SpaceMakeUnshareable:output_type -> coordinator.SpaceMakeUnshareableResponse + 29, // 53: coordinator.Coordinator.NetworkConfiguration:output_type -> coordinator.NetworkConfigurationResponse + 34, // 54: coordinator.Coordinator.DeletionLog:output_type -> coordinator.DeletionLogResponse + 37, // 55: coordinator.Coordinator.SpaceDelete:output_type -> coordinator.SpaceDeleteResponse + 40, // 56: coordinator.Coordinator.AccountDelete:output_type -> coordinator.AccountDeleteResponse + 42, // 57: coordinator.Coordinator.AccountRevertDeletion:output_type -> coordinator.AccountRevertDeletionResponse + 44, // 58: coordinator.Coordinator.AclAddRecord:output_type -> coordinator.AclAddRecordResponse + 46, // 59: coordinator.Coordinator.AclGetRecords:output_type -> coordinator.AclGetRecordsResponse + 48, // 60: coordinator.Coordinator.AccountLimitsSet:output_type -> coordinator.AccountLimitsSetResponse + 50, // 61: coordinator.Coordinator.AclEventLog:output_type -> coordinator.AclEventLogResponse + 56, // 62: coordinator.Coordinator.InboxFetch:output_type -> coordinator.InboxFetchResponse + 58, // 63: coordinator.Coordinator.InboxAddMessage:output_type -> coordinator.InboxAddMessageResponse + 60, // 64: coordinator.Coordinator.NotifySubscribe:output_type -> coordinator.NotifySubscribeEvent + 62, // 65: coordinator.Coordinator.AclUploadInvite:output_type -> coordinator.AclUploadInviteResponse + 64, // 66: coordinator.Coordinator.AclDeleteInvite:output_type -> coordinator.AclDeleteInviteResponse + 66, // 67: coordinator.Coordinator.FileLimitsGet:output_type -> coordinator.FileLimitsGetResponse + 69, // 68: coordinator.Coordinator.FileUsageReport:output_type -> coordinator.FileUsageReportResponse + 47, // [47:69] is the sub-list for method output_type + 25, // [25:47] is the sub-list for method input_type + 25, // [25:25] is the sub-list for extension type_name + 25, // [25:25] is the sub-list for extension extendee + 0, // [0:25] is the sub-list for field type_name } func init() { file_coordinator_coordinatorproto_protos_coordinator_proto_init() } @@ -4019,7 +4338,7 @@ func file_coordinator_coordinatorproto_protos_coordinator_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_coordinator_coordinatorproto_protos_coordinator_proto_rawDesc), len(file_coordinator_coordinatorproto_protos_coordinator_proto_rawDesc)), NumEnums: 11, - NumMessages: 54, + NumMessages: 59, NumExtensions: 0, NumServices: 1, }, diff --git a/coordinator/coordinatorproto/coordinator_drpc.pb.go b/coordinator/coordinatorproto/coordinator_drpc.pb.go index 1975cb2dc..82085085f 100644 --- a/coordinator/coordinatorproto/coordinator_drpc.pb.go +++ b/coordinator/coordinatorproto/coordinator_drpc.pb.go @@ -53,6 +53,8 @@ type DRPCCoordinatorClient interface { NotifySubscribe(ctx context.Context, in *NotifySubscribeRequest) (DRPCCoordinator_NotifySubscribeClient, error) AclUploadInvite(ctx context.Context, in *AclUploadInviteRequest) (*AclUploadInviteResponse, error) AclDeleteInvite(ctx context.Context, in *AclDeleteInviteRequest) (*AclDeleteInviteResponse, error) + FileLimitsGet(ctx context.Context, in *FileLimitsGetRequest) (*FileLimitsGetResponse, error) + FileUsageReport(ctx context.Context, in *FileUsageReportRequest) (*FileUsageReportResponse, error) } type drpcCoordinatorClient struct { @@ -276,6 +278,24 @@ func (c *drpcCoordinatorClient) AclDeleteInvite(ctx context.Context, in *AclDele return out, nil } +func (c *drpcCoordinatorClient) FileLimitsGet(ctx context.Context, in *FileLimitsGetRequest) (*FileLimitsGetResponse, error) { + out := new(FileLimitsGetResponse) + err := c.cc.Invoke(ctx, "/coordinator.Coordinator/FileLimitsGet", drpcEncoding_File_coordinator_coordinatorproto_protos_coordinator_proto{}, in, out) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *drpcCoordinatorClient) FileUsageReport(ctx context.Context, in *FileUsageReportRequest) (*FileUsageReportResponse, error) { + out := new(FileUsageReportResponse) + err := c.cc.Invoke(ctx, "/coordinator.Coordinator/FileUsageReport", drpcEncoding_File_coordinator_coordinatorproto_protos_coordinator_proto{}, in, out) + if err != nil { + return nil, err + } + return out, nil +} + type DRPCCoordinatorServer interface { SpaceSign(context.Context, *SpaceSignRequest) (*SpaceSignResponse, error) SpaceStatusCheck(context.Context, *SpaceStatusCheckRequest) (*SpaceStatusCheckResponse, error) @@ -297,6 +317,8 @@ type DRPCCoordinatorServer interface { NotifySubscribe(*NotifySubscribeRequest, DRPCCoordinator_NotifySubscribeStream) error AclUploadInvite(context.Context, *AclUploadInviteRequest) (*AclUploadInviteResponse, error) AclDeleteInvite(context.Context, *AclDeleteInviteRequest) (*AclDeleteInviteResponse, error) + FileLimitsGet(context.Context, *FileLimitsGetRequest) (*FileLimitsGetResponse, error) + FileUsageReport(context.Context, *FileUsageReportRequest) (*FileUsageReportResponse, error) } type DRPCCoordinatorUnimplementedServer struct{} @@ -381,9 +403,17 @@ func (s *DRPCCoordinatorUnimplementedServer) AclDeleteInvite(context.Context, *A return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) } +func (s *DRPCCoordinatorUnimplementedServer) FileLimitsGet(context.Context, *FileLimitsGetRequest) (*FileLimitsGetResponse, error) { + return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) +} + +func (s *DRPCCoordinatorUnimplementedServer) FileUsageReport(context.Context, *FileUsageReportRequest) (*FileUsageReportResponse, error) { + return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) +} + type DRPCCoordinatorDescription struct{} -func (DRPCCoordinatorDescription) NumMethods() int { return 20 } +func (DRPCCoordinatorDescription) NumMethods() int { return 22 } func (DRPCCoordinatorDescription) Method(n int) (string, drpc.Encoding, drpc.Receiver, interface{}, bool) { switch n { @@ -567,6 +597,24 @@ func (DRPCCoordinatorDescription) Method(n int) (string, drpc.Encoding, drpc.Rec in1.(*AclDeleteInviteRequest), ) }, DRPCCoordinatorServer.AclDeleteInvite, true + case 20: + return "/coordinator.Coordinator/FileLimitsGet", drpcEncoding_File_coordinator_coordinatorproto_protos_coordinator_proto{}, + func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) { + return srv.(DRPCCoordinatorServer). + FileLimitsGet( + ctx, + in1.(*FileLimitsGetRequest), + ) + }, DRPCCoordinatorServer.FileLimitsGet, true + case 21: + return "/coordinator.Coordinator/FileUsageReport", drpcEncoding_File_coordinator_coordinatorproto_protos_coordinator_proto{}, + func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) { + return srv.(DRPCCoordinatorServer). + FileUsageReport( + ctx, + in1.(*FileUsageReportRequest), + ) + }, DRPCCoordinatorServer.FileUsageReport, true default: return "", nil, nil, nil, false } @@ -972,3 +1020,43 @@ func (x *drpcCoordinator_AclDeleteInviteStream) SendAndClose(m *AclDeleteInviteR } return x.CloseSend() } + +type DRPCCoordinator_FileLimitsGetStream interface { + drpc.Stream + SendAndClose(*FileLimitsGetResponse) error +} + +type drpcCoordinator_FileLimitsGetStream struct { + drpc.Stream +} + +func (x *drpcCoordinator_FileLimitsGetStream) GetStream() drpc.Stream { + return x.Stream +} + +func (x *drpcCoordinator_FileLimitsGetStream) SendAndClose(m *FileLimitsGetResponse) error { + if err := x.MsgSend(m, drpcEncoding_File_coordinator_coordinatorproto_protos_coordinator_proto{}); err != nil { + return err + } + return x.CloseSend() +} + +type DRPCCoordinator_FileUsageReportStream interface { + drpc.Stream + SendAndClose(*FileUsageReportResponse) error +} + +type drpcCoordinator_FileUsageReportStream struct { + drpc.Stream +} + +func (x *drpcCoordinator_FileUsageReportStream) GetStream() drpc.Stream { + return x.Stream +} + +func (x *drpcCoordinator_FileUsageReportStream) SendAndClose(m *FileUsageReportResponse) error { + if err := x.MsgSend(m, drpcEncoding_File_coordinator_coordinatorproto_protos_coordinator_proto{}); err != nil { + return err + } + return x.CloseSend() +} diff --git a/coordinator/coordinatorproto/coordinator_vtproto.pb.go b/coordinator/coordinatorproto/coordinator_vtproto.pb.go index 5b5561fb7..b5cafeb01 100644 --- a/coordinator/coordinatorproto/coordinator_vtproto.pb.go +++ b/coordinator/coordinatorproto/coordinator_vtproto.pb.go @@ -899,6 +899,18 @@ func (m *NetworkConfigurationResponse) MarshalToSizedBufferVT(dAtA []byte) (int, i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if len(m.FileNetworkId) > 0 { + i -= len(m.FileNetworkId) + copy(dAtA[i:], m.FileNetworkId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.FileNetworkId))) + i-- + dAtA[i] = 0x32 + } + if m.Epoch != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Epoch)) + i-- + dAtA[i] = 0x28 + } if m.CreationTimeUnix != 0 { i = protohelpers.EncodeVarint(dAtA, i, uint64(m.CreationTimeUnix)) i-- @@ -2650,6 +2662,236 @@ func (m *AclDeleteInviteResponse) MarshalToSizedBufferVT(dAtA []byte) (int, erro return len(dAtA) - i, nil } +func (m *FileLimitsGetRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FileLimitsGetRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *FileLimitsGetRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Identity) > 0 { + i -= len(m.Identity) + copy(dAtA[i:], m.Identity) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Identity))) + i-- + dAtA[i] = 0x12 + } + if len(m.SpaceId) > 0 { + i -= len(m.SpaceId) + copy(dAtA[i:], m.SpaceId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *FileLimitsGetResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FileLimitsGetResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *FileLimitsGetResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.SpaceLimitBytes != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.SpaceLimitBytes)) + i-- + dAtA[i] = 0x18 + } + if m.AccountTotalUsageBytes != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.AccountTotalUsageBytes)) + i-- + dAtA[i] = 0x10 + } + if m.AccountLimitBytes != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.AccountLimitBytes)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *FileUsageReportRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FileUsageReportRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *FileUsageReportRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Rows) > 0 { + for iNdEx := len(m.Rows) - 1; iNdEx >= 0; iNdEx-- { + size, err := m.Rows[iNdEx].MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *FileUsageRow) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FileUsageRow) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *FileUsageRow) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.DurableFilesCount != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.DurableFilesCount)) + i-- + dAtA[i] = 0x20 + } + if m.DurableBytes != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.DurableBytes)) + i-- + dAtA[i] = 0x18 + } + if len(m.Identity) > 0 { + i -= len(m.Identity) + copy(dAtA[i:], m.Identity) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Identity))) + i-- + dAtA[i] = 0x12 + } + if len(m.SpaceId) > 0 { + i -= len(m.SpaceId) + copy(dAtA[i:], m.SpaceId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *FileUsageReportResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FileUsageReportResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *FileUsageReportResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + return len(dAtA) - i, nil +} + func (m *SpaceSignRequest) SizeVT() (n int) { if m == nil { return 0 @@ -2987,6 +3229,13 @@ func (m *NetworkConfigurationResponse) SizeVT() (n int) { if m.CreationTimeUnix != 0 { n += 1 + protohelpers.SizeOfVarint(uint64(m.CreationTimeUnix)) } + if m.Epoch != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Epoch)) + } + l = len(m.FileNetworkId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } n += len(m.unknownFields) return n } @@ -3646,50 +3895,137 @@ func (m *AclDeleteInviteResponse) SizeVT() (n int) { return n } -func (m *SpaceSignRequest) UnmarshalVT(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: SpaceSignRequest: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: SpaceSignRequest: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field SpaceId", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift +func (m *FileLimitsGetRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.SpaceId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Identity) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + +func (m *FileLimitsGetResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.AccountLimitBytes != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.AccountLimitBytes)) + } + if m.AccountTotalUsageBytes != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.AccountTotalUsageBytes)) + } + if m.SpaceLimitBytes != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.SpaceLimitBytes)) + } + n += len(m.unknownFields) + return n +} + +func (m *FileUsageReportRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Rows) > 0 { + for _, e := range m.Rows { + l = e.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *FileUsageRow) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.SpaceId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Identity) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.DurableBytes != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.DurableBytes)) + } + if m.DurableFilesCount != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.DurableFilesCount)) + } + n += len(m.unknownFields) + return n +} + +func (m *FileUsageReportResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + n += len(m.unknownFields) + return n +} + +func (m *SpaceSignRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SpaceSignRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SpaceSignRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } @@ -5738,6 +6074,57 @@ func (m *NetworkConfigurationResponse) UnmarshalVT(dAtA []byte) error { break } } + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Epoch", wireType) + } + m.Epoch = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Epoch |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field FileNetworkId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.FileNetworkId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) @@ -9849,3 +10236,515 @@ func (m *AclDeleteInviteResponse) UnmarshalVT(dAtA []byte) error { } return nil } +func (m *FileLimitsGetRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FileLimitsGetRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FileLimitsGetRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Identity", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Identity = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *FileLimitsGetResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FileLimitsGetResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FileLimitsGetResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AccountLimitBytes", wireType) + } + m.AccountLimitBytes = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.AccountLimitBytes |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AccountTotalUsageBytes", wireType) + } + m.AccountTotalUsageBytes = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.AccountTotalUsageBytes |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceLimitBytes", wireType) + } + m.SpaceLimitBytes = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.SpaceLimitBytes |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *FileUsageReportRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FileUsageReportRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FileUsageReportRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Rows", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Rows = append(m.Rows, &FileUsageRow{}) + if err := m.Rows[len(m.Rows)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *FileUsageRow) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FileUsageRow: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FileUsageRow: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Identity", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Identity = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field DurableBytes", wireType) + } + m.DurableBytes = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.DurableBytes |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field DurableFilesCount", wireType) + } + m.DurableFilesCount = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.DurableFilesCount |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *FileUsageReportResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FileUsageReportResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FileUsageReportResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} diff --git a/coordinator/coordinatorproto/protos/coordinator.proto b/coordinator/coordinatorproto/protos/coordinator.proto index 2c54265e9..90d2047dd 100644 --- a/coordinator/coordinatorproto/protos/coordinator.proto +++ b/coordinator/coordinatorproto/protos/coordinator.proto @@ -60,6 +60,12 @@ service Coordinator { // AclDeleteInvite permanently removes an invite file from the filenodes. // The caller must be the space owner and the invite must be revoked in the acl. rpc AclDeleteInvite(AclDeleteInviteRequest) returns (AclDeleteInviteResponse); + // FileLimitsGet returns the file-storage pool bounds for an uploader identity in a space (files v2; fileV2 nodes only). + // The response carries absolute bounds — the caller computes effective headroom locally. + rpc FileLimitsGet(FileLimitsGetRequest) returns (FileLimitsGetResponse); + // FileUsageReport accepts per-(space, identity) durable file usage reports from fileV2 nodes. + // The reporting node is taken from the authenticated peer context. Values are absolute (idempotent re-reports). + rpc FileUsageReport(FileUsageReportRequest) returns (FileUsageReportResponse); } @@ -214,6 +220,11 @@ message NetworkConfigurationResponse { repeated Node nodes = 3; // unix timestamp of the creation time of configuration uint64 creationTimeUnix = 4; + // monotonically increasing version of the configuration, 0 if the coordinator predates epoch support + uint64 epoch = 5; + // identity of the fileV2 fleet's shared receipt-signing key (network + // string encoding); empty if the network has no fileV2 fleet + string fileNetworkId = 6; } // NodeType determines the type of API that a node supports @@ -232,6 +243,8 @@ enum NodeType { // PaymentProcessingAPI supports payment processing api // (see any-pp-node repository) PaymentProcessingAPI = 5; + // FileV2API supports the v2 file api (filenode v2 broker pool) + FileV2API = 6; } // DeletionChangeType determines the type of deletion payload @@ -515,3 +528,32 @@ message AclDeleteInviteRequest { } message AclDeleteInviteResponse {} +message FileLimitsGetRequest { + string spaceId = 1; + // Identity is the uploader account identity whose pool bounds are requested + string identity = 2; +} + +message FileLimitsGetResponse { + // AccountLimitBytes is the identity's total file-storage cap (the common account pool) + uint64 accountLimitBytes = 1; + // AccountTotalUsageBytes is the coordinator-aggregated durable usage across all the identity's spaces + uint64 accountTotalUsageBytes = 2; + // SpaceLimitBytes is an optional space-scoped cap; 0 = unset + uint64 spaceLimitBytes = 3; +} + +message FileUsageReportRequest { + repeated FileUsageRow rows = 1; +} + +message FileUsageRow { + string spaceId = 1; + // Identity is the charged uploader identity (the author of the root's earliest bind row) + string identity = 2; + // DurableBytes is the HEAD-measured durable usage of this (space, identity) slice; in-flight is never reported + uint64 durableBytes = 3; + uint64 durableFilesCount = 4; +} + +message FileUsageReportResponse {} diff --git a/coordinator/inboxclient/clienttestutil_test.go b/coordinator/inboxclient/clienttestutil_test.go index 100c32033..fb54a8999 100644 --- a/coordinator/inboxclient/clienttestutil_test.go +++ b/coordinator/inboxclient/clienttestutil_test.go @@ -181,6 +181,8 @@ func (m *mockConf) Name() (name string) { return nodeconf.CName } +func (m *mockConf) ObserveChanges(observer nodeconf.ChangeObserver) {} + func (m *mockConf) Run(ctx context.Context) (err error) { return nil } @@ -210,6 +212,14 @@ func (m *mockConf) FilePeers() []string { return nil } +func (m *mockConf) FileV2Peers() []string { + return nil +} + +func (m *mockConf) FileV2NodeIds(spaceId string) []string { + return nil +} + func (m *mockConf) ConsensusPeers() []string { return nil } diff --git a/coordinator/nodeconfsource/nodeconfsource.go b/coordinator/nodeconfsource/nodeconfsource.go index b2130b4f3..c7fb61ec9 100644 --- a/coordinator/nodeconfsource/nodeconfsource.go +++ b/coordinator/nodeconfsource/nodeconfsource.go @@ -48,6 +48,8 @@ func (n *nodeConfSource) GetLast(ctx context.Context, currentId string) (c nodec switch nt { case coordinatorproto.NodeType_FileAPI: types = append(types, nodeconf.NodeTypeFile) + case coordinatorproto.NodeType_FileV2API: + types = append(types, nodeconf.NodeTypeFileV2) case coordinatorproto.NodeType_CoordinatorAPI: types = append(types, nodeconf.NodeTypeCoordinator) case coordinatorproto.NodeType_TreeAPI: @@ -68,9 +70,11 @@ func (n *nodeConfSource) GetLast(ctx context.Context, currentId string) (c nodec } return nodeconf.Configuration{ - Id: res.ConfigurationId, - NetworkId: res.NetworkId, - Nodes: nodes, - CreationTime: time.Unix(int64(res.CreationTimeUnix), 0), + Id: res.ConfigurationId, + NetworkId: res.NetworkId, + FileNetworkId: res.FileNetworkId, + Nodes: nodes, + CreationTime: time.Unix(int64(res.CreationTimeUnix), 0), + Epoch: res.Epoch, }, nil } diff --git a/coordinator/nodeconfsource/nodeconfsource_test.go b/coordinator/nodeconfsource/nodeconfsource_test.go new file mode 100644 index 000000000..7553ce980 --- /dev/null +++ b/coordinator/nodeconfsource/nodeconfsource_test.go @@ -0,0 +1,68 @@ +package nodeconfsource + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/anyproto/any-sync/coordinator/coordinatorclient/mock_coordinatorclient" + "github.com/anyproto/any-sync/coordinator/coordinatorproto" + "github.com/anyproto/any-sync/nodeconf" +) + +func newTestSource(t *testing.T, resp *coordinatorproto.NetworkConfigurationResponse) *nodeConfSource { + ctrl := gomock.NewController(t) + t.Cleanup(ctrl.Finish) + cl := mock_coordinatorclient.NewMockCoordinatorClient(ctrl) + cl.EXPECT().NetworkConfiguration(gomock.Any(), gomock.Any()).Return(resp, nil) + return &nodeConfSource{cl: cl} +} + +func TestNodeConfSource_GetLast_FileV2(t *testing.T) { + t.Run("maps FileV2API to NodeTypeFileV2", func(t *testing.T) { + src := newTestSource(t, &coordinatorproto.NetworkConfigurationResponse{ + ConfigurationId: "new", + NetworkId: "net", + Nodes: []*coordinatorproto.Node{ + {PeerId: "n1", Types: []coordinatorproto.NodeType{coordinatorproto.NodeType_FileV2API}}, + {PeerId: "n2", Types: []coordinatorproto.NodeType{coordinatorproto.NodeType_TreeAPI, coordinatorproto.NodeType_FileV2API}}, + }, + }) + conf, err := src.GetLast(context.Background(), "old") + require.NoError(t, err) + require.Len(t, conf.Nodes, 2) + assert.Equal(t, []nodeconf.NodeType{nodeconf.NodeTypeFileV2}, conf.Nodes[0].Types) + assert.Equal(t, []nodeconf.NodeType{nodeconf.NodeTypeTree, nodeconf.NodeTypeFileV2}, conf.Nodes[1].Types) + }) + + // Simulates an OLD client receiving a node type its proto/switch does not know: + // the unknown enum value must be silently dropped while known types are kept, + // so the node stays in the sync pool and routing is unchanged. + t.Run("unknown enum type is dropped, known types preserved", func(t *testing.T) { + const unknownType = coordinatorproto.NodeType(9999) + src := newTestSource(t, &coordinatorproto.NetworkConfigurationResponse{ + ConfigurationId: "new", + NetworkId: "net", + Nodes: []*coordinatorproto.Node{ + {PeerId: "n1", Types: []coordinatorproto.NodeType{coordinatorproto.NodeType_TreeAPI, unknownType}}, + {PeerId: "n2", Types: []coordinatorproto.NodeType{unknownType}}, + }, + }) + conf, err := src.GetLast(context.Background(), "old") + require.NoError(t, err) + require.Len(t, conf.Nodes, 2) + // known type kept + assert.Equal(t, []nodeconf.NodeType{nodeconf.NodeTypeTree}, conf.Nodes[0].Types) + // unknown-only node ends up with no recognized role (empty), i.e. inert — no panic, no misroute + assert.Empty(t, conf.Nodes[1].Types) + }) + + t.Run("unchanged configuration returns ErrConfigurationNotChanged", func(t *testing.T) { + src := newTestSource(t, &coordinatorproto.NetworkConfigurationResponse{ConfigurationId: "same"}) + _, err := src.GetLast(context.Background(), "same") + assert.ErrorIs(t, err, nodeconf.ErrConfigurationNotChanged) + }) +} diff --git a/coordinator/subscribeclient/client.go b/coordinator/subscribeclient/client.go index 289067a8e..b72080f3c 100644 --- a/coordinator/subscribeclient/client.go +++ b/coordinator/subscribeclient/client.go @@ -43,6 +43,7 @@ type subscribeClient struct { ctx context.Context ctxCancel context.CancelFunc close chan struct{} + running bool } func (s *subscribeClient) Init(a *app.App) (err error) { @@ -60,18 +61,26 @@ func (s *subscribeClient) Name() (name string) { } func (s *subscribeClient) Run(ctx context.Context) error { + s.mu.Lock() + s.running = true + s.mu.Unlock() go s.streamWatcher() return nil } func (s *subscribeClient) Close(_ context.Context) (err error) { s.mu.Lock() + running := s.running if s.stream != nil { _ = s.stream.Close() } s.mu.Unlock() s.ctxCancel() - <-s.close + // s.close is closed by streamWatcher, which only exists after Run: app.Start + // closes components it never ran + if running { + <-s.close + } return nil } diff --git a/coordinator/subscribeclient/client_test.go b/coordinator/subscribeclient/client_test.go new file mode 100644 index 000000000..ecbec0d6d --- /dev/null +++ b/coordinator/subscribeclient/client_test.go @@ -0,0 +1,32 @@ +package subscribeclient + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/anyproto/any-sync/coordinator/coordinatorproto" +) + +// app.Start closes components it never ran: a failing Init makes closeServices +// call Close on every component registered before it. s.close is only closed by +// streamWatcher, which Run starts. +func TestSubscribeClient_CloseWithoutRun(t *testing.T) { + s := &subscribeClient{} + s.ctx, s.ctxCancel = context.WithCancel(context.Background()) + s.close = make(chan struct{}) + s.callbacks = make(map[coordinatorproto.NotifyEventType]EventCallback) + + closed := make(chan struct{}) + go func() { + require.NoError(t, s.Close(context.Background())) + close(closed) + }() + select { + case <-closed: + case <-time.After(time.Second): + require.Fail(t, "Close deadlocked when Run was never called") + } +} diff --git a/docs/rpc-error-offsets.md b/docs/rpc-error-offsets.md new file mode 100644 index 000000000..84e9d17c0 --- /dev/null +++ b/docs/rpc-error-offsets.md @@ -0,0 +1,111 @@ +# RPC Error Code Offsets + +## Overview + +any-sync maps typed RPC errors onto numeric [dRPC](https://storj.io/drpc) error codes +through a single **process-global** registry in +[`net/rpc/rpcerr`](../net/rpc/rpcerr/registry.go). + +```go +// net/rpc/rpcerr/registry.go +var errsMap = make(map[uint64]error) // global, one per process + +func RegisterErr(err error, code uint64) error { + if e, ok := errsMap[code]; ok { + panic(fmt.Errorf("attempt to register error with existing code: %d ...", code)) + } + // ... +} + +type ErrGroup int64 +func (g ErrGroup) Register(err error, code uint64) error { + return RegisterErr(err, uint64(g)+code) // registers at offset+code +} +``` + +Each proto/service declares an `ErrorOffset` constant and registers its errors +through `rpcerr.ErrGroup(ErrorOffset)`; a concrete error's wire code is +`ErrorOffset + localCode`. + +### Why this needs coordinating across repos + +`errsMap` is **global to the running binary**, and registration happens in +`var`/`init` blocks at import time. A node binary routinely links **any-sync +plus one or more downstream repos** (e.g. `any-sync-node` also registers +`nodesync` errors). If any two groups ever register the same `offset + code`, +the program **panics at startup** — before any request is served. + +So offsets are a shared namespace across the whole anyproto ecosystem, not just +within one repo. **This file is the source of truth for who owns which offset.** + +## Allocation rules + +1. **Offsets are multiples of 100.** Each group owns the band + `[offset, offset+99]`. +2. **Local codes must stay in `0..99`** so a group never bleeds into the next + band. (The `ErrorOffset` enum value itself is only the base — it is never + registered as a code.) +3. **Local code `0` is the group's `Unexpected`/fallback** (registers at + `offset+0`), mirroring `filesync.ErrCodes.Unexpected = 0`. +4. **any-sync core historically used `< 1000`.** That range is now fully + allocated (100–900), so new **core** groups take the next free `>= 1000` + offset (e.g. `pubsub` at 1100). Downstream repos that import any-sync also + use **`>= 1000`**; the two now share that space, so always claim the next + free offset from the tables below and add a row. +5. When you add a group, **pick the next free offset and add a row below.** + +### Globally reserved low codes + +The base registry claims two codes directly (outside any group), so **no group +may use offset `0`**: + +| Code | Name | Source | +|------|------|--------| +| 1 | `Unexpected` | `net/rpc/rpcerr/registry.go` | +| 2 | `Closed` | `net/rpc/rpcerr/registry.go` | + +## Reserved offsets + +### any-sync (core, `< 1000`) + +| Offset | Band | Proto / service | Package | +|--------|------|-----------------|---------| +| 100 | 100–199 | `spacesync` | `commonspace/spacesyncproto` | +| 200 | 200–299 | `filesync` (File, v1) | `commonfile/fileproto` | +| 300 | 300–399 | `coordinator` | `coordinator/coordinatorproto` | +| 400 | 400–499 | `treechange` | `commonspace/object/tree/treechangeproto` | +| 500 | 500–599 | `consensus` | `consensus/consensusproto` | +| 600 | 600–699 | `payment` | `paymentservice/paymentserviceproto` | +| 700 | 700–799 | `limiter` | `net/rpc/limiter/limiterproto` | +| 800 | 800–899 | `filesyncv2` (FileV2, v2 broker) | `commonfile/fileproto/fileprotov2` | +| 900 | 900–999 | `filesyncp2p` (FileP2P, p2p file transfer) | `commonfile/fileproto/filep2p` | + +The `< 1000` range is now full; further **core** groups continue in the +`>= 1000` tables below (see `pubsub` at 1100). + +### Core (any-sync) & downstream repos (`>= 1000`) + +Downstream groups live in other repositories but share this global registry +whenever their binary also links any-sync. Core any-sync groups appear here too +once the `< 1000` range is exhausted. + +| Offset | Band | Proto / service | Repo · package | +|--------|------|-----------------|----------------| +| 1000 | 1000–1099 | `nodesync` | `any-sync-node` · `nodesync/nodesyncproto` | +| 1100 | 1100–1199 | `pubsub` (space-scoped stateless pub/sub) | any-sync **(core)** · `commonspace/pubsub/pubsubproto` | +| 1200 | 1200–1299 | `push` | `anytype-push-server` · `pushclient/pushapi` | +| 1300 | 1300–1399 | `bobrik` | `bobrik-clusterctl` · `bobrikclient/bobrikapi` | +| 1400+ | — | _free_ | — | + +## Adding a new group + +1. Choose the next free offset from the tables above (core: next free `< 1000`; + downstream: next free `>= 1000`). +2. In the `.proto`, add `ErrorOffset = ;` to the service's `ErrCodes` + enum, with local codes `0..N` (`0` = `Unexpected`). +3. Register in a Go errors file via `rpcerr.ErrGroup(_ErrorOffset)` + (see `commonfile/fileproto/fileprotov2/fileprotov2err/fileprotov2err.go` for + the current template). +4. **Add a row to the correct table in this file.** +5. Build and run any test — a colliding offset panics at init, so a green test + run proves the new band is clear. diff --git a/docs/stateless-pubsub.md b/docs/stateless-pubsub.md new file mode 100644 index 000000000..ab10564b9 --- /dev/null +++ b/docs/stateless-pubsub.md @@ -0,0 +1,750 @@ +# Space-Scoped Stateless Pub/Sub over any-sync + +Status: **IMPLEMENTED (v1)** — shipped in this repo as `commonspace/pubsub` (engine) plus +additions to `net/streampool`. This document is the as-built reference: it describes what +the code does, why the design is shaped this way, and what is deliberately left for +downstream repos or a later version. Grounded against the current tree +(`commonspace/pubsub/`, `net/streampool/`). + +Downstream wiring — the any-sync-node relay component and the anytype-heart client — lives +in separate repos and is tracked in [§13](#13-implementation-status). + +--- + +## 1. Summary + +An ephemeral, fire-and-forget, at-most-once publish/subscribe channel scoped to a space, +carried over a **dedicated DRPC bidi stream** fully isolated from the sync engine. Topics +are plaintext `/`-separated hierarchies inside a space; subscriptions may use NATS-style +wildcards (`*` one segment, `>` trailing segments). Any space member (Reader/Guest +included) may publish and subscribe. Payloads are end-to-end encrypted with the space +ReadKey and signed with the sender's account key; relay nodes route ciphertext they cannot +read. Fan-out goes through the responsible sync nodes **and** directly to LAN-discovered +peers, with a small bounded msgId dedup cache on receivers. No message is ever persisted; +the only routing state is transient in-memory interest state (stream tags + a pattern trie) +that dies with the connection. + +The engine (`commonspace/pubsub.Service`, `CName = "common.commonspace.pubsub"`) is a single +component used by **both** sides: nodes wire it with a `Relay` (relay role), clients wire it +with a `PeerProvider` (client role). One long-lived `PubSubStream` per peer pair multiplexes +all spaces and topics. + +### Design decisions + +| Question | Decision | +|---|---| +| Meaning of "stateless" | No message persistence anywhere. Transient in-memory interest tables (stream tags + trie) are allowed and used. | +| Stream reuse vs dedicated | Dedicated `PubSub` DRPC service + its own streampool instance; never touches `ObjectSyncStream` or the sync dispatch path. | +| New service vs new RPC on SpaceSync | New service, own proto package — independent versioning, unknown-service fallback for old peers. | +| Topic model | `/`-separated segment hierarchy within a space. Subscriptions may use NATS-style wildcards: `*` matches exactly one segment, `>` matches one-or-more trailing segments (tail-only). Publishers must use fully-qualified topics. Matching runs on a bounded per-space pattern trie at the relay ([§5.1](#51-interest-matching-wildcards)). | +| Topic privacy | Plaintext to relays. Comparable exposure to today's plaintext `objectId`s on the sync path. | +| Subscribe permission | Any space member: `!NoPermissions()` (`commonspace/object/acl/list/models.go:90`). | +| Publish permission | Any space member. Attribution via per-message account-key signature; abuse contained by per-peer rate limits. Plus a reserved **self-owned namespace**: topics `acc/…/` accept publishes only from `accId` ([§7.2](#72-access-control)). | +| Payload confidentiality | Encrypted client-side with the current space ReadKey + `keyId` indirection. Removed member loses access at next key rotation. | +| Authenticity | Per-message account-key signature, verified by receivers. Protects against a semi-trusted node forging or reattributing messages. | +| Topology | Publisher → its responsible node (+ direct LAN peers). Node relays once to the other responsible nodes (`relayed` flag, never re-forwarded → loop-free). | +| Duplicate suppression | Receiver-side bounded FIFO ring keyed by `msgId` (duplicate paths exist by design: LAN + node). | +| Backpressure | Bounded queues, `TryAdd` drop-on-overflow end to end — the existing streampool discipline. Slow subscriber ⇒ dropped messages, never memory growth. | +| Queue groups (1-of-N) | Non-goal v1. | +| Catch-up / replay / retained messages | Non-goal, by definition of stateless. Reconnect ⇒ resubscribe ⇒ only new messages. | +| Presence lifecycle | Not in the protocol. v1 is a generic app API; presence (heartbeat/timeout/leave, Yjs-awareness style) is an app pattern on top ([Appendix A](#appendix-a--presence-as-an-app-pattern-non-normative)). | + +### Requirements compliance + +The feature was specified against three hard requirements from the original ask ("users of +the same space can subscribe/publish topics within a space; work effectively over the +any-sync protocol; be memory-effective; respect ACL access"): + +- **R1 (on-protocol):** rides existing transports, the secureservice handshake, the DRPC + mux, and a second streampool instance. A dedicated sub-stream on the existing `MultiConn` + — no new transport or TLS handshake. +- **R2 (memory-effective):** per-stream bounded `mb.MB` queues with drop-on-overflow + (`net/streampool/stream.go:32-44`), interest state = stream tags + pattern trie (both + O(active subscriptions)), a fixed-size dedup FIFO ring, a per-peer publish token bucket, + and caps on pattern count and topic/payload size. No path grows with message volume or + offline duration. +- **R3 (ACL-aware):** the serving peer gates subscribe *and* publish on space membership at + the current ACL head (new, additive enforcement — the sync path has no such check at + subscribe time); confidentiality holds against the relay via ReadKey encryption; membership + removal cuts new traffic at key rotation and actively drops subscriptions ([§7.4](#74-confidentiality--membership-change)). + +--- + +## 2. Background & rationale + +This section distills the research that shaped the design, for readers who want the *why* +without the original problem-framing document. + +### 2.1 Where this sits in the pub/sub landscape + +"Stateless pub/sub" here means the **core-NATS / Redis-pub-sub** family: fire-and-forget, +at-most-once, no persistence, full decoupling of publishers and subscribers. The relay may +hold a small *transient* in-memory interest table (as core NATS does) — that is not +"stateful" in the sense that matters; only durable message state is excluded. + +| System | Topic model | Wildcards | Delivery | Persistence | Notable for us | +|---|---|---|---|---|---| +| **NATS core** | dot-hierarchy subjects | `*` (1 token), `>` (tail multi) | fan-out + queue-group (1-of-N) | none (JetStream is separate) | the canonical model; subject wildcards; sublist trie | +| **Redis pub/sub** | flat channels + patterns | `*`, `?`, `[..]` glob | fan-out | none | simplest fire-and-forget; subscriber must be connected | +| **MQTT** | `/`-hierarchy topics | `+` (1 level), `#` (tail multi) | QoS 0/1/2 | retained msg + sessions ⇒ *stateful* | wildcard syntax variant; retained message is the anti-pattern to avoid | +| **libp2p GossipSub** | flat topics | none | epidemic mesh fan-out | none | closest to any-sync's decentralization ethos; peer-scoring for abuse | +| **Yjs awareness** | one implicit channel/doc | n/a | state-based CRDT diff | ephemeral, auto-GC | the presence reference: `(clientID, clock, state\|null)`, re-announce/timeout | +| **Phoenix Channels** | `topic:subtopic` strings | none | fan-out; Presence on top | ephemeral | presence = minimal ephemeral metadata over pub/sub | +| **Matrix EDUs** | per-room | n/a | fan-out to room servers | ephemeral | typing/presence as Ephemeral Data Units, distinct from the persistent event DAG | + +Three cross-cutting lessons drove the design: + +- **Topology is the fork.** Central-broker systems keep an interest table; GossipSub is a + brokerless mesh. any-sync sits between: clients reach a small set of **responsible sync + nodes** (semi-trusted relays that store ciphertext they cannot read) and *may* also reach + other clients directly (LAN). The relay-through-node model is broker-like, but the broker + is **untrusted for content** — which forces the encryption + signing model in [§7](#7-security-model). +- **Presence is the killer use case**, and it is *always* kept separate from the persistent + document layer (Yjs awareness, Phoenix Presence, Matrix EDUs). v1 keeps presence out of + the protocol and provides it as an app recipe ([Appendix A](#appendix-a--presence-as-an-app-pattern-non-normative)). +- **Wildcards cost the relay** a per-message trie walk; flat topics do not. We accept that + cost for expressiveness, bounded by per-space pattern caps ([§5.1](#51-interest-matching-wildcards)). + +### 2.2 The any-sync substrate we build on + +The load-bearing facts about this codebase that the design leans on: + +- **StreamPool is the fan-out core.** `net/streampool` caches `drpc.Stream`s, indexes them + by peer and by **tag**, opens them lazily, and pushes messages onto **bounded** per-stream + write queues (`mb.MB`, `TryAdd`, drop-on-overflow — `net/streampool/stream.go:32-44`). + `Broadcast(msg, tags...)` is the existing tag-keyed fan-out primitive; `AddTagsCtx` / + `RemoveTagsCtx` are the existing dynamic (un)subscribe hooks. Pub/sub instantiates its own + pool rather than layering onto the sync pool. +- **Responsible nodes via consistent hash.** `NodeIds(spaceId)` returns the tree nodes on + the chash ring for a space; `ReplicationFactor = 3` ⇒ up to 3 responsible sync nodes per + space. These are the always-reachable fan-out hubs. +- **ACL is cryptographic, not a per-message live check.** Writes are enforced when a signed + record is validated on apply; reads are enforced by **encryption** — content is encrypted + with a per-space `ReadKey` handed only to members, and the key rotates on membership + change. The permission ladder is `None=0, Owner=1, Admin=2, Writer=3, Reader=4, Guest=5`; + there is no `CanRead()` — "can read" is `!NoPermissions()`. The shared sync layer performs + **no** per-message reader-authorization, so pub/sub's ACL gating at subscribe/publish time + is new, additive enforcement. +- **Don't reuse `ObjectSyncStream`.** Every frame on it is routed into the sync engine and + shares one bounded queue; multiplexing ephemeral pub/sub there would couple the two + (head-of-line blocking, intertwined backpressure) and muddle fire-and-forget ephemera with + DAG anti-entropy. A separate multiplexed DRPC stream is cheap — another sub-stream on the + existing `MultiConn`, not a new handshake — so pub/sub gets its own stream, tags, and + queues, fully isolated from sync flow control. + +### 2.3 Why the big forks landed where they did + +- **Dedicated service, not a new SpaceSync RPC** — independent lifecycle and versioning; + old peers that don't know `PubSub` fail the stream open cleanly and the client treats the + peer as pubsub-unavailable. +- **Relay-through-node, not pure mesh** — nodes are the only always-reachable members; mesh + fits the ethos but has no guaranteed connectivity. LAN peers are used *in addition*, not + instead. +- **Per-message signing on top of encryption** — encryption alone gives confidentiality but + not authenticity: a Reader (or the relay) could spoof or reattribute a message that other + members can still decrypt. Ed25519 sign/verify (~30–80 µs) is negligible at + ephemeral-signal rates and buys spoof-proof attribution, which the self-owned `acc/` + namespace then builds on. +- **No queue groups, no interest propagation between nodes in v1** — see [§14](#14-non-goals-v1). + +--- + +## 3. Semantics contract + +- **At-most-once.** A publish reaching the relay is copied into bounded per-subscriber + queues; overflow drops. No acks, no retransmit, no ordering across publishers, no dedup + beyond the duplicate-path ring. +- **Fire-and-forget.** A publish with no subscribers is discarded. Nothing is stored. +- **Decoupled.** Publishers don't know subscribers. The subscriber set is whoever holds a + live tagged stream at the instant of fan-out. +- **Subscribe is unacknowledged.** Success is silent ([§6](#6-what-happens-on-each-frame-serving-peer)); interest takes effect when the serving + peer processes the frame, so messages published concurrently by others may be missed. + There is no "subscribed as of time T" guarantee — the same race NATS documents for + cluster-wide subscription visibility, inherent to at-most-once. Apps needing a consistent + starting state combine pub/sub with a snapshot read (e.g. presence: subscribe, then + announce yourself, which prompts others' next heartbeat). +- **Reconnect = clean slate.** Subscriptions die with the stream; the client re-subscribes on + reconnect (via the resync loop, [§9](#9-lifecycle--reconnect)) and sees only new traffic. +- **Echo.** A publisher whose own interest set matches the topic would receive its own + message back from the relay. The client suppresses this by pre-recording its own `msgId` in + the dedup ring at publish time and delivering to local handlers directly (bounded, + drop-on-overflow dispatch queue — so local delivery is best-effort too). Net effect = NATS + `no_echo` semantics without a wire flag. +- **Duplicates possible in theory** (ring eviction under extreme rates), so payload design + must be idempotent / last-write-wins at the app level. In practice the ring makes + duplicates vanishingly rare. + +--- + +## 4. Wire protocol + +Proto package `commonspace/pubsub/pubsubproto/protos/pubsub.proto` (generated via the +Makefile pipeline): + +```proto +syntax = "proto3"; +package pubsub; + +service PubSub { + // One long-lived bidi stream per peer pair, multiplexing all spaces/topics. + rpc PubSubStream(stream PubSubMessage) returns (stream PubSubMessage); +} + +message PubSubMessage { + oneof content { + Subscribe subscribe = 1; + Unsubscribe unsubscribe = 2; + Publish publish = 3; + Status status = 4; + } +} + +// Delta semantics: adds topic patterns to the stream's interest set for spaceId. +// Patterns may contain wildcards: '*' (one segment), '>' (trailing segments, tail-only). +message Subscribe { + string spaceId = 1; + repeated string topics = 2; +} + +// Removes patterns (matched verbatim against the interest set, not expanded); +// empty topics = remove all patterns of spaceId. +message Unsubscribe { + string spaceId = 1; + repeated string topics = 2; +} + +message Publish { + string spaceId = 1; + string topic = 2; // fully-qualified; wildcards not allowed + bytes msgId = 3; // 16 random bytes, generated by publisher (dedup key) + string keyId = 4; // space ReadKey id used for payload; "" = plaintext (keyless spaces) + bytes payload = 5; // ciphertext (or plaintext iff keyId == "") + bytes identity = 6; // sender account pubkey (marshalled) + bytes signature = 7; // account-key sig, see §7.3 + int64 timestampMilli = 8; // sender wall clock, informational + bool relayed = 9; // set by a node when forwarding node→node; never re-forwarded; excluded from signature +} + +// Sent by the serving peer on a rejected subscribe/publish. Success is silent. +message Status { + string spaceId = 1; + repeated string topics = 2; // echo of the offending request (topic of a publish goes here too) + ErrCodes code = 3; + bytes msgId = 4; // echoes a rejected publish's id for correlation; empty for subscribe rejections +} + +enum ErrCodes { + Unexpected = 0; // zero value: a zeroed Status reads as an error, not success + NotAMember = 1; // identity has no permissions in the space's ACL + NotResponsible = 2; // the serving node is not responsible for the space + RateLimited = 3; + TooManyTopics = 4; + InvalidMessage = 5; // malformed frame, oversized payload, identity mismatch, bad signature + TopicNotOwned = 6; // publish into acc/…/ by an identity other than accId + InvalidTopic = 7; // malformed topic/pattern: bad wildcard placement, reserved chars, non-canonical form + ErrorOffset = 1100; // core bands 100..900 are full; pubsub takes 1100 (see docs/rpc-error-offsets.md) +} +``` + +Constraints enforced by the serving peer (config defaults, [§10](#10-configuration--defaults)). A rejected +Subscribe/Publish gets a `Status` and is otherwise ignored — the stream **stays open** (the +NATS precedent: a limit violation is an error reply, not a disconnect): + +- **topic:** UTF-8, 1..256 bytes, `/`-separated segments (≤16 segments, no empty segments); + recommended segment charset alnum + `.-_`. Canonical form has **no leading `/`** (`acc/x` + and `/acc/x` would silently differ — rejected as `InvalidTopic`). The 256-byte limit is a + compile-time constant (`topic.go`), not a config knob. +- **wildcards (subscription patterns only):** `*` matches exactly one segment + (`chat/*/typing` ⇒ `chat/abc/typing`, not `chat/typing` or `chat/a/b/typing`); `>` matches + one-or-more trailing segments and is valid only as the final segment (`chat/>` ⇒ everything + under `chat/`). Wildcards must be complete segments (`chat/ty*` is invalid). A `Publish` + topic containing `*` or `>` is rejected — publishers always use fully-qualified topics. +- **reserved self-owned namespace:** a topic whose first segment is `acc` is publishable + **only** by the account whose id equals the topic's *last* segment — e.g. + `acc/online/`, `acc/cursor/`. Subscribe stays open to all members (including + patterns like `acc/online/*`). Violations ⇒ `Status{TopicNotOwned}`. +- **payload:** ≤ 64 KiB (enforced after DRPC decode). +- **per-stream interest set:** ≤ 100 patterns per space, ≤ 1000 total. +- **identity in a Publish must equal the connection-context identity** (`net/peer/context.go:88`) + when arriving from a client (`relayed == false`). This binds attribution to the + handshake-proven account without the node verifying the signature per message. + +Interest tag format inside the pool: `spaceId + "/" + pattern`, verbatim (spaceId contains no +`/`; first separator wins). Wildcard resolution happens in the pubsub engine's matcher, not +in the pool ([§5.1](#51-interest-matching-wildcards)). + +--- + +## 5. Topology & relay rules + +``` + publisher client ──publish──▶ responsible node A ──relayed=true──▶ node B ──▶ its subscribers + │ │ node C ──▶ its subscribers + │ └──▶ A's local subscribers (tag fan-out) + └──────direct publish──▶ LAN peers (same space, discovered via mDNS) +``` + +1. **Clients never forward.** A client receiving a `Publish` (from a node or a LAN peer) + delivers it locally only. +2. **A node forwards only client-originated messages** (`relayed == false` arriving on a + client stream): it stamps `relayed = true` and sends one copy to each *other* responsible + node for the space, plus fans out to its local subscribers via the tag index. +3. **A node never forwards `relayed == true`** — it only fans out locally, and only if the + sending peer is itself a responsible node for the space (`Relay.IsResponsibleNode`). With + `ReplicationFactor = 3`, every message traverses at most client → node → node, hop limit + 2, loop-free without any dedup state on nodes. (This is the NATS cluster rule: messages + from a route are distributed only to local clients.) +4. **A node rejects subscribe/publish for spaces it is not responsible for** + (`Status{NotResponsible}`, via `Relay.IsResponsible`). +5. **LAN peers are symmetric.** Both sides run the same pubsub component; a LAN peer's stream + carries Subscribe frames like a node's does, and publishes to LAN peers go direct. Clients + apply the member-check on LAN subscribes the same way nodes do (they hold the ACL). + +Duplicate paths are expected (a subscriber may get the same message from a LAN peer and from +its node). The **receiver** suppresses via a fixed-size FIFO ring keyed by `msgId` (default +4096 entries). Publishers self-suppress echoes by msgId too ([§3](#3-semantics-contract), Echo). + +Client → node selection piggybacks on the host's existing responsible-peer choice (one node +at a time), so the pubsub stream goes to the same node the client already syncs with. + +### 5.1 Interest matching (wildcards) + +The streampool tag index is exact-match (`streamIdsByTag`, `net/streampool/streampool.go`), +so the pubsub engine layers a matcher on top rather than replacing the pool: + +- Each accepted subscription registers its **pattern string verbatim as the stream tag** + (`spaceId + "/" + pattern`) — the pool keeps doing stream bookkeeping (add/remove/GC on + stream close) exactly as today. +- In parallel, the engine maintains a **per-space segment trie** of live patterns, mirroring + the NATS sublist shape (`trie.go`): one level per segment, a `map[segment]` for literals + plus two dedicated wildcard slots per level (`pwc` for `*`, `fwc` for `>`), each terminal + holding a refcount of subscribing streams. Match order follows NATS `matchLevel`: at each + level add `fwc` matches, branch through `pwc`, hash-lookup the literal. +- On publish to concrete topic `T`: walk the trie with `T`'s segments, collecting every + matching pattern (O(segments × matched branches), segments ≤ 16). Then fan out once per + matched pattern tag, **deduplicating stream ids across patterns**: a stream subscribed to + both `chat/>` and `chat/*/typing` receives one copy, not two. `streampool.Broadcast` dedups + stream ids across the tag set it is given (it builds a `seen` set when handed more than one + tag — `net/streampool/streampool.go:411-439`), so the engine passes all matched pattern + tags to a single `Broadcast` call and the pool guarantees one copy per stream. +- Trie cleanup: refcount decrement on Unsubscribe; on stream close the engine reconciles via + the pool's `WithStreamCloseHook` callback (`net/streampool/streampool.go:43`), keyed by + `streamId`, dropping the stream's patterns from the trie. +- Exact-topic subscriptions are just patterns without wildcard segments — one code path. + +The trie is bounded by the same caps as the interest set (≤ 1000 patterns/stream, ≤ 100/space/ +stream), so relay memory stays O(active subscriptions), and matching cost is paid only per +publish within that space. + +**Deliberately no match-result cache.** NATS fronts its sublist with a literal-subject → +result cache, but its own history (`<0.5%` hit rates, lock contention, latency spikes under +sub/unsub churn) led to `NoCache` sublists for exactly our analog: small, churny, +per-connection tries. Our tries are per-space (small) and ephemeral interest is churny, so a +direct walk of a ≤ 16-level trie beats a cache we'd constantly flush. Revisit only with +profiling evidence. + +--- + +## 6. What happens on each frame (serving peer) + +**Subscribe** — validate `spaceId` (non-empty, no `/`); resolve space ACL state; reject with +`Status{NotResponsible}` if the node isn't responsible; check membership +(`MembershipChecker.CheckMember`); validate patterns (`InvalidTopic` on bad wildcard +placement / reserved chars / non-canonical form); check pattern-count caps; then register in +the trie and add the tags to the stream. Interest and tag are committed under one lock with +rollback if the stream vanished mid-subscribe. Reject ⇒ `Status`, no tag. + +**Unsubscribe** — remove tags + trie refcount decrement. No checks needed. + +**Publish (from a client stream, `relayed == false`)** — +1. Size/shape checks (a topic containing `*`/`>` ⇒ `InvalidTopic`); `identity == ctxIdentity` + with a non-empty guard; responsibility check; membership check against cached ACL state; + self-owned-namespace check (topic `acc/…` ⇒ last segment must equal the sender's account — + one string compare); per-peer token bucket ([§8](#8-memory--abuse-bounds)). A rejected publish returns a + `Status` (rate-limited publishes are notified per rejection, not once per window). +2. Match the topic against the space's pattern trie ([§5.1](#51-interest-matching-wildcards)), dedup stream ids across + matched patterns, then `Broadcast` on the pubsub pool — the existing tag-index fan-out, + which per-stream `TryAdd`s and drops on overflow. +3. Stamp `relayed = true` and send to the other responsible nodes (`Relay.OtherResponsiblePeers`, + lazy stream open via the pool). + +**Publish (`relayed == true`, from a node stream)** — verify the sending peer is a +responsible node for the space (`Relay.IsResponsibleNode`); fan out locally only (same trie +match). The `identity == ctxIdentity` rule does not apply (the forwarding node is not the +author); authenticity is the receiver's signature check ([§7.3](#73-authenticity)). + +**Receive (client)** — the order is cheap-filters-first, verify, then dedup: +local-interest match → membership (`identity` is a member at the local ACL head) → if the +topic is in the `acc/` namespace, check its last segment equals `identity.Account()` +(end-to-end: the signature covers the topic, so not even a malicious relay can inject into +someone else's self-owned topic) → timestamp staleness → **Ed25519 signature verify** → +record `msgId` in the dedup ring → resolve ReadKey by `keyId`, decrypt → dispatch to local +topic handlers on the bounded dispatch queue. Any failure ⇒ drop + debug metric, never an +error to the peer. Recording the ring only *after* verify means a forged-signature flood is +shed cheaply and cannot evict legitimate ids to reopen a replay window. + +--- + +## 7. Security model + +### 7.1 Threat model — the semi-trusted relay + +The node **can**: observe spaceIds, topics, sender identities, timing, sizes (accepted — +comparable to sync-path metadata today); drop, delay, reorder messages (accepted — +at-most-once contract). The node **cannot**: read payloads (no ReadKey); forge or reattribute +messages (signature); replay effectively. Replay defense is two-layer: the `msgId` dedup ring +catches the short window, and the receiver enforces a **signed-timestamp staleness window** +(`Config.MaxTimestampSkew`, default 5 min) so that even after the ring evicts an id, a relay +replaying an old signed frame is rejected on its stale timestamp. A relay cannot forge a fresh +timestamp — it is covered by the signature. + +### 7.2 Access control + +Both directions gate on **space membership at the current ACL head**, checked by the serving +peer from its local ACL copy — a cached in-memory lookup, no coordinator round trip. This is +*new* enforcement: today's sync path has none at subscribe time. Publish and subscribe both +require `!NoPermissions()`; there is no `CanWrite` requirement (any member publishes — +presence/typing-style uses need Readers to emit). + +**Self-owned topics.** The `acc/` namespace adds a per-topic ownership rule on top of +membership: only the account named by the topic's last segment may publish there. It is +enforced in **three** places — the publisher's own `Publish` call fails fast before sending; +the serving peer rejects it (cheap, because `identity == ctxIdentity` is bound by the +handshake); and every receiver re-checks it (via the signature, which covers the topic +string). This gives apps spoof-proof per-account channels — e.g. `acc/online/` — where +consumers can trust the topic itself, not just the message attribution. It generalizes the +push-server's "silent self-channel" restriction and matches the proven NATS pattern +(per-identity subject prefixes rather than dynamic per-message grants). Wildcards make the +fan-in side cheap: one `acc/online/*` subscription covers every member's online topic, and +each received message is still individually ownership-checked against its concrete topic and +verified signature. + +### 7.3 Authenticity + +``` +signature = accountKey.Sign( + "anysync:pubsub:v1" | le32‖spaceId | le32‖topic | le32‖msgId | le32‖keyId | + le64(timestampMilli) | payload) +``` + +Each variable-length field is length-prefixed (le32) so field boundaries can't shift (e.g. +`spaceId="ab",topic="c"` vs `spaceId="a",topic="bc"` sign differently); `payload` trails +unprefixed as the final field. `relayed` is excluded (mutated in transit). Receivers verify; +nodes don't need to (attribution from clients is already bound by `identity == ctxIdentity`, +and verifying per-message on the relay buys little at real CPU cost). Ed25519 sign/verify is +~30–80 µs — negligible at ephemeral-signal rates. + +Receivers run the cheap filters (local-interest match, membership, `acc/` ownership, timestamp +staleness) *before* the Ed25519 verify, and record the dedup ring only after verify succeeds +([§6](#6-what-happens-on-each-frame-serving-peer)) — so a relay/LAN-peer flood of forged-signature messages is shed cheaply and +cannot evict legitimate ids from the ring to reopen a replay window ([§7.1](#71-threat-model--the-semi-trusted-relay)). + +### 7.4 Confidentiality & membership change + +Payloads encrypt with the space's current ReadKey, carrying its key id as `keyId`. Receivers +hold historical keys via the ACL, so rotation mid-flight is safe. On member removal the +existing ACL key rotation cuts decryption of new traffic automatically. Additionally the +engine exposes two active-eviction entry points that nodes wire to their ACL-update hook: + +- **`RevalidateMembers(spaceId, isMember func(account string) bool)`** — the ACL-change hook. + On an ACL update the node calls it once; it evicts *every* subscriber of the space whose + account no longer passes `isMember`, in a single pass. +- **`EvictMember(spaceId, identity)`** — the single-identity variant, for when exactly one + account is being removed. + +Both strip the target's per-stream tags (via `streampool.RemoveTagsById`, so delivery — +which is tag-keyed — stops even while the stream stays open) and decrement the match trie. +Bounded work: one scan of the streams subscribed to that space per ACL change. + +Spaces without a ReadKey (keyless/public spaces): `keyId = ""`, plaintext payload, signature +still required. A nil `Crypto` in `Deps` selects this mode. + +--- + +## 8. Memory & abuse bounds + +| Resource | Bound | Mechanism | +|---|---|---| +| Outbound per-stream buffer | `WriteQueueSize` msgs (default 100) | `mb.MB` + `TryAdd` drop (`net/streampool/stream.go:39`) | +| Interest table | ≤ 1000 patterns/stream, ≤ 100/space/stream | reject with `TooManyTopics`; state dies with stream | +| Pattern trie ([§5.1](#51-interest-matching-wildcards)) | O(total live patterns × segments), segments ≤ 16 | same caps as interest table; pruned via stream-close hook | +| Publish rate | token bucket **per peer** (default 30 msg/s, burst 60) | checked in the pubsub handler — *inside* the stream, because the RPC limiter only gates stream-open. One bucket per peer (all of a peer's streams share it), so opening more streams can't multiply the budget. Deliberate divergence from core NATS, which has no publish rate limiting; acceptable for trusted clients but not for semi-trusted multi-tenant relays | +| Local dispatch queue | `DispatchQueueSize` msgs (default 100) | `mb.MB`, drop-on-overflow — local handler delivery is best-effort | +| Payload size | ≤ 64 KiB | reject `InvalidMessage` (after DRPC decode) | +| Dedup cache | fixed FIFO ring, 4096 msgIds | evicts oldest, O(1) | +| Fan-out amplification | 1 upload → ≤ 2 node-node copies → N subscriber queues | node-side copy (publisher uploads once); per-destination stamping reuses the streampool copy path | +| Idle streams | closed with the sub-connection; tags GC'd on stream removal; `PeerTTL` keeps a quiet subscriber's peer from idle-GC reaping | existing pool behavior + `Config.PeerTTL` | + +No persistence, no unbounded map on the message path, no per-message allocation beyond pooled +message structs. + +--- + +## 9. Lifecycle & reconnect + +Because streampool opens streams lazily and never re-sends interest on a fresh stream, a pure +subscriber would go silent after its stream drops (a routine event: idle pool GC reaps quiet +streams). The engine handles this: + +- **`SyncInterest(ctx, spaceId)`** re-sends all local interest for a space to its current + peers. Hosts call it after (re)connecting to a space's peers (e.g. on + `rebuildResponsiblePeers`). +- A **resync loop** (client role only) re-pushes all local interest every + `Config.ResyncInterval` (default 20 s) and immediately on a client-side stream close. + Re-sends are idempotent (the serving side dedups per stream) and bounded by the local + subscription set. Each pass fires one dial task per local space through the bounded dial + queue (`DialQueueSize`); a client with more open spaces than that drops the overflow for a + tick and restores it on a later tick (self-correcting, no leak). +- **`CloseSpace(spaceId)`** drops all local and serving-side interest for a space and + withdraws local interest from its peers. Hosts call it on space unload so a global service + doesn't retain closed-space state. + +Interest on the serving side is keyed by **streamId, not peerId**: two streams from the same +peer are independent, so a reconnecting peer's fresh stream keeps its interest when the stale +stream closes, and a stream that dies mid-subscribe can't orphan interest. + +--- + +## 10. Configuration & defaults + +`pubsub.Config` (zero values take the defaults below, via `Config.withDefaults()`): + +| Knob | Field | Default | +|---|---|---| +| max payload | `MaxPayloadSize` | 64 KiB | +| max patterns per stream | `MaxPatternsPerStream` | 1000 | +| max patterns per space | `MaxPatternsPerSpace` | 100 | +| publish rate per peer | `PublishRps` / `PublishBurst` | 30 msg/s / burst 60 | +| per-stream write queue | `WriteQueueSize` | 100 (both client and node) | +| local dispatch queue | `DispatchQueueSize` | 100 | +| dedup ring | `DedupSize` | 4096 msgIds | +| received-timestamp staleness window | `MaxTimestampSkew` | 5 min | +| client interest resync interval | `ResyncInterval` | 20 s | +| pubsub stream peer TTL | `PeerTTL` | 1 h | +| outbound dial pool | `DialQueueWorkers` / `DialQueueSize` | 4 workers / 100 queued | + +Max topic length (256 B) and the signature prefix are compile-time constants, not config +fields. + +--- + +## 11. Public API & package layout + +### 11.1 `commonspace/pubsub` (the engine) + +`Service` is the single component used by both node and client hosts: + +```go +type Service interface { + app.ComponentRunnable + + // Publish encrypts, signs and fire-and-forgets payload to the topic within the space. + Publish(ctx context.Context, spaceId, topic string, payload []byte) error + + // Subscribe registers a local handler for a pattern and pushes the interest to the + // space's peers. The returned func unregisters and unsubscribes. + Subscribe(spaceId, pattern string, h Handler) (unsubscribe func(), err error) + + // SyncInterest (re)sends all local interest for the space to its current peers; + // call after (re)connecting to a space's peers. + SyncInterest(ctx context.Context, spaceId string) error + + // CloseSpace drops all local and serving-side interest for the space and withdraws + // the local interest from its peers. + CloseSpace(spaceId string) + + // EvictMember drops all serving-side interest of one identity in a space (§7.4). + EvictMember(spaceId string, identity crypto.PubKey) + + // RevalidateMembers evicts every subscriber of the space whose account no longer + // passes isMember — the ACL-change hook (§7.4). + RevalidateMembers(spaceId string, isMember func(account string) bool) + + // HandleStream serves an inbound PubSubStream; blocks for the stream lifetime. + HandleStream(stream drpc.Stream) error +} + +func New(deps Deps) Service + +// Handler receives a decrypted, signature-verified message on a subscribed topic. +// Handlers run on a bounded dispatch queue and MUST NOT block. +type Handler func(spaceId, topic string, identity crypto.PubKey, payload []byte) +``` + +`Deps` carries the pluggable pieces the host wires in: + +| Field | Role | Node | Client | +|---|---|---|---| +| `Membership MembershipChecker` | gate subscribe/publish on ACL membership | ✓ | ✓ (LAN serving) | +| `Crypto Crypto` | ReadKey encrypt/decrypt; nil ⇒ plaintext (keyless spaces) | — | ✓ | +| `Peers PeerProvider` | resolve a space's peers (responsible node + LAN) | — | ✓ | +| `Relay Relay` | responsibility + other-node resolution; nil on clients | ✓ | — | +| `OnStatus StatusHandler` | observe rejection `Status` frames from serving peers | optional | optional | +| `Metric metric.Metric` | register the private pool's prometheus gauges | optional | optional | +| `Config Config` | bounds ([§10](#10-configuration--defaults)) | ✓ | ✓ | + +`RegisterRpc(mux, service)` registers the `PubSub` DRPC service on a node's mux. + +### 11.2 `net/streampool` additions + +The pubsub engine reuses the streampool but needs a **second, independent instance**, so the +pool grew a few seams (all backward compatible; the sync-side pool is untouched): + +- **`NewStreamPool(handler streamhandler.StreamHandler, cfg StreamConfig, opts ...Option) StreamPool`** + — a non-component constructor. The existing component `New()` remains the base; the pubsub + service embeds its own pool with its own handler, queues, and tags. +- **`WithStreamCloseHook(func(streamId uint32, peerId string, tags []string))`** — a + post-removal callback the engine uses to reconcile trie interest on stream close, keyed by + `streamId`. +- **`WithMetric(m, prefix)`** — registers namespaced gauges (the pubsub pool uses prefix + `"pubsub"`), deliberately *not* the shared single-slot sync metric. +- **`RemoveTagsById(streamId, tags...)`** — removes tags from a specific stream, used by + member eviction ([§7.4](#74-confidentiality--membership-change)) to stop delivery while the stream stays open. +- **`Broadcast` cross-tag dedup** — when handed more than one tag, `Broadcast` builds a `seen` + set so a stream matching multiple patterns receives one copy. + +--- + +## 12. Tests + +The engine ships with a multi-peer in-memory fixture (in the synctest style) covering fan-out, +ACL rejection, relay rules, drop-on-overflow, dedup, echo suppression, ownership, and the +lifecycle edge cases. Notable regression tests: + +- `TestReconnectKeepsInterest`, `TestStreamCloseDrainsInterest` — interest keyed by streamId. +- `TestResyncRestoresDeliveryAfterDrop` — resync loop revives a dropped subscriber. +- `TestCloseSpaceDropsInterest`, `TestEvictMemberStopsDelivery`, + `TestRevalidateMembersEvictsNonMembers` — teardown and active eviction. +- `TestPubSubTopicOwnership`, `TestPubSubEchoSuppression` — `acc/` ownership at all layers, + echo self-suppression. + +Plus focused unit tests: `trie_test.go` (wildcard matching), `dedup_test.go` (ring), +`sign_test.go` (signature format), `topic_test.go` (validation). + +--- + +## 13. Implementation status + +**Done (this repo, `commonspace/pubsub` + `net/streampool`):** full wire protocol, flat + +wildcard topic model with the NATS-sublist trie, ACL-gated subscribe/publish, signed & +encrypted payloads, node relay with the one-hop rule, LAN symmetry, echo/duplicate-path +suppression, per-peer publish rate limiting, receive-path filter ordering + timestamp +staleness window + empty-identity guard, streamId-keyed interest with reconnect resync, and +active member eviction (`EvictMember` / `RevalidateMembers`). + +**Deferred (documented, not silent):** + +- **Zombie shedding** — close a stream stuck in queue-overflow for a sustained window. Needs + per-stream drop stats from the pool; the drop-on-overflow bound already holds without it. +- **Subscribe-rate limiting / per-space lock sharding.** Serving-side interest is guarded by a + single global mutex; a member can thrash subscribe/unsubscribe within the caps. Fine at + expected scale; shard or rate-limit if a multi-tenant relay shows contention. +- **Reader-level payload cap.** The 64 KiB cap is enforced after DRPC decode; enforcing it at + the stream reader (smaller buffer) needs per-stream buffer plumbing. +- **Rate-limiter size cap.** Per-peer buckets are time-GC'd but uncapped; bounded by + authenticated members in practice. +- **Pubsub-specific metrics** — per-stream drop counter, slow-subscriber flag, one event per + overflow episode, close-reason strings. The private pool is already observable via + `Deps.Metric`; these are the additional counters. Per-topic counters are deliberately not + added (unbounded cardinality); per-space is safe. +- **`MembershipChecker` returning a permission level** rather than a bare member/not — a + one-way door kept simple for v1 (current-head `!NoPermissions()`). +- **`Close` blocks on in-flight dispatch handlers** with no timeout; a handler that violates + the "must not block" contract wedges shutdown. Acceptable given the contract. + +**Downstream wiring (separate repos, not in this repo):** + +- **any-sync-node** — a `pubsubrelay` component: register `PubSub` next to SpaceSync via + `RegisterRpc`; wire `Relay`/`Membership` from nodeconf + the hosted space's ACL; call + `RevalidateMembers` (or `EvictMember`) from the ACL-update hook. +- **anytype-heart** — the client component: wire `Crypto` from the space ReadKey, `Peers` + from the per-space peer manager, `SyncInterest` on `rebuildResponsiblePeers`, `CloseSpace` + on space unload; serve inbound LAN pubsub streams from the existing client server; surface + to apps via middleware commands (`PubsubPublish`, `PubsubSubscribe`/`Unsubscribe`) emitting + `pb.Event`s through the local event bus. This side gets its own design doc. + +**Compatibility & rollout.** Old peers don't know the `PubSub` service → DRPC unknown-RPC +error on stream open; the client treats the peer as "pubsub unavailable", backs off, and +retries opportunistically. No protoVersion bump, no coordinator/nodeconf changes. Ship order: +any-sync (this lib) → any-sync-node deploy → heart. Until nodes deploy, LAN-only pubsub still +works between updated clients. + +--- + +## 14. Non-goals (v1) + +Queue groups (1-of-N); persistence, replay, retained messages, catch-up after reconnect; +delivery receipts/acks; cross-space topics (patterns never span spaces — `spaceId` is a +separate field, not a topic segment); protocol-level presence; WAN client↔client (only +LAN-discovered direct peers); interest propagation between nodes (a node always forwards +client publishes to the other responsible nodes, which drop them if nothing matches locally — +2 bounded copies beats holding cross-node subscription state). + +On that last non-goal, the NATS prior art maps cleanly onto our choice. NATS *clusters* +propagate interest (refcounted per subject) because a cluster may span many servers and +unnecessary fan-out is expensive at that scale — at the cost of every server holding the full +cluster interest map and an inherent propagation race. NATS *gateways* (WAN) instead default +to **optimistic sends**: forward without interest knowledge, let the receiver reply "no +interest", and switch to interest-only mode only after ~1000 rejections per account. With a +fixed fan-out of 2 peer nodes per space and shared-space traffic being likely-relevant to all +replicas, our always-forward is the optimistic-send strategy at the scale where it wins, and +it sidesteps NATS's biggest documented scaling pain (interest churn = a cluster-wide broadcast +plus a global cache flush per first-subscribe/last-unsubscribe). If inter-node waste ever +becomes measurable, the proven incremental fix is the gateway one: a bounded per-topic +no-interest map with a switch-to-interest-only threshold — not full interest replication. + +--- + +## Appendix A — presence as an app pattern (non-normative) + +Presence is deliberately **not** in the protocol. This is the recommended app recipe, modeled +on Yjs awareness with corrections where its semantics depend on a trusted relay. The critical +difference: y-websocket presence relies on the *server synthesizing* `state:null` for a dead +connection's clients. Our relay cannot forge signed messages, so that path does not exist +here — TTL expiry is the normative leave mechanism. + +**Entry & lifecycle.** Each device publishes its full presence entry +`{sessionId, clock, state|null}` on state change and as a heartbeat every ~15 s (TTL/2); +receivers expire an entry after ~30 s (TTL) without re-announce, measured by +**receiver-local receipt time** (immune to sender clock skew). Full state per message, no +diffs: every message stands alone, so any at-most-once loss self-heals within one heartbeat, +and per-message signing composes cleanly. + +- **TTL expiry is the normative leave.** Explicit leave (`state=null`) is a best-effort + latency optimization on graceful shutdown. Worst-case ghost duration = TTL. (Matrix + `m.typing` runs entirely on refresh-or-expire; that is the baseline.) +- **Key = `(accountId, sessionId)`, fresh random `sessionId` per app session.** An account + with two devices is two entries; a rebooted client never needs to out-clock its dead + predecessor — the old entry just times out. Never persist clocks across sessions. + `accountId` comes from the verified message `identity`, never from the payload — signing + closes the identity-hijack hole Yjs punts on. +- **Clock: bump before every publish** — state changes, heartbeats, leaves, reconnect + re-announces alike, starting at 1. (Starting at 1 avoids the Yjs clock-0 trap where an + unknown client's first entry is dead on arrival.) +- **Accept rule:** accept iff `clock > knownClock`, or (`clock == knownClock` and + `state == null` and a live state exists) — equal-clock-null keeps leave idempotent under + duplicate/multi-path delivery. On removal, keep a `sessionId → lastClock` tombstone for + ≥ TTL so reordered pre-leave messages can't resurrect a ghost. +- **Join snapshot:** a stateless relay cannot push room state to a new joiner. Recipe: + subscribe first, then announce yourself; peers treat an unknown-session announce as a cue to + re-announce early (with random jitter ≤ heartbeat/2 to avoid an answer storm). Alternatively + accept ≤ one heartbeat of join blindness. +- **Fan-out hygiene:** never re-publish an applied remote update (Yjs's origin-blind echo + handler caused a documented N² storm at ~20 users/room); keep state small (identity + + cursor); throttle high-frequency fields app-side (~10 Hz cursor max, further coalesced by + the publish token bucket); consider separate topics and cadences for slow presence vs fast + cursors. Idle-room budget is ≈ N²/heartbeat deliveries — scale the heartbeat up for large + spaces. + +Two topic layouts, both valid: + +- **Shared topic** `presence` (or `presence/{objectId}`): one subscription covers the whole + space; attribution comes from the verified `identity`. +- **Self-owned topics** `acc/online/`: spoof-proof per-account channels ([§7.2](#72-access-control)). + Subscribe with `acc/online/*` to follow everyone at the cost of a single interest entry, or + with concrete topics to follow specific accounts. + +If sub-TTL leave latency ever matters, a v2 option is relay-emitted **unsigned transport +hints** ("subscriber stream closed") that clients may use only to shorten their local expiry +check for that peer — never as an authoritative removal; authority stays with signed messages +and TTL. diff --git a/net/peer/peer.go b/net/peer/peer.go index 56d193177..100e1c4d3 100644 --- a/net/peer/peer.go +++ b/net/peer/peer.go @@ -380,6 +380,15 @@ func (p *peer) TryClose(objectTTL time.Duration) (res bool, err error) { } func (p *peer) gc(ttl time.Duration) (aliveCount int) { + // drpc conn Close blocks until its reader unwinds, which on a stalled stream + // takes until the yamux stream close timeout: collect the doomed conns and + // close them after releasing the lock + var toClose []*subConn + defer func() { + for _, conn := range toClose { + _ = conn.Close() + } + }() p.mu.Lock() defer p.mu.Unlock() minLastUsage := time.Now().Add(-ttl) @@ -392,7 +401,7 @@ func (p *peer) gc(ttl time.Duration) (aliveCount int) { default: } if in.LastUsage().Before(minLastUsage) { - _ = in.Close() + toClose = append(toClose, in) p.inactive[i] = nil hasClosed = true } @@ -415,7 +424,7 @@ func (p *peer) gc(ttl time.Duration) (aliveCount int) { } if act.LastUsage().Before(minLastUsage) { log.Warn("close active connection because no activity", zap.String("peerId", p.id), zap.String("addr", p.Addr())) - _ = act.Close() + toClose = append(toClose, act) delete(p.active, act) continue } diff --git a/net/rpc/rpcerr/registry.go b/net/rpc/rpcerr/registry.go index 0860ee2a5..9dc301a52 100644 --- a/net/rpc/rpcerr/registry.go +++ b/net/rpc/rpcerr/registry.go @@ -48,6 +48,10 @@ func Unwrap(e error) error { return err } +// ErrGroup is the base offset for a service's error codes. Every concrete +// error registers at offset+localCode into the process-global errsMap, which +// panics on a duplicate. Offsets are a namespace shared across all repos that +// link any-sync — reserve a new one in docs/rpc-error-offsets.md. type ErrGroup int64 func (g ErrGroup) Register(err error, code uint64) error { diff --git a/net/secureservice/secureservice.go b/net/secureservice/secureservice.go index ab85933d2..632fd813c 100644 --- a/net/secureservice/secureservice.go +++ b/net/secureservice/secureservice.go @@ -38,12 +38,14 @@ var ( // ProtoVersion 9 - nested derived objects, delete restrictions (bridge release v0.11.Y) // ProtoVersion 10 - reserved (not used, version alignment gap) // ProtoVersion 11 - reserved (not used, version alignment gap) - // ProtoVersion 12 will align with v0.12.X - ProtoVersion = uint32(12) + // ProtoVersion 12 aligns with v0.12.X + // ProtoVersion 13 - files v2 (v2 filenodes, chash routing, space header fileprotoVersion), space exchange v2 (token-based LAN discovery), aligns with v0.13.X + ProtoVersion = uint32(13) ) var ( - defaultCompatibleVersions = []uint32{9, 12, 13} + // v0.13.X: accept self ±1 (V-1, V, V+1). Bridge v9 is dropped now that v0.12 has shipped. + defaultCompatibleVersions = []uint32{12, 13, 14} ) func New() SecureService { diff --git a/net/streampool/closewithoutrun_test.go b/net/streampool/closewithoutrun_test.go new file mode 100644 index 000000000..d5f1ae2c3 --- /dev/null +++ b/net/streampool/closewithoutrun_test.go @@ -0,0 +1,20 @@ +package streampool + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/anyproto/any-sync/app/debugstat" +) + +// app.Start closes components it never ran: a failing Init makes closeServices +// call Close on every component registered before it. +func TestStreamPool_CloseWithoutRun(t *testing.T) { + s := New().(*streamPool) + s.statService = debugstat.NewNoOp() + require.NotPanics(t, func() { + require.NoError(t, s.Close(context.Background())) + }) +} diff --git a/net/streampool/context.go b/net/streampool/context.go index 724c172f8..2b12d7466 100644 --- a/net/streampool/context.go +++ b/net/streampool/context.go @@ -15,3 +15,12 @@ func streamCtx(ctx context.Context, streamId uint32, peerId string) context.Cont ctx = peer.CtxWithPeerId(ctx, peerId) return context.WithValue(ctx, streamCtxKeyStreamId, streamId) } + +// CtxStreamId returns the id of the stream that delivered the current message. +// It is set on the context passed to StreamHandler.HandleMessage, letting a +// handler key per-stream state (e.g. subscription interest) so that two streams +// from the same peer stay independent. +func CtxStreamId(ctx context.Context) (streamId uint32, ok bool) { + streamId, ok = ctx.Value(streamCtxKeyStreamId).(uint32) + return +} diff --git a/net/streampool/streampool.go b/net/streampool/streampool.go index d7a72c843..2f310429c 100644 --- a/net/streampool/streampool.go +++ b/net/streampool/streampool.go @@ -33,6 +33,53 @@ func New() StreamPool { } } +// Option configures a standalone pool created with NewStreamPool. +type Option func(*streamPool) + +// WithStreamCloseHook registers a callback invoked after a stream is removed from +// the pool, outside the pool lock, with the closed stream's id, peerId and the +// tags it carried. The streamId lets a handler clean up per-stream state keyed on +// CtxStreamId even when several streams share a peerId. +func WithStreamCloseHook(hook func(streamId uint32, peerId string, tags []string)) Option { + return func(s *streamPool) { + s.closeHook = hook + } +} + +// WithMetric registers the standalone pool's prometheus gauges (stream_count, +// tag_count, dial_queue) under the given namespace prefix, so a service that owns +// a private pool (e.g. pubsub) is observable even though it never runs the +// app-component Init. +// +// It deliberately does NOT call RegisterStreamPoolSyncMetric: that feeds a single +// shared slot on the metric component meant for the one app-level sync streampool, +// and registering a second pool there would overwrite (and, on Close, null) the +// sync pool's OutgoingMsg telemetry. The namespaced gauges below give the private +// pool independent observability without touching that slot. +func WithMetric(m metric.Metric, prefix string) Option { + return func(s *streamPool) { + if m == nil { + return + } + registerMetrics(m.Registry(), s, prefix) + } +} + +// NewStreamPool creates a standalone pool with explicit dependencies, for services +// that own a private pool (e.g. pubsub) instead of sharing the app-level component. +// The caller must not register it in the app and is responsible for calling +// Run(ctx) and Close(ctx); Init must not be called. +func NewStreamPool(handler streamhandler.StreamHandler, cfg StreamConfig, opts ...Option) StreamPool { + s := New().(*streamPool) + s.handler = handler + s.streamConfig = cfg + s.statService = debugstat.NewNoOp() + for _, opt := range opts { + opt(s) + } + return s +} + type configGetter interface { GetStreamConfig() StreamConfig } @@ -64,6 +111,9 @@ type StreamPool interface { AddTagsCtx(ctx context.Context, tags ...string) error // RemoveTagsCtx removes tags from stream, stream will be extracted from ctx RemoveTagsCtx(ctx context.Context, tags ...string) error + // RemoveTagsById removes tags from a specific stream by id, for callers that + // track streamId out of band. Missing streams and tags are ignored. + RemoveTagsById(streamId uint32, tags ...string) error // Streams gets all streams for specific tags Streams(tags ...string) (streams []drpc.Stream) } @@ -78,6 +128,7 @@ type streamPool struct { opening map[string]*openingProcess streamConfig StreamConfig dial *ExecPool + closeHook func(streamId uint32, peerId string, tags []string) mu sync.Mutex writeQueueSize int lastStreamId uint32 @@ -360,8 +411,19 @@ func (s *streamPool) openStream(ctx context.Context, p peer.Peer) *openingProces func (s *streamPool) Broadcast(ctx context.Context, msg drpc.Message, tags ...string) (err error) { s.mu.Lock() var streams []*stream + var seen map[uint32]struct{} + if len(tags) > 1 { + seen = make(map[uint32]struct{}) + } for _, tag := range tags { for _, streamId := range s.streamIdsByTag[tag] { + if seen != nil { + if _, ok := seen[streamId]; ok { + // a stream subscribed to several matching tags gets one copy + continue + } + seen[streamId] = struct{}{} + } streams = append(streams, s.streams[streamId]) } } @@ -428,12 +490,36 @@ func (s *streamPool) RemoveTagsCtx(ctx context.Context, tags ...string) error { return nil } -func (s *streamPool) removeStream(streamId uint32) { +func (s *streamPool) RemoveTagsById(streamId uint32, tags ...string) error { s.mu.Lock() defer s.mu.Unlock() + st, ok := s.streams[streamId] + if !ok { + return nil + } + var filtered = st.tags[:0] + var toRemove = make([]string, 0, len(tags)) + for _, t := range st.tags { + if slices.Contains(tags, t) { + toRemove = append(toRemove, t) + } else { + filtered = append(filtered, t) + } + } + st.tags = filtered + for _, t := range toRemove { + removeStream(s.streamIdsByTag, t, streamId) + } + return nil +} + +func (s *streamPool) removeStream(streamId uint32) { + s.mu.Lock() st := s.streams[streamId] if st == nil { + s.mu.Unlock() log.Fatal("removeStream: stream does not exist", zap.Uint32("streamId", streamId)) + return } removeStream(s.streamIdsByPeer, st.peerId, streamId) @@ -442,7 +528,12 @@ func (s *streamPool) removeStream(streamId uint32) { } delete(s.streams, streamId) - st.l.Debug("stream removed", zap.Strings("tags", st.tags)) + closedTags := slices.Clone(st.tags) + s.mu.Unlock() + st.l.Debug("stream removed", zap.Strings("tags", closedTags)) + if s.closeHook != nil { + s.closeHook(streamId, st.peerId, closedTags) + } } func (s *streamPool) Close(ctx context.Context) (err error) { @@ -450,7 +541,10 @@ func (s *streamPool) Close(ctx context.Context) (err error) { if s.metric != nil { s.metric.UnregisterStreamPoolSyncMetric() } - return s.dial.Close() + if s.dial != nil { + return s.dial.Close() + } + return nil } func removeStream(m map[string][]uint32, key string, streamId uint32) { diff --git a/net/transport/webtransport/webtransport_native.go b/net/transport/webtransport/webtransport_native.go index 4d56b101c..ad1440b4f 100644 --- a/net/transport/webtransport/webtransport_native.go +++ b/net/transport/webtransport/webtransport_native.go @@ -5,10 +5,12 @@ package webtransport import ( "context" "crypto/tls" + "errors" "fmt" "net" "net/http" "net/url" + "sync" "time" "github.com/quic-go/quic-go" @@ -36,8 +38,10 @@ type wtTransport struct { accepter transport.Accepter conf Config - server *wt.Server - udpConns []net.PacketConn + server *wt.Server + udpConns []net.PacketConn + listeners []*quic.EarlyListener + serveWg sync.WaitGroup listCtx context.Context listCtxCancel context.CancelFunc @@ -126,19 +130,45 @@ func (t *wtTransport) Run(ctx context.Context) (err error) { } t.udpConns = append(t.udpConns, udpConn) + // listen and accept ourselves instead of wt.Server.Serve: Serve registers itself + // in the server's internal WaitGroup from this goroutine, which races with + // Close when the transport is shut down before the goroutine is scheduled + ln, err := quic.ListenEarly(udpConn, tlsConf, quicConf) + if err != nil { + return fmt.Errorf("listen quic %q: %w", addr, err) + } + t.listeners = append(t.listeners, ln) + log.Info("webtransport listener started", zap.String("addr", udpConn.LocalAddr().String()), zap.String("path", t.conf.Path), ) + t.serveWg.Add(1) go func() { - if err := t.server.Serve(udpConn); err != nil { - log.Debug("webtransport server stopped", zap.Error(err)) - } + defer t.serveWg.Done() + t.serveListener(ln) }() } return nil } +func (t *wtTransport) serveListener(ln *quic.EarlyListener) { + for { + qconn, err := ln.Accept(t.listCtx) + if err != nil { + log.Debug("webtransport server stopped", zap.Error(err)) + return + } + t.serveWg.Add(1) + go func() { + defer t.serveWg.Done() + if err := t.server.ServeQUICConn(qconn); err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Debug("webtransport connection closed", zap.Error(err)) + } + }() + } +} + func (t *wtTransport) handleUpgrade(w http.ResponseWriter, r *http.Request) { // CORS headers w.Header().Set("Access-Control-Allow-Origin", "*") @@ -206,6 +236,10 @@ func (t *wtTransport) Dial(ctx context.Context, addr string) (transport.MultiCon if expectedPeerId == "" { return nil, fmt.Errorf("no expected peer id in context for WebTransport dial") } + // a peer that completes the quic handshake and then stalls keeps the idle + // timeout from firing, so bound the dial like yamux does + ctx, cancel := context.WithTimeout(ctx, time.Duration(t.conf.DialTimeoutSec)*time.Second) + defer cancel() dialer := wt.Transport{ TLSClientConfig: &tls.Config{ @@ -255,8 +289,14 @@ func (t *wtTransport) Close(ctx context.Context) error { t.listCtxCancel() } if t.server != nil { + // closes established sessions gracefully while the udp sockets are still open + // and cancels the server context, unblocking ServeQUICConn calls _ = t.server.Close() } + t.serveWg.Wait() + for _, ln := range t.listeners { + _ = ln.Close() + } for _, c := range t.udpConns { _ = c.Close() } diff --git a/net/transport/yamux/yamux.go b/net/transport/yamux/yamux.go index 0530ca8be..ef921ad7a 100644 --- a/net/transport/yamux/yamux.go +++ b/net/transport/yamux/yamux.go @@ -63,6 +63,9 @@ func (y *yamuxTransport) Init(a *app.App) (err error) { } } y.yamuxConf.StreamOpenTimeout = time.Duration(y.conf.DialTimeoutSec) * time.Second + // yamux defaults to 5 minutes, longer than the app close deadline: a stream + // whose peer never sends FIN back would keep drpc conn Close blocked + y.yamuxConf.StreamCloseTimeout = time.Duration(y.conf.WriteTimeoutSec) * time.Second y.yamuxConf.ConnectionWriteTimeout = time.Duration(y.conf.WriteTimeoutSec) * time.Second y.listCtx, y.listCtxCancel = context.WithCancel(context.Background()) return diff --git a/nodeconf/config.go b/nodeconf/config.go index 3caea78e5..d98ee3f4e 100644 --- a/nodeconf/config.go +++ b/nodeconf/config.go @@ -23,6 +23,7 @@ const ( NodeTypeTree NodeType = "tree" NodeTypeConsensus NodeType = "consensus" NodeTypeFile NodeType = "file" + NodeTypeFileV2 NodeType = "fileV2" NodeTypeCoordinator NodeType = "coordinator" NodeTypeNamingNode NodeType = "namingNode" @@ -53,8 +54,17 @@ func (n Node) HasType(t NodeType) bool { } type Configuration struct { - Id string `yaml:"id"` - NetworkId string `yaml:"networkId"` - Nodes []Node `yaml:"nodes"` - CreationTime time.Time `yaml:"creationTime"` + Id string `yaml:"id"` + NetworkId string `yaml:"networkId"` + // FileNetworkId is the identity of the NodeTypeFileV2 fleet's shared + // signing key (network string encoding, crypto.DecodeNetworkId) — + // the key fileV2 durability receipts (networkSign) verify against. + // Empty on networks without a fileV2 fleet. + FileNetworkId string `yaml:"fileNetworkId,omitempty"` + Nodes []Node `yaml:"nodes"` + CreationTime time.Time `yaml:"creationTime"` + // Epoch is a monotonically increasing version of the network configuration. + // It is incremented on every published topology change and is 0 for + // configurations that predate epoch support. + Epoch uint64 `yaml:"epoch,omitempty"` } diff --git a/nodeconf/mock_nodeconf/mock_nodeconf.go b/nodeconf/mock_nodeconf/mock_nodeconf.go index 459f654d0..41cdc8823 100644 --- a/nodeconf/mock_nodeconf/mock_nodeconf.go +++ b/nodeconf/mock_nodeconf/mock_nodeconf.go @@ -127,6 +127,34 @@ func (mr *MockServiceMockRecorder) FilePeers() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FilePeers", reflect.TypeOf((*MockService)(nil).FilePeers)) } +// FileV2NodeIds mocks base method. +func (m *MockService) FileV2NodeIds(spaceId string) []string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FileV2NodeIds", spaceId) + ret0, _ := ret[0].([]string) + return ret0 +} + +// FileV2NodeIds indicates an expected call of FileV2NodeIds. +func (mr *MockServiceMockRecorder) FileV2NodeIds(spaceId any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FileV2NodeIds", reflect.TypeOf((*MockService)(nil).FileV2NodeIds), spaceId) +} + +// FileV2Peers mocks base method. +func (m *MockService) FileV2Peers() []string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FileV2Peers") + ret0, _ := ret[0].([]string) + return ret0 +} + +// FileV2Peers indicates an expected call of FileV2Peers. +func (mr *MockServiceMockRecorder) FileV2Peers() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FileV2Peers", reflect.TypeOf((*MockService)(nil).FileV2Peers)) +} + // Id mocks base method. func (m *MockService) Id() string { m.ctrl.T.Helper() @@ -239,6 +267,18 @@ func (mr *MockServiceMockRecorder) NodeTypes(nodeId any) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NodeTypes", reflect.TypeOf((*MockService)(nil).NodeTypes), nodeId) } +// ObserveChanges mocks base method. +func (m *MockService) ObserveChanges(observer nodeconf.ChangeObserver) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "ObserveChanges", observer) +} + +// ObserveChanges indicates an expected call of ObserveChanges. +func (mr *MockServiceMockRecorder) ObserveChanges(observer any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ObserveChanges", reflect.TypeOf((*MockService)(nil).ObserveChanges), observer) +} + // Partition mocks base method. func (m *MockService) Partition(spaceId string) int { m.ctrl.T.Helper() diff --git a/nodeconf/nodeconf.go b/nodeconf/nodeconf.go index dcef87cfc..4f1334c44 100644 --- a/nodeconf/nodeconf.go +++ b/nodeconf/nodeconf.go @@ -17,6 +17,12 @@ type NodeConf interface { IsResponsible(spaceId string) bool // FilePeers returns list of filenodes FilePeers() []string + // FileV2Peers returns the list of v2 filenodes (the NodeTypeFileV2 pool) + FileV2Peers() []string + // FileV2NodeIds returns the RF=2 responsible v2 filenodes for a given spaceId. + // Unlike NodeIds, the current account is NOT excluded: the full responsible pair is + // returned so a v2 filenode can see its partner (e.g. to derive the leader). + FileV2NodeIds(spaceId string) []string // ConsensusPeers returns list of consensusnodes ConsensusPeers() []string // CoordinatorPeers returns list of coordinator nodes @@ -40,11 +46,13 @@ type nodeConf struct { id string accountId string filePeers []string + fileV2Peers []string consensusPeers []string coordinatorPeers []string namingNodePeers []string paymentProcessingNodePeers []string chash chash.CHash + chashFileV2 chash.CHash allMembers []Node c Configuration addrs map[string][]string @@ -82,6 +90,19 @@ func (c *nodeConf) FilePeers() []string { return c.filePeers } +func (c *nodeConf) FileV2Peers() []string { + return c.fileV2Peers +} + +func (c *nodeConf) FileV2NodeIds(spaceId string) []string { + members := c.chashFileV2.GetMembers(ReplKey(spaceId)) + res := make([]string, 0, len(members)) + for _, m := range members { + res = append(res, m.Id()) + } + return res +} + func (c *nodeConf) ConsensusPeers() []string { return c.consensusPeers } @@ -146,8 +167,15 @@ func сonfigurationToNodeConf(c Configuration) (nc *nodeConf, err error) { }); err != nil { return } + if nc.chashFileV2, err = chash.New(chash.Config{ + PartitionCount: PartitionCount, + ReplicationFactor: FileV2ReplicationFactor, + }); err != nil { + return + } members := make([]chash.Member, 0, len(c.Nodes)) + fileV2Members := make([]chash.Member, 0, len(c.Nodes)) for _, n := range c.Nodes { if n.HasType(NodeTypeTree) { members = append(members, n) @@ -158,6 +186,10 @@ func сonfigurationToNodeConf(c Configuration) (nc *nodeConf, err error) { if n.HasType(NodeTypeFile) { nc.filePeers = append(nc.filePeers, n.PeerId) } + if n.HasType(NodeTypeFileV2) { + nc.fileV2Peers = append(nc.fileV2Peers, n.PeerId) + fileV2Members = append(fileV2Members, n) + } if n.HasType(NodeTypeCoordinator) { nc.coordinatorPeers = append(nc.coordinatorPeers, n.PeerId) } @@ -171,6 +203,13 @@ func сonfigurationToNodeConf(c Configuration) (nc *nodeConf, err error) { nc.allMembers = append(nc.allMembers, n) nc.addrs[n.PeerId] = n.Addresses } - err = nc.chash.AddMembers(members...) + if err = nc.chash.AddMembers(members...); err != nil { + return + } + if len(fileV2Members) > 0 { + if err = nc.chashFileV2.AddMembers(fileV2Members...); err != nil { + return + } + } return } diff --git a/nodeconf/nodeconf_test.go b/nodeconf/nodeconf_test.go index 9298e1006..b22536fa8 100644 --- a/nodeconf/nodeconf_test.go +++ b/nodeconf/nodeconf_test.go @@ -165,6 +165,127 @@ creationTime: 2023-07-21T11:51:12.970882+01:00 assert.Equal(t, []string{"12D3KooXXXEXV3KjBxEU5EnsPfneLx84vMWAtTBQBeyooN82KSuS"}, nodeConf.PaymentProcessingNodePeers()) } +func TestConfiguration_FileV2NodeIds(t *testing.T) { + chFileV2, err := chash.New(chash.Config{ + PartitionCount: PartitionCount, + ReplicationFactor: FileV2ReplicationFactor, + }) + require.NoError(t, err) + conf := &nodeConf{ + id: "last", + accountId: "1", + chashFileV2: chFileV2, + } + for i := 0; i < 5; i++ { + require.NoError(t, conf.chashFileV2.AddMembers(testMember(fmt.Sprint(i+1)))) + } + + t.Run("returns exactly RF=2 responsible nodes", func(t *testing.T) { + for i := 0; i < 20; i++ { + spaceId := fmt.Sprintf("%d.%d", rand.Int(), rand.Int()) + assert.Len(t, conf.FileV2NodeIds(spaceId), FileV2ReplicationFactor) + } + }) + + t.Run("stable for same repl key", func(t *testing.T) { + var prev []string + for i := 0; i < 10; i++ { + spaceId := fmt.Sprintf("%d.%d", rand.Int(), 42) + members := conf.FileV2NodeIds(spaceId) + assert.Len(t, members, FileV2ReplicationFactor) + if i != 0 { + assert.Equal(t, prev, members) + } + prev = members + } + }) + + t.Run("does not exclude self (full pair returned)", func(t *testing.T) { + // unlike NodeIds, the current account stays in the result: find a space for + // which account "1" is responsible and assert it is still present. + selfSeen := false + for i := 0; i < 2000 && !selfSeen; i++ { + members := conf.FileV2NodeIds(fmt.Sprintf("%d.%d", i, i)) + require.Len(t, members, FileV2ReplicationFactor) + for _, m := range members { + if m == conf.accountId { + selfSeen = true + } + } + } + assert.True(t, selfSeen, "self must be included when responsible") + }) +} + +func TestNewNodeConfFromYaml_FileV2(t *testing.T) { + yamlData := ` +id: net1 +networkId: net1 +nodes: + - peerId: file-v1 + addresses: [127.0.0.1:1] + types: [file] + - peerId: filev2-1 + addresses: [127.0.0.1:2] + types: [fileV2] + - peerId: filev2-2 + addresses: [127.0.0.1:3] + types: [fileV2] + - peerId: tree-and-filev2 + addresses: [127.0.0.1:4] + types: [tree, fileV2] + - peerId: tree-1 + addresses: [127.0.0.1:5] + types: [tree] +creationTime: 2023-07-21T11:51:12.970882+01:00 +` + var conf Configuration + require.NoError(t, yaml.Unmarshal([]byte(yamlData), &conf)) + nc, err := сonfigurationToNodeConf(conf) + require.NoError(t, err) + + // v2 pool = exactly the fileV2-typed nodes; the v1 file node is excluded + assert.Equal(t, []string{"filev2-1", "filev2-2", "tree-and-filev2"}, nc.FileV2Peers()) + // v1 file pool unaffected by the new type + assert.Equal(t, []string{"file-v1"}, nc.FilePeers()) + + // resolution returns RF=2 responsible nodes, all from the v2 pool + v2set := map[string]bool{"filev2-1": true, "filev2-2": true, "tree-and-filev2": true} + ids := nc.FileV2NodeIds("space.repl") + assert.Len(t, ids, FileV2ReplicationFactor) + for _, id := range ids { + assert.Truef(t, v2set[id], "resolved node %s must be a v2 file node", id) + } + + // the tree/sync container is unaffected: it resolves only from tree-typed nodes + treeSet := map[string]bool{"tree-and-filev2": true, "tree-1": true} + for _, id := range nc.NodeIds("space.repl") { + assert.Truef(t, treeSet[id], "sync node %s must be a tree node", id) + } +} + +func TestNewNodeConfFromYaml_NoFileV2(t *testing.T) { + // a config without any fileV2 nodes must still build and resolve to an empty pair + yamlData := ` +id: net1 +networkId: net1 +nodes: + - peerId: tree-1 + addresses: [127.0.0.1:1] + types: [tree] + - peerId: file-v1 + addresses: [127.0.0.1:2] + types: [file] +creationTime: 2023-07-21T11:51:12.970882+01:00 +` + var conf Configuration + require.NoError(t, yaml.Unmarshal([]byte(yamlData), &conf)) + nc, err := сonfigurationToNodeConf(conf) + require.NoError(t, err) + assert.Empty(t, nc.FileV2Peers()) + assert.Empty(t, nc.FileV2NodeIds("space.repl")) +} + type testMember string func (t testMember) Id() string { diff --git a/nodeconf/nodeconfstore/nodeconfstore.go b/nodeconf/nodeconfstore/nodeconfstore.go index e9866c755..2f2534c19 100644 --- a/nodeconf/nodeconfstore/nodeconfstore.go +++ b/nodeconf/nodeconfstore/nodeconfstore.go @@ -2,8 +2,12 @@ package nodeconfstore import ( "context" + "fmt" "os" "path/filepath" + "sort" + "strconv" + "strings" "sync" "github.com/anyproto/any-sync/app" @@ -11,13 +15,16 @@ import ( "gopkg.in/yaml.v3" ) +// historyLimit is the number of historical configurations retained per network. +const historyLimit = 100 + func New() NodeConfStore { return new(nodeConfStore) } type NodeConfStore interface { app.Component - nodeconf.Store + nodeconf.HistoryStore } type nodeConfStore struct { @@ -62,5 +69,74 @@ func (n *nodeConfStore) SaveLast(ctx context.Context, c nodeconf.Configuration) if err != nil { return } - return os.WriteFile(path, data, 0o644) + if err = os.WriteFile(path, data, 0o644); err != nil { + return + } + if c.Epoch > 0 { + if err = os.WriteFile(n.epochPath(c.NetworkId, c.Epoch), data, 0o644); err != nil { + return + } + err = n.pruneHistory(c.NetworkId) + } + return +} + +func (n *nodeConfStore) GetByEpoch(ctx context.Context, netId string, epoch uint64) (c nodeconf.Configuration, err error) { + n.mu.Lock() + defer n.mu.Unlock() + data, err := os.ReadFile(n.epochPath(netId, epoch)) + if os.IsNotExist(err) { + err = nodeconf.ErrConfigurationNotFound + return + } + if err != nil { + return + } + err = yaml.Unmarshal(data, &c) + return +} + +func (n *nodeConfStore) Epochs(ctx context.Context, netId string) (epochs []uint64, err error) { + n.mu.Lock() + defer n.mu.Unlock() + return n.epochs(netId) +} + +func (n *nodeConfStore) epochPath(netId string, epoch uint64) string { + return filepath.Join(n.path, fmt.Sprintf("%s.e%d.yml", netId, epoch)) +} + +func (n *nodeConfStore) epochs(netId string) (epochs []uint64, err error) { + entries, err := os.ReadDir(n.path) + if err != nil { + return + } + prefix := netId + ".e" + for _, entry := range entries { + name := entry.Name() + if !strings.HasPrefix(name, prefix) || !strings.HasSuffix(name, ".yml") { + continue + } + epoch, pErr := strconv.ParseUint(strings.TrimSuffix(strings.TrimPrefix(name, prefix), ".yml"), 10, 64) + if pErr != nil { + continue + } + epochs = append(epochs, epoch) + } + sort.Slice(epochs, func(i, j int) bool { return epochs[i] < epochs[j] }) + return +} + +func (n *nodeConfStore) pruneHistory(netId string) (err error) { + epochs, err := n.epochs(netId) + if err != nil { + return + } + for len(epochs) > historyLimit { + if err = os.Remove(n.epochPath(netId, epochs[0])); err != nil { + return + } + epochs = epochs[1:] + } + return } diff --git a/nodeconf/nodeconfstore/nodeconfstore_test.go b/nodeconf/nodeconfstore/nodeconfstore_test.go index ecb9114fa..b5109a00f 100644 --- a/nodeconf/nodeconfstore/nodeconfstore_test.go +++ b/nodeconf/nodeconfstore/nodeconfstore_test.go @@ -85,3 +85,54 @@ func (c config) Init(a *app.App) (err error) { func (c config) Name() (name string) { return "config" } + +func TestNodeConfStore_History(t *testing.T) { + t.Run("get by epoch", func(t *testing.T) { + fx := newFixture(t) + defer fx.finish(t) + for epoch := uint64(1); epoch <= 3; epoch++ { + c := nodeconf.Configuration{ + Id: "id" + string(rune('0'+epoch)), + NetworkId: "net1", + Epoch: epoch, + } + require.NoError(t, fx.SaveLast(ctx, c)) + } + + c, err := fx.GetByEpoch(ctx, "net1", 2) + require.NoError(t, err) + assert.Equal(t, "id2", c.Id) + assert.Equal(t, uint64(2), c.Epoch) + + last, err := fx.GetLast(ctx, "net1") + require.NoError(t, err) + assert.Equal(t, uint64(3), last.Epoch) + + epochs, err := fx.Epochs(ctx, "net1") + require.NoError(t, err) + assert.Equal(t, []uint64{1, 2, 3}, epochs) + + _, err = fx.GetByEpoch(ctx, "net1", 10) + assert.EqualError(t, err, nodeconf.ErrConfigurationNotFound.Error()) + }) + t.Run("no history for epochless configs", func(t *testing.T) { + fx := newFixture(t) + defer fx.finish(t) + require.NoError(t, fx.SaveLast(ctx, nodeconf.Configuration{Id: "1", NetworkId: "net2"})) + epochs, err := fx.Epochs(ctx, "net2") + require.NoError(t, err) + assert.Empty(t, epochs) + }) + t.Run("prune", func(t *testing.T) { + fx := newFixture(t) + defer fx.finish(t) + for epoch := uint64(1); epoch <= historyLimit+3; epoch++ { + require.NoError(t, fx.SaveLast(ctx, nodeconf.Configuration{Id: "x", NetworkId: "net3", Epoch: epoch})) + } + epochs, err := fx.Epochs(ctx, "net3") + require.NoError(t, err) + require.Len(t, epochs, historyLimit) + assert.Equal(t, uint64(4), epochs[0]) + assert.Equal(t, uint64(historyLimit+3), epochs[len(epochs)-1]) + }) +} diff --git a/nodeconf/service.go b/nodeconf/service.go index 418ef395a..36761d9eb 100644 --- a/nodeconf/service.go +++ b/nodeconf/service.go @@ -22,6 +22,8 @@ const CName = "common.nodeconf" const ( PartitionCount = 3000 ReplicationFactor = 3 + // FileV2ReplicationFactor is the replication factor of the v2 file node pool (07c: RF=2) + FileV2ReplicationFactor = 2 ) var log = logger.NewNamed(CName) @@ -43,9 +45,17 @@ func New() Service { type Service interface { NodeConf NetworkCompatibilityStatus() NetworkCompatibilityStatus + // ObserveChanges registers an observer called every time the active network + // configuration is replaced. It is not called for the initial configuration. + // Observers are called sequentially from the configuration update flow and + // must not block for long. + ObserveChanges(observer ChangeObserver) app.ComponentRunnable } +// ChangeObserver is called with the previous and the new active configuration. +type ChangeObserver func(prev, cur NodeConf) + type NetworkProtoVersionChecker interface { IsNetworkNeedsUpdate(ctx context.Context) (bool, error) } @@ -58,6 +68,7 @@ type service struct { last NodeConf mu sync.RWMutex sync periodicsync.PeriodicSync + observers []ChangeObserver compatibilityStatus NetworkCompatibilityStatus networkProtoVersionChecker NetworkProtoVersionChecker @@ -236,30 +247,47 @@ func (s *service) setCompatibilityStatusByErr(err error) { func (s *service) setLastConfiguration(c Configuration) (err error) { s.mu.Lock() - defer s.mu.Unlock() if s.last != nil && s.last.Id() == c.Id { + s.mu.Unlock() return } nc, err := сonfigurationToNodeConf(c) if err != nil { + s.mu.Unlock() return } nc.accountId = s.accountId - var beforeId = "" - if s.last != nil { - beforeId = s.last.Id() - } - if s.last != nil { - log.Info("net configuration changed", zap.String("before", beforeId), zap.String("after", nc.Id())) + prev := s.last + if prev != nil { + log.Info("net configuration changed", + zap.String("before", prev.Id()), + zap.String("after", nc.Id()), + zap.Uint64("beforeEpoch", prev.Configuration().Epoch), + zap.Uint64("afterEpoch", nc.Configuration().Epoch)) } else { log.Info("net configuration applied", zap.String("netId", nc.Configuration().NetworkId), zap.String("id", nc.Id())) } s.last = nc + observers := make([]ChangeObserver, len(s.observers)) + copy(observers, s.observers) + s.mu.Unlock() + + if prev != nil { + for _, observer := range observers { + observer(prev, nc) + } + } return } +func (s *service) ObserveChanges(observer ChangeObserver) { + s.mu.Lock() + defer s.mu.Unlock() + s.observers = append(s.observers, observer) +} + func (s *service) Id() string { s.mu.RLock() defer s.mu.RUnlock() @@ -290,6 +318,18 @@ func (s *service) FilePeers() []string { return s.last.FilePeers() } +func (s *service) FileV2Peers() []string { + s.mu.RLock() + defer s.mu.RUnlock() + return s.last.FileV2Peers() +} + +func (s *service) FileV2NodeIds(spaceId string) []string { + s.mu.RLock() + defer s.mu.RUnlock() + return s.last.FileV2NodeIds(spaceId) +} + func (s *service) ConsensusPeers() []string { s.mu.RLock() defer s.mu.RUnlock() diff --git a/nodeconf/service_test.go b/nodeconf/service_test.go index 841c1b965..aad5fd0ba 100644 --- a/nodeconf/service_test.go +++ b/nodeconf/service_test.go @@ -289,3 +289,43 @@ func TestService_mergeCoordinatorAddrs(t *testing.T) { assert.Equal(t, "192.168.1.1:8833", confStored.Nodes[0].Addresses[1]) }) } + +func TestService_ObserveChanges(t *testing.T) { + fx := newFixture(t) + defer fx.finish(t) + + newConf := newTestConf().Configuration + newConf.Id = "updated" + newConf.Epoch = 2 + fx.testSource.conf = newConf + + var ( + mu sync.Mutex + prevId string + curId string + curEpoch uint64 + calls int + ) + fx.ObserveChanges(func(prev, cur NodeConf) { + mu.Lock() + defer mu.Unlock() + prevId = prev.Id() + curId = cur.Id() + curEpoch = cur.Configuration().Epoch + calls++ + }) + + fx.run(t) + require.Eventually(t, func() bool { + mu.Lock() + defer mu.Unlock() + return calls > 0 + }, time.Second*5, time.Millisecond*5) + + mu.Lock() + defer mu.Unlock() + require.Equal(t, 1, calls, "observer must fire exactly once, not for the initial configuration") + assert.Equal(t, "test", prevId) + assert.Equal(t, "updated", curId) + assert.Equal(t, uint64(2), curEpoch) +} diff --git a/nodeconf/store.go b/nodeconf/store.go index 70b1e644d..4788222a4 100644 --- a/nodeconf/store.go +++ b/nodeconf/store.go @@ -8,3 +8,15 @@ type Store interface { GetLast(ctx context.Context, netId string) (c Configuration, err error) SaveLast(ctx context.Context, c Configuration) (err error) } + +// HistoryStore is an optional extension of Store that additionally retains +// previously applied configurations keyed by epoch. It allows a node restarted +// mid-migration to recompute the ring of an earlier epoch. +type HistoryStore interface { + Store + // GetByEpoch returns a previously saved configuration of the given network. + // Returns ErrConfigurationNotFound if the epoch is not retained. + GetByEpoch(ctx context.Context, netId string, epoch uint64) (c Configuration, err error) + // Epochs returns the retained epochs for the given network in ascending order. + Epochs(ctx context.Context, netId string) (epochs []uint64, err error) +} diff --git a/nodeconf/testconf/nodeconf.go b/nodeconf/testconf/nodeconf.go index e14663dbb..adc78e980 100644 --- a/nodeconf/testconf/nodeconf.go +++ b/nodeconf/testconf/nodeconf.go @@ -36,10 +36,13 @@ func (m *StubConf) Init(a *app.App) (err error) { NetworkId: networkId, Nodes: []nodeconf.Node{node}, CreationTime: time.Now(), + Epoch: 1, } return nil } +func (m *StubConf) ObserveChanges(observer nodeconf.ChangeObserver) {} + func (m *StubConf) Name() (name string) { return nodeconf.CName } @@ -76,6 +79,14 @@ func (m *StubConf) FilePeers() []string { return nil } +func (m *StubConf) FileV2Peers() []string { + return nil +} + +func (m *StubConf) FileV2NodeIds(spaceId string) []string { + return nil +} + func (m *StubConf) ConsensusPeers() []string { return nil } diff --git a/util/crypto/aes.go b/util/crypto/aes.go index 8790ad242..8efade4b6 100644 --- a/util/crypto/aes.go +++ b/util/crypto/aes.go @@ -123,6 +123,9 @@ func (k *AESKey) Decrypt(ciphertext []byte) ([]byte, error) { // DecryptReuse is like Decrypt but reuses dst's underlying array to avoid allocation. func (k *AESKey) DecryptReuse(dst, ciphertext []byte) ([]byte, error) { + if len(ciphertext) < NonceBytes { + return nil, fmt.Errorf("ciphertext too short: %d bytes, need at least %d", len(ciphertext), NonceBytes) + } block, err := aes.NewCipher(k.raw[:KeyBytes]) if err != nil { return nil, err diff --git a/util/crypto/aes_test.go b/util/crypto/aes_test.go index 6bd727e36..219f2cce3 100644 --- a/util/crypto/aes_test.go +++ b/util/crypto/aes_test.go @@ -43,6 +43,18 @@ func TestAESKey_DecryptReuse_NilDst(t *testing.T) { require.Equal(t, msg, result) } +func TestAESKey_Decrypt_ShortInput(t *testing.T) { + key := NewAES() + // wire-controlled input shorter than the GCM nonce must error, not panic + for l := 0; l < NonceBytes; l++ { + _, err := key.Decrypt(make([]byte, l)) + require.Error(t, err, "len %d", l) + } + // nonce-only input has no room for the GCM tag: auth error, not panic + _, err := key.Decrypt(make([]byte, NonceBytes)) + require.Error(t, err) +} + func TestAESKey_DecryptReuse_BufferReuse(t *testing.T) { key := NewAES() msg := make([]byte, 256) diff --git a/util/syncqueues/actionpool.go b/util/syncqueues/actionpool.go index e80a204a8..bca25ae75 100644 --- a/util/syncqueues/actionpool.go +++ b/util/syncqueues/actionpool.go @@ -109,7 +109,12 @@ func (rp *actionPool) Add(peerId, objectId string, action func(ctx context.Conte } func (rp *actionPool) Close() { - rp.periodicLoop.Close() + // cancels the ctx handed to every queued action, otherwise in-flight peer + // requests keep running after close + rp.cancel() + if rp.periodicLoop != nil { + rp.periodicLoop.Close() + } rp.mu.Lock() defer rp.mu.Unlock() rp.isClosed = true diff --git a/util/syncqueues/actionpool_test.go b/util/syncqueues/actionpool_test.go index 81b482f13..434159789 100644 --- a/util/syncqueues/actionpool_test.go +++ b/util/syncqueues/actionpool_test.go @@ -152,3 +152,33 @@ func TestRequestPool(t *testing.T) { rp.Close() }) } + +// app.Start closes components it never ran: a failing Init makes closeServices +// call Close on every component registered before it. +func TestActionPool_CloseWithoutRun(t *testing.T) { + rp := NewActionPool(time.Minute, time.Minute, func(peerId string) *replaceableQueue { + return newReplaceableQueue(1, 1) + }) + require.NotPanics(t, rp.Close) +} + +func TestActionPool_CloseCancelsActionCtx(t *testing.T) { + rp := NewActionPool(time.Minute, time.Minute, func(peerId string) *replaceableQueue { + return newReplaceableQueue(1, 1) + }) + rp.Run() + started := make(chan struct{}) + cancelled := make(chan struct{}) + rp.Add("peerId", "objectId", func(ctx context.Context) { + close(started) + <-ctx.Done() + close(cancelled) + }, func() {}) + <-started + rp.Close() + select { + case <-cancelled: + case <-time.After(time.Second): + require.Fail(t, "in-flight action kept running after Close") + } +}