From 34743c1ec1c957a89af2884bf8da1025acbd4202 Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Wed, 1 Jul 2026 13:10:53 +0200 Subject: [PATCH 01/20] feat(space): default new spaces to v1 header Add StoragePayloadForSpaceDeriveV1 (deterministic v1 derive builder) and switch CreateSpace, DeriveId and DeriveSpace to the v1 builders. v1 headers embed the acl and settings payloads and validate via byte-compare (needCheckSpaceId=false), matching the one-to-one path already shipping in production. Note: v1 derive produces a different derived space id than v0 for the same keys (the header bytes, hence the CID, differ). Safe on greenfield; existing v0-derived spaces would need an id-continuity strategy before an in-place upgrade. Co-Authored-By: Claude Opus 4.8 (1M context) --- commonspace/spacepayloads/payloads.go | 83 ++++++++++++++++++++++ commonspace/spacepayloads/payloads_test.go | 46 ++++++++++++ commonspace/spaceservice.go | 8 +-- 3 files changed, 132 insertions(+), 5 deletions(-) diff --git a/commonspace/spacepayloads/payloads.go b/commonspace/spacepayloads/payloads.go index 9a30f09ec..5fcc54b35 100644 --- a/commonspace/spacepayloads/payloads.go +++ b/commonspace/spacepayloads/payloads.go @@ -315,6 +315,89 @@ func StoragePayloadForSpaceDerive(payload SpaceDerivePayload) (storagePayload sp return } +func StoragePayloadForSpaceDeriveV1(payload SpaceDerivePayload) (storagePayload spacestorage.SpaceStorageCreatePayload, err error) { + // marshalling keys + identity, err := payload.SigningKey.GetPublic().Marshall() + if err != nil { + return + } + pubKey, err := payload.SigningKey.GetPublic().Raw() + if err != nil { + return + } + + // preparing replication key + hasher := fnv.New64() + _, err = hasher.Write(pubKey) + if err != nil { + return + } + repKey := hasher.Sum64() + + // preparing header (acl and settings payloads are embedded below, before signing) + header := &spacesyncproto.SpaceHeader{ + Identity: identity, + SpaceType: payload.SpaceType, + SpaceHeaderPayload: payload.SpacePayload, + ReplicationKey: repKey, + Version: spacesyncproto.SpaceHeaderVersion_SpaceHeaderVersion1, + } + + // building acl root (no SpaceId: for v1 the space id derives from the header that embeds this payload) + keyStorage := crypto.NewKeyStorage() + aclBuilder := list.NewAclRecordBuilder("", keyStorage, nil, recordverifier.NewValidateFull()) + aclRoot, err := aclBuilder.BuildRoot(list.RootContent{ + PrivKey: payload.SigningKey, + MasterKey: payload.MasterKey, + }) + if err != nil { + return + } + + // building settings (deterministic: no seed, no timestamp) + builder := objecttree.NewChangeBuilder(keyStorage, nil) + _, settingsRoot, err := builder.BuildRoot(objecttree.InitialContent{ + AclHeadId: aclRoot.Id, + PrivKey: payload.SigningKey, + ChangeType: SpaceReserved, + }) + if err != nil { + return + } + + // build header + header.AclPayload = aclRoot.Payload + header.SettingPayload = settingsRoot.RawChange + + marshalled, err := header.MarshalVT() + if err != nil { + return + } + signature, err := payload.SigningKey.Sign(marshalled) + if err != nil { + return + } + rawHeader := &spacesyncproto.RawSpaceHeader{SpaceHeader: marshalled, Signature: signature} + marshalled, err = rawHeader.MarshalVT() + if err != nil { + return + } + id, err := cidutil.NewCidFromBytes(marshalled) + spaceId := NewSpaceId(id, repKey) + rawHeaderWithId := &spacesyncproto.RawSpaceHeaderWithId{ + RawHeader: marshalled, + Id: spaceId, + } + + // creating storage + storagePayload = spacestorage.SpaceStorageCreatePayload{ + AclWithId: aclRoot, + SpaceHeaderWithId: rawHeaderWithId, + SpaceSettingsWithId: settingsRoot, + } + return +} + func makeOneToOneInfo(sharedSk crypto.PrivKey, aPk, bPk crypto.PubKey) (oneToOneInfo aclrecordproto.AclOneToOneInfo, err error) { writers := make([][]byte, 2) diff --git a/commonspace/spacepayloads/payloads_test.go b/commonspace/spacepayloads/payloads_test.go index d7f68cbd1..63502d696 100644 --- a/commonspace/spacepayloads/payloads_test.go +++ b/commonspace/spacepayloads/payloads_test.go @@ -878,6 +878,52 @@ func TestStoragePayloadForSpaceDerive(t *testing.T) { }) } +func TestStoragePayloadForSpaceDeriveV1(t *testing.T) { + t.Run("success: builds valid v1 derived space payload", func(t *testing.T) { + acc, err := accountdata.NewRandom() + require.NoError(t, err) + + master, _, err := crypto.GenerateRandomEd25519KeyPair() + require.NoError(t, err) + + pl := SpaceDerivePayload{ + SigningKey: acc.SignKey, + MasterKey: master, + SpaceType: "derived.space", + SpacePayload: randBytes(10), + } + + out, err := StoragePayloadForSpaceDeriveV1(pl) + require.NoError(t, err) + + // v1 payload should validate as a set (acl + settings embedded in header). + require.NoError(t, ValidateSpaceStorageCreatePayload(out)) + }) + + t.Run("deterministic: same keys yield same space id", func(t *testing.T) { + acc, err := accountdata.NewRandom() + require.NoError(t, err) + + master, _, err := crypto.GenerateRandomEd25519KeyPair() + require.NoError(t, err) + + pl := SpaceDerivePayload{ + SigningKey: acc.SignKey, + MasterKey: master, + SpaceType: "derived.space", + SpacePayload: []byte("stable-payload"), + } + + out1, err := StoragePayloadForSpaceDeriveV1(pl) + require.NoError(t, err) + out2, err := StoragePayloadForSpaceDeriveV1(pl) + require.NoError(t, err) + + require.Equal(t, out1.SpaceHeaderWithId.Id, out2.SpaceHeaderWithId.Id) + require.Equal(t, out1.SpaceHeaderWithId.RawHeader, out2.SpaceHeaderWithId.RawHeader) + }) +} + func randBytes(n int) []byte { b := make([]byte, n) rand.Read(b) diff --git a/commonspace/spaceservice.go b/commonspace/spaceservice.go index d6384f2d9..8aa84d627 100644 --- a/commonspace/spaceservice.go +++ b/commonspace/spaceservice.go @@ -127,9 +127,7 @@ func (s *spaceService) buildRecordVerifier() (recordverifier.RecordVerifier, err } func (s *spaceService) CreateSpace(ctx context.Context, payload spacepayloads.SpaceCreatePayload) (id string, err error) { - // TODO: switch to SpaceCreatePayloadV1 after next proto update - // storageCreate, err := spacepayloads.StoragePayloadForSpaceCreateV1(payload) - storageCreate, err := spacepayloads.StoragePayloadForSpaceCreate(payload) + storageCreate, err := spacepayloads.StoragePayloadForSpaceCreateV1(payload) if err != nil { return } @@ -145,7 +143,7 @@ func (s *spaceService) CreateSpace(ctx context.Context, payload spacepayloads.Sp } func (s *spaceService) DeriveId(ctx context.Context, payload spacepayloads.SpaceDerivePayload) (id string, err error) { - storageCreate, err := spacepayloads.StoragePayloadForSpaceDerive(payload) + storageCreate, err := spacepayloads.StoragePayloadForSpaceDeriveV1(payload) if err != nil { return } @@ -154,7 +152,7 @@ func (s *spaceService) DeriveId(ctx context.Context, payload spacepayloads.Space } func (s *spaceService) DeriveSpace(ctx context.Context, payload spacepayloads.SpaceDerivePayload) (id string, err error) { - storageCreate, err := spacepayloads.StoragePayloadForSpaceDerive(payload) + storageCreate, err := spacepayloads.StoragePayloadForSpaceDeriveV1(payload) if err != nil { return } From 2c1a8b802511769e8078cae415a53a4688fcf7ca Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Wed, 1 Jul 2026 13:14:40 +0200 Subject: [PATCH 02/20] feat(space): add fileprotoVersion to space header Add the SpaceFileProtoVersion enum (Unspecified=0, V2=2) and a fileprotoVersion field to the SpaceHeader proto, and plumb it through SpaceCreatePayload and SpaceDerivePayload into every builder. The field lives in the signed, immutable header, so it is tamper-evident and recovered verbatim on cold restore. Gates the v2 file protocol (filenode v2 + S3-direct + networkSign) per SYN-15. Callers set the value (e.g. the SDK mints spaces at V2); unset stays Unspecified, so legacy any-sync consumers are unaffected. OneToOne spaces stay Unspecified. Co-Authored-By: Claude Opus 4.8 (1M context) --- commonspace/spacepayloads/payloads.go | 8 + commonspace/spacepayloads/payloads_test.go | 94 +++++++ .../spacesyncproto/protos/spacesync.proto | 10 + commonspace/spacesyncproto/spacesync.pb.go | 259 +++++++++++------- .../spacesyncproto/spacesync_vtproto.pb.go | 27 ++ 5 files changed, 300 insertions(+), 98 deletions(-) diff --git a/commonspace/spacepayloads/payloads.go b/commonspace/spacepayloads/payloads.go index 5fcc54b35..9517a70cb 100644 --- a/commonspace/spacepayloads/payloads.go +++ b/commonspace/spacepayloads/payloads.go @@ -42,6 +42,8 @@ type SpaceCreatePayload struct { Metadata []byte // Options is the ACL space options (e.g. delete restrictions) Options *aclrecordproto.AclSpaceOptions + // FileProtoVersion gates the file protocol the space uses (embedded in the signed header) + FileProtoVersion spacesyncproto.SpaceFileProtoVersion } type SpaceDerivePayload struct { @@ -49,6 +51,8 @@ type SpaceDerivePayload struct { MasterKey crypto.PrivKey SpaceType string SpacePayload []byte + // FileProtoVersion gates the file protocol the space uses (embedded in the signed header) + FileProtoVersion spacesyncproto.SpaceFileProtoVersion } const ( @@ -78,6 +82,7 @@ func StoragePayloadForSpaceCreate(payload SpaceCreatePayload) (storagePayload sp SpaceHeaderPayload: payload.SpacePayload, ReplicationKey: payload.ReplicationKey, Seed: spaceHeaderSeed, + FileprotoVersion: payload.FileProtoVersion, } marshalled, err := header.MarshalVT() if err != nil { @@ -165,6 +170,7 @@ func StoragePayloadForSpaceCreateV1(payload SpaceCreatePayload) (storagePayload SpaceHeaderPayload: payload.SpacePayload, ReplicationKey: payload.ReplicationKey, Seed: spaceHeaderSeed, + FileprotoVersion: payload.FileProtoVersion, Version: spacesyncproto.SpaceHeaderVersion_SpaceHeaderVersion1, } @@ -261,6 +267,7 @@ func StoragePayloadForSpaceDerive(payload SpaceDerivePayload) (storagePayload sp SpaceType: payload.SpaceType, SpaceHeaderPayload: payload.SpacePayload, ReplicationKey: repKey, + FileprotoVersion: payload.FileProtoVersion, } marshalled, err := header.MarshalVT() if err != nil { @@ -340,6 +347,7 @@ func StoragePayloadForSpaceDeriveV1(payload SpaceDerivePayload) (storagePayload SpaceType: payload.SpaceType, SpaceHeaderPayload: payload.SpacePayload, ReplicationKey: repKey, + FileprotoVersion: payload.FileProtoVersion, Version: spacesyncproto.SpaceHeaderVersion_SpaceHeaderVersion1, } diff --git a/commonspace/spacepayloads/payloads_test.go b/commonspace/spacepayloads/payloads_test.go index 63502d696..8508a869a 100644 --- a/commonspace/spacepayloads/payloads_test.go +++ b/commonspace/spacepayloads/payloads_test.go @@ -924,6 +924,100 @@ func TestStoragePayloadForSpaceDeriveV1(t *testing.T) { }) } +func decodeSpaceHeader(t *testing.T, out spacestorage.SpaceStorageCreatePayload) *spacesyncproto.SpaceHeader { + var raw spacesyncproto.RawSpaceHeader + require.NoError(t, raw.UnmarshalVT(out.SpaceHeaderWithId.RawHeader)) + var h spacesyncproto.SpaceHeader + require.NoError(t, h.UnmarshalVT(raw.SpaceHeader)) + return &h +} + +func TestSpaceHeaderFileProtoVersion(t *testing.T) { + newCreatePayload := func(t *testing.T, fpv spacesyncproto.SpaceFileProtoVersion) SpaceCreatePayload { + acc, err := accountdata.NewRandom() + require.NoError(t, err) + master, _, err := crypto.GenerateRandomEd25519KeyPair() + require.NoError(t, err) + metaKey, _, err := crypto.GenerateRandomEd25519KeyPair() + require.NoError(t, err) + readKey, _ := crypto.NewRandomAES() + return SpaceCreatePayload{ + SigningKey: acc.SignKey, + SpaceType: "test.space", + ReplicationKey: mrand.Uint64(), + SpacePayload: randBytes(12), + MasterKey: master, + ReadKey: readKey, + MetadataKey: metaKey, + Metadata: randBytes(6), + FileProtoVersion: fpv, + } + } + + t.Run("create v1 carries fileprotoVersion=V2 and survives round-trip", func(t *testing.T) { + out, err := StoragePayloadForSpaceCreateV1(newCreatePayload(t, spacesyncproto.SpaceFileProtoVersion_SpaceFileProtoVersionV2)) + require.NoError(t, err) + require.NoError(t, ValidateSpaceStorageCreatePayload(out)) + require.Equal(t, spacesyncproto.SpaceFileProtoVersion_SpaceFileProtoVersionV2, decodeSpaceHeader(t, out).GetFileprotoVersion()) + }) + + t.Run("create v0 carries fileprotoVersion=V2", func(t *testing.T) { + out, err := StoragePayloadForSpaceCreate(newCreatePayload(t, spacesyncproto.SpaceFileProtoVersion_SpaceFileProtoVersionV2)) + require.NoError(t, err) + require.NoError(t, ValidateSpaceStorageCreatePayload(out)) + require.Equal(t, spacesyncproto.SpaceFileProtoVersion_SpaceFileProtoVersionV2, decodeSpaceHeader(t, out).GetFileprotoVersion()) + }) + + t.Run("derive v1 carries fileprotoVersion=V2", func(t *testing.T) { + acc, err := accountdata.NewRandom() + require.NoError(t, err) + master, _, err := crypto.GenerateRandomEd25519KeyPair() + require.NoError(t, err) + out, err := StoragePayloadForSpaceDeriveV1(SpaceDerivePayload{ + SigningKey: acc.SignKey, + MasterKey: master, + SpaceType: "derived.space", + SpacePayload: randBytes(10), + FileProtoVersion: spacesyncproto.SpaceFileProtoVersion_SpaceFileProtoVersionV2, + }) + require.NoError(t, err) + require.NoError(t, ValidateSpaceStorageCreatePayload(out)) + require.Equal(t, spacesyncproto.SpaceFileProtoVersion_SpaceFileProtoVersionV2, decodeSpaceHeader(t, out).GetFileprotoVersion()) + }) + + t.Run("unset defaults to Unspecified", func(t *testing.T) { + out, err := StoragePayloadForSpaceCreateV1(newCreatePayload(t, spacesyncproto.SpaceFileProtoVersion_SpaceFileProtoVersionUnspecified)) + require.NoError(t, err) + require.Equal(t, spacesyncproto.SpaceFileProtoVersion_SpaceFileProtoVersionUnspecified, decodeSpaceHeader(t, out).GetFileprotoVersion()) + }) + + t.Run("fileprotoVersion is covered by the header signature (tamper rejected)", func(t *testing.T) { + out, err := StoragePayloadForSpaceCreateV1(newCreatePayload(t, spacesyncproto.SpaceFileProtoVersion_SpaceFileProtoVersionUnspecified)) + require.NoError(t, err) + // original validates + _, err = ValidateSpaceHeader(out.SpaceHeaderWithId, nil, out.AclWithId.Payload, out.SpaceSettingsWithId.RawChange) + require.NoError(t, err) + + // flip fileprotoVersion inside the signed header body, keep the original signature + var raw spacesyncproto.RawSpaceHeader + require.NoError(t, raw.UnmarshalVT(out.SpaceHeaderWithId.RawHeader)) + var h spacesyncproto.SpaceHeader + require.NoError(t, h.UnmarshalVT(raw.SpaceHeader)) + h.FileprotoVersion = spacesyncproto.SpaceFileProtoVersion_SpaceFileProtoVersionV2 + raw.SpaceHeader, err = h.MarshalVT() + require.NoError(t, err) + tamperedRaw, err := raw.MarshalVT() + require.NoError(t, err) + + // recompute the cid so we pass the cid check and reach the signature check + newCid, err := cidutil.NewCidFromBytes(tamperedRaw) + require.NoError(t, err) + tampered := &spacesyncproto.RawSpaceHeaderWithId{RawHeader: tamperedRaw, Id: NewSpaceId(newCid, h.ReplicationKey)} + _, err = ValidateSpaceHeader(tampered, nil, out.AclWithId.Payload, out.SpaceSettingsWithId.RawChange) + require.ErrorIs(t, err, spacestorage.ErrIncorrectSpaceHeader) + }) +} + func randBytes(n int) []byte { b := make([]byte, n) rand.Read(b) diff --git a/commonspace/spacesyncproto/protos/spacesync.proto b/commonspace/spacesyncproto/protos/spacesync.proto index 92f1d8b70..228fb0a55 100644 --- a/commonspace/spacesyncproto/protos/spacesync.proto +++ b/commonspace/spacesyncproto/protos/spacesync.proto @@ -125,6 +125,13 @@ enum SpaceHeaderVersion { SpaceHeaderVersion1 = 1; } +// SpaceFileProtoVersion gates the file protocol a space uses. It lives in the signed, +// immutable header so it can't be stripped/downgraded without invalidating the owner signature. +enum SpaceFileProtoVersion { + SpaceFileProtoVersionUnspecified = 0; + SpaceFileProtoVersionV2 = 2; +} + // SpaceHeader is a header for a space message SpaceHeader { bytes identity = 1; @@ -138,6 +145,9 @@ message SpaceHeader { bytes aclPayload = 7; bytes settingPayload = 8; + // fileprotoVersion gates the file protocol (v2 = filenode v2 + S3-direct + networkSign) + SpaceFileProtoVersion fileprotoVersion = 9; + SpaceHeaderVersion version = 100; } diff --git a/commonspace/spacesyncproto/spacesync.pb.go b/commonspace/spacesyncproto/spacesync.pb.go index 7907ba194..f18393ebd 100644 --- a/commonspace/spacesyncproto/spacesync.pb.go +++ b/commonspace/spacesyncproto/spacesync.pb.go @@ -140,6 +140,54 @@ func (SpaceHeaderVersion) EnumDescriptor() ([]byte, []int) { return file_commonspace_spacesyncproto_protos_spacesync_proto_rawDescGZIP(), []int{1} } +// SpaceFileProtoVersion gates the file protocol a space uses. It lives in the signed, +// immutable header so it can't be stripped/downgraded without invalidating the owner signature. +type SpaceFileProtoVersion int32 + +const ( + SpaceFileProtoVersion_SpaceFileProtoVersionUnspecified SpaceFileProtoVersion = 0 + SpaceFileProtoVersion_SpaceFileProtoVersionV2 SpaceFileProtoVersion = 2 +) + +// Enum value maps for SpaceFileProtoVersion. +var ( + SpaceFileProtoVersion_name = map[int32]string{ + 0: "SpaceFileProtoVersionUnspecified", + 2: "SpaceFileProtoVersionV2", + } + SpaceFileProtoVersion_value = map[string]int32{ + "SpaceFileProtoVersionUnspecified": 0, + "SpaceFileProtoVersionV2": 2, + } +) + +func (x SpaceFileProtoVersion) Enum() *SpaceFileProtoVersion { + p := new(SpaceFileProtoVersion) + *p = x + return p +} + +func (x SpaceFileProtoVersion) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (SpaceFileProtoVersion) Descriptor() protoreflect.EnumDescriptor { + return file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes[2].Descriptor() +} + +func (SpaceFileProtoVersion) Type() protoreflect.EnumType { + return &file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes[2] +} + +func (x SpaceFileProtoVersion) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use SpaceFileProtoVersion.Descriptor instead. +func (SpaceFileProtoVersion) EnumDescriptor() ([]byte, []int) { + return file_commonspace_spacesyncproto_protos_spacesync_proto_rawDescGZIP(), []int{2} +} + // SpaceSubscription contains in ObjectSyncMessage.Payload and indicates that we need to subscribe or unsubscribe the current stream to this space type SpaceSubscriptionAction int32 @@ -171,11 +219,11 @@ func (x SpaceSubscriptionAction) String() string { } func (SpaceSubscriptionAction) Descriptor() protoreflect.EnumDescriptor { - return file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes[2].Descriptor() + return file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes[3].Descriptor() } func (SpaceSubscriptionAction) Type() protoreflect.EnumType { - return &file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes[2] + return &file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes[3] } func (x SpaceSubscriptionAction) Number() protoreflect.EnumNumber { @@ -184,7 +232,7 @@ func (x SpaceSubscriptionAction) Number() protoreflect.EnumNumber { // Deprecated: Use SpaceSubscriptionAction.Descriptor instead. func (SpaceSubscriptionAction) EnumDescriptor() ([]byte, []int) { - return file_commonspace_spacesyncproto_protos_spacesync_proto_rawDescGZIP(), []int{2} + return file_commonspace_spacesyncproto_protos_spacesync_proto_rawDescGZIP(), []int{3} } // DiffType is a type of diff @@ -224,11 +272,11 @@ func (x DiffType) String() string { } func (DiffType) Descriptor() protoreflect.EnumDescriptor { - return file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes[3].Descriptor() + return file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes[4].Descriptor() } func (DiffType) Type() protoreflect.EnumType { - return &file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes[3] + return &file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes[4] } func (x DiffType) Number() protoreflect.EnumNumber { @@ -237,7 +285,7 @@ func (x DiffType) Number() protoreflect.EnumNumber { // Deprecated: Use DiffType.Descriptor instead. func (DiffType) EnumDescriptor() ([]byte, []int) { - return file_commonspace_spacesyncproto_protos_spacesync_proto_rawDescGZIP(), []int{3} + return file_commonspace_spacesyncproto_protos_spacesync_proto_rawDescGZIP(), []int{4} } // ObjectType is a type of object @@ -274,11 +322,11 @@ func (x ObjectType) String() string { } func (ObjectType) Descriptor() protoreflect.EnumDescriptor { - return file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes[4].Descriptor() + return file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes[5].Descriptor() } func (ObjectType) Type() protoreflect.EnumType { - return &file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes[4] + return &file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes[5] } func (x ObjectType) Number() protoreflect.EnumNumber { @@ -287,7 +335,7 @@ func (x ObjectType) Number() protoreflect.EnumNumber { // Deprecated: Use ObjectType.Descriptor instead. func (ObjectType) EnumDescriptor() ([]byte, []int) { - return file_commonspace_spacesyncproto_protos_spacesync_proto_rawDescGZIP(), []int{4} + return file_commonspace_spacesyncproto_protos_spacesync_proto_rawDescGZIP(), []int{5} } // HeadSyncRange presenting a request for one range @@ -999,11 +1047,13 @@ type SpaceHeader struct { Seed []byte `protobuf:"bytes,5,opt,name=seed,proto3" json:"seed,omitempty"` SpaceHeaderPayload []byte `protobuf:"bytes,6,opt,name=spaceHeaderPayload,proto3" json:"spaceHeaderPayload,omitempty"` // since v1 fields - AclPayload []byte `protobuf:"bytes,7,opt,name=aclPayload,proto3" json:"aclPayload,omitempty"` - SettingPayload []byte `protobuf:"bytes,8,opt,name=settingPayload,proto3" json:"settingPayload,omitempty"` - Version SpaceHeaderVersion `protobuf:"varint,100,opt,name=version,proto3,enum=spacesync.SpaceHeaderVersion" json:"version,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + AclPayload []byte `protobuf:"bytes,7,opt,name=aclPayload,proto3" json:"aclPayload,omitempty"` + SettingPayload []byte `protobuf:"bytes,8,opt,name=settingPayload,proto3" json:"settingPayload,omitempty"` + // fileprotoVersion gates the file protocol (v2 = filenode v2 + S3-direct + networkSign) + FileprotoVersion SpaceFileProtoVersion `protobuf:"varint,9,opt,name=fileprotoVersion,proto3,enum=spacesync.SpaceFileProtoVersion" json:"fileprotoVersion,omitempty"` + Version SpaceHeaderVersion `protobuf:"varint,100,opt,name=version,proto3,enum=spacesync.SpaceHeaderVersion" json:"version,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SpaceHeader) Reset() { @@ -1092,6 +1142,13 @@ func (x *SpaceHeader) GetSettingPayload() []byte { return nil } +func (x *SpaceHeader) GetFileprotoVersion() SpaceFileProtoVersion { + if x != nil { + return x.FileprotoVersion + } + return SpaceFileProtoVersion_SpaceFileProtoVersionUnspecified +} + func (x *SpaceHeader) GetVersion() SpaceHeaderVersion { if x != nil { return x.Version @@ -2204,7 +2261,7 @@ const file_commonspace_spacesyncproto_protos_spacesync_proto_rawDesc = "" + "aclPayload\x12\"\n" + "\faclPayloadId\x18\x03 \x01(\tR\faclPayloadId\x122\n" + "\x14spaceSettingsPayload\x18\x04 \x01(\fR\x14spaceSettingsPayload\x126\n" + - "\x16spaceSettingsPayloadId\x18\x05 \x01(\tR\x16spaceSettingsPayloadId\"\xd2\x02\n" + + "\x16spaceSettingsPayloadId\x18\x05 \x01(\tR\x16spaceSettingsPayloadId\"\xa0\x03\n" + "\vSpaceHeader\x12\x1a\n" + "\bidentity\x18\x01 \x01(\fR\bidentity\x12\x1c\n" + "\ttimestamp\x18\x02 \x01(\x03R\ttimestamp\x12\x1c\n" + @@ -2215,7 +2272,8 @@ const file_commonspace_spacesyncproto_protos_spacesync_proto_rawDesc = "" + "\n" + "aclPayload\x18\a \x01(\fR\n" + "aclPayload\x12&\n" + - "\x0esettingPayload\x18\b \x01(\fR\x0esettingPayload\x127\n" + + "\x0esettingPayload\x18\b \x01(\fR\x0esettingPayload\x12L\n" + + "\x10fileprotoVersion\x18\t \x01(\x0e2 .spacesync.SpaceFileProtoVersionR\x10fileprotoVersion\x127\n" + "\aversion\x18d \x01(\x0e2\x1d.spacesync.SpaceHeaderVersionR\aversion\"P\n" + "\x0eRawSpaceHeader\x12 \n" + "\vspaceHeader\x18\x01 \x01(\fR\vspaceHeader\x12\x1c\n" + @@ -2294,7 +2352,10 @@ const file_commonspace_spacesyncproto_protos_spacesync_proto_rawDesc = "" + "\vErrorOffset\x10d*F\n" + "\x12SpaceHeaderVersion\x12\x17\n" + "\x13SpaceHeaderVersion0\x10\x00\x12\x17\n" + - "\x13SpaceHeaderVersion1\x10\x01*9\n" + + "\x13SpaceHeaderVersion1\x10\x01*Z\n" + + "\x15SpaceFileProtoVersion\x12$\n" + + " SpaceFileProtoVersionUnspecified\x10\x00\x12\x1b\n" + + "\x17SpaceFileProtoVersionV2\x10\x02*9\n" + "\x17SpaceSubscriptionAction\x12\r\n" + "\tSubscribe\x10\x00\x12\x0f\n" + "\vUnsubscribe\x10\x01*/\n" + @@ -2333,92 +2394,94 @@ func file_commonspace_spacesyncproto_protos_spacesync_proto_rawDescGZIP() []byte return file_commonspace_spacesyncproto_protos_spacesync_proto_rawDescData } -var file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes = make([]protoimpl.EnumInfo, 5) +var file_commonspace_spacesyncproto_protos_spacesync_proto_enumTypes = make([]protoimpl.EnumInfo, 6) var file_commonspace_spacesyncproto_protos_spacesync_proto_msgTypes = make([]protoimpl.MessageInfo, 32) var file_commonspace_spacesyncproto_protos_spacesync_proto_goTypes = []any{ (ErrCodes)(0), // 0: spacesync.ErrCodes (SpaceHeaderVersion)(0), // 1: spacesync.SpaceHeaderVersion - (SpaceSubscriptionAction)(0), // 2: spacesync.SpaceSubscriptionAction - (DiffType)(0), // 3: spacesync.DiffType - (ObjectType)(0), // 4: spacesync.ObjectType - (*HeadSyncRange)(nil), // 5: spacesync.HeadSyncRange - (*HeadSyncResult)(nil), // 6: spacesync.HeadSyncResult - (*HeadSyncResultElement)(nil), // 7: spacesync.HeadSyncResultElement - (*HeadSyncRequest)(nil), // 8: spacesync.HeadSyncRequest - (*HeadSyncResponse)(nil), // 9: spacesync.HeadSyncResponse - (*ObjectSyncMessage)(nil), // 10: spacesync.ObjectSyncMessage - (*SpacePushRequest)(nil), // 11: spacesync.SpacePushRequest - (*SpacePushResponse)(nil), // 12: spacesync.SpacePushResponse - (*SpacePullRequest)(nil), // 13: spacesync.SpacePullRequest - (*SpacePullResponse)(nil), // 14: spacesync.SpacePullResponse - (*AclRecord)(nil), // 15: spacesync.AclRecord - (*SpacePayload)(nil), // 16: spacesync.SpacePayload - (*SpaceHeader)(nil), // 17: spacesync.SpaceHeader - (*RawSpaceHeader)(nil), // 18: spacesync.RawSpaceHeader - (*RawSpaceHeaderWithId)(nil), // 19: spacesync.RawSpaceHeaderWithId - (*SpaceSettingsContent)(nil), // 20: spacesync.SpaceSettingsContent - (*ObjectDelete)(nil), // 21: spacesync.ObjectDelete - (*StoreHeader)(nil), // 22: spacesync.StoreHeader - (*SpaceDelete)(nil), // 23: spacesync.SpaceDelete - (*SpaceSettingsSnapshot)(nil), // 24: spacesync.SpaceSettingsSnapshot - (*SettingsData)(nil), // 25: spacesync.SettingsData - (*SpaceSubscription)(nil), // 26: spacesync.SpaceSubscription - (*AclAddRecordRequest)(nil), // 27: spacesync.AclAddRecordRequest - (*AclAddRecordResponse)(nil), // 28: spacesync.AclAddRecordResponse - (*AclGetRecordsRequest)(nil), // 29: spacesync.AclGetRecordsRequest - (*AclGetRecordsResponse)(nil), // 30: spacesync.AclGetRecordsResponse - (*StoreDiffRequest)(nil), // 31: spacesync.StoreDiffRequest - (*StoreDiffResponse)(nil), // 32: spacesync.StoreDiffResponse - (*StoreKeyValue)(nil), // 33: spacesync.StoreKeyValue - (*StoreKeyValues)(nil), // 34: spacesync.StoreKeyValues - (*StoreKeyInner)(nil), // 35: spacesync.StoreKeyInner - (*StorageHeader)(nil), // 36: spacesync.StorageHeader + (SpaceFileProtoVersion)(0), // 2: spacesync.SpaceFileProtoVersion + (SpaceSubscriptionAction)(0), // 3: spacesync.SpaceSubscriptionAction + (DiffType)(0), // 4: spacesync.DiffType + (ObjectType)(0), // 5: spacesync.ObjectType + (*HeadSyncRange)(nil), // 6: spacesync.HeadSyncRange + (*HeadSyncResult)(nil), // 7: spacesync.HeadSyncResult + (*HeadSyncResultElement)(nil), // 8: spacesync.HeadSyncResultElement + (*HeadSyncRequest)(nil), // 9: spacesync.HeadSyncRequest + (*HeadSyncResponse)(nil), // 10: spacesync.HeadSyncResponse + (*ObjectSyncMessage)(nil), // 11: spacesync.ObjectSyncMessage + (*SpacePushRequest)(nil), // 12: spacesync.SpacePushRequest + (*SpacePushResponse)(nil), // 13: spacesync.SpacePushResponse + (*SpacePullRequest)(nil), // 14: spacesync.SpacePullRequest + (*SpacePullResponse)(nil), // 15: spacesync.SpacePullResponse + (*AclRecord)(nil), // 16: spacesync.AclRecord + (*SpacePayload)(nil), // 17: spacesync.SpacePayload + (*SpaceHeader)(nil), // 18: spacesync.SpaceHeader + (*RawSpaceHeader)(nil), // 19: spacesync.RawSpaceHeader + (*RawSpaceHeaderWithId)(nil), // 20: spacesync.RawSpaceHeaderWithId + (*SpaceSettingsContent)(nil), // 21: spacesync.SpaceSettingsContent + (*ObjectDelete)(nil), // 22: spacesync.ObjectDelete + (*StoreHeader)(nil), // 23: spacesync.StoreHeader + (*SpaceDelete)(nil), // 24: spacesync.SpaceDelete + (*SpaceSettingsSnapshot)(nil), // 25: spacesync.SpaceSettingsSnapshot + (*SettingsData)(nil), // 26: spacesync.SettingsData + (*SpaceSubscription)(nil), // 27: spacesync.SpaceSubscription + (*AclAddRecordRequest)(nil), // 28: spacesync.AclAddRecordRequest + (*AclAddRecordResponse)(nil), // 29: spacesync.AclAddRecordResponse + (*AclGetRecordsRequest)(nil), // 30: spacesync.AclGetRecordsRequest + (*AclGetRecordsResponse)(nil), // 31: spacesync.AclGetRecordsResponse + (*StoreDiffRequest)(nil), // 32: spacesync.StoreDiffRequest + (*StoreDiffResponse)(nil), // 33: spacesync.StoreDiffResponse + (*StoreKeyValue)(nil), // 34: spacesync.StoreKeyValue + (*StoreKeyValues)(nil), // 35: spacesync.StoreKeyValues + (*StoreKeyInner)(nil), // 36: spacesync.StoreKeyInner + (*StorageHeader)(nil), // 37: spacesync.StorageHeader } var file_commonspace_spacesyncproto_protos_spacesync_proto_depIdxs = []int32{ - 7, // 0: spacesync.HeadSyncResult.elements:type_name -> spacesync.HeadSyncResultElement - 5, // 1: spacesync.HeadSyncRequest.ranges:type_name -> spacesync.HeadSyncRange - 3, // 2: spacesync.HeadSyncRequest.diffType:type_name -> spacesync.DiffType - 6, // 3: spacesync.HeadSyncResponse.results:type_name -> spacesync.HeadSyncResult - 3, // 4: spacesync.HeadSyncResponse.diffType:type_name -> spacesync.DiffType - 4, // 5: spacesync.ObjectSyncMessage.objectType:type_name -> spacesync.ObjectType - 16, // 6: spacesync.SpacePushRequest.payload:type_name -> spacesync.SpacePayload - 16, // 7: spacesync.SpacePullResponse.payload:type_name -> spacesync.SpacePayload - 15, // 8: spacesync.SpacePullResponse.aclRecords:type_name -> spacesync.AclRecord - 19, // 9: spacesync.SpacePayload.spaceHeader:type_name -> spacesync.RawSpaceHeaderWithId - 1, // 10: spacesync.SpaceHeader.version:type_name -> spacesync.SpaceHeaderVersion - 21, // 11: spacesync.SpaceSettingsContent.objectDelete:type_name -> spacesync.ObjectDelete - 23, // 12: spacesync.SpaceSettingsContent.spaceDelete:type_name -> spacesync.SpaceDelete - 20, // 13: spacesync.SettingsData.content:type_name -> spacesync.SpaceSettingsContent - 24, // 14: spacesync.SettingsData.snapshot:type_name -> spacesync.SpaceSettingsSnapshot - 2, // 15: spacesync.SpaceSubscription.action:type_name -> spacesync.SpaceSubscriptionAction - 5, // 16: spacesync.StoreDiffRequest.ranges:type_name -> spacesync.HeadSyncRange - 6, // 17: spacesync.StoreDiffResponse.results:type_name -> spacesync.HeadSyncResult - 33, // 18: spacesync.StoreKeyValues.keyValues:type_name -> spacesync.StoreKeyValue - 8, // 19: spacesync.SpaceSync.HeadSync:input_type -> spacesync.HeadSyncRequest - 31, // 20: spacesync.SpaceSync.StoreDiff:input_type -> spacesync.StoreDiffRequest - 33, // 21: spacesync.SpaceSync.StoreElements:input_type -> spacesync.StoreKeyValue - 11, // 22: spacesync.SpaceSync.SpacePush:input_type -> spacesync.SpacePushRequest - 13, // 23: spacesync.SpaceSync.SpacePull:input_type -> spacesync.SpacePullRequest - 10, // 24: spacesync.SpaceSync.ObjectSyncStream:input_type -> spacesync.ObjectSyncMessage - 10, // 25: spacesync.SpaceSync.ObjectSync:input_type -> spacesync.ObjectSyncMessage - 10, // 26: spacesync.SpaceSync.ObjectSyncRequestStream:input_type -> spacesync.ObjectSyncMessage - 27, // 27: spacesync.SpaceSync.AclAddRecord:input_type -> spacesync.AclAddRecordRequest - 29, // 28: spacesync.SpaceSync.AclGetRecords:input_type -> spacesync.AclGetRecordsRequest - 9, // 29: spacesync.SpaceSync.HeadSync:output_type -> spacesync.HeadSyncResponse - 32, // 30: spacesync.SpaceSync.StoreDiff:output_type -> spacesync.StoreDiffResponse - 33, // 31: spacesync.SpaceSync.StoreElements:output_type -> spacesync.StoreKeyValue - 12, // 32: spacesync.SpaceSync.SpacePush:output_type -> spacesync.SpacePushResponse - 14, // 33: spacesync.SpaceSync.SpacePull:output_type -> spacesync.SpacePullResponse - 10, // 34: spacesync.SpaceSync.ObjectSyncStream:output_type -> spacesync.ObjectSyncMessage - 10, // 35: spacesync.SpaceSync.ObjectSync:output_type -> spacesync.ObjectSyncMessage - 10, // 36: spacesync.SpaceSync.ObjectSyncRequestStream:output_type -> spacesync.ObjectSyncMessage - 28, // 37: spacesync.SpaceSync.AclAddRecord:output_type -> spacesync.AclAddRecordResponse - 30, // 38: spacesync.SpaceSync.AclGetRecords:output_type -> spacesync.AclGetRecordsResponse - 29, // [29:39] is the sub-list for method output_type - 19, // [19:29] is the sub-list for method input_type - 19, // [19:19] is the sub-list for extension type_name - 19, // [19:19] is the sub-list for extension extendee - 0, // [0:19] is the sub-list for field type_name + 8, // 0: spacesync.HeadSyncResult.elements:type_name -> spacesync.HeadSyncResultElement + 6, // 1: spacesync.HeadSyncRequest.ranges:type_name -> spacesync.HeadSyncRange + 4, // 2: spacesync.HeadSyncRequest.diffType:type_name -> spacesync.DiffType + 7, // 3: spacesync.HeadSyncResponse.results:type_name -> spacesync.HeadSyncResult + 4, // 4: spacesync.HeadSyncResponse.diffType:type_name -> spacesync.DiffType + 5, // 5: spacesync.ObjectSyncMessage.objectType:type_name -> spacesync.ObjectType + 17, // 6: spacesync.SpacePushRequest.payload:type_name -> spacesync.SpacePayload + 17, // 7: spacesync.SpacePullResponse.payload:type_name -> spacesync.SpacePayload + 16, // 8: spacesync.SpacePullResponse.aclRecords:type_name -> spacesync.AclRecord + 20, // 9: spacesync.SpacePayload.spaceHeader:type_name -> spacesync.RawSpaceHeaderWithId + 2, // 10: spacesync.SpaceHeader.fileprotoVersion:type_name -> spacesync.SpaceFileProtoVersion + 1, // 11: spacesync.SpaceHeader.version:type_name -> spacesync.SpaceHeaderVersion + 22, // 12: spacesync.SpaceSettingsContent.objectDelete:type_name -> spacesync.ObjectDelete + 24, // 13: spacesync.SpaceSettingsContent.spaceDelete:type_name -> spacesync.SpaceDelete + 21, // 14: spacesync.SettingsData.content:type_name -> spacesync.SpaceSettingsContent + 25, // 15: spacesync.SettingsData.snapshot:type_name -> spacesync.SpaceSettingsSnapshot + 3, // 16: spacesync.SpaceSubscription.action:type_name -> spacesync.SpaceSubscriptionAction + 6, // 17: spacesync.StoreDiffRequest.ranges:type_name -> spacesync.HeadSyncRange + 7, // 18: spacesync.StoreDiffResponse.results:type_name -> spacesync.HeadSyncResult + 34, // 19: spacesync.StoreKeyValues.keyValues:type_name -> spacesync.StoreKeyValue + 9, // 20: spacesync.SpaceSync.HeadSync:input_type -> spacesync.HeadSyncRequest + 32, // 21: spacesync.SpaceSync.StoreDiff:input_type -> spacesync.StoreDiffRequest + 34, // 22: spacesync.SpaceSync.StoreElements:input_type -> spacesync.StoreKeyValue + 12, // 23: spacesync.SpaceSync.SpacePush:input_type -> spacesync.SpacePushRequest + 14, // 24: spacesync.SpaceSync.SpacePull:input_type -> spacesync.SpacePullRequest + 11, // 25: spacesync.SpaceSync.ObjectSyncStream:input_type -> spacesync.ObjectSyncMessage + 11, // 26: spacesync.SpaceSync.ObjectSync:input_type -> spacesync.ObjectSyncMessage + 11, // 27: spacesync.SpaceSync.ObjectSyncRequestStream:input_type -> spacesync.ObjectSyncMessage + 28, // 28: spacesync.SpaceSync.AclAddRecord:input_type -> spacesync.AclAddRecordRequest + 30, // 29: spacesync.SpaceSync.AclGetRecords:input_type -> spacesync.AclGetRecordsRequest + 10, // 30: spacesync.SpaceSync.HeadSync:output_type -> spacesync.HeadSyncResponse + 33, // 31: spacesync.SpaceSync.StoreDiff:output_type -> spacesync.StoreDiffResponse + 34, // 32: spacesync.SpaceSync.StoreElements:output_type -> spacesync.StoreKeyValue + 13, // 33: spacesync.SpaceSync.SpacePush:output_type -> spacesync.SpacePushResponse + 15, // 34: spacesync.SpaceSync.SpacePull:output_type -> spacesync.SpacePullResponse + 11, // 35: spacesync.SpaceSync.ObjectSyncStream:output_type -> spacesync.ObjectSyncMessage + 11, // 36: spacesync.SpaceSync.ObjectSync:output_type -> spacesync.ObjectSyncMessage + 11, // 37: spacesync.SpaceSync.ObjectSyncRequestStream:output_type -> spacesync.ObjectSyncMessage + 29, // 38: spacesync.SpaceSync.AclAddRecord:output_type -> spacesync.AclAddRecordResponse + 31, // 39: spacesync.SpaceSync.AclGetRecords:output_type -> spacesync.AclGetRecordsResponse + 30, // [30:40] is the sub-list for method output_type + 20, // [20:30] is the sub-list for method input_type + 20, // [20:20] is the sub-list for extension type_name + 20, // [20:20] is the sub-list for extension extendee + 0, // [0:20] is the sub-list for field type_name } func init() { file_commonspace_spacesyncproto_protos_spacesync_proto_init() } @@ -2435,7 +2498,7 @@ func file_commonspace_spacesyncproto_protos_spacesync_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_commonspace_spacesyncproto_protos_spacesync_proto_rawDesc), len(file_commonspace_spacesyncproto_protos_spacesync_proto_rawDesc)), - NumEnums: 5, + NumEnums: 6, NumMessages: 32, NumExtensions: 0, NumServices: 1, diff --git a/commonspace/spacesyncproto/spacesync_vtproto.pb.go b/commonspace/spacesyncproto/spacesync_vtproto.pb.go index 6ce78ed40..1f1ae6093 100644 --- a/commonspace/spacesyncproto/spacesync_vtproto.pb.go +++ b/commonspace/spacesyncproto/spacesync_vtproto.pb.go @@ -693,6 +693,11 @@ func (m *SpaceHeader) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i-- dAtA[i] = 0xa0 } + if m.FileprotoVersion != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.FileprotoVersion)) + i-- + dAtA[i] = 0x48 + } if len(m.SettingPayload) > 0 { i -= len(m.SettingPayload) copy(dAtA[i:], m.SettingPayload) @@ -1997,6 +2002,9 @@ func (m *SpaceHeader) SizeVT() (n int) { if l > 0 { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } + if m.FileprotoVersion != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.FileprotoVersion)) + } if m.Version != 0 { n += 2 + protohelpers.SizeOfVarint(uint64(m.Version)) } @@ -4219,6 +4227,25 @@ func (m *SpaceHeader) UnmarshalVT(dAtA []byte) error { m.SettingPayload = []byte{} } iNdEx = postIndex + case 9: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FileprotoVersion", wireType) + } + m.FileprotoVersion = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.FileprotoVersion |= SpaceFileProtoVersion(b&0x7F) << shift + if b < 0x80 { + break + } + } case 100: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType) From 0a7a691912ad94640b3916fce404fcd2631c11fd Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Wed, 1 Jul 2026 13:34:54 +0200 Subject: [PATCH 03/20] feat(nodeconf): add NodeTypeFileV2 + RF=2 chash container (SYN-16) Add the NodeTypeFileV2 node type ("fileV2") and a second, independent chash container built from the v2 file-node pool at RF=2 (FileV2ReplicationFactor). New NodeConf methods: - FileV2Peers() lists the v2 filenodes (mirrors FilePeers). - FileV2NodeIds(spaceId) returns the RF=2 responsible v2 filenodes for a space. Unlike NodeIds, the current account is NOT excluded, so the full responsible pair is visible (e.g. to derive the leader). The tree (sync) and v1-file containers are untouched. Leader selection and per-space responsibility consumers are out of scope. Propagated to service, testconf StubConf and the regenerated mock. Co-Authored-By: Claude Opus 4.8 (1M context) --- nodeconf/config.go | 1 + nodeconf/mock_nodeconf/mock_nodeconf.go | 28 ++++++ nodeconf/nodeconf.go | 41 +++++++- nodeconf/nodeconf_test.go | 121 ++++++++++++++++++++++++ nodeconf/service.go | 14 +++ nodeconf/testconf/nodeconf.go | 8 ++ 6 files changed, 212 insertions(+), 1 deletion(-) diff --git a/nodeconf/config.go b/nodeconf/config.go index 3caea78e5..bcf9df455 100644 --- a/nodeconf/config.go +++ b/nodeconf/config.go @@ -23,6 +23,7 @@ const ( NodeTypeTree NodeType = "tree" NodeTypeConsensus NodeType = "consensus" NodeTypeFile NodeType = "file" + NodeTypeFileV2 NodeType = "fileV2" NodeTypeCoordinator NodeType = "coordinator" NodeTypeNamingNode NodeType = "namingNode" diff --git a/nodeconf/mock_nodeconf/mock_nodeconf.go b/nodeconf/mock_nodeconf/mock_nodeconf.go index 459f654d0..2d4a81d2a 100644 --- a/nodeconf/mock_nodeconf/mock_nodeconf.go +++ b/nodeconf/mock_nodeconf/mock_nodeconf.go @@ -127,6 +127,34 @@ func (mr *MockServiceMockRecorder) FilePeers() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FilePeers", reflect.TypeOf((*MockService)(nil).FilePeers)) } +// FileV2NodeIds mocks base method. +func (m *MockService) FileV2NodeIds(spaceId string) []string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FileV2NodeIds", spaceId) + ret0, _ := ret[0].([]string) + return ret0 +} + +// FileV2NodeIds indicates an expected call of FileV2NodeIds. +func (mr *MockServiceMockRecorder) FileV2NodeIds(spaceId any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FileV2NodeIds", reflect.TypeOf((*MockService)(nil).FileV2NodeIds), spaceId) +} + +// FileV2Peers mocks base method. +func (m *MockService) FileV2Peers() []string { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FileV2Peers") + ret0, _ := ret[0].([]string) + return ret0 +} + +// FileV2Peers indicates an expected call of FileV2Peers. +func (mr *MockServiceMockRecorder) FileV2Peers() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FileV2Peers", reflect.TypeOf((*MockService)(nil).FileV2Peers)) +} + // Id mocks base method. func (m *MockService) Id() string { m.ctrl.T.Helper() diff --git a/nodeconf/nodeconf.go b/nodeconf/nodeconf.go index dcef87cfc..4f1334c44 100644 --- a/nodeconf/nodeconf.go +++ b/nodeconf/nodeconf.go @@ -17,6 +17,12 @@ type NodeConf interface { IsResponsible(spaceId string) bool // FilePeers returns list of filenodes FilePeers() []string + // FileV2Peers returns the list of v2 filenodes (the NodeTypeFileV2 pool) + FileV2Peers() []string + // FileV2NodeIds returns the RF=2 responsible v2 filenodes for a given spaceId. + // Unlike NodeIds, the current account is NOT excluded: the full responsible pair is + // returned so a v2 filenode can see its partner (e.g. to derive the leader). + FileV2NodeIds(spaceId string) []string // ConsensusPeers returns list of consensusnodes ConsensusPeers() []string // CoordinatorPeers returns list of coordinator nodes @@ -40,11 +46,13 @@ type nodeConf struct { id string accountId string filePeers []string + fileV2Peers []string consensusPeers []string coordinatorPeers []string namingNodePeers []string paymentProcessingNodePeers []string chash chash.CHash + chashFileV2 chash.CHash allMembers []Node c Configuration addrs map[string][]string @@ -82,6 +90,19 @@ func (c *nodeConf) FilePeers() []string { return c.filePeers } +func (c *nodeConf) FileV2Peers() []string { + return c.fileV2Peers +} + +func (c *nodeConf) FileV2NodeIds(spaceId string) []string { + members := c.chashFileV2.GetMembers(ReplKey(spaceId)) + res := make([]string, 0, len(members)) + for _, m := range members { + res = append(res, m.Id()) + } + return res +} + func (c *nodeConf) ConsensusPeers() []string { return c.consensusPeers } @@ -146,8 +167,15 @@ func сonfigurationToNodeConf(c Configuration) (nc *nodeConf, err error) { }); err != nil { return } + if nc.chashFileV2, err = chash.New(chash.Config{ + PartitionCount: PartitionCount, + ReplicationFactor: FileV2ReplicationFactor, + }); err != nil { + return + } members := make([]chash.Member, 0, len(c.Nodes)) + fileV2Members := make([]chash.Member, 0, len(c.Nodes)) for _, n := range c.Nodes { if n.HasType(NodeTypeTree) { members = append(members, n) @@ -158,6 +186,10 @@ func сonfigurationToNodeConf(c Configuration) (nc *nodeConf, err error) { if n.HasType(NodeTypeFile) { nc.filePeers = append(nc.filePeers, n.PeerId) } + if n.HasType(NodeTypeFileV2) { + nc.fileV2Peers = append(nc.fileV2Peers, n.PeerId) + fileV2Members = append(fileV2Members, n) + } if n.HasType(NodeTypeCoordinator) { nc.coordinatorPeers = append(nc.coordinatorPeers, n.PeerId) } @@ -171,6 +203,13 @@ func сonfigurationToNodeConf(c Configuration) (nc *nodeConf, err error) { nc.allMembers = append(nc.allMembers, n) nc.addrs[n.PeerId] = n.Addresses } - err = nc.chash.AddMembers(members...) + if err = nc.chash.AddMembers(members...); err != nil { + return + } + if len(fileV2Members) > 0 { + if err = nc.chashFileV2.AddMembers(fileV2Members...); err != nil { + return + } + } return } diff --git a/nodeconf/nodeconf_test.go b/nodeconf/nodeconf_test.go index 9298e1006..b22536fa8 100644 --- a/nodeconf/nodeconf_test.go +++ b/nodeconf/nodeconf_test.go @@ -165,6 +165,127 @@ creationTime: 2023-07-21T11:51:12.970882+01:00 assert.Equal(t, []string{"12D3KooXXXEXV3KjBxEU5EnsPfneLx84vMWAtTBQBeyooN82KSuS"}, nodeConf.PaymentProcessingNodePeers()) } +func TestConfiguration_FileV2NodeIds(t *testing.T) { + chFileV2, err := chash.New(chash.Config{ + PartitionCount: PartitionCount, + ReplicationFactor: FileV2ReplicationFactor, + }) + require.NoError(t, err) + conf := &nodeConf{ + id: "last", + accountId: "1", + chashFileV2: chFileV2, + } + for i := 0; i < 5; i++ { + require.NoError(t, conf.chashFileV2.AddMembers(testMember(fmt.Sprint(i+1)))) + } + + t.Run("returns exactly RF=2 responsible nodes", func(t *testing.T) { + for i := 0; i < 20; i++ { + spaceId := fmt.Sprintf("%d.%d", rand.Int(), rand.Int()) + assert.Len(t, conf.FileV2NodeIds(spaceId), FileV2ReplicationFactor) + } + }) + + t.Run("stable for same repl key", func(t *testing.T) { + var prev []string + for i := 0; i < 10; i++ { + spaceId := fmt.Sprintf("%d.%d", rand.Int(), 42) + members := conf.FileV2NodeIds(spaceId) + assert.Len(t, members, FileV2ReplicationFactor) + if i != 0 { + assert.Equal(t, prev, members) + } + prev = members + } + }) + + t.Run("does not exclude self (full pair returned)", func(t *testing.T) { + // unlike NodeIds, the current account stays in the result: find a space for + // which account "1" is responsible and assert it is still present. + selfSeen := false + for i := 0; i < 2000 && !selfSeen; i++ { + members := conf.FileV2NodeIds(fmt.Sprintf("%d.%d", i, i)) + require.Len(t, members, FileV2ReplicationFactor) + for _, m := range members { + if m == conf.accountId { + selfSeen = true + } + } + } + assert.True(t, selfSeen, "self must be included when responsible") + }) +} + +func TestNewNodeConfFromYaml_FileV2(t *testing.T) { + yamlData := ` +id: net1 +networkId: net1 +nodes: + - peerId: file-v1 + addresses: [127.0.0.1:1] + types: [file] + - peerId: filev2-1 + addresses: [127.0.0.1:2] + types: [fileV2] + - peerId: filev2-2 + addresses: [127.0.0.1:3] + types: [fileV2] + - peerId: tree-and-filev2 + addresses: [127.0.0.1:4] + types: [tree, fileV2] + - peerId: tree-1 + addresses: [127.0.0.1:5] + types: [tree] +creationTime: 2023-07-21T11:51:12.970882+01:00 +` + var conf Configuration + require.NoError(t, yaml.Unmarshal([]byte(yamlData), &conf)) + nc, err := сonfigurationToNodeConf(conf) + require.NoError(t, err) + + // v2 pool = exactly the fileV2-typed nodes; the v1 file node is excluded + assert.Equal(t, []string{"filev2-1", "filev2-2", "tree-and-filev2"}, nc.FileV2Peers()) + // v1 file pool unaffected by the new type + assert.Equal(t, []string{"file-v1"}, nc.FilePeers()) + + // resolution returns RF=2 responsible nodes, all from the v2 pool + v2set := map[string]bool{"filev2-1": true, "filev2-2": true, "tree-and-filev2": true} + ids := nc.FileV2NodeIds("space.repl") + assert.Len(t, ids, FileV2ReplicationFactor) + for _, id := range ids { + assert.Truef(t, v2set[id], "resolved node %s must be a v2 file node", id) + } + + // the tree/sync container is unaffected: it resolves only from tree-typed nodes + treeSet := map[string]bool{"tree-and-filev2": true, "tree-1": true} + for _, id := range nc.NodeIds("space.repl") { + assert.Truef(t, treeSet[id], "sync node %s must be a tree node", id) + } +} + +func TestNewNodeConfFromYaml_NoFileV2(t *testing.T) { + // a config without any fileV2 nodes must still build and resolve to an empty pair + yamlData := ` +id: net1 +networkId: net1 +nodes: + - peerId: tree-1 + addresses: [127.0.0.1:1] + types: [tree] + - peerId: file-v1 + addresses: [127.0.0.1:2] + types: [file] +creationTime: 2023-07-21T11:51:12.970882+01:00 +` + var conf Configuration + require.NoError(t, yaml.Unmarshal([]byte(yamlData), &conf)) + nc, err := сonfigurationToNodeConf(conf) + require.NoError(t, err) + assert.Empty(t, nc.FileV2Peers()) + assert.Empty(t, nc.FileV2NodeIds("space.repl")) +} + type testMember string func (t testMember) Id() string { diff --git a/nodeconf/service.go b/nodeconf/service.go index 418ef395a..9f40660d5 100644 --- a/nodeconf/service.go +++ b/nodeconf/service.go @@ -22,6 +22,8 @@ const CName = "common.nodeconf" const ( PartitionCount = 3000 ReplicationFactor = 3 + // FileV2ReplicationFactor is the replication factor of the v2 file node pool (07c: RF=2) + FileV2ReplicationFactor = 2 ) var log = logger.NewNamed(CName) @@ -290,6 +292,18 @@ func (s *service) FilePeers() []string { return s.last.FilePeers() } +func (s *service) FileV2Peers() []string { + s.mu.RLock() + defer s.mu.RUnlock() + return s.last.FileV2Peers() +} + +func (s *service) FileV2NodeIds(spaceId string) []string { + s.mu.RLock() + defer s.mu.RUnlock() + return s.last.FileV2NodeIds(spaceId) +} + func (s *service) ConsensusPeers() []string { s.mu.RLock() defer s.mu.RUnlock() diff --git a/nodeconf/testconf/nodeconf.go b/nodeconf/testconf/nodeconf.go index e14663dbb..d436b8819 100644 --- a/nodeconf/testconf/nodeconf.go +++ b/nodeconf/testconf/nodeconf.go @@ -76,6 +76,14 @@ func (m *StubConf) FilePeers() []string { return nil } +func (m *StubConf) FileV2Peers() []string { + return nil +} + +func (m *StubConf) FileV2NodeIds(spaceId string) []string { + return nil +} + func (m *StubConf) ConsensusPeers() []string { return nil } From 748dc29e01e2ff0f520c85ca28597d3715df320f Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Wed, 1 Jul 2026 13:42:53 +0200 Subject: [PATCH 04/20] feat(nodeconf): plumb NodeTypeFileV2 through the coordinator config path (SYN-16) Add FileV2API=6 to the coordinator NodeType proto enum and map it to nodeconf.NodeTypeFileV2 in nodeconfsource. Without this the v2 file type only survived local YAML configs and was dropped on the coordinator-distributed path. The enum->string switch keeps its no-default behavior: an older client that does not know FileV2API (or any future type) silently drops the unknown value while keeping the node's known types, so it stays correctly routed in the tree/file pools and never crashes. New enum values are therefore backward-compatible. Note: the coordinator server (any-sync-coordinator) still needs the reverse string->enum mapping to actually serve fileV2 nodes. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../coordinatorproto/coordinator.pb.go | 9 ++- .../coordinatorproto/protos/coordinator.proto | 2 + coordinator/nodeconfsource/nodeconfsource.go | 2 + .../nodeconfsource/nodeconfsource_test.go | 68 +++++++++++++++++++ 4 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 coordinator/nodeconfsource/nodeconfsource_test.go diff --git a/coordinator/coordinatorproto/coordinator.pb.go b/coordinator/coordinatorproto/coordinator.pb.go index c26d48178..02575d311 100644 --- a/coordinator/coordinatorproto/coordinator.pb.go +++ b/coordinator/coordinatorproto/coordinator.pb.go @@ -222,6 +222,8 @@ const ( // PaymentProcessingAPI supports payment processing api // (see any-pp-node repository) NodeType_PaymentProcessingAPI NodeType = 5 + // FileV2API supports the v2 file api (filenode v2 broker pool) + NodeType_FileV2API NodeType = 6 ) // Enum value maps for NodeType. @@ -233,6 +235,7 @@ var ( 3: "ConsensusAPI", 4: "NamingNodeAPI", 5: "PaymentProcessingAPI", + 6: "FileV2API", } NodeType_value = map[string]int32{ "TreeAPI": 0, @@ -241,6 +244,7 @@ var ( "ConsensusAPI": 3, "NamingNodeAPI": 4, "PaymentProcessingAPI": 5, + "FileV2API": 6, } ) @@ -3677,14 +3681,15 @@ const file_coordinator_coordinatorproto_protos_coordinator_proto_rawDesc = "" + "\x14SpaceStatusNotExists\x10\x04*J\n" + "\x10SpacePermissions\x12\x1b\n" + "\x17SpacePermissionsUnknown\x10\x00\x12\x19\n" + - "\x15SpacePermissionsOwner\x10\x01*w\n" + + "\x15SpacePermissionsOwner\x10\x01*\x86\x01\n" + "\bNodeType\x12\v\n" + "\aTreeAPI\x10\x00\x12\v\n" + "\aFileAPI\x10\x01\x12\x12\n" + "\x0eCoordinatorAPI\x10\x02\x12\x10\n" + "\fConsensusAPI\x10\x03\x12\x11\n" + "\rNamingNodeAPI\x10\x04\x12\x18\n" + - "\x14PaymentProcessingAPI\x10\x05*9\n" + + "\x14PaymentProcessingAPI\x10\x05\x12\r\n" + + "\tFileV2API\x10\x06*9\n" + "\x13DeletionPayloadType\x12\b\n" + "\x04Tree\x10\x00\x12\v\n" + "\aConfirm\x10\x01\x12\v\n" + diff --git a/coordinator/coordinatorproto/protos/coordinator.proto b/coordinator/coordinatorproto/protos/coordinator.proto index 38d720d16..2ddb801d5 100644 --- a/coordinator/coordinatorproto/protos/coordinator.proto +++ b/coordinator/coordinatorproto/protos/coordinator.proto @@ -228,6 +228,8 @@ enum NodeType { // PaymentProcessingAPI supports payment processing api // (see any-pp-node repository) PaymentProcessingAPI = 5; + // FileV2API supports the v2 file api (filenode v2 broker pool) + FileV2API = 6; } // DeletionChangeType determines the type of deletion payload diff --git a/coordinator/nodeconfsource/nodeconfsource.go b/coordinator/nodeconfsource/nodeconfsource.go index b2130b4f3..536c4b8d9 100644 --- a/coordinator/nodeconfsource/nodeconfsource.go +++ b/coordinator/nodeconfsource/nodeconfsource.go @@ -48,6 +48,8 @@ func (n *nodeConfSource) GetLast(ctx context.Context, currentId string) (c nodec switch nt { case coordinatorproto.NodeType_FileAPI: types = append(types, nodeconf.NodeTypeFile) + case coordinatorproto.NodeType_FileV2API: + types = append(types, nodeconf.NodeTypeFileV2) case coordinatorproto.NodeType_CoordinatorAPI: types = append(types, nodeconf.NodeTypeCoordinator) case coordinatorproto.NodeType_TreeAPI: diff --git a/coordinator/nodeconfsource/nodeconfsource_test.go b/coordinator/nodeconfsource/nodeconfsource_test.go new file mode 100644 index 000000000..7553ce980 --- /dev/null +++ b/coordinator/nodeconfsource/nodeconfsource_test.go @@ -0,0 +1,68 @@ +package nodeconfsource + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + + "github.com/anyproto/any-sync/coordinator/coordinatorclient/mock_coordinatorclient" + "github.com/anyproto/any-sync/coordinator/coordinatorproto" + "github.com/anyproto/any-sync/nodeconf" +) + +func newTestSource(t *testing.T, resp *coordinatorproto.NetworkConfigurationResponse) *nodeConfSource { + ctrl := gomock.NewController(t) + t.Cleanup(ctrl.Finish) + cl := mock_coordinatorclient.NewMockCoordinatorClient(ctrl) + cl.EXPECT().NetworkConfiguration(gomock.Any(), gomock.Any()).Return(resp, nil) + return &nodeConfSource{cl: cl} +} + +func TestNodeConfSource_GetLast_FileV2(t *testing.T) { + t.Run("maps FileV2API to NodeTypeFileV2", func(t *testing.T) { + src := newTestSource(t, &coordinatorproto.NetworkConfigurationResponse{ + ConfigurationId: "new", + NetworkId: "net", + Nodes: []*coordinatorproto.Node{ + {PeerId: "n1", Types: []coordinatorproto.NodeType{coordinatorproto.NodeType_FileV2API}}, + {PeerId: "n2", Types: []coordinatorproto.NodeType{coordinatorproto.NodeType_TreeAPI, coordinatorproto.NodeType_FileV2API}}, + }, + }) + conf, err := src.GetLast(context.Background(), "old") + require.NoError(t, err) + require.Len(t, conf.Nodes, 2) + assert.Equal(t, []nodeconf.NodeType{nodeconf.NodeTypeFileV2}, conf.Nodes[0].Types) + assert.Equal(t, []nodeconf.NodeType{nodeconf.NodeTypeTree, nodeconf.NodeTypeFileV2}, conf.Nodes[1].Types) + }) + + // Simulates an OLD client receiving a node type its proto/switch does not know: + // the unknown enum value must be silently dropped while known types are kept, + // so the node stays in the sync pool and routing is unchanged. + t.Run("unknown enum type is dropped, known types preserved", func(t *testing.T) { + const unknownType = coordinatorproto.NodeType(9999) + src := newTestSource(t, &coordinatorproto.NetworkConfigurationResponse{ + ConfigurationId: "new", + NetworkId: "net", + Nodes: []*coordinatorproto.Node{ + {PeerId: "n1", Types: []coordinatorproto.NodeType{coordinatorproto.NodeType_TreeAPI, unknownType}}, + {PeerId: "n2", Types: []coordinatorproto.NodeType{unknownType}}, + }, + }) + conf, err := src.GetLast(context.Background(), "old") + require.NoError(t, err) + require.Len(t, conf.Nodes, 2) + // known type kept + assert.Equal(t, []nodeconf.NodeType{nodeconf.NodeTypeTree}, conf.Nodes[0].Types) + // unknown-only node ends up with no recognized role (empty), i.e. inert — no panic, no misroute + assert.Empty(t, conf.Nodes[1].Types) + }) + + t.Run("unchanged configuration returns ErrConfigurationNotChanged", func(t *testing.T) { + src := newTestSource(t, &coordinatorproto.NetworkConfigurationResponse{ConfigurationId: "same"}) + _, err := src.GetLast(context.Background(), "same") + assert.ErrorIs(t, err, nodeconf.ErrConfigurationNotChanged) + }) +} From 2debdd41e48d43674889c6a33a17b7cb342578e4 Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Wed, 1 Jul 2026 14:59:23 +0200 Subject: [PATCH 05/20] feat(fileproto): add v2 broker protocol (SYN-17) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Define the filenode v2 (FileV2) broker dRPC contract: a byteless broker that authorizes uploads, signs durable-custody receipts, and issues signed download URLs — never transferring bytes itself. All four RPCs are batch (Upload / RequestSign / RequestDownload / SpaceInfo): each request carries a shared spaceId + repeated items, and each response is a repeated per-item result keyed by the echoed rootCid/spaceId with a flat status enum, so one bad item never fails the whole batch. A two-layer error model keeps whole-RPC failures (auth/malformed/batch-too-large) on the drpc error while per-item business outcomes travel in-body. New proto package filesyncv2 / service FileV2 (route /filesyncv2.FileV2/*), distinct from v1 filesync/File, plus the fileprotov2err registry at error offset 800 (see docs/rpc-error-offsets.md). Co-Authored-By: Claude Opus 4.8 --- Makefile | 1 + .../fileprotov2err/fileprotov2err.go | 37 + commonfile/fileproto/fileprotov2/filev2.pb.go | 1310 +++++++ .../fileproto/fileprotov2/filev2_drpc.pb.go | 226 ++ .../fileprotov2/filev2_vtproto.pb.go | 3330 +++++++++++++++++ .../fileproto/fileprotov2/protos/filev2.proto | 188 + 6 files changed, 5092 insertions(+) create mode 100644 commonfile/fileproto/fileprotov2/fileprotov2err/fileprotov2err.go create mode 100644 commonfile/fileproto/fileprotov2/filev2.pb.go create mode 100644 commonfile/fileproto/fileprotov2/filev2_drpc.pb.go create mode 100644 commonfile/fileproto/fileprotov2/filev2_vtproto.pb.go create mode 100644 commonfile/fileproto/fileprotov2/protos/filev2.proto diff --git a/Makefile b/Makefile index 5fdf6cfa3..91b23fa6b 100644 --- a/Makefile +++ b/Makefile @@ -47,6 +47,7 @@ proto: $(call generate_drpc,$(PKGMAP),commonspace/spacesyncproto/protos) $(call generate_drpc,$(PKGMAP),commonspace/clientspaceproto/protos) $(call generate_drpc,$(PKGMAP),commonfile/fileproto/protos) + $(call generate_drpc,,commonfile/fileproto/fileprotov2/protos) $(call generate_drpc,$(PKGMAP),net/streampool/testservice/protos) $(call generate_drpc,$(PKGMAP),net/endtoendtest/testpeer/testproto/protos) diff --git a/commonfile/fileproto/fileprotov2/fileprotov2err/fileprotov2err.go b/commonfile/fileproto/fileprotov2/fileprotov2err/fileprotov2err.go new file mode 100644 index 000000000..a9d19617c --- /dev/null +++ b/commonfile/fileproto/fileprotov2/fileprotov2err/fileprotov2err.go @@ -0,0 +1,37 @@ +package fileprotov2err + +import ( + "fmt" + + "github.com/anyproto/any-sync/commonfile/fileproto/fileprotov2" + "github.com/anyproto/any-sync/net/rpc/rpcerr" +) + +// errGroup registers the v2 WHOLE-RPC (drpc-level) errors in the offset-800 +// block, distinct from v1 filesync's 200 block. Only the cross-cutting +// ErrCodes values are registered here; per-item ErrCode outcomes travel in the +// response body and are NOT registered with rpcerr. Mirrors the v1 +// commonfile/fileproto/fileprotoerr package. +var ( + errGroup = rpcerr.ErrGroup(fileprotov2.ErrCodes_ErrorOffset) + ErrUnexpected = errGroup.Register(fmt.Errorf("unexpected filenode-v2 error"), uint64(fileprotov2.ErrCodes_Unexpected)) + ErrForbidden = errGroup.Register(fmt.Errorf("forbidden"), uint64(fileprotov2.ErrCodes_Forbidden)) + ErrInvalidRequest = errGroup.Register(fmt.Errorf("invalid request"), uint64(fileprotov2.ErrCodes_InvalidRequest)) + ErrQuerySizeExceeded = errGroup.Register(fmt.Errorf("query size exceeded"), uint64(fileprotov2.ErrCodes_QuerySizeExceeded)) +) + +// FromCode maps a whole-RPC ErrCodes value to its registered drpc error so +// handlers can uniformly `return fileprotov2err.FromCode(code)` at the RPC +// boundary. Unknown/zero codes fall back to ErrUnexpected. +func FromCode(code fileprotov2.ErrCodes) error { + switch code { + case fileprotov2.ErrCodes_Forbidden: + return ErrForbidden + case fileprotov2.ErrCodes_InvalidRequest: + return ErrInvalidRequest + case fileprotov2.ErrCodes_QuerySizeExceeded: + return ErrQuerySizeExceeded + default: + return ErrUnexpected + } +} diff --git a/commonfile/fileproto/fileprotov2/filev2.pb.go b/commonfile/fileproto/fileprotov2/filev2.pb.go new file mode 100644 index 000000000..6066437cf --- /dev/null +++ b/commonfile/fileproto/fileprotov2/filev2.pb.go @@ -0,0 +1,1310 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v3.21.12 +// source: commonfile/fileproto/fileprotov2/protos/filev2.proto + +package fileprotov2 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// ErrCodes are WHOLE-RPC (drpc-level) errors, mirroring v1 filesync.ErrCodes. +// Wired through rpcerr.ErrGroup(ErrCodes_ErrorOffset) in the fileprotov2err +// package. Returned as a NON-nil drpc error ONLY for auth / malformed / +// batch-too-large. Never used to report the outcome of an individual item +// (that is the per-item ErrCode below). Zero value Unexpected=0 registers at +// offset+0 (=800), so an uncoded server fault reads as Unexpected — exactly as +// v1 does at 200. Offset 800 is the only free rpcerr block (v1=200, 100/300/ +// 400/500/600/700 are taken). +type ErrCodes int32 + +const ( + ErrCodes_Unexpected ErrCodes = 0 // unclassified server fault (registry fallback) + ErrCodes_Forbidden ErrCodes = 1 // connection identity is not permitted at all + ErrCodes_InvalidRequest ErrCodes = 2 // malformed: empty spaceId, empty items, bad rootCid bytes + ErrCodes_QuerySizeExceeded ErrCodes = 3 // batch too large (too many items in one request) + ErrCodes_ErrorOffset ErrCodes = 800 // rpcerr.ErrGroup base for v2 (constant, never an error) +) + +// Enum value maps for ErrCodes. +var ( + ErrCodes_name = map[int32]string{ + 0: "Unexpected", + 1: "Forbidden", + 2: "InvalidRequest", + 3: "QuerySizeExceeded", + 800: "ErrorOffset", + } + ErrCodes_value = map[string]int32{ + "Unexpected": 0, + "Forbidden": 1, + "InvalidRequest": 2, + "QuerySizeExceeded": 3, + "ErrorOffset": 800, + } +) + +func (x ErrCodes) Enum() *ErrCodes { + p := new(ErrCodes) + *p = x + return p +} + +func (x ErrCodes) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ErrCodes) Descriptor() protoreflect.EnumDescriptor { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_enumTypes[0].Descriptor() +} + +func (ErrCodes) Type() protoreflect.EnumType { + return &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_enumTypes[0] +} + +func (x ErrCodes) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ErrCodes.Descriptor instead. +func (ErrCodes) EnumDescriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{0} +} + +// ErrCode is the PER-ITEM, in-body business outcome (err == nil at the RPC +// layer). Flat status enum with an explicit success sentinel Ok=0. The typed +// payload on a *Result is valid ONLY when code == Ok. Deliberately a SEPARATE +// enum from ErrCodes: its zero value means success (not error), it never flows +// through rpcerr, and its value short-names are kept disjoint from ErrCodes so +// proto3's package-level enum-value scoping stays clean. Servers MUST set +// `code` explicitly on every result (Ok is the proto3 default, so an unset +// code would be misread as a success with an empty payload). +type ErrCode int32 + +const ( + ErrCode_Ok ErrCode = 0 // item succeeded; the typed payload is populated + ErrCode_ErrUnexpected ErrCode = 1 // internal error handling this single item + ErrCode_ErrCidNotFound ErrCode = 2 // sign/download: no such stored file for this rootCid + ErrCode_ErrForbidden ErrCode = 3 // identity may not access this item's space/file + ErrCode_ErrLimitExceeded ErrCode = 4 // upload rejected: space/account storage limit reached + ErrCode_ErrSpaceNotFound ErrCode = 5 // quota-info: unknown/deleted space id +) + +// Enum value maps for ErrCode. +var ( + ErrCode_name = map[int32]string{ + 0: "Ok", + 1: "ErrUnexpected", + 2: "ErrCidNotFound", + 3: "ErrForbidden", + 4: "ErrLimitExceeded", + 5: "ErrSpaceNotFound", + } + ErrCode_value = map[string]int32{ + "Ok": 0, + "ErrUnexpected": 1, + "ErrCidNotFound": 2, + "ErrForbidden": 3, + "ErrLimitExceeded": 4, + "ErrSpaceNotFound": 5, + } +) + +func (x ErrCode) Enum() *ErrCode { + p := new(ErrCode) + *p = x + return p +} + +func (x ErrCode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ErrCode) Descriptor() protoreflect.EnumDescriptor { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_enumTypes[1].Descriptor() +} + +func (ErrCode) Type() protoreflect.EnumType { + return &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_enumTypes[1] +} + +func (x ErrCode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ErrCode.Descriptor instead. +func (ErrCode) EnumDescriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{1} +} + +type UploadRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + SpaceId string `protobuf:"bytes,1,opt,name=spaceId,proto3" json:"spaceId,omitempty"` // shared space for every item (auth scope) + Items []*UploadRequestItem `protobuf:"bytes,2,rep,name=items,proto3" json:"items,omitempty"` // files to obtain an upload target for + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *UploadRequest) Reset() { + *x = UploadRequest{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *UploadRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UploadRequest) ProtoMessage() {} + +func (x *UploadRequest) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UploadRequest.ProtoReflect.Descriptor instead. +func (*UploadRequest) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{0} +} + +func (x *UploadRequest) GetSpaceId() string { + if x != nil { + return x.SpaceId + } + return "" +} + +func (x *UploadRequest) GetItems() []*UploadRequestItem { + if x != nil { + return x.Items + } + return nil +} + +type UploadRequestItem struct { + state protoimpl.MessageState `protogen:"open.v1"` + RootCid []byte `protobuf:"bytes,1,opt,name=rootCid,proto3" json:"rootCid,omitempty"` // CID of the file root (result match key) + Size uint64 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"` // declared total byte size of the file + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *UploadRequestItem) Reset() { + *x = UploadRequestItem{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *UploadRequestItem) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UploadRequestItem) ProtoMessage() {} + +func (x *UploadRequestItem) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UploadRequestItem.ProtoReflect.Descriptor instead. +func (*UploadRequestItem) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{1} +} + +func (x *UploadRequestItem) GetRootCid() []byte { + if x != nil { + return x.RootCid + } + return nil +} + +func (x *UploadRequestItem) GetSize() uint64 { + if x != nil { + return x.Size + } + return 0 +} + +type UploadResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Results []*UploadResult `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` // one per requested item, keyed by rootCid + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *UploadResponse) Reset() { + *x = UploadResponse{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *UploadResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UploadResponse) ProtoMessage() {} + +func (x *UploadResponse) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UploadResponse.ProtoReflect.Descriptor instead. +func (*UploadResponse) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{2} +} + +func (x *UploadResponse) GetResults() []*UploadResult { + if x != nil { + return x.Results + } + return nil +} + +type UploadResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + RootCid []byte `protobuf:"bytes,1,opt,name=rootCid,proto3" json:"rootCid,omitempty"` // echoes UploadRequestItem.rootCid (match key) + Code ErrCode `protobuf:"varint,2,opt,name=code,proto3,enum=filesyncv2.ErrCode" json:"code,omitempty"` // per-item outcome; `upload` valid only when Ok + Upload *PresignedUpload `protobuf:"bytes,3,opt,name=upload,proto3" json:"upload,omitempty"` // set ONLY when code == Ok + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *UploadResult) Reset() { + *x = UploadResult{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *UploadResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*UploadResult) ProtoMessage() {} + +func (x *UploadResult) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use UploadResult.ProtoReflect.Descriptor instead. +func (*UploadResult) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{3} +} + +func (x *UploadResult) GetRootCid() []byte { + if x != nil { + return x.RootCid + } + return nil +} + +func (x *UploadResult) GetCode() ErrCode { + if x != nil { + return x.Code + } + return ErrCode_Ok +} + +func (x *UploadResult) GetUpload() *PresignedUpload { + if x != nil { + return x.Upload + } + return nil +} + +// PresignedUpload is an OPAQUE presigned upload target. No S3 vendor structs +// leak into the shared proto: the client POSTs `fields` + the object body to +// `url` verbatim. Modeled as a presigned POST form (fields are ordered +// key/value pairs rather than a map) so the signing order is deterministic and +// reproducible. +type PresignedUpload struct { + state protoimpl.MessageState `protogen:"open.v1"` + Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` // presigned POST endpoint + Fields []*PresignField `protobuf:"bytes,2,rep,name=fields,proto3" json:"fields,omitempty"` // multipart form fields to send verbatim, in order + MaxContentLength uint64 `protobuf:"varint,3,opt,name=maxContentLength,proto3" json:"maxContentLength,omitempty"` // hard upper bound the presign enforces (bytes) + Expiry int64 `protobuf:"varint,4,opt,name=expiry,proto3" json:"expiry,omitempty"` // unix seconds; POST must complete before this + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PresignedUpload) Reset() { + *x = PresignedUpload{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PresignedUpload) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PresignedUpload) ProtoMessage() {} + +func (x *PresignedUpload) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PresignedUpload.ProtoReflect.Descriptor instead. +func (*PresignedUpload) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{4} +} + +func (x *PresignedUpload) GetUrl() string { + if x != nil { + return x.Url + } + return "" +} + +func (x *PresignedUpload) GetFields() []*PresignField { + if x != nil { + return x.Fields + } + return nil +} + +func (x *PresignedUpload) GetMaxContentLength() uint64 { + if x != nil { + return x.MaxContentLength + } + return 0 +} + +func (x *PresignedUpload) GetExpiry() int64 { + if x != nil { + return x.Expiry + } + return 0 +} + +type PresignField struct { + state protoimpl.MessageState `protogen:"open.v1"` + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PresignField) Reset() { + *x = PresignField{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PresignField) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PresignField) ProtoMessage() {} + +func (x *PresignField) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PresignField.ProtoReflect.Descriptor instead. +func (*PresignField) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{5} +} + +func (x *PresignField) GetKey() string { + if x != nil { + return x.Key + } + return "" +} + +func (x *PresignField) GetValue() string { + if x != nil { + return x.Value + } + return "" +} + +type RequestSignRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + SpaceId string `protobuf:"bytes,1,opt,name=spaceId,proto3" json:"spaceId,omitempty"` // shared space for every rootCid (auth scope) + RootCids [][]byte `protobuf:"bytes,2,rep,name=rootCids,proto3" json:"rootCids,omitempty"` // files to obtain a durable-custody receipt for + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RequestSignRequest) Reset() { + *x = RequestSignRequest{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RequestSignRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RequestSignRequest) ProtoMessage() {} + +func (x *RequestSignRequest) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RequestSignRequest.ProtoReflect.Descriptor instead. +func (*RequestSignRequest) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{6} +} + +func (x *RequestSignRequest) GetSpaceId() string { + if x != nil { + return x.SpaceId + } + return "" +} + +func (x *RequestSignRequest) GetRootCids() [][]byte { + if x != nil { + return x.RootCids + } + return nil +} + +type RequestSignResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Results []*RequestSignResult `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` // one per rootCid, keyed by rootCid + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RequestSignResponse) Reset() { + *x = RequestSignResponse{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RequestSignResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RequestSignResponse) ProtoMessage() {} + +func (x *RequestSignResponse) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RequestSignResponse.ProtoReflect.Descriptor instead. +func (*RequestSignResponse) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{7} +} + +func (x *RequestSignResponse) GetResults() []*RequestSignResult { + if x != nil { + return x.Results + } + return nil +} + +type RequestSignResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + RootCid []byte `protobuf:"bytes,1,opt,name=rootCid,proto3" json:"rootCid,omitempty"` // echoes the requested rootCid (match key) + Code ErrCode `protobuf:"varint,2,opt,name=code,proto3,enum=filesyncv2.ErrCode" json:"code,omitempty"` // per-item outcome; `receipt` valid only when Ok + Receipt *NetworkSignReceipt `protobuf:"bytes,3,opt,name=receipt,proto3" json:"receipt,omitempty"` // set ONLY when code == Ok + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RequestSignResult) Reset() { + *x = RequestSignResult{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RequestSignResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RequestSignResult) ProtoMessage() {} + +func (x *RequestSignResult) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RequestSignResult.ProtoReflect.Descriptor instead. +func (*RequestSignResult) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{8} +} + +func (x *RequestSignResult) GetRootCid() []byte { + if x != nil { + return x.RootCid + } + return nil +} + +func (x *RequestSignResult) GetCode() ErrCode { + if x != nil { + return x.Code + } + return ErrCode_Ok +} + +func (x *RequestSignResult) GetReceipt() *NetworkSignReceipt { + if x != nil { + return x.Receipt + } + return nil +} + +// NetworkSignReceipt mirrors coordinator SpaceReceiptWithSignature: an opaque +// protobuf-encoded payload plus the network-key signature over it. The payload +// schema is intentionally not defined here (opaque blob forwarded verbatim). +type NetworkSignReceipt struct { + state protoimpl.MessageState `protogen:"open.v1"` + ReceiptPayload []byte `protobuf:"bytes,1,opt,name=receiptPayload,proto3" json:"receiptPayload,omitempty"` // protobuf-encoded signed receipt payload (opaque) + Signature []byte `protobuf:"bytes,2,opt,name=signature,proto3" json:"signature,omitempty"` // network key signature over receiptPayload + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *NetworkSignReceipt) Reset() { + *x = NetworkSignReceipt{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *NetworkSignReceipt) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NetworkSignReceipt) ProtoMessage() {} + +func (x *NetworkSignReceipt) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NetworkSignReceipt.ProtoReflect.Descriptor instead. +func (*NetworkSignReceipt) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{9} +} + +func (x *NetworkSignReceipt) GetReceiptPayload() []byte { + if x != nil { + return x.ReceiptPayload + } + return nil +} + +func (x *NetworkSignReceipt) GetSignature() []byte { + if x != nil { + return x.Signature + } + return nil +} + +type RequestDownloadRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + SpaceId string `protobuf:"bytes,1,opt,name=spaceId,proto3" json:"spaceId,omitempty"` // shared space for every rootCid (auth scope) + RootCids [][]byte `protobuf:"bytes,2,rep,name=rootCids,proto3" json:"rootCids,omitempty"` // files to obtain a signed GET target for + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RequestDownloadRequest) Reset() { + *x = RequestDownloadRequest{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RequestDownloadRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RequestDownloadRequest) ProtoMessage() {} + +func (x *RequestDownloadRequest) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RequestDownloadRequest.ProtoReflect.Descriptor instead. +func (*RequestDownloadRequest) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{10} +} + +func (x *RequestDownloadRequest) GetSpaceId() string { + if x != nil { + return x.SpaceId + } + return "" +} + +func (x *RequestDownloadRequest) GetRootCids() [][]byte { + if x != nil { + return x.RootCids + } + return nil +} + +type RequestDownloadResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Results []*RequestDownloadResult `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` // one per rootCid, keyed by rootCid + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RequestDownloadResponse) Reset() { + *x = RequestDownloadResponse{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RequestDownloadResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RequestDownloadResponse) ProtoMessage() {} + +func (x *RequestDownloadResponse) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RequestDownloadResponse.ProtoReflect.Descriptor instead. +func (*RequestDownloadResponse) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{11} +} + +func (x *RequestDownloadResponse) GetResults() []*RequestDownloadResult { + if x != nil { + return x.Results + } + return nil +} + +type RequestDownloadResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + RootCid []byte `protobuf:"bytes,1,opt,name=rootCid,proto3" json:"rootCid,omitempty"` // echoes the requested rootCid (match key) + Code ErrCode `protobuf:"varint,2,opt,name=code,proto3,enum=filesyncv2.ErrCode" json:"code,omitempty"` // per-item outcome; `download` valid only when Ok + Download *SignedDownload `protobuf:"bytes,3,opt,name=download,proto3" json:"download,omitempty"` // set ONLY when code == Ok + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RequestDownloadResult) Reset() { + *x = RequestDownloadResult{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RequestDownloadResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RequestDownloadResult) ProtoMessage() {} + +func (x *RequestDownloadResult) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RequestDownloadResult.ProtoReflect.Descriptor instead. +func (*RequestDownloadResult) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{12} +} + +func (x *RequestDownloadResult) GetRootCid() []byte { + if x != nil { + return x.RootCid + } + return nil +} + +func (x *RequestDownloadResult) GetCode() ErrCode { + if x != nil { + return x.Code + } + return ErrCode_Ok +} + +func (x *RequestDownloadResult) GetDownload() *SignedDownload { + if x != nil { + return x.Download + } + return nil +} + +// SignedDownload is an OPAQUE signed GET target (kept as a message, symmetric +// with PresignedUpload, so download can grow response headers etc. without a +// wire break on the result envelope). +type SignedDownload struct { + state protoimpl.MessageState `protogen:"open.v1"` + Url string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` // opaque signed GET url + Expiry int64 `protobuf:"varint,2,opt,name=expiry,proto3" json:"expiry,omitempty"` // unix seconds the url is valid until + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SignedDownload) Reset() { + *x = SignedDownload{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SignedDownload) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SignedDownload) ProtoMessage() {} + +func (x *SignedDownload) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SignedDownload.ProtoReflect.Descriptor instead. +func (*SignedDownload) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{13} +} + +func (x *SignedDownload) GetUrl() string { + if x != nil { + return x.Url + } + return "" +} + +func (x *SignedDownload) GetExpiry() int64 { + if x != nil { + return x.Expiry + } + return 0 +} + +type SpaceInfoRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + SpaceIds []string `protobuf:"bytes,1,rep,name=spaceIds,proto3" json:"spaceIds,omitempty"` // batch of spaces to report on (result match key) + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SpaceInfoRequest) Reset() { + *x = SpaceInfoRequest{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SpaceInfoRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SpaceInfoRequest) ProtoMessage() {} + +func (x *SpaceInfoRequest) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[14] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SpaceInfoRequest.ProtoReflect.Descriptor instead. +func (*SpaceInfoRequest) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{14} +} + +func (x *SpaceInfoRequest) GetSpaceIds() []string { + if x != nil { + return x.SpaceIds + } + return nil +} + +type SpaceInfoResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Results []*SpaceInfoResult `protobuf:"bytes,1,rep,name=results,proto3" json:"results,omitempty"` // one per requested space, keyed by spaceId + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SpaceInfoResponse) Reset() { + *x = SpaceInfoResponse{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SpaceInfoResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SpaceInfoResponse) ProtoMessage() {} + +func (x *SpaceInfoResponse) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SpaceInfoResponse.ProtoReflect.Descriptor instead. +func (*SpaceInfoResponse) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{15} +} + +func (x *SpaceInfoResponse) GetResults() []*SpaceInfoResult { + if x != nil { + return x.Results + } + return nil +} + +type SpaceInfoResult struct { + state protoimpl.MessageState `protogen:"open.v1"` + SpaceId string `protobuf:"bytes,1,opt,name=spaceId,proto3" json:"spaceId,omitempty"` // echoes the requested spaceId (match key) + Code ErrCode `protobuf:"varint,2,opt,name=code,proto3,enum=filesyncv2.ErrCode" json:"code,omitempty"` // per-item outcome; `quota` valid only when Ok + Quota *SpaceQuota `protobuf:"bytes,3,opt,name=quota,proto3" json:"quota,omitempty"` // set ONLY when code == Ok + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SpaceInfoResult) Reset() { + *x = SpaceInfoResult{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SpaceInfoResult) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SpaceInfoResult) ProtoMessage() {} + +func (x *SpaceInfoResult) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SpaceInfoResult.ProtoReflect.Descriptor instead. +func (*SpaceInfoResult) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{16} +} + +func (x *SpaceInfoResult) GetSpaceId() string { + if x != nil { + return x.SpaceId + } + return "" +} + +func (x *SpaceInfoResult) GetCode() ErrCode { + if x != nil { + return x.Code + } + return ErrCode_Ok +} + +func (x *SpaceInfoResult) GetQuota() *SpaceQuota { + if x != nil { + return x.Quota + } + return nil +} + +type SpaceQuota struct { + state protoimpl.MessageState `protogen:"open.v1"` + LimitBytes uint64 `protobuf:"varint,1,opt,name=limitBytes,proto3" json:"limitBytes,omitempty"` // effective storage limit for this space + TotalUsageBytes uint64 `protobuf:"varint,2,opt,name=totalUsageBytes,proto3" json:"totalUsageBytes,omitempty"` // counted usage == durableUsageBytes + inflightUsageBytes + DurableUsageBytes uint64 `protobuf:"varint,3,opt,name=durableUsageBytes,proto3" json:"durableUsageBytes,omitempty"` // bytes fully committed to durable storage + InflightUsageBytes uint64 `protobuf:"varint,4,opt,name=inflightUsageBytes,proto3" json:"inflightUsageBytes,omitempty"` // bytes reserved by presigned/not-yet-committed uploads + FilesCount uint64 `protobuf:"varint,5,opt,name=filesCount,proto3" json:"filesCount,omitempty"` // number of file roots stored in the space + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SpaceQuota) Reset() { + *x = SpaceQuota{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SpaceQuota) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SpaceQuota) ProtoMessage() {} + +func (x *SpaceQuota) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[17] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SpaceQuota.ProtoReflect.Descriptor instead. +func (*SpaceQuota) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{17} +} + +func (x *SpaceQuota) GetLimitBytes() uint64 { + if x != nil { + return x.LimitBytes + } + return 0 +} + +func (x *SpaceQuota) GetTotalUsageBytes() uint64 { + if x != nil { + return x.TotalUsageBytes + } + return 0 +} + +func (x *SpaceQuota) GetDurableUsageBytes() uint64 { + if x != nil { + return x.DurableUsageBytes + } + return 0 +} + +func (x *SpaceQuota) GetInflightUsageBytes() uint64 { + if x != nil { + return x.InflightUsageBytes + } + return 0 +} + +func (x *SpaceQuota) GetFilesCount() uint64 { + if x != nil { + return x.FilesCount + } + return 0 +} + +var File_commonfile_fileproto_fileprotov2_protos_filev2_proto protoreflect.FileDescriptor + +const file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDesc = "" + + "\n" + + "4commonfile/fileproto/fileprotov2/protos/filev2.proto\x12\n" + + "filesyncv2\"^\n" + + "\rUploadRequest\x12\x18\n" + + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x123\n" + + "\x05items\x18\x02 \x03(\v2\x1d.filesyncv2.UploadRequestItemR\x05items\"A\n" + + "\x11UploadRequestItem\x12\x18\n" + + "\arootCid\x18\x01 \x01(\fR\arootCid\x12\x12\n" + + "\x04size\x18\x02 \x01(\x04R\x04size\"D\n" + + "\x0eUploadResponse\x122\n" + + "\aresults\x18\x01 \x03(\v2\x18.filesyncv2.UploadResultR\aresults\"\x86\x01\n" + + "\fUploadResult\x12\x18\n" + + "\arootCid\x18\x01 \x01(\fR\arootCid\x12'\n" + + "\x04code\x18\x02 \x01(\x0e2\x13.filesyncv2.ErrCodeR\x04code\x123\n" + + "\x06upload\x18\x03 \x01(\v2\x1b.filesyncv2.PresignedUploadR\x06upload\"\x99\x01\n" + + "\x0fPresignedUpload\x12\x10\n" + + "\x03url\x18\x01 \x01(\tR\x03url\x120\n" + + "\x06fields\x18\x02 \x03(\v2\x18.filesyncv2.PresignFieldR\x06fields\x12*\n" + + "\x10maxContentLength\x18\x03 \x01(\x04R\x10maxContentLength\x12\x16\n" + + "\x06expiry\x18\x04 \x01(\x03R\x06expiry\"6\n" + + "\fPresignField\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value\"J\n" + + "\x12RequestSignRequest\x12\x18\n" + + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x12\x1a\n" + + "\brootCids\x18\x02 \x03(\fR\brootCids\"N\n" + + "\x13RequestSignResponse\x127\n" + + "\aresults\x18\x01 \x03(\v2\x1d.filesyncv2.RequestSignResultR\aresults\"\x90\x01\n" + + "\x11RequestSignResult\x12\x18\n" + + "\arootCid\x18\x01 \x01(\fR\arootCid\x12'\n" + + "\x04code\x18\x02 \x01(\x0e2\x13.filesyncv2.ErrCodeR\x04code\x128\n" + + "\areceipt\x18\x03 \x01(\v2\x1e.filesyncv2.NetworkSignReceiptR\areceipt\"Z\n" + + "\x12NetworkSignReceipt\x12&\n" + + "\x0ereceiptPayload\x18\x01 \x01(\fR\x0ereceiptPayload\x12\x1c\n" + + "\tsignature\x18\x02 \x01(\fR\tsignature\"N\n" + + "\x16RequestDownloadRequest\x12\x18\n" + + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x12\x1a\n" + + "\brootCids\x18\x02 \x03(\fR\brootCids\"V\n" + + "\x17RequestDownloadResponse\x12;\n" + + "\aresults\x18\x01 \x03(\v2!.filesyncv2.RequestDownloadResultR\aresults\"\x92\x01\n" + + "\x15RequestDownloadResult\x12\x18\n" + + "\arootCid\x18\x01 \x01(\fR\arootCid\x12'\n" + + "\x04code\x18\x02 \x01(\x0e2\x13.filesyncv2.ErrCodeR\x04code\x126\n" + + "\bdownload\x18\x03 \x01(\v2\x1a.filesyncv2.SignedDownloadR\bdownload\":\n" + + "\x0eSignedDownload\x12\x10\n" + + "\x03url\x18\x01 \x01(\tR\x03url\x12\x16\n" + + "\x06expiry\x18\x02 \x01(\x03R\x06expiry\".\n" + + "\x10SpaceInfoRequest\x12\x1a\n" + + "\bspaceIds\x18\x01 \x03(\tR\bspaceIds\"J\n" + + "\x11SpaceInfoResponse\x125\n" + + "\aresults\x18\x01 \x03(\v2\x1b.filesyncv2.SpaceInfoResultR\aresults\"\x82\x01\n" + + "\x0fSpaceInfoResult\x12\x18\n" + + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x12'\n" + + "\x04code\x18\x02 \x01(\x0e2\x13.filesyncv2.ErrCodeR\x04code\x12,\n" + + "\x05quota\x18\x03 \x01(\v2\x16.filesyncv2.SpaceQuotaR\x05quota\"\xd4\x01\n" + + "\n" + + "SpaceQuota\x12\x1e\n" + + "\n" + + "limitBytes\x18\x01 \x01(\x04R\n" + + "limitBytes\x12(\n" + + "\x0ftotalUsageBytes\x18\x02 \x01(\x04R\x0ftotalUsageBytes\x12,\n" + + "\x11durableUsageBytes\x18\x03 \x01(\x04R\x11durableUsageBytes\x12.\n" + + "\x12inflightUsageBytes\x18\x04 \x01(\x04R\x12inflightUsageBytes\x12\x1e\n" + + "\n" + + "filesCount\x18\x05 \x01(\x04R\n" + + "filesCount*f\n" + + "\bErrCodes\x12\x0e\n" + + "\n" + + "Unexpected\x10\x00\x12\r\n" + + "\tForbidden\x10\x01\x12\x12\n" + + "\x0eInvalidRequest\x10\x02\x12\x15\n" + + "\x11QuerySizeExceeded\x10\x03\x12\x10\n" + + "\vErrorOffset\x10\xa0\x06*v\n" + + "\aErrCode\x12\x06\n" + + "\x02Ok\x10\x00\x12\x11\n" + + "\rErrUnexpected\x10\x01\x12\x12\n" + + "\x0eErrCidNotFound\x10\x02\x12\x10\n" + + "\fErrForbidden\x10\x03\x12\x14\n" + + "\x10ErrLimitExceeded\x10\x04\x12\x14\n" + + "\x10ErrSpaceNotFound\x10\x052\xbf\x02\n" + + "\x06FileV2\x12?\n" + + "\x06Upload\x12\x19.filesyncv2.UploadRequest\x1a\x1a.filesyncv2.UploadResponse\x12N\n" + + "\vRequestSign\x12\x1e.filesyncv2.RequestSignRequest\x1a\x1f.filesyncv2.RequestSignResponse\x12Z\n" + + "\x0fRequestDownload\x12\".filesyncv2.RequestDownloadRequest\x1a#.filesyncv2.RequestDownloadResponse\x12H\n" + + "\tSpaceInfo\x12\x1c.filesyncv2.SpaceInfoRequest\x1a\x1d.filesyncv2.SpaceInfoResponseB\"Z commonfile/fileproto/fileprotov2b\x06proto3" + +var ( + file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescOnce sync.Once + file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescData []byte +) + +func file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP() []byte { + file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescOnce.Do(func() { + file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDesc), len(file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDesc))) + }) + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescData +} + +var file_commonfile_fileproto_fileprotov2_protos_filev2_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes = make([]protoimpl.MessageInfo, 18) +var file_commonfile_fileproto_fileprotov2_protos_filev2_proto_goTypes = []any{ + (ErrCodes)(0), // 0: filesyncv2.ErrCodes + (ErrCode)(0), // 1: filesyncv2.ErrCode + (*UploadRequest)(nil), // 2: filesyncv2.UploadRequest + (*UploadRequestItem)(nil), // 3: filesyncv2.UploadRequestItem + (*UploadResponse)(nil), // 4: filesyncv2.UploadResponse + (*UploadResult)(nil), // 5: filesyncv2.UploadResult + (*PresignedUpload)(nil), // 6: filesyncv2.PresignedUpload + (*PresignField)(nil), // 7: filesyncv2.PresignField + (*RequestSignRequest)(nil), // 8: filesyncv2.RequestSignRequest + (*RequestSignResponse)(nil), // 9: filesyncv2.RequestSignResponse + (*RequestSignResult)(nil), // 10: filesyncv2.RequestSignResult + (*NetworkSignReceipt)(nil), // 11: filesyncv2.NetworkSignReceipt + (*RequestDownloadRequest)(nil), // 12: filesyncv2.RequestDownloadRequest + (*RequestDownloadResponse)(nil), // 13: filesyncv2.RequestDownloadResponse + (*RequestDownloadResult)(nil), // 14: filesyncv2.RequestDownloadResult + (*SignedDownload)(nil), // 15: filesyncv2.SignedDownload + (*SpaceInfoRequest)(nil), // 16: filesyncv2.SpaceInfoRequest + (*SpaceInfoResponse)(nil), // 17: filesyncv2.SpaceInfoResponse + (*SpaceInfoResult)(nil), // 18: filesyncv2.SpaceInfoResult + (*SpaceQuota)(nil), // 19: filesyncv2.SpaceQuota +} +var file_commonfile_fileproto_fileprotov2_protos_filev2_proto_depIdxs = []int32{ + 3, // 0: filesyncv2.UploadRequest.items:type_name -> filesyncv2.UploadRequestItem + 5, // 1: filesyncv2.UploadResponse.results:type_name -> filesyncv2.UploadResult + 1, // 2: filesyncv2.UploadResult.code:type_name -> filesyncv2.ErrCode + 6, // 3: filesyncv2.UploadResult.upload:type_name -> filesyncv2.PresignedUpload + 7, // 4: filesyncv2.PresignedUpload.fields:type_name -> filesyncv2.PresignField + 10, // 5: filesyncv2.RequestSignResponse.results:type_name -> filesyncv2.RequestSignResult + 1, // 6: filesyncv2.RequestSignResult.code:type_name -> filesyncv2.ErrCode + 11, // 7: filesyncv2.RequestSignResult.receipt:type_name -> filesyncv2.NetworkSignReceipt + 14, // 8: filesyncv2.RequestDownloadResponse.results:type_name -> filesyncv2.RequestDownloadResult + 1, // 9: filesyncv2.RequestDownloadResult.code:type_name -> filesyncv2.ErrCode + 15, // 10: filesyncv2.RequestDownloadResult.download:type_name -> filesyncv2.SignedDownload + 18, // 11: filesyncv2.SpaceInfoResponse.results:type_name -> filesyncv2.SpaceInfoResult + 1, // 12: filesyncv2.SpaceInfoResult.code:type_name -> filesyncv2.ErrCode + 19, // 13: filesyncv2.SpaceInfoResult.quota:type_name -> filesyncv2.SpaceQuota + 2, // 14: filesyncv2.FileV2.Upload:input_type -> filesyncv2.UploadRequest + 8, // 15: filesyncv2.FileV2.RequestSign:input_type -> filesyncv2.RequestSignRequest + 12, // 16: filesyncv2.FileV2.RequestDownload:input_type -> filesyncv2.RequestDownloadRequest + 16, // 17: filesyncv2.FileV2.SpaceInfo:input_type -> filesyncv2.SpaceInfoRequest + 4, // 18: filesyncv2.FileV2.Upload:output_type -> filesyncv2.UploadResponse + 9, // 19: filesyncv2.FileV2.RequestSign:output_type -> filesyncv2.RequestSignResponse + 13, // 20: filesyncv2.FileV2.RequestDownload:output_type -> filesyncv2.RequestDownloadResponse + 17, // 21: filesyncv2.FileV2.SpaceInfo:output_type -> filesyncv2.SpaceInfoResponse + 18, // [18:22] is the sub-list for method output_type + 14, // [14:18] is the sub-list for method input_type + 14, // [14:14] is the sub-list for extension type_name + 14, // [14:14] is the sub-list for extension extendee + 0, // [0:14] is the sub-list for field type_name +} + +func init() { file_commonfile_fileproto_fileprotov2_protos_filev2_proto_init() } +func file_commonfile_fileproto_fileprotov2_protos_filev2_proto_init() { + if File_commonfile_fileproto_fileprotov2_protos_filev2_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDesc), len(file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDesc)), + NumEnums: 2, + NumMessages: 18, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_commonfile_fileproto_fileprotov2_protos_filev2_proto_goTypes, + DependencyIndexes: file_commonfile_fileproto_fileprotov2_protos_filev2_proto_depIdxs, + EnumInfos: file_commonfile_fileproto_fileprotov2_protos_filev2_proto_enumTypes, + MessageInfos: file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes, + }.Build() + File_commonfile_fileproto_fileprotov2_protos_filev2_proto = out.File + file_commonfile_fileproto_fileprotov2_protos_filev2_proto_goTypes = nil + file_commonfile_fileproto_fileprotov2_protos_filev2_proto_depIdxs = nil +} diff --git a/commonfile/fileproto/fileprotov2/filev2_drpc.pb.go b/commonfile/fileproto/fileprotov2/filev2_drpc.pb.go new file mode 100644 index 000000000..be7fbe385 --- /dev/null +++ b/commonfile/fileproto/fileprotov2/filev2_drpc.pb.go @@ -0,0 +1,226 @@ +// Code generated by protoc-gen-go-drpc. DO NOT EDIT. +// protoc-gen-go-drpc version: v0.0.34 +// source: commonfile/fileproto/fileprotov2/protos/filev2.proto + +package fileprotov2 + +import ( + context "context" + errors "errors" + drpc1 "github.com/planetscale/vtprotobuf/codec/drpc" + drpc "storj.io/drpc" + drpcerr "storj.io/drpc/drpcerr" +) + +type drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto struct{} + +func (drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto) Marshal(msg drpc.Message) ([]byte, error) { + return drpc1.Marshal(msg) +} + +func (drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto) Unmarshal(buf []byte, msg drpc.Message) error { + return drpc1.Unmarshal(buf, msg) +} + +func (drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto) JSONMarshal(msg drpc.Message) ([]byte, error) { + return drpc1.JSONMarshal(msg) +} + +func (drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto) JSONUnmarshal(buf []byte, msg drpc.Message) error { + return drpc1.JSONUnmarshal(buf, msg) +} + +type DRPCFileV2Client interface { + DRPCConn() drpc.Conn + + Upload(ctx context.Context, in *UploadRequest) (*UploadResponse, error) + RequestSign(ctx context.Context, in *RequestSignRequest) (*RequestSignResponse, error) + RequestDownload(ctx context.Context, in *RequestDownloadRequest) (*RequestDownloadResponse, error) + SpaceInfo(ctx context.Context, in *SpaceInfoRequest) (*SpaceInfoResponse, error) +} + +type drpcFileV2Client struct { + cc drpc.Conn +} + +func NewDRPCFileV2Client(cc drpc.Conn) DRPCFileV2Client { + return &drpcFileV2Client{cc} +} + +func (c *drpcFileV2Client) DRPCConn() drpc.Conn { return c.cc } + +func (c *drpcFileV2Client) Upload(ctx context.Context, in *UploadRequest) (*UploadResponse, error) { + out := new(UploadResponse) + err := c.cc.Invoke(ctx, "/filesyncv2.FileV2/Upload", drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}, in, out) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *drpcFileV2Client) RequestSign(ctx context.Context, in *RequestSignRequest) (*RequestSignResponse, error) { + out := new(RequestSignResponse) + err := c.cc.Invoke(ctx, "/filesyncv2.FileV2/RequestSign", drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}, in, out) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *drpcFileV2Client) RequestDownload(ctx context.Context, in *RequestDownloadRequest) (*RequestDownloadResponse, error) { + out := new(RequestDownloadResponse) + err := c.cc.Invoke(ctx, "/filesyncv2.FileV2/RequestDownload", drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}, in, out) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *drpcFileV2Client) SpaceInfo(ctx context.Context, in *SpaceInfoRequest) (*SpaceInfoResponse, error) { + out := new(SpaceInfoResponse) + err := c.cc.Invoke(ctx, "/filesyncv2.FileV2/SpaceInfo", drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}, in, out) + if err != nil { + return nil, err + } + return out, nil +} + +type DRPCFileV2Server interface { + Upload(context.Context, *UploadRequest) (*UploadResponse, error) + RequestSign(context.Context, *RequestSignRequest) (*RequestSignResponse, error) + RequestDownload(context.Context, *RequestDownloadRequest) (*RequestDownloadResponse, error) + SpaceInfo(context.Context, *SpaceInfoRequest) (*SpaceInfoResponse, error) +} + +type DRPCFileV2UnimplementedServer struct{} + +func (s *DRPCFileV2UnimplementedServer) Upload(context.Context, *UploadRequest) (*UploadResponse, error) { + return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) +} + +func (s *DRPCFileV2UnimplementedServer) RequestSign(context.Context, *RequestSignRequest) (*RequestSignResponse, error) { + return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) +} + +func (s *DRPCFileV2UnimplementedServer) RequestDownload(context.Context, *RequestDownloadRequest) (*RequestDownloadResponse, error) { + return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) +} + +func (s *DRPCFileV2UnimplementedServer) SpaceInfo(context.Context, *SpaceInfoRequest) (*SpaceInfoResponse, error) { + return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) +} + +type DRPCFileV2Description struct{} + +func (DRPCFileV2Description) NumMethods() int { return 4 } + +func (DRPCFileV2Description) Method(n int) (string, drpc.Encoding, drpc.Receiver, interface{}, bool) { + switch n { + case 0: + return "/filesyncv2.FileV2/Upload", drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}, + func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) { + return srv.(DRPCFileV2Server). + Upload( + ctx, + in1.(*UploadRequest), + ) + }, DRPCFileV2Server.Upload, true + case 1: + return "/filesyncv2.FileV2/RequestSign", drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}, + func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) { + return srv.(DRPCFileV2Server). + RequestSign( + ctx, + in1.(*RequestSignRequest), + ) + }, DRPCFileV2Server.RequestSign, true + case 2: + return "/filesyncv2.FileV2/RequestDownload", drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}, + func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) { + return srv.(DRPCFileV2Server). + RequestDownload( + ctx, + in1.(*RequestDownloadRequest), + ) + }, DRPCFileV2Server.RequestDownload, true + case 3: + return "/filesyncv2.FileV2/SpaceInfo", drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}, + func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) { + return srv.(DRPCFileV2Server). + SpaceInfo( + ctx, + in1.(*SpaceInfoRequest), + ) + }, DRPCFileV2Server.SpaceInfo, true + default: + return "", nil, nil, nil, false + } +} + +func DRPCRegisterFileV2(mux drpc.Mux, impl DRPCFileV2Server) error { + return mux.Register(impl, DRPCFileV2Description{}) +} + +type DRPCFileV2_UploadStream interface { + drpc.Stream + SendAndClose(*UploadResponse) error +} + +type drpcFileV2_UploadStream struct { + drpc.Stream +} + +func (x *drpcFileV2_UploadStream) SendAndClose(m *UploadResponse) error { + if err := x.MsgSend(m, drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}); err != nil { + return err + } + return x.CloseSend() +} + +type DRPCFileV2_RequestSignStream interface { + drpc.Stream + SendAndClose(*RequestSignResponse) error +} + +type drpcFileV2_RequestSignStream struct { + drpc.Stream +} + +func (x *drpcFileV2_RequestSignStream) SendAndClose(m *RequestSignResponse) error { + if err := x.MsgSend(m, drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}); err != nil { + return err + } + return x.CloseSend() +} + +type DRPCFileV2_RequestDownloadStream interface { + drpc.Stream + SendAndClose(*RequestDownloadResponse) error +} + +type drpcFileV2_RequestDownloadStream struct { + drpc.Stream +} + +func (x *drpcFileV2_RequestDownloadStream) SendAndClose(m *RequestDownloadResponse) error { + if err := x.MsgSend(m, drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}); err != nil { + return err + } + return x.CloseSend() +} + +type DRPCFileV2_SpaceInfoStream interface { + drpc.Stream + SendAndClose(*SpaceInfoResponse) error +} + +type drpcFileV2_SpaceInfoStream struct { + drpc.Stream +} + +func (x *drpcFileV2_SpaceInfoStream) SendAndClose(m *SpaceInfoResponse) error { + if err := x.MsgSend(m, drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}); err != nil { + return err + } + return x.CloseSend() +} diff --git a/commonfile/fileproto/fileprotov2/filev2_vtproto.pb.go b/commonfile/fileproto/fileprotov2/filev2_vtproto.pb.go new file mode 100644 index 000000000..49d19b483 --- /dev/null +++ b/commonfile/fileproto/fileprotov2/filev2_vtproto.pb.go @@ -0,0 +1,3330 @@ +// Code generated by protoc-gen-go-vtproto. DO NOT EDIT. +// protoc-gen-go-vtproto version: v0.6.0 +// source: commonfile/fileproto/fileprotov2/protos/filev2.proto + +package fileprotov2 + +import ( + fmt "fmt" + protohelpers "github.com/planetscale/vtprotobuf/protohelpers" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + io "io" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +func (m *UploadRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *UploadRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *UploadRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Items) > 0 { + for iNdEx := len(m.Items) - 1; iNdEx >= 0; iNdEx-- { + size, err := m.Items[iNdEx].MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x12 + } + } + if len(m.SpaceId) > 0 { + i -= len(m.SpaceId) + copy(dAtA[i:], m.SpaceId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *UploadRequestItem) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *UploadRequestItem) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *UploadRequestItem) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.Size != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Size)) + i-- + dAtA[i] = 0x10 + } + if len(m.RootCid) > 0 { + i -= len(m.RootCid) + copy(dAtA[i:], m.RootCid) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.RootCid))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *UploadResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *UploadResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *UploadResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Results) > 0 { + for iNdEx := len(m.Results) - 1; iNdEx >= 0; iNdEx-- { + size, err := m.Results[iNdEx].MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *UploadResult) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *UploadResult) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *UploadResult) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.Upload != nil { + size, err := m.Upload.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x1a + } + if m.Code != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Code)) + i-- + dAtA[i] = 0x10 + } + if len(m.RootCid) > 0 { + i -= len(m.RootCid) + copy(dAtA[i:], m.RootCid) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.RootCid))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *PresignedUpload) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PresignedUpload) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *PresignedUpload) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.Expiry != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Expiry)) + i-- + dAtA[i] = 0x20 + } + if m.MaxContentLength != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.MaxContentLength)) + i-- + dAtA[i] = 0x18 + } + if len(m.Fields) > 0 { + for iNdEx := len(m.Fields) - 1; iNdEx >= 0; iNdEx-- { + size, err := m.Fields[iNdEx].MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x12 + } + } + if len(m.Url) > 0 { + i -= len(m.Url) + copy(dAtA[i:], m.Url) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Url))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *PresignField) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *PresignField) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *PresignField) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Value) > 0 { + i -= len(m.Value) + copy(dAtA[i:], m.Value) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Value))) + i-- + dAtA[i] = 0x12 + } + if len(m.Key) > 0 { + i -= len(m.Key) + copy(dAtA[i:], m.Key) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Key))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *RequestSignRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RequestSignRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *RequestSignRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.RootCids) > 0 { + for iNdEx := len(m.RootCids) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.RootCids[iNdEx]) + copy(dAtA[i:], m.RootCids[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.RootCids[iNdEx]))) + i-- + dAtA[i] = 0x12 + } + } + if len(m.SpaceId) > 0 { + i -= len(m.SpaceId) + copy(dAtA[i:], m.SpaceId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *RequestSignResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RequestSignResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *RequestSignResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Results) > 0 { + for iNdEx := len(m.Results) - 1; iNdEx >= 0; iNdEx-- { + size, err := m.Results[iNdEx].MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *RequestSignResult) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RequestSignResult) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *RequestSignResult) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.Receipt != nil { + size, err := m.Receipt.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x1a + } + if m.Code != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Code)) + i-- + dAtA[i] = 0x10 + } + if len(m.RootCid) > 0 { + i -= len(m.RootCid) + copy(dAtA[i:], m.RootCid) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.RootCid))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *NetworkSignReceipt) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *NetworkSignReceipt) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *NetworkSignReceipt) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Signature) > 0 { + i -= len(m.Signature) + copy(dAtA[i:], m.Signature) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Signature))) + i-- + dAtA[i] = 0x12 + } + if len(m.ReceiptPayload) > 0 { + i -= len(m.ReceiptPayload) + copy(dAtA[i:], m.ReceiptPayload) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.ReceiptPayload))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *RequestDownloadRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RequestDownloadRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *RequestDownloadRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.RootCids) > 0 { + for iNdEx := len(m.RootCids) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.RootCids[iNdEx]) + copy(dAtA[i:], m.RootCids[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.RootCids[iNdEx]))) + i-- + dAtA[i] = 0x12 + } + } + if len(m.SpaceId) > 0 { + i -= len(m.SpaceId) + copy(dAtA[i:], m.SpaceId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *RequestDownloadResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RequestDownloadResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *RequestDownloadResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Results) > 0 { + for iNdEx := len(m.Results) - 1; iNdEx >= 0; iNdEx-- { + size, err := m.Results[iNdEx].MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *RequestDownloadResult) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *RequestDownloadResult) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *RequestDownloadResult) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.Download != nil { + size, err := m.Download.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x1a + } + if m.Code != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Code)) + i-- + dAtA[i] = 0x10 + } + if len(m.RootCid) > 0 { + i -= len(m.RootCid) + copy(dAtA[i:], m.RootCid) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.RootCid))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *SignedDownload) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SignedDownload) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *SignedDownload) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.Expiry != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Expiry)) + i-- + dAtA[i] = 0x10 + } + if len(m.Url) > 0 { + i -= len(m.Url) + copy(dAtA[i:], m.Url) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Url))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *SpaceInfoRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SpaceInfoRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *SpaceInfoRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.SpaceIds) > 0 { + for iNdEx := len(m.SpaceIds) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.SpaceIds[iNdEx]) + copy(dAtA[i:], m.SpaceIds[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceIds[iNdEx]))) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *SpaceInfoResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SpaceInfoResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *SpaceInfoResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Results) > 0 { + for iNdEx := len(m.Results) - 1; iNdEx >= 0; iNdEx-- { + size, err := m.Results[iNdEx].MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *SpaceInfoResult) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SpaceInfoResult) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *SpaceInfoResult) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.Quota != nil { + size, err := m.Quota.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x1a + } + if m.Code != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Code)) + i-- + dAtA[i] = 0x10 + } + if len(m.SpaceId) > 0 { + i -= len(m.SpaceId) + copy(dAtA[i:], m.SpaceId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *SpaceQuota) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SpaceQuota) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *SpaceQuota) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.FilesCount != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.FilesCount)) + i-- + dAtA[i] = 0x28 + } + if m.InflightUsageBytes != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.InflightUsageBytes)) + i-- + dAtA[i] = 0x20 + } + if m.DurableUsageBytes != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.DurableUsageBytes)) + i-- + dAtA[i] = 0x18 + } + if m.TotalUsageBytes != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.TotalUsageBytes)) + i-- + dAtA[i] = 0x10 + } + if m.LimitBytes != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.LimitBytes)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *UploadRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.SpaceId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if len(m.Items) > 0 { + for _, e := range m.Items { + l = e.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *UploadRequestItem) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.RootCid) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.Size != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Size)) + } + n += len(m.unknownFields) + return n +} + +func (m *UploadResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Results) > 0 { + for _, e := range m.Results { + l = e.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *UploadResult) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.RootCid) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.Code != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Code)) + } + if m.Upload != nil { + l = m.Upload.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + +func (m *PresignedUpload) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Url) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if len(m.Fields) > 0 { + for _, e := range m.Fields { + l = e.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + if m.MaxContentLength != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.MaxContentLength)) + } + if m.Expiry != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Expiry)) + } + n += len(m.unknownFields) + return n +} + +func (m *PresignField) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Key) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Value) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + +func (m *RequestSignRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.SpaceId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if len(m.RootCids) > 0 { + for _, b := range m.RootCids { + l = len(b) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *RequestSignResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Results) > 0 { + for _, e := range m.Results { + l = e.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *RequestSignResult) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.RootCid) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.Code != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Code)) + } + if m.Receipt != nil { + l = m.Receipt.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + +func (m *NetworkSignReceipt) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.ReceiptPayload) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Signature) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + +func (m *RequestDownloadRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.SpaceId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if len(m.RootCids) > 0 { + for _, b := range m.RootCids { + l = len(b) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *RequestDownloadResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Results) > 0 { + for _, e := range m.Results { + l = e.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *RequestDownloadResult) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.RootCid) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.Code != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Code)) + } + if m.Download != nil { + l = m.Download.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + +func (m *SignedDownload) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Url) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.Expiry != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Expiry)) + } + n += len(m.unknownFields) + return n +} + +func (m *SpaceInfoRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.SpaceIds) > 0 { + for _, s := range m.SpaceIds { + l = len(s) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *SpaceInfoResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Results) > 0 { + for _, e := range m.Results { + l = e.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *SpaceInfoResult) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.SpaceId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.Code != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Code)) + } + if m.Quota != nil { + l = m.Quota.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + +func (m *SpaceQuota) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.LimitBytes != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.LimitBytes)) + } + if m.TotalUsageBytes != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.TotalUsageBytes)) + } + if m.DurableUsageBytes != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.DurableUsageBytes)) + } + if m.InflightUsageBytes != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.InflightUsageBytes)) + } + if m.FilesCount != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.FilesCount)) + } + n += len(m.unknownFields) + return n +} + +func (m *UploadRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: UploadRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: UploadRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Items", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Items = append(m.Items, &UploadRequestItem{}) + if err := m.Items[len(m.Items)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *UploadRequestItem) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: UploadRequestItem: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: UploadRequestItem: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RootCid", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RootCid = append(m.RootCid[:0], dAtA[iNdEx:postIndex]...) + if m.RootCid == nil { + m.RootCid = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Size", wireType) + } + m.Size = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Size |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *UploadResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: UploadResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: UploadResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Results", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Results = append(m.Results, &UploadResult{}) + if err := m.Results[len(m.Results)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *UploadResult) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: UploadResult: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: UploadResult: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RootCid", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RootCid = append(m.RootCid[:0], dAtA[iNdEx:postIndex]...) + if m.RootCid == nil { + m.RootCid = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Code", wireType) + } + m.Code = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Code |= ErrCode(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Upload", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Upload == nil { + m.Upload = &PresignedUpload{} + } + if err := m.Upload.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *PresignedUpload) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PresignedUpload: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PresignedUpload: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Url", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Url = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Fields", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Fields = append(m.Fields, &PresignField{}) + if err := m.Fields[len(m.Fields)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field MaxContentLength", wireType) + } + m.MaxContentLength = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.MaxContentLength |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Expiry", wireType) + } + m.Expiry = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Expiry |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *PresignField) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: PresignField: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: PresignField: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Key = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Value", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Value = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *RequestSignRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RequestSignRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RequestSignRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RootCids", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RootCids = append(m.RootCids, make([]byte, postIndex-iNdEx)) + copy(m.RootCids[len(m.RootCids)-1], dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *RequestSignResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RequestSignResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RequestSignResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Results", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Results = append(m.Results, &RequestSignResult{}) + if err := m.Results[len(m.Results)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *RequestSignResult) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RequestSignResult: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RequestSignResult: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RootCid", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RootCid = append(m.RootCid[:0], dAtA[iNdEx:postIndex]...) + if m.RootCid == nil { + m.RootCid = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Code", wireType) + } + m.Code = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Code |= ErrCode(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Receipt", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Receipt == nil { + m.Receipt = &NetworkSignReceipt{} + } + if err := m.Receipt.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *NetworkSignReceipt) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: NetworkSignReceipt: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: NetworkSignReceipt: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ReceiptPayload", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ReceiptPayload = append(m.ReceiptPayload[:0], dAtA[iNdEx:postIndex]...) + if m.ReceiptPayload == nil { + m.ReceiptPayload = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Signature", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Signature = append(m.Signature[:0], dAtA[iNdEx:postIndex]...) + if m.Signature == nil { + m.Signature = []byte{} + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *RequestDownloadRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RequestDownloadRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RequestDownloadRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RootCids", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RootCids = append(m.RootCids, make([]byte, postIndex-iNdEx)) + copy(m.RootCids[len(m.RootCids)-1], dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *RequestDownloadResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RequestDownloadResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RequestDownloadResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Results", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Results = append(m.Results, &RequestDownloadResult{}) + if err := m.Results[len(m.Results)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *RequestDownloadResult) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: RequestDownloadResult: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: RequestDownloadResult: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RootCid", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RootCid = append(m.RootCid[:0], dAtA[iNdEx:postIndex]...) + if m.RootCid == nil { + m.RootCid = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Code", wireType) + } + m.Code = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Code |= ErrCode(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Download", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Download == nil { + m.Download = &SignedDownload{} + } + if err := m.Download.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SignedDownload) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SignedDownload: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SignedDownload: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Url", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Url = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Expiry", wireType) + } + m.Expiry = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Expiry |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SpaceInfoRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SpaceInfoRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SpaceInfoRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceIds", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceIds = append(m.SpaceIds, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SpaceInfoResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SpaceInfoResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SpaceInfoResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Results", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Results = append(m.Results, &SpaceInfoResult{}) + if err := m.Results[len(m.Results)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SpaceInfoResult) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SpaceInfoResult: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SpaceInfoResult: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Code", wireType) + } + m.Code = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Code |= ErrCode(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Quota", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Quota == nil { + m.Quota = &SpaceQuota{} + } + if err := m.Quota.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SpaceQuota) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SpaceQuota: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SpaceQuota: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field LimitBytes", wireType) + } + m.LimitBytes = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.LimitBytes |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field TotalUsageBytes", wireType) + } + m.TotalUsageBytes = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.TotalUsageBytes |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field DurableUsageBytes", wireType) + } + m.DurableUsageBytes = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.DurableUsageBytes |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field InflightUsageBytes", wireType) + } + m.InflightUsageBytes = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.InflightUsageBytes |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FilesCount", wireType) + } + m.FilesCount = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.FilesCount |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} diff --git a/commonfile/fileproto/fileprotov2/protos/filev2.proto b/commonfile/fileproto/fileprotov2/protos/filev2.proto new file mode 100644 index 000000000..337e618ce --- /dev/null +++ b/commonfile/fileproto/fileprotov2/protos/filev2.proto @@ -0,0 +1,188 @@ +syntax = "proto3"; +package filesyncv2; + +option go_package = "commonfile/fileproto/fileprotov2"; + +// FileV2 is the v2 filenode broker (S3-direct uploads/downloads). +// +// drpc route prefix is "/filesyncv2.FileV2/..." — DISTINCT from v1 +// "/filesync.File/..." so both services can be mounted on the same drpc mux +// without a Register() panic. The package/service names are load-bearing: +// method paths are derived from `package filesyncv2` + `service FileV2`. +// +// Conventions (ALL four RPCs are BATCH): +// * request carries a single shared spaceId (or repeated spaceIds for +// SpaceInfo) + a repeated list of items; +// * identity is NEVER a request field — it is taken from the authenticated +// connection, and all per-space/per-cid authorization derives from it; +// * rootCid is bytes (raw self-describing CID, as in v1's `bytes cid`); +// * every result echoes its own key (rootCid bytes, or spaceId string) so +// clients build map[string(key)]->result in one pass — results are matched +// by key, NEVER by positional index; the server may reorder/dedupe; +// * per-item result = flat status enum (ErrCode, Ok=0) + exactly one typed +// payload message that is valid ONLY when code == Ok; +// * one bad/forbidden item never fails the whole batch. +// +// Two error layers: +// 1. whole-RPC drpc error (err != nil), carrying an ErrCodes value from the +// offset-800 rpcerr registry, returned ONLY for cross-cutting failures: +// failed/absent auth (Forbidden), malformed input (InvalidRequest), or an +// over-sized batch (QuerySizeExceeded); +// 2. per-item business outcome lives in the in-body ErrCode `code` field +// (err == nil for the RPC as a whole). +service FileV2 { + // Upload issues a presigned upload target per requested file root. + rpc Upload(UploadRequest) returns (UploadResponse); + // RequestSign returns a network-signed durable-custody receipt per rootCid. + rpc RequestSign(RequestSignRequest) returns (RequestSignResponse); + // RequestDownload returns a signed GET target per rootCid. + rpc RequestDownload(RequestDownloadRequest) returns (RequestDownloadResponse); + // SpaceInfo returns quota/usage per requested space (batch quota-info). + rpc SpaceInfo(SpaceInfoRequest) returns (SpaceInfoResponse); +} + +// ErrCodes are WHOLE-RPC (drpc-level) errors, mirroring v1 filesync.ErrCodes. +// Wired through rpcerr.ErrGroup(ErrCodes_ErrorOffset) in the fileprotov2err +// package. Returned as a NON-nil drpc error ONLY for auth / malformed / +// batch-too-large. Never used to report the outcome of an individual item +// (that is the per-item ErrCode below). Zero value Unexpected=0 registers at +// offset+0 (=800), so an uncoded server fault reads as Unexpected — exactly as +// v1 does at 200. Offset 800 is the only free rpcerr block (v1=200, 100/300/ +// 400/500/600/700 are taken). +enum ErrCodes { + Unexpected = 0; // unclassified server fault (registry fallback) + Forbidden = 1; // connection identity is not permitted at all + InvalidRequest = 2; // malformed: empty spaceId, empty items, bad rootCid bytes + QuerySizeExceeded = 3; // batch too large (too many items in one request) + ErrorOffset = 800; // rpcerr.ErrGroup base for v2 (constant, never an error) +} + +// ErrCode is the PER-ITEM, in-body business outcome (err == nil at the RPC +// layer). Flat status enum with an explicit success sentinel Ok=0. The typed +// payload on a *Result is valid ONLY when code == Ok. Deliberately a SEPARATE +// enum from ErrCodes: its zero value means success (not error), it never flows +// through rpcerr, and its value short-names are kept disjoint from ErrCodes so +// proto3's package-level enum-value scoping stays clean. Servers MUST set +// `code` explicitly on every result (Ok is the proto3 default, so an unset +// code would be misread as a success with an empty payload). +enum ErrCode { + Ok = 0; // item succeeded; the typed payload is populated + ErrUnexpected = 1; // internal error handling this single item + ErrCidNotFound = 2; // sign/download: no such stored file for this rootCid + ErrForbidden = 3; // identity may not access this item's space/file + ErrLimitExceeded = 4; // upload rejected: space/account storage limit reached + ErrSpaceNotFound = 5; // quota-info: unknown/deleted space id +} + +// ---- Upload --------------------------------------------------------------- + +message UploadRequest { + string spaceId = 1; // shared space for every item (auth scope) + repeated UploadRequestItem items = 2; // files to obtain an upload target for +} + +message UploadRequestItem { + bytes rootCid = 1; // CID of the file root (result match key) + uint64 size = 2; // declared total byte size of the file +} + +message UploadResponse { + repeated UploadResult results = 1; // one per requested item, keyed by rootCid +} + +message UploadResult { + bytes rootCid = 1; // echoes UploadRequestItem.rootCid (match key) + ErrCode code = 2; // per-item outcome; `upload` valid only when Ok + PresignedUpload upload = 3; // set ONLY when code == Ok +} + +// PresignedUpload is an OPAQUE presigned upload target. No S3 vendor structs +// leak into the shared proto: the client POSTs `fields` + the object body to +// `url` verbatim. Modeled as a presigned POST form (fields are ordered +// key/value pairs rather than a map) so the signing order is deterministic and +// reproducible. +message PresignedUpload { + string url = 1; // presigned POST endpoint + repeated PresignField fields = 2; // multipart form fields to send verbatim, in order + uint64 maxContentLength = 3; // hard upper bound the presign enforces (bytes) + int64 expiry = 4; // unix seconds; POST must complete before this +} + +message PresignField { + string key = 1; + string value = 2; +} + +// ---- RequestSign ---------------------------------------------------------- + +message RequestSignRequest { + string spaceId = 1; // shared space for every rootCid (auth scope) + repeated bytes rootCids = 2; // files to obtain a durable-custody receipt for +} + +message RequestSignResponse { + repeated RequestSignResult results = 1; // one per rootCid, keyed by rootCid +} + +message RequestSignResult { + bytes rootCid = 1; // echoes the requested rootCid (match key) + ErrCode code = 2; // per-item outcome; `receipt` valid only when Ok + NetworkSignReceipt receipt = 3; // set ONLY when code == Ok +} + +// NetworkSignReceipt mirrors coordinator SpaceReceiptWithSignature: an opaque +// protobuf-encoded payload plus the network-key signature over it. The payload +// schema is intentionally not defined here (opaque blob forwarded verbatim). +message NetworkSignReceipt { + bytes receiptPayload = 1; // protobuf-encoded signed receipt payload (opaque) + bytes signature = 2; // network key signature over receiptPayload +} + +// ---- RequestDownload ------------------------------------------------------ + +message RequestDownloadRequest { + string spaceId = 1; // shared space for every rootCid (auth scope) + repeated bytes rootCids = 2; // files to obtain a signed GET target for +} + +message RequestDownloadResponse { + repeated RequestDownloadResult results = 1; // one per rootCid, keyed by rootCid +} + +message RequestDownloadResult { + bytes rootCid = 1; // echoes the requested rootCid (match key) + ErrCode code = 2; // per-item outcome; `download` valid only when Ok + SignedDownload download = 3; // set ONLY when code == Ok +} + +// SignedDownload is an OPAQUE signed GET target (kept as a message, symmetric +// with PresignedUpload, so download can grow response headers etc. without a +// wire break on the result envelope). +message SignedDownload { + string url = 1; // opaque signed GET url + int64 expiry = 2; // unix seconds the url is valid until +} + +// ---- SpaceInfo (batch quota-info) ----------------------------------------- + +message SpaceInfoRequest { + repeated string spaceIds = 1; // batch of spaces to report on (result match key) +} + +message SpaceInfoResponse { + repeated SpaceInfoResult results = 1; // one per requested space, keyed by spaceId +} + +message SpaceInfoResult { + string spaceId = 1; // echoes the requested spaceId (match key) + ErrCode code = 2; // per-item outcome; `quota` valid only when Ok + SpaceQuota quota = 3; // set ONLY when code == Ok +} + +message SpaceQuota { + uint64 limitBytes = 1; // effective storage limit for this space + uint64 totalUsageBytes = 2; // counted usage == durableUsageBytes + inflightUsageBytes + uint64 durableUsageBytes = 3; // bytes fully committed to durable storage + uint64 inflightUsageBytes = 4; // bytes reserved by presigned/not-yet-committed uploads + uint64 filesCount = 5; // number of file roots stored in the space +} From d1368faf0884bf387ef5798e5d85ad272532f0bf Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Wed, 1 Jul 2026 14:59:24 +0200 Subject: [PATCH 06/20] docs(rpcerr): reserve error-code offsets across repos Document the process-global rpcerr registry as a shared cross-repo namespace: the mechanism, allocation rules (offsets are multiples of 100, local codes 0-99, any-sync core < 1000 / downstream repos >= 1000), the globally reserved low codes, and a full table of offsets in use across the ecosystem (any-sync 100-800, any-sync-node 1000, anytype-push-server 1200, bobrik-clusterctl 1300). Add an ErrGroup doc comment pointing to it. Co-Authored-By: Claude Opus 4.8 --- docs/rpc-error-offsets.md | 104 +++++++++++++++++++++++++++++++++++++ net/rpc/rpcerr/registry.go | 4 ++ 2 files changed, 108 insertions(+) create mode 100644 docs/rpc-error-offsets.md diff --git a/docs/rpc-error-offsets.md b/docs/rpc-error-offsets.md new file mode 100644 index 000000000..cd06944f6 --- /dev/null +++ b/docs/rpc-error-offsets.md @@ -0,0 +1,104 @@ +# RPC Error Code Offsets + +## Overview + +any-sync maps typed RPC errors onto numeric [dRPC](https://storj.io/drpc) error codes +through a single **process-global** registry in +[`net/rpc/rpcerr`](../net/rpc/rpcerr/registry.go). + +```go +// net/rpc/rpcerr/registry.go +var errsMap = make(map[uint64]error) // global, one per process + +func RegisterErr(err error, code uint64) error { + if e, ok := errsMap[code]; ok { + panic(fmt.Errorf("attempt to register error with existing code: %d ...", code)) + } + // ... +} + +type ErrGroup int64 +func (g ErrGroup) Register(err error, code uint64) error { + return RegisterErr(err, uint64(g)+code) // registers at offset+code +} +``` + +Each proto/service declares an `ErrorOffset` constant and registers its errors +through `rpcerr.ErrGroup(ErrorOffset)`; a concrete error's wire code is +`ErrorOffset + localCode`. + +### Why this needs coordinating across repos + +`errsMap` is **global to the running binary**, and registration happens in +`var`/`init` blocks at import time. A node binary routinely links **any-sync +plus one or more downstream repos** (e.g. `any-sync-node` also registers +`nodesync` errors). If any two groups ever register the same `offset + code`, +the program **panics at startup** — before any request is served. + +So offsets are a shared namespace across the whole anyproto ecosystem, not just +within one repo. **This file is the source of truth for who owns which offset.** + +## Allocation rules + +1. **Offsets are multiples of 100.** Each group owns the band + `[offset, offset+99]`. +2. **Local codes must stay in `0..99`** so a group never bleeds into the next + band. (The `ErrorOffset` enum value itself is only the base — it is never + registered as a code.) +3. **Local code `0` is the group's `Unexpected`/fallback** (registers at + `offset+0`), mirroring `filesync.ErrCodes.Unexpected = 0`. +4. **any-sync core uses `< 1000`.** Downstream repos that import any-sync must + use **`>= 1000`** to stay clear of the core range. +5. When you add a group, **pick the next free offset and add a row below.** + +### Globally reserved low codes + +The base registry claims two codes directly (outside any group), so **no group +may use offset `0`**: + +| Code | Name | Source | +|------|------|--------| +| 1 | `Unexpected` | `net/rpc/rpcerr/registry.go` | +| 2 | `Closed` | `net/rpc/rpcerr/registry.go` | + +## Reserved offsets + +### any-sync (core, `< 1000`) + +| Offset | Band | Proto / service | Package | +|--------|------|-----------------|---------| +| 100 | 100–199 | `spacesync` | `commonspace/spacesyncproto` | +| 200 | 200–299 | `filesync` (File, v1) | `commonfile/fileproto` | +| 300 | 300–399 | `coordinator` | `coordinator/coordinatorproto` | +| 400 | 400–499 | `treechange` | `commonspace/object/tree/treechangeproto` | +| 500 | 500–599 | `consensus` | `consensus/consensusproto` | +| 600 | 600–699 | `payment` | `paymentservice/paymentserviceproto` | +| 700 | 700–799 | `limiter` | `net/rpc/limiter/limiterproto` | +| 800 | 800–899 | `filesyncv2` (FileV2, v2 broker) | `commonfile/fileproto/fileprotov2` | +| 900 | 900–999 | _free_ | — | + +### Downstream repos (`>= 1000`) + +These live in other repositories but share this global registry whenever their +binary also links any-sync. + +| Offset | Band | Proto / service | Repo · package | +|--------|------|-----------------|----------------| +| 1000 | 1000–1099 | `nodesync` | `any-sync-node` · `nodesync/nodesyncproto` | +| 1100 | 1100–1199 | _free_ | — | +| 1200 | 1200–1299 | `push` | `anytype-push-server` · `pushclient/pushapi` | +| 1300 | 1300–1399 | `bobrik` | `bobrik-clusterctl` · `bobrikclient/bobrikapi` | +| 1400+ | — | _free_ | — | + +## Adding a new group + +1. Choose the next free offset from the tables above (core: next free `< 1000`; + downstream: next free `>= 1000`). +2. In the `.proto`, add `ErrorOffset = ;` to the service's `ErrCodes` + enum, with local codes `0..N` (`0` = `Unexpected`). +3. Register in a Go errors file via `rpcerr.ErrGroup(_ErrorOffset)` + (see `commonfile/fileproto/fileprotov2/fileprotov2err/fileprotov2err.go` for + the current template). +4. **Add a row to the correct table in this file.** +5. Build and run any test — a colliding offset panics at init, so a green test + run proves the new band is clear. diff --git a/net/rpc/rpcerr/registry.go b/net/rpc/rpcerr/registry.go index 0860ee2a5..9dc301a52 100644 --- a/net/rpc/rpcerr/registry.go +++ b/net/rpc/rpcerr/registry.go @@ -48,6 +48,10 @@ func Unwrap(e error) error { return err } +// ErrGroup is the base offset for a service's error codes. Every concrete +// error registers at offset+localCode into the process-global errsMap, which +// panics on a duplicate. Offsets are a namespace shared across all repos that +// link any-sync — reserve a new one in docs/rpc-error-offsets.md. type ErrGroup int64 func (g ErrGroup) Register(err error, code uint64) error { From 93a00425281ea5e9e959bb0736efebac10818612 Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Thu, 2 Jul 2026 01:52:55 +0200 Subject: [PATCH 07/20] =?UTF-8?q?feat(sync):=20selective-sync=20hooks=20?= =?UTF-8?q?=E2=80=94=20tree=20probe=20requests=20+=20optional=20PullFilter?= =?UTF-8?q?=20(SYN-18)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two opt-in, backward-compatible hooks for participants that head-sync a whole space but materialize only selected tree types (filenode-v2 broker): - TreeFullSyncRequest.probe: the responder replies with the tree's root and current heads only, no change bodies. Old responders ignore the flag and stream the full tree. Plumbed via BuildTreeOpts.Probe. - treesyncer.PullFilter: optional extension of the registered TreeSyncer, consulted by objectsync on a head update for a locally-missing tree; head updates already carry the raw root change, so the filter can classify the tree (cleartext changeType) and decline the pull with no extra round trips. Co-Authored-By: Claude Fable 5 --- commonspace/object/tree/synctree/request.go | 12 ++ .../object/tree/synctree/synchandler.go | 25 ++++ .../object/tree/synctree/synchandler_test.go | 56 +++++++++ commonspace/object/tree/synctree/synctree.go | 6 + .../object/tree/synctree/treeremotegetter.go | 8 +- .../tree/synctree/treeremotegetter_test.go | 30 +++++ .../treechangeproto/protos/treechange.proto | 4 + .../tree/treechangeproto/treechange.pb.go | 24 +++- .../treechangeproto/treechange_vtproto.pb.go | 33 ++++++ commonspace/object/treesyncer/treesyncer.go | 12 ++ commonspace/objecttreebuilder/treebuilder.go | 5 + .../sync/objectsync/objectsync_test.go | 108 ++++++++++++++++++ commonspace/sync/objectsync/synchandler.go | 32 +++++- 13 files changed, 343 insertions(+), 12 deletions(-) diff --git a/commonspace/object/tree/synctree/request.go b/commonspace/object/tree/synctree/request.go index 3f092915d..a978e1590 100644 --- a/commonspace/object/tree/synctree/request.go +++ b/commonspace/object/tree/synctree/request.go @@ -9,6 +9,7 @@ type InnerRequest struct { heads []string snapshotPath []string root *treechangeproto.RawTreeChangeWithId + probe bool } func (r *InnerRequest) MsgSize() uint64 { @@ -29,10 +30,21 @@ func NewRequest(peerId, spaceId, objectId string, heads []string, snapshotPath [ }) } +// NewProbeRequest creates a request for the tree's root and current heads +// only — the responder sends no change bodies. Old responders ignore the +// probe flag and stream the full tree, so the caller's response collector +// must handle both shapes. +func NewProbeRequest(peerId, spaceId, objectId string) *objectmessages.Request { + return objectmessages.NewRequest(peerId, spaceId, objectId, &InnerRequest{ + probe: true, + }) +} + func (r *InnerRequest) Marshall() ([]byte, error) { msg := &treechangeproto.TreeFullSyncRequest{ Heads: r.heads, SnapshotPath: r.snapshotPath, + Probe: r.probe, } req := treechangeproto.WrapFullRequest(msg, r.root) return req.MarshalVT() diff --git a/commonspace/object/tree/synctree/synchandler.go b/commonspace/object/tree/synctree/synchandler.go index 0edeb7d91..7c62954c6 100644 --- a/commonspace/object/tree/synctree/synchandler.go +++ b/commonspace/object/tree/synctree/synchandler.go @@ -113,6 +113,31 @@ func (s *syncHandler) HandleStreamRequest(ctx context.Context, rq syncdeps.Reque if request == nil { return nil, ErrUnexpectedRequestType } + if request.Probe { + // A probe declares nothing about the requester's state, so no + // counter-request — just root + current heads, no change bodies. + s.tree.Lock() + heads := make([]string, len(s.tree.Heads())) + copy(heads, s.tree.Heads()) + snapshotPath, err := s.tree.SnapshotPath() + root := s.tree.Header() + s.tree.Unlock() + if err != nil { + return nil, err + } + resp := &response.Response{ + SpaceId: s.spaceId, + ObjectId: req.ObjectId(), + Heads: heads, + SnapshotPath: snapshotPath, + Root: root, + } + protoResp, err := resp.ProtoMessage() + if err != nil { + return nil, err + } + return nil, send(protoResp) + } s.tree.Lock() curHeads := s.tree.Heads() log.Debug("got stream request", diff --git a/commonspace/object/tree/synctree/synchandler_test.go b/commonspace/object/tree/synctree/synchandler_test.go index 1bbbcbb06..caa3d039e 100644 --- a/commonspace/object/tree/synctree/synchandler_test.go +++ b/commonspace/object/tree/synctree/synchandler_test.go @@ -12,6 +12,7 @@ import ( "github.com/anyproto/any-sync/commonspace/object/tree/synctree/response" "github.com/anyproto/any-sync/commonspace/object/tree/synctree/response/mock_response" "github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto" + "github.com/anyproto/any-sync/commonspace/spacesyncproto" "github.com/anyproto/any-sync/commonspace/sync/objectsync/objectmessages" "github.com/anyproto/any-sync/commonspace/syncstatus/mock_syncstatus" "github.com/anyproto/any-sync/net/peer" @@ -337,6 +338,61 @@ func TestSyncHandler_HandleStreamRequest(t *testing.T) { }) } +func TestSyncHandler_HandleStreamRequest_Probe(t *testing.T) { + t.Run("probe request, we send root and heads without changes, no request", func(t *testing.T) { + fx := newSyncHandlerFixture(t) + defer fx.finish() + fullRequest := &treechangeproto.TreeFullSyncRequest{ + Probe: true, + } + wrapped := treechangeproto.WrapFullRequest(fullRequest, nil) + marshaled, err := wrapped.MarshalVT() + require.NoError(t, err) + request := objectmessages.NewByteRequest("peerId", "spaceId", "objectId", marshaled) + root := &treechangeproto.RawTreeChangeWithId{ + RawChange: []byte("rootChange"), + Id: "objectId", + } + fx.tree.EXPECT().Heads().AnyTimes().Return([]string{"curHead"}) + fx.tree.EXPECT().SnapshotPath().Return([]string{"objectId"}, nil) + fx.tree.EXPECT().Header().Return(root) + ctx = peer.CtxWithPeerId(ctx, "peerId") + callCount := 0 + req, err := fx.syncHandler.HandleStreamRequest(ctx, request, testUpdater{}, func(resp proto.Message) error { + callCount++ + msg, ok := resp.(*spacesyncproto.ObjectSyncMessage) + require.True(t, ok) + treeMsg := &treechangeproto.TreeSyncMessage{} + require.NoError(t, treeMsg.UnmarshalVT(msg.Payload)) + fullResp := treeMsg.GetContent().GetFullSyncResponse() + require.NotNil(t, fullResp) + require.Equal(t, []string{"curHead"}, fullResp.Heads) + require.Empty(t, fullResp.Changes) + require.Equal(t, root.Id, treeMsg.RootChange.Id) + require.Equal(t, root.RawChange, treeMsg.RootChange.RawChange) + return nil + }) + require.NoError(t, err) + require.Nil(t, req) + require.Equal(t, 1, callCount) + }) +} + +func TestNewProbeRequest(t *testing.T) { + req := NewProbeRequest("peerId", "spaceId", "objectId") + protoMsg, err := req.Proto() + require.NoError(t, err) + msg, ok := protoMsg.(*spacesyncproto.ObjectSyncMessage) + require.True(t, ok) + treeMsg := &treechangeproto.TreeSyncMessage{} + require.NoError(t, treeMsg.UnmarshalVT(msg.Payload)) + fullReq := treeMsg.GetContent().GetFullSyncRequest() + require.NotNil(t, fullReq) + require.True(t, fullReq.Probe) + require.Empty(t, fullReq.Heads) + require.Empty(t, fullReq.Changes) +} + func TestSyncHandler_HandleResponse(t *testing.T) { t.Run("handle response with changes", func(t *testing.T) { fx := newSyncHandlerFixture(t) diff --git a/commonspace/object/tree/synctree/synctree.go b/commonspace/object/tree/synctree/synctree.go index dd597d502..33816a902 100644 --- a/commonspace/object/tree/synctree/synctree.go +++ b/commonspace/object/tree/synctree/synctree.go @@ -85,6 +85,12 @@ type BuildDeps struct { BuildObjectTree objecttree.BuildObjectTreeFunc ValidateObjectTree objecttree.ValidatorFunc StatsCollector *TreeStatsCollector + // Probe makes the remote fetch request the tree's root and current + // heads only (no change bodies). The response never yields a usable + // tree by itself, so Probe callers must supply a ValidateObjectTree + // that inspects the payload and returns an error to abort the build. + // Old responders ignore the flag and stream the full tree. + Probe bool } var newTreeGetter = func(deps BuildDeps, treeId string) treeGetter { diff --git a/commonspace/object/tree/synctree/treeremotegetter.go b/commonspace/object/tree/synctree/treeremotegetter.go index 0de5399a9..60d11825b 100644 --- a/commonspace/object/tree/synctree/treeremotegetter.go +++ b/commonspace/object/tree/synctree/treeremotegetter.go @@ -6,6 +6,7 @@ import ( "github.com/anyproto/any-sync/commonspace/object/tree/objecttree" "github.com/anyproto/any-sync/commonspace/object/tree/treestorage" + "github.com/anyproto/any-sync/commonspace/sync/objectsync/objectmessages" "github.com/anyproto/any-sync/net/peer" ) @@ -51,7 +52,12 @@ func (t treeRemoteGetter) getPeers(ctx context.Context) (peerIds []string, err e func (t treeRemoteGetter) treeRequest(ctx context.Context, peerId string) (collector *fullResponseCollector, err error) { collector = createCollector(t.deps) - req := t.deps.SyncClient.CreateNewTreeRequest(peerId, t.treeId) + var req *objectmessages.Request + if t.deps.Probe { + req = NewProbeRequest(peerId, t.deps.SpaceId, t.treeId) + } else { + req = t.deps.SyncClient.CreateNewTreeRequest(peerId, t.treeId) + } err = t.deps.SyncClient.SendTreeRequest(ctx, req, collector) if err != nil { return nil, err diff --git a/commonspace/object/tree/synctree/treeremotegetter_test.go b/commonspace/object/tree/synctree/treeremotegetter_test.go index d622df5ac..650fe3552 100644 --- a/commonspace/object/tree/synctree/treeremotegetter_test.go +++ b/commonspace/object/tree/synctree/treeremotegetter_test.go @@ -9,7 +9,9 @@ import ( "go.uber.org/mock/gomock" "github.com/anyproto/any-sync/commonspace/object/tree/synctree/mock_synctree" + "github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto" "github.com/anyproto/any-sync/commonspace/peermanager/mock_peermanager" + "github.com/anyproto/any-sync/commonspace/spacesyncproto" "github.com/anyproto/any-sync/commonspace/sync/objectsync/objectmessages" "github.com/anyproto/any-sync/net/peer" "github.com/anyproto/any-sync/net/peer/mock_peer" @@ -103,4 +105,32 @@ func TestTreeRemoteGetter(t *testing.T) { _, _, err := fx.treeGetter.treeRequestLoop(tCtx) require.Error(t, err) }) + + t.Run("probe deps send a probe request instead of a full one", func(t *testing.T) { + fx := newTreeRemoteGetterFixture(t) + defer fx.stop() + fx.treeGetter.deps.Probe = true + coll := newFullResponseCollector(BuildDeps{}) + createCollector = func(deps BuildDeps) *fullResponseCollector { + return coll + } + tCtx := peer.CtxWithPeerId(ctx, peerId) + fx.syncClientMock.EXPECT().SendTreeRequest(tCtx, gomock.Any(), coll).DoAndReturn( + func(_ context.Context, rq any, _ any) error { + req, ok := rq.(*objectmessages.Request) + require.True(t, ok) + protoMsg, err := req.Proto() + require.NoError(t, err) + treeMsg := &treechangeproto.TreeSyncMessage{} + require.NoError(t, treeMsg.UnmarshalVT(protoMsg.(*spacesyncproto.ObjectSyncMessage).Payload)) + fullReq := treeMsg.GetContent().GetFullSyncRequest() + require.NotNil(t, fullReq) + require.True(t, fullReq.Probe) + return nil + }) + retColl, pId, err := fx.treeGetter.treeRequestLoop(tCtx) + require.NoError(t, err) + require.Equal(t, peerId, pId) + require.Equal(t, coll, retColl) + }) } diff --git a/commonspace/object/tree/treechangeproto/protos/treechange.proto b/commonspace/object/tree/treechangeproto/protos/treechange.proto index c52ede9a7..f8e57becb 100644 --- a/commonspace/object/tree/treechangeproto/protos/treechange.proto +++ b/commonspace/object/tree/treechangeproto/protos/treechange.proto @@ -121,6 +121,10 @@ message TreeFullSyncRequest { repeated string heads = 1; repeated RawTreeChangeWithId changes = 2; repeated string snapshotPath = 3; + // probe asks the responder for the tree's root and current heads only, + // with no change bodies. Old responders ignore it and stream the full + // tree; a probing requester must be ready for both response shapes. + bool probe = 4; } // TreeFullSyncResponse is a message sent as a response for a specific full sync diff --git a/commonspace/object/tree/treechangeproto/treechange.pb.go b/commonspace/object/tree/treechangeproto/treechange.pb.go index a59611635..fd7057bca 100644 --- a/commonspace/object/tree/treechangeproto/treechange.pb.go +++ b/commonspace/object/tree/treechangeproto/treechange.pb.go @@ -803,10 +803,14 @@ func (x *TreeHeadUpdate) GetSnapshotPath() []string { // TreeHeadUpdate is a message sent when document needs full sync type TreeFullSyncRequest struct { - state protoimpl.MessageState `protogen:"open.v1"` - Heads []string `protobuf:"bytes,1,rep,name=heads,proto3" json:"heads,omitempty"` - Changes []*RawTreeChangeWithId `protobuf:"bytes,2,rep,name=changes,proto3" json:"changes,omitempty"` - SnapshotPath []string `protobuf:"bytes,3,rep,name=snapshotPath,proto3" json:"snapshotPath,omitempty"` + state protoimpl.MessageState `protogen:"open.v1"` + Heads []string `protobuf:"bytes,1,rep,name=heads,proto3" json:"heads,omitempty"` + Changes []*RawTreeChangeWithId `protobuf:"bytes,2,rep,name=changes,proto3" json:"changes,omitempty"` + SnapshotPath []string `protobuf:"bytes,3,rep,name=snapshotPath,proto3" json:"snapshotPath,omitempty"` + // probe asks the responder for the tree's root and current heads only, + // with no change bodies. Old responders ignore it and stream the full + // tree; a probing requester must be ready for both response shapes. + Probe bool `protobuf:"varint,4,opt,name=probe,proto3" json:"probe,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -862,6 +866,13 @@ func (x *TreeFullSyncRequest) GetSnapshotPath() []string { return nil } +func (x *TreeFullSyncRequest) GetProbe() bool { + if x != nil { + return x.Probe + } + return false +} + // TreeFullSyncResponse is a message sent as a response for a specific full sync type TreeFullSyncResponse struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -1096,11 +1107,12 @@ const file_treechange_proto_rawDesc = "" + "\x0eTreeHeadUpdate\x12\x14\n" + "\x05heads\x18\x01 \x03(\tR\x05heads\x129\n" + "\achanges\x18\x02 \x03(\v2\x1f.treechange.RawTreeChangeWithIdR\achanges\x12\"\n" + - "\fsnapshotPath\x18\x03 \x03(\tR\fsnapshotPath\"\x8a\x01\n" + + "\fsnapshotPath\x18\x03 \x03(\tR\fsnapshotPath\"\xa0\x01\n" + "\x13TreeFullSyncRequest\x12\x14\n" + "\x05heads\x18\x01 \x03(\tR\x05heads\x129\n" + "\achanges\x18\x02 \x03(\v2\x1f.treechange.RawTreeChangeWithIdR\achanges\x12\"\n" + - "\fsnapshotPath\x18\x03 \x03(\tR\fsnapshotPath\"\x8b\x01\n" + + "\fsnapshotPath\x18\x03 \x03(\tR\fsnapshotPath\x12\x14\n" + + "\x05probe\x18\x04 \x01(\bR\x05probe\"\x8b\x01\n" + "\x14TreeFullSyncResponse\x12\x14\n" + "\x05heads\x18\x01 \x03(\tR\x05heads\x129\n" + "\achanges\x18\x02 \x03(\v2\x1f.treechange.RawTreeChangeWithIdR\achanges\x12\"\n" + diff --git a/commonspace/object/tree/treechangeproto/treechange_vtproto.pb.go b/commonspace/object/tree/treechangeproto/treechange_vtproto.pb.go index 467552b47..2725bb80c 100644 --- a/commonspace/object/tree/treechangeproto/treechange_vtproto.pb.go +++ b/commonspace/object/tree/treechangeproto/treechange_vtproto.pb.go @@ -706,6 +706,16 @@ func (m *TreeFullSyncRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if m.Probe { + i-- + if m.Probe { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x20 + } if len(m.SnapshotPath) > 0 { for iNdEx := len(m.SnapshotPath) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.SnapshotPath[iNdEx]) @@ -1209,6 +1219,9 @@ func (m *TreeFullSyncRequest) SizeVT() (n int) { n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) } } + if m.Probe { + n += 2 + } n += len(m.unknownFields) return n } @@ -3131,6 +3144,26 @@ func (m *TreeFullSyncRequest) UnmarshalVT(dAtA []byte) error { } m.SnapshotPath = append(m.SnapshotPath, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Probe", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Probe = bool(v != 0) default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) diff --git a/commonspace/object/treesyncer/treesyncer.go b/commonspace/object/treesyncer/treesyncer.go index 1ebe8267b..463c42568 100644 --- a/commonspace/object/treesyncer/treesyncer.go +++ b/commonspace/object/treesyncer/treesyncer.go @@ -5,6 +5,7 @@ import ( "context" "github.com/anyproto/any-sync/app" + "github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto" "github.com/anyproto/any-sync/net/peer" ) @@ -17,3 +18,14 @@ type TreeSyncer interface { ShouldSync(peerId string) bool SyncAll(ctx context.Context, p peer.Peer, existing, missing []string) error } + +// PullFilter is an optional extension of the registered TreeSyncer. When a +// head update arrives for a tree that does not exist locally, objectsync +// consults it before queueing the fetch: returning false swallows the update +// and the tree is not pulled. rootChange is the tree's raw root (carried by +// every head update; its header — including changeType — is cleartext) and +// heads are the sender's current heads. TreeSyncers that do not implement +// the interface keep the default always-pull behavior. +type PullFilter interface { + ShouldPull(ctx context.Context, objectId string, rootChange *treechangeproto.RawTreeChangeWithId, heads []string) bool +} diff --git a/commonspace/objecttreebuilder/treebuilder.go b/commonspace/objecttreebuilder/treebuilder.go index a414f25a3..10723522f 100644 --- a/commonspace/objecttreebuilder/treebuilder.go +++ b/commonspace/objecttreebuilder/treebuilder.go @@ -30,6 +30,10 @@ type BuildTreeOpts struct { Listener updatelistener.UpdateListener TreeBuilder objecttree.BuildObjectTreeFunc TreeValidator objecttree.ValidatorFunc + // Probe requests the tree's root and current heads only (no change + // bodies) when the tree is fetched remotely. Requires a TreeValidator + // that inspects the payload and aborts — see synctree.BuildDeps.Probe. + Probe bool } const CName = "common.commonspace.objecttreebuilder" @@ -155,6 +159,7 @@ func (t *treeBuilder) BuildTree(ctx context.Context, id string, opts BuildTreeOp BuildObjectTree: treeBuilder, ValidateObjectTree: opts.TreeValidator, StatsCollector: t.treeStats, + Probe: opts.Probe, } t.treesUsed.Add(1) t.log.Debug("incrementing counter", zap.String("id", id), zap.Int32("trees", t.treesUsed.Load())) diff --git a/commonspace/sync/objectsync/objectsync_test.go b/commonspace/sync/objectsync/objectsync_test.go index 1062538eb..3ac7a5e9a 100644 --- a/commonspace/sync/objectsync/objectsync_test.go +++ b/commonspace/sync/objectsync/objectsync_test.go @@ -15,6 +15,7 @@ import ( "github.com/anyproto/any-sync/commonspace/object/tree/synctree" "github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto" "github.com/anyproto/any-sync/commonspace/object/treemanager" + "github.com/anyproto/any-sync/commonspace/object/treesyncer" "github.com/anyproto/any-sync/commonspace/objectmanager/mock_objectmanager" "github.com/anyproto/any-sync/commonspace/spacestate" "github.com/anyproto/any-sync/commonspace/sync/objectsync/objectmessages" @@ -79,6 +80,85 @@ func TestObjectSync_HandleHeadUpdate(t *testing.T) { req := synctree.NewRequest(update.Meta.PeerId, update.Meta.SpaceId, update.Meta.ObjectId, nil, nil, nil) require.Equal(t, req, r) }) + t.Run("handle head update object missing, pull filter declines, update swallowed", func(t *testing.T) { + root := &treechangeproto.RawTreeChangeWithId{ + RawChange: []byte("rootChange"), + Id: "objectId", + } + var gotRoot *treechangeproto.RawTreeChangeWithId + var gotHeads []string + fx := newFixtureWithPullFilter(t, func(_ context.Context, objectId string, rootChange *treechangeproto.RawTreeChangeWithId, heads []string) bool { + require.Equal(t, "objectId", objectId) + gotRoot = rootChange + gotHeads = heads + return false + }) + defer fx.close(t) + update := headUpdateWithRoot(t, root, []string{"headId"}) + ctx = peer.CtxWithPeerId(ctx, "peerId") + fx.objectManager.EXPECT().GetObject(context.Background(), "objectId").Return(nil, fmt.Errorf("no object")) + r, err := fx.objectSync.HandleHeadUpdate(ctx, update) + require.NoError(t, err) + require.Nil(t, r) + require.Equal(t, root.Id, gotRoot.Id) + require.Equal(t, root.RawChange, gotRoot.RawChange) + require.Equal(t, []string{"headId"}, gotHeads) + }) + t.Run("handle head update object missing, pull filter accepts, return request", func(t *testing.T) { + root := &treechangeproto.RawTreeChangeWithId{ + RawChange: []byte("rootChange"), + Id: "objectId", + } + fx := newFixtureWithPullFilter(t, func(_ context.Context, _ string, _ *treechangeproto.RawTreeChangeWithId, _ []string) bool { + return true + }) + defer fx.close(t) + update := headUpdateWithRoot(t, root, []string{"headId"}) + ctx = peer.CtxWithPeerId(ctx, "peerId") + fx.objectManager.EXPECT().GetObject(context.Background(), "objectId").Return(nil, fmt.Errorf("no object")) + r, err := fx.objectSync.HandleHeadUpdate(ctx, update) + require.NoError(t, err) + req := synctree.NewRequest("peerId", "spaceId", "objectId", nil, nil, nil) + require.Equal(t, req, r) + }) + t.Run("handle head update object missing, no root in update, pull filter skipped, return request", func(t *testing.T) { + fx := newFixtureWithPullFilter(t, func(_ context.Context, _ string, _ *treechangeproto.RawTreeChangeWithId, _ []string) bool { + require.Fail(t, "filter must not be consulted without a root change") + return false + }) + defer fx.close(t) + update := &objectmessages.HeadUpdate{ + Meta: objectmessages.ObjectMeta{ + PeerId: "peerId", + ObjectId: "objectId", + SpaceId: "spaceId", + }, + } + ctx = peer.CtxWithPeerId(ctx, "peerId") + fx.objectManager.EXPECT().GetObject(context.Background(), "objectId").Return(nil, fmt.Errorf("no object")) + r, err := fx.objectSync.HandleHeadUpdate(ctx, update) + require.NoError(t, err) + req := synctree.NewRequest("peerId", "spaceId", "objectId", nil, nil, nil) + require.Equal(t, req, r) + }) +} + +func headUpdateWithRoot(t *testing.T, root *treechangeproto.RawTreeChangeWithId, heads []string) *objectmessages.HeadUpdate { + treeHeadUpdate := &treechangeproto.TreeHeadUpdate{ + Heads: heads, + SnapshotPath: []string{root.Id}, + } + wrapped := treechangeproto.WrapHeadUpdate(treeHeadUpdate, root) + marshaled, err := wrapped.MarshalVT() + require.NoError(t, err) + return &objectmessages.HeadUpdate{ + Bytes: marshaled, + Meta: objectmessages.ObjectMeta{ + PeerId: "peerId", + ObjectId: "objectId", + SpaceId: "spaceId", + }, + } } func TestObjectSync_HandleStreamRequest(t *testing.T) { @@ -168,6 +248,31 @@ type fixture struct { } func newFixture(t *testing.T) *fixture { + return newFixtureWithPullFilter(t, nil) +} + +// filteringTreeSyncer is a minimal TreeSyncer that also implements the +// optional treesyncer.PullFilter extension. +type filteringTreeSyncer struct { + shouldPull func(ctx context.Context, objectId string, rootChange *treechangeproto.RawTreeChangeWithId, heads []string) bool +} + +func (f *filteringTreeSyncer) Init(a *app.App) error { return nil } +func (f *filteringTreeSyncer) Name() string { return treesyncer.CName } +func (f *filteringTreeSyncer) Run(ctx context.Context) error { return nil } +func (f *filteringTreeSyncer) Close(ctx context.Context) error { return nil } +func (f *filteringTreeSyncer) StartSync() {} +func (f *filteringTreeSyncer) StopSync() {} +func (f *filteringTreeSyncer) ShouldSync(string) bool { return true } +func (f *filteringTreeSyncer) SyncAll(ctx context.Context, p peer.Peer, existing, missing []string) error { + return nil +} + +func (f *filteringTreeSyncer) ShouldPull(ctx context.Context, objectId string, rootChange *treechangeproto.RawTreeChangeWithId, heads []string) bool { + return f.shouldPull(ctx, objectId, rootChange, heads) +} + +func newFixtureWithPullFilter(t *testing.T, shouldPull func(ctx context.Context, objectId string, rootChange *treechangeproto.RawTreeChangeWithId, heads []string) bool) *fixture { fx := &fixture{ a: &app.App{}, } @@ -186,6 +291,9 @@ func newFixture(t *testing.T) *fixture { Register(fx.keyValue). Register(syncstatus.NewNoOpSyncStatus()). Register(fx.objectSync) + if shouldPull != nil { + fx.a.Register(&filteringTreeSyncer{shouldPull: shouldPull}) + } require.NoError(t, fx.a.Start(context.Background())) return fx } diff --git a/commonspace/sync/objectsync/synchandler.go b/commonspace/sync/objectsync/synchandler.go index 89509bf69..73c1aebee 100644 --- a/commonspace/sync/objectsync/synchandler.go +++ b/commonspace/sync/objectsync/synchandler.go @@ -15,6 +15,7 @@ import ( "github.com/anyproto/any-sync/commonspace/object/tree/synctree" "github.com/anyproto/any-sync/commonspace/object/tree/treechangeproto" "github.com/anyproto/any-sync/commonspace/object/treemanager" + "github.com/anyproto/any-sync/commonspace/object/treesyncer" "github.com/anyproto/any-sync/commonspace/objectmanager" "github.com/anyproto/any-sync/commonspace/spacestate" "github.com/anyproto/any-sync/commonspace/spacesyncproto" @@ -31,11 +32,12 @@ var ErrUnexpectedHeadUpdateType = errors.New("unexpected head update type") var log = logger.NewNamed(syncdeps.CName) type objectSync struct { - spaceId string - pool pool.Service - manager objectmanager.ObjectManager - status syncstatus.StatusUpdater - keyValue kvinterfaces.KeyValueService + spaceId string + pool pool.Service + manager objectmanager.ObjectManager + status syncstatus.StatusUpdater + keyValue kvinterfaces.KeyValueService + pullFilter treesyncer.PullFilter } func New() syncdeps.SyncHandler { @@ -48,6 +50,8 @@ func (o *objectSync) Init(a *app.App) (err error) { o.keyValue = a.MustComponent(kvinterfaces.CName).(kvinterfaces.KeyValueService) o.status = a.MustComponent(syncstatus.CName).(syncstatus.StatusUpdater) o.spaceId = a.MustComponent(spacestate.CName).(*spacestate.SpaceState).SpaceId + // optional extension of the registered tree syncer + o.pullFilter, _ = a.Component(treesyncer.CName).(treesyncer.PullFilter) return } @@ -69,6 +73,9 @@ func (o *objectSync) HandleHeadUpdate(ctx context.Context, headUpdate drpc.Messa } obj, err := o.manager.GetObject(context.Background(), update.Meta.ObjectId) if err != nil { + if o.pullFilter != nil && !o.shouldPull(ctx, update) { + return nil, nil + } return synctree.NewRequest(peerId, update.Meta.SpaceId, update.Meta.ObjectId, nil, nil, nil), nil } objHandler, ok := obj.(syncdeps.ObjectSyncHandler) @@ -78,6 +85,21 @@ func (o *objectSync) HandleHeadUpdate(ctx context.Context, headUpdate drpc.Messa return objHandler.HandleHeadUpdate(ctx, o.status, update) } +// shouldPull consults the pull filter for a locally-missing tree. Head +// updates carry the tree's raw root change, so the filter can classify the +// tree without fetching anything. Malformed updates default to pull. +func (o *objectSync) shouldPull(ctx context.Context, update *objectmessages.HeadUpdate) bool { + treeSyncMsg := &treechangeproto.TreeSyncMessage{} + if err := treeSyncMsg.UnmarshalVT(update.Bytes); err != nil { + return true + } + contentUpdate := treeSyncMsg.GetContent().GetHeadUpdate() + if contentUpdate == nil || treeSyncMsg.RootChange == nil { + return true + } + return o.pullFilter.ShouldPull(ctx, update.Meta.ObjectId, treeSyncMsg.RootChange, contentUpdate.Heads) +} + func (o *objectSync) HandleStreamRequest(ctx context.Context, rq syncdeps.Request, updater syncdeps.QueueSizeUpdater, sendResponse func(resp proto.Message) error) (syncdeps.Request, error) { obj, err := o.manager.GetObject(context.Background(), rq.ObjectId()) if err != nil { From b3d23a5badc8f94b83044e4f68b6b40476aad630 Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Thu, 2 Jul 2026 13:47:07 +0200 Subject: [PATCH 08/20] feat(fileprotov2): add NotResponsible error codes for chash routing (SYN-31) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Spaces chash-shard across the NodeTypeFileV2 pool (RF=2), and a filenode must reject — and never activate/sync — spaces outside its responsibility set. Give that rejection a dedicated retryable routing signal instead of overloading Forbidden, so client SDKs can re-route to the responsible pair: * whole-RPC ErrCodes.NotResponsible = 4 (registered in fileprotov2err at offset 800+4) — for the single-space RPCs where the entire request is misrouted; * per-item ErrCode.ErrNotResponsible = 6 — for SpaceInfo, whose batch may legally mix spaces owned by different pairs. Co-Authored-By: Claude Fable 5 --- .../fileprotov2err/fileprotov2err.go | 3 ++ commonfile/fileproto/fileprotov2/filev2.pb.go | 50 +++++++++++-------- .../fileproto/fileprotov2/protos/filev2.proto | 17 ++++--- 3 files changed, 42 insertions(+), 28 deletions(-) diff --git a/commonfile/fileproto/fileprotov2/fileprotov2err/fileprotov2err.go b/commonfile/fileproto/fileprotov2/fileprotov2err/fileprotov2err.go index a9d19617c..6905073e5 100644 --- a/commonfile/fileproto/fileprotov2/fileprotov2err/fileprotov2err.go +++ b/commonfile/fileproto/fileprotov2/fileprotov2err/fileprotov2err.go @@ -18,6 +18,7 @@ var ( ErrForbidden = errGroup.Register(fmt.Errorf("forbidden"), uint64(fileprotov2.ErrCodes_Forbidden)) ErrInvalidRequest = errGroup.Register(fmt.Errorf("invalid request"), uint64(fileprotov2.ErrCodes_InvalidRequest)) ErrQuerySizeExceeded = errGroup.Register(fmt.Errorf("query size exceeded"), uint64(fileprotov2.ErrCodes_QuerySizeExceeded)) + ErrNotResponsible = errGroup.Register(fmt.Errorf("node is not responsible for the space"), uint64(fileprotov2.ErrCodes_NotResponsible)) ) // FromCode maps a whole-RPC ErrCodes value to its registered drpc error so @@ -31,6 +32,8 @@ func FromCode(code fileprotov2.ErrCodes) error { return ErrInvalidRequest case fileprotov2.ErrCodes_QuerySizeExceeded: return ErrQuerySizeExceeded + case fileprotov2.ErrCodes_NotResponsible: + return ErrNotResponsible default: return ErrUnexpected } diff --git a/commonfile/fileproto/fileprotov2/filev2.pb.go b/commonfile/fileproto/fileprotov2/filev2.pb.go index 6066437cf..6e432fad4 100644 --- a/commonfile/fileproto/fileprotov2/filev2.pb.go +++ b/commonfile/fileproto/fileprotov2/filev2.pb.go @@ -24,11 +24,11 @@ const ( // ErrCodes are WHOLE-RPC (drpc-level) errors, mirroring v1 filesync.ErrCodes. // Wired through rpcerr.ErrGroup(ErrCodes_ErrorOffset) in the fileprotov2err // package. Returned as a NON-nil drpc error ONLY for auth / malformed / -// batch-too-large. Never used to report the outcome of an individual item -// (that is the per-item ErrCode below). Zero value Unexpected=0 registers at -// offset+0 (=800), so an uncoded server fault reads as Unexpected — exactly as -// v1 does at 200. Offset 800 is the only free rpcerr block (v1=200, 100/300/ -// 400/500/600/700 are taken). +// batch-too-large / misrouted. Never used to report the outcome of an +// individual item (that is the per-item ErrCode below). Zero value +// Unexpected=0 registers at offset+0 (=800), so an uncoded server fault reads +// as Unexpected — exactly as v1 does at 200. Offset 800 is the only free +// rpcerr block (v1=200, 100/300/400/500/600/700 are taken). type ErrCodes int32 const ( @@ -36,6 +36,7 @@ const ( ErrCodes_Forbidden ErrCodes = 1 // connection identity is not permitted at all ErrCodes_InvalidRequest ErrCodes = 2 // malformed: empty spaceId, empty items, bad rootCid bytes ErrCodes_QuerySizeExceeded ErrCodes = 3 // batch too large (too many items in one request) + ErrCodes_NotResponsible ErrCodes = 4 // retryable routing signal: this node is not in the space's chash pair ErrCodes_ErrorOffset ErrCodes = 800 // rpcerr.ErrGroup base for v2 (constant, never an error) ) @@ -46,6 +47,7 @@ var ( 1: "Forbidden", 2: "InvalidRequest", 3: "QuerySizeExceeded", + 4: "NotResponsible", 800: "ErrorOffset", } ErrCodes_value = map[string]int32{ @@ -53,6 +55,7 @@ var ( "Forbidden": 1, "InvalidRequest": 2, "QuerySizeExceeded": 3, + "NotResponsible": 4, "ErrorOffset": 800, } ) @@ -95,12 +98,13 @@ func (ErrCodes) EnumDescriptor() ([]byte, []int) { type ErrCode int32 const ( - ErrCode_Ok ErrCode = 0 // item succeeded; the typed payload is populated - ErrCode_ErrUnexpected ErrCode = 1 // internal error handling this single item - ErrCode_ErrCidNotFound ErrCode = 2 // sign/download: no such stored file for this rootCid - ErrCode_ErrForbidden ErrCode = 3 // identity may not access this item's space/file - ErrCode_ErrLimitExceeded ErrCode = 4 // upload rejected: space/account storage limit reached - ErrCode_ErrSpaceNotFound ErrCode = 5 // quota-info: unknown/deleted space id + ErrCode_Ok ErrCode = 0 // item succeeded; the typed payload is populated + ErrCode_ErrUnexpected ErrCode = 1 // internal error handling this single item + ErrCode_ErrCidNotFound ErrCode = 2 // sign/download: no such stored file for this rootCid + ErrCode_ErrForbidden ErrCode = 3 // identity may not access this item's space/file + ErrCode_ErrLimitExceeded ErrCode = 4 // upload rejected: space/account storage limit reached + ErrCode_ErrSpaceNotFound ErrCode = 5 // quota-info: unknown/deleted space id + ErrCode_ErrNotResponsible ErrCode = 6 // retryable routing signal: this node is not in this item's space chash pair ) // Enum value maps for ErrCode. @@ -112,14 +116,16 @@ var ( 3: "ErrForbidden", 4: "ErrLimitExceeded", 5: "ErrSpaceNotFound", + 6: "ErrNotResponsible", } ErrCode_value = map[string]int32{ - "Ok": 0, - "ErrUnexpected": 1, - "ErrCidNotFound": 2, - "ErrForbidden": 3, - "ErrLimitExceeded": 4, - "ErrSpaceNotFound": 5, + "Ok": 0, + "ErrUnexpected": 1, + "ErrCidNotFound": 2, + "ErrForbidden": 3, + "ErrLimitExceeded": 4, + "ErrSpaceNotFound": 5, + "ErrNotResponsible": 6, } ) @@ -1197,21 +1203,23 @@ const file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDesc = "" + "\x12inflightUsageBytes\x18\x04 \x01(\x04R\x12inflightUsageBytes\x12\x1e\n" + "\n" + "filesCount\x18\x05 \x01(\x04R\n" + - "filesCount*f\n" + + "filesCount*z\n" + "\bErrCodes\x12\x0e\n" + "\n" + "Unexpected\x10\x00\x12\r\n" + "\tForbidden\x10\x01\x12\x12\n" + "\x0eInvalidRequest\x10\x02\x12\x15\n" + - "\x11QuerySizeExceeded\x10\x03\x12\x10\n" + - "\vErrorOffset\x10\xa0\x06*v\n" + + "\x11QuerySizeExceeded\x10\x03\x12\x12\n" + + "\x0eNotResponsible\x10\x04\x12\x10\n" + + "\vErrorOffset\x10\xa0\x06*\x8d\x01\n" + "\aErrCode\x12\x06\n" + "\x02Ok\x10\x00\x12\x11\n" + "\rErrUnexpected\x10\x01\x12\x12\n" + "\x0eErrCidNotFound\x10\x02\x12\x10\n" + "\fErrForbidden\x10\x03\x12\x14\n" + "\x10ErrLimitExceeded\x10\x04\x12\x14\n" + - "\x10ErrSpaceNotFound\x10\x052\xbf\x02\n" + + "\x10ErrSpaceNotFound\x10\x05\x12\x15\n" + + "\x11ErrNotResponsible\x10\x062\xbf\x02\n" + "\x06FileV2\x12?\n" + "\x06Upload\x12\x19.filesyncv2.UploadRequest\x1a\x1a.filesyncv2.UploadResponse\x12N\n" + "\vRequestSign\x12\x1e.filesyncv2.RequestSignRequest\x1a\x1f.filesyncv2.RequestSignResponse\x12Z\n" + diff --git a/commonfile/fileproto/fileprotov2/protos/filev2.proto b/commonfile/fileproto/fileprotov2/protos/filev2.proto index 337e618ce..7d4928168 100644 --- a/commonfile/fileproto/fileprotov2/protos/filev2.proto +++ b/commonfile/fileproto/fileprotov2/protos/filev2.proto @@ -26,8 +26,9 @@ option go_package = "commonfile/fileproto/fileprotov2"; // Two error layers: // 1. whole-RPC drpc error (err != nil), carrying an ErrCodes value from the // offset-800 rpcerr registry, returned ONLY for cross-cutting failures: -// failed/absent auth (Forbidden), malformed input (InvalidRequest), or an -// over-sized batch (QuerySizeExceeded); +// failed/absent auth (Forbidden), malformed input (InvalidRequest), an +// over-sized batch (QuerySizeExceeded), or a misrouted single-space +// request (NotResponsible — retry against the space's responsible pair); // 2. per-item business outcome lives in the in-body ErrCode `code` field // (err == nil for the RPC as a whole). service FileV2 { @@ -44,16 +45,17 @@ service FileV2 { // ErrCodes are WHOLE-RPC (drpc-level) errors, mirroring v1 filesync.ErrCodes. // Wired through rpcerr.ErrGroup(ErrCodes_ErrorOffset) in the fileprotov2err // package. Returned as a NON-nil drpc error ONLY for auth / malformed / -// batch-too-large. Never used to report the outcome of an individual item -// (that is the per-item ErrCode below). Zero value Unexpected=0 registers at -// offset+0 (=800), so an uncoded server fault reads as Unexpected — exactly as -// v1 does at 200. Offset 800 is the only free rpcerr block (v1=200, 100/300/ -// 400/500/600/700 are taken). +// batch-too-large / misrouted. Never used to report the outcome of an +// individual item (that is the per-item ErrCode below). Zero value +// Unexpected=0 registers at offset+0 (=800), so an uncoded server fault reads +// as Unexpected — exactly as v1 does at 200. Offset 800 is the only free +// rpcerr block (v1=200, 100/300/400/500/600/700 are taken). enum ErrCodes { Unexpected = 0; // unclassified server fault (registry fallback) Forbidden = 1; // connection identity is not permitted at all InvalidRequest = 2; // malformed: empty spaceId, empty items, bad rootCid bytes QuerySizeExceeded = 3; // batch too large (too many items in one request) + NotResponsible = 4; // retryable routing signal: this node is not in the space's chash pair ErrorOffset = 800; // rpcerr.ErrGroup base for v2 (constant, never an error) } @@ -72,6 +74,7 @@ enum ErrCode { ErrForbidden = 3; // identity may not access this item's space/file ErrLimitExceeded = 4; // upload rejected: space/account storage limit reached ErrSpaceNotFound = 5; // quota-info: unknown/deleted space id + ErrNotResponsible = 6; // retryable routing signal: this node is not in this item's space chash pair } // ---- Upload --------------------------------------------------------------- From 70c6397cc63bcee93f4e083b9b02264b306e54a1 Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Thu, 2 Jul 2026 13:59:34 +0200 Subject: [PATCH 09/20] feat(fileprotov2): add node-level Info RPC (SYN-31) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Downloads start PUBLIC-read: chat-message images must render with zero broker round-trips, so the client constructs durable object URLs straight from the rootCid — {publicReadBaseUrl}/blob/{spaceId}/{rootCid} — and GETs the CDN. The base URL is distributed by the filenode itself (not nodeconf) via a new node-level Info RPC that clients cache: no spaceId, no responsibility gate, no auth beyond the connection. InfoRequest is deliberately empty so future node metadata (capabilities, limits) extends it without a wire break; an empty publicReadBaseUrl means no public read — use RequestDownload. Co-Authored-By: Claude Fable 5 --- commonfile/fileproto/fileprotov2/filev2.pb.go | 115 ++++++++- .../fileproto/fileprotov2/filev2_drpc.pb.go | 42 +++- .../fileprotov2/filev2_vtproto.pb.go | 231 ++++++++++++++++++ .../fileproto/fileprotov2/protos/filev2.proto | 18 +- 4 files changed, 393 insertions(+), 13 deletions(-) diff --git a/commonfile/fileproto/fileprotov2/filev2.pb.go b/commonfile/fileproto/fileprotov2/filev2.pb.go index 6e432fad4..64f747220 100644 --- a/commonfile/fileproto/fileprotov2/filev2.pb.go +++ b/commonfile/fileproto/fileprotov2/filev2.pb.go @@ -1135,6 +1135,91 @@ func (x *SpaceQuota) GetFilesCount() uint64 { return 0 } +// InfoRequest is deliberately empty: Info is static node metadata, +// extensible later (capabilities, limits) without a wire break. +type InfoRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *InfoRequest) Reset() { + *x = InfoRequest{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *InfoRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*InfoRequest) ProtoMessage() {} + +func (x *InfoRequest) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[18] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use InfoRequest.ProtoReflect.Descriptor instead. +func (*InfoRequest) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{18} +} + +type InfoResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // publicReadBaseUrl, when non-empty, lets clients construct durable object + // URLs as {publicReadBaseUrl}/blob/{spaceId}/{rootCid} with zero broker + // round-trips; clients cache it. Empty = no public read: use RequestDownload. + PublicReadBaseUrl string `protobuf:"bytes,1,opt,name=publicReadBaseUrl,proto3" json:"publicReadBaseUrl,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *InfoResponse) Reset() { + *x = InfoResponse{} + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *InfoResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*InfoResponse) ProtoMessage() {} + +func (x *InfoResponse) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes[19] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use InfoResponse.ProtoReflect.Descriptor instead. +func (*InfoResponse) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP(), []int{19} +} + +func (x *InfoResponse) GetPublicReadBaseUrl() string { + if x != nil { + return x.PublicReadBaseUrl + } + return "" +} + var File_commonfile_fileproto_fileprotov2_protos_filev2_proto protoreflect.FileDescriptor const file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDesc = "" + @@ -1203,7 +1288,10 @@ const file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDesc = "" + "\x12inflightUsageBytes\x18\x04 \x01(\x04R\x12inflightUsageBytes\x12\x1e\n" + "\n" + "filesCount\x18\x05 \x01(\x04R\n" + - "filesCount*z\n" + + "filesCount\"\r\n" + + "\vInfoRequest\"<\n" + + "\fInfoResponse\x12,\n" + + "\x11publicReadBaseUrl\x18\x01 \x01(\tR\x11publicReadBaseUrl*z\n" + "\bErrCodes\x12\x0e\n" + "\n" + "Unexpected\x10\x00\x12\r\n" + @@ -1219,12 +1307,13 @@ const file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDesc = "" + "\fErrForbidden\x10\x03\x12\x14\n" + "\x10ErrLimitExceeded\x10\x04\x12\x14\n" + "\x10ErrSpaceNotFound\x10\x05\x12\x15\n" + - "\x11ErrNotResponsible\x10\x062\xbf\x02\n" + + "\x11ErrNotResponsible\x10\x062\xfa\x02\n" + "\x06FileV2\x12?\n" + "\x06Upload\x12\x19.filesyncv2.UploadRequest\x1a\x1a.filesyncv2.UploadResponse\x12N\n" + "\vRequestSign\x12\x1e.filesyncv2.RequestSignRequest\x1a\x1f.filesyncv2.RequestSignResponse\x12Z\n" + "\x0fRequestDownload\x12\".filesyncv2.RequestDownloadRequest\x1a#.filesyncv2.RequestDownloadResponse\x12H\n" + - "\tSpaceInfo\x12\x1c.filesyncv2.SpaceInfoRequest\x1a\x1d.filesyncv2.SpaceInfoResponseB\"Z commonfile/fileproto/fileprotov2b\x06proto3" + "\tSpaceInfo\x12\x1c.filesyncv2.SpaceInfoRequest\x1a\x1d.filesyncv2.SpaceInfoResponse\x129\n" + + "\x04Info\x12\x17.filesyncv2.InfoRequest\x1a\x18.filesyncv2.InfoResponseB\"Z commonfile/fileproto/fileprotov2b\x06proto3" var ( file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescOnce sync.Once @@ -1239,7 +1328,7 @@ func file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDescGZIP() []b } var file_commonfile_fileproto_fileprotov2_protos_filev2_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes = make([]protoimpl.MessageInfo, 18) +var file_commonfile_fileproto_fileprotov2_protos_filev2_proto_msgTypes = make([]protoimpl.MessageInfo, 20) var file_commonfile_fileproto_fileprotov2_protos_filev2_proto_goTypes = []any{ (ErrCodes)(0), // 0: filesyncv2.ErrCodes (ErrCode)(0), // 1: filesyncv2.ErrCode @@ -1261,6 +1350,8 @@ var file_commonfile_fileproto_fileprotov2_protos_filev2_proto_goTypes = []any{ (*SpaceInfoResponse)(nil), // 17: filesyncv2.SpaceInfoResponse (*SpaceInfoResult)(nil), // 18: filesyncv2.SpaceInfoResult (*SpaceQuota)(nil), // 19: filesyncv2.SpaceQuota + (*InfoRequest)(nil), // 20: filesyncv2.InfoRequest + (*InfoResponse)(nil), // 21: filesyncv2.InfoResponse } var file_commonfile_fileproto_fileprotov2_protos_filev2_proto_depIdxs = []int32{ 3, // 0: filesyncv2.UploadRequest.items:type_name -> filesyncv2.UploadRequestItem @@ -1281,12 +1372,14 @@ var file_commonfile_fileproto_fileprotov2_protos_filev2_proto_depIdxs = []int32{ 8, // 15: filesyncv2.FileV2.RequestSign:input_type -> filesyncv2.RequestSignRequest 12, // 16: filesyncv2.FileV2.RequestDownload:input_type -> filesyncv2.RequestDownloadRequest 16, // 17: filesyncv2.FileV2.SpaceInfo:input_type -> filesyncv2.SpaceInfoRequest - 4, // 18: filesyncv2.FileV2.Upload:output_type -> filesyncv2.UploadResponse - 9, // 19: filesyncv2.FileV2.RequestSign:output_type -> filesyncv2.RequestSignResponse - 13, // 20: filesyncv2.FileV2.RequestDownload:output_type -> filesyncv2.RequestDownloadResponse - 17, // 21: filesyncv2.FileV2.SpaceInfo:output_type -> filesyncv2.SpaceInfoResponse - 18, // [18:22] is the sub-list for method output_type - 14, // [14:18] is the sub-list for method input_type + 20, // 18: filesyncv2.FileV2.Info:input_type -> filesyncv2.InfoRequest + 4, // 19: filesyncv2.FileV2.Upload:output_type -> filesyncv2.UploadResponse + 9, // 20: filesyncv2.FileV2.RequestSign:output_type -> filesyncv2.RequestSignResponse + 13, // 21: filesyncv2.FileV2.RequestDownload:output_type -> filesyncv2.RequestDownloadResponse + 17, // 22: filesyncv2.FileV2.SpaceInfo:output_type -> filesyncv2.SpaceInfoResponse + 21, // 23: filesyncv2.FileV2.Info:output_type -> filesyncv2.InfoResponse + 19, // [19:24] is the sub-list for method output_type + 14, // [14:19] is the sub-list for method input_type 14, // [14:14] is the sub-list for extension type_name 14, // [14:14] is the sub-list for extension extendee 0, // [0:14] is the sub-list for field type_name @@ -1303,7 +1396,7 @@ func file_commonfile_fileproto_fileprotov2_protos_filev2_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDesc), len(file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDesc)), NumEnums: 2, - NumMessages: 18, + NumMessages: 20, NumExtensions: 0, NumServices: 1, }, diff --git a/commonfile/fileproto/fileprotov2/filev2_drpc.pb.go b/commonfile/fileproto/fileprotov2/filev2_drpc.pb.go index be7fbe385..9da77b7e2 100644 --- a/commonfile/fileproto/fileprotov2/filev2_drpc.pb.go +++ b/commonfile/fileproto/fileprotov2/filev2_drpc.pb.go @@ -37,6 +37,7 @@ type DRPCFileV2Client interface { RequestSign(ctx context.Context, in *RequestSignRequest) (*RequestSignResponse, error) RequestDownload(ctx context.Context, in *RequestDownloadRequest) (*RequestDownloadResponse, error) SpaceInfo(ctx context.Context, in *SpaceInfoRequest) (*SpaceInfoResponse, error) + Info(ctx context.Context, in *InfoRequest) (*InfoResponse, error) } type drpcFileV2Client struct { @@ -85,11 +86,21 @@ func (c *drpcFileV2Client) SpaceInfo(ctx context.Context, in *SpaceInfoRequest) return out, nil } +func (c *drpcFileV2Client) Info(ctx context.Context, in *InfoRequest) (*InfoResponse, error) { + out := new(InfoResponse) + err := c.cc.Invoke(ctx, "/filesyncv2.FileV2/Info", drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}, in, out) + if err != nil { + return nil, err + } + return out, nil +} + type DRPCFileV2Server interface { Upload(context.Context, *UploadRequest) (*UploadResponse, error) RequestSign(context.Context, *RequestSignRequest) (*RequestSignResponse, error) RequestDownload(context.Context, *RequestDownloadRequest) (*RequestDownloadResponse, error) SpaceInfo(context.Context, *SpaceInfoRequest) (*SpaceInfoResponse, error) + Info(context.Context, *InfoRequest) (*InfoResponse, error) } type DRPCFileV2UnimplementedServer struct{} @@ -110,9 +121,13 @@ func (s *DRPCFileV2UnimplementedServer) SpaceInfo(context.Context, *SpaceInfoReq return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) } +func (s *DRPCFileV2UnimplementedServer) Info(context.Context, *InfoRequest) (*InfoResponse, error) { + return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) +} + type DRPCFileV2Description struct{} -func (DRPCFileV2Description) NumMethods() int { return 4 } +func (DRPCFileV2Description) NumMethods() int { return 5 } func (DRPCFileV2Description) Method(n int) (string, drpc.Encoding, drpc.Receiver, interface{}, bool) { switch n { @@ -152,6 +167,15 @@ func (DRPCFileV2Description) Method(n int) (string, drpc.Encoding, drpc.Receiver in1.(*SpaceInfoRequest), ) }, DRPCFileV2Server.SpaceInfo, true + case 4: + return "/filesyncv2.FileV2/Info", drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}, + func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) { + return srv.(DRPCFileV2Server). + Info( + ctx, + in1.(*InfoRequest), + ) + }, DRPCFileV2Server.Info, true default: return "", nil, nil, nil, false } @@ -224,3 +248,19 @@ func (x *drpcFileV2_SpaceInfoStream) SendAndClose(m *SpaceInfoResponse) error { } return x.CloseSend() } + +type DRPCFileV2_InfoStream interface { + drpc.Stream + SendAndClose(*InfoResponse) error +} + +type drpcFileV2_InfoStream struct { + drpc.Stream +} + +func (x *drpcFileV2_InfoStream) SendAndClose(m *InfoResponse) error { + if err := x.MsgSend(m, drpcEncoding_File_commonfile_fileproto_fileprotov2_protos_filev2_proto{}); err != nil { + return err + } + return x.CloseSend() +} diff --git a/commonfile/fileproto/fileprotov2/filev2_vtproto.pb.go b/commonfile/fileproto/fileprotov2/filev2_vtproto.pb.go index 49d19b483..c31a407d7 100644 --- a/commonfile/fileproto/fileprotov2/filev2_vtproto.pb.go +++ b/commonfile/fileproto/fileprotov2/filev2_vtproto.pb.go @@ -914,6 +914,79 @@ func (m *SpaceQuota) MarshalToSizedBufferVT(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *InfoRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *InfoRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *InfoRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + return len(dAtA) - i, nil +} + +func (m *InfoResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *InfoResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *InfoResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.PublicReadBaseUrl) > 0 { + i -= len(m.PublicReadBaseUrl) + copy(dAtA[i:], m.PublicReadBaseUrl) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.PublicReadBaseUrl))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func (m *UploadRequest) SizeVT() (n int) { if m == nil { return 0 @@ -1259,6 +1332,30 @@ func (m *SpaceQuota) SizeVT() (n int) { return n } +func (m *InfoRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + n += len(m.unknownFields) + return n +} + +func (m *InfoResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.PublicReadBaseUrl) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + func (m *UploadRequest) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -3328,3 +3425,137 @@ func (m *SpaceQuota) UnmarshalVT(dAtA []byte) error { } return nil } +func (m *InfoRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: InfoRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: InfoRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *InfoResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: InfoResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: InfoResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PublicReadBaseUrl", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PublicReadBaseUrl = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} diff --git a/commonfile/fileproto/fileprotov2/protos/filev2.proto b/commonfile/fileproto/fileprotov2/protos/filev2.proto index 7d4928168..0fd7f2cfe 100644 --- a/commonfile/fileproto/fileprotov2/protos/filev2.proto +++ b/commonfile/fileproto/fileprotov2/protos/filev2.proto @@ -10,7 +10,7 @@ option go_package = "commonfile/fileproto/fileprotov2"; // without a Register() panic. The package/service names are load-bearing: // method paths are derived from `package filesyncv2` + `service FileV2`. // -// Conventions (ALL four RPCs are BATCH): +// Conventions (all four SPACE-SCOPED RPCs are BATCH; Info is node-level): // * request carries a single shared spaceId (or repeated spaceIds for // SpaceInfo) + a repeated list of items; // * identity is NEVER a request field — it is taken from the authenticated @@ -40,6 +40,9 @@ service FileV2 { rpc RequestDownload(RequestDownloadRequest) returns (RequestDownloadResponse); // SpaceInfo returns quota/usage per requested space (batch quota-info). rpc SpaceInfo(SpaceInfoRequest) returns (SpaceInfoResponse); + // Info returns static node-level metadata (no spaceId, no responsibility + // gate, no auth beyond the connection); clients cache it. + rpc Info(InfoRequest) returns (InfoResponse); } // ErrCodes are WHOLE-RPC (drpc-level) errors, mirroring v1 filesync.ErrCodes. @@ -189,3 +192,16 @@ message SpaceQuota { uint64 inflightUsageBytes = 4; // bytes reserved by presigned/not-yet-committed uploads uint64 filesCount = 5; // number of file roots stored in the space } + +// ---- Info (node-level metadata) --------------------------------------------- + +// InfoRequest is deliberately empty: Info is static node metadata, +// extensible later (capabilities, limits) without a wire break. +message InfoRequest {} + +message InfoResponse { + // publicReadBaseUrl, when non-empty, lets clients construct durable object + // URLs as {publicReadBaseUrl}/blob/{spaceId}/{rootCid} with zero broker + // round-trips; clients cache it. Empty = no public read: use RequestDownload. + string publicReadBaseUrl = 1; +} From b9bece9f1a2f86dea61cd52bf8cb28a4fb369067 Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Fri, 3 Jul 2026 11:20:28 +0200 Subject: [PATCH 10/20] test(inboxclient): implement FileV2 methods on mockConf mockConf did not implement the FileV2Peers/FileV2NodeIds methods added to nodeconf.NodeConf on this branch, causing a panic in TestInbox_Fetch. Co-Authored-By: Claude Opus 4.8 --- coordinator/inboxclient/clienttestutil_test.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/coordinator/inboxclient/clienttestutil_test.go b/coordinator/inboxclient/clienttestutil_test.go index 100c32033..825b5ff42 100644 --- a/coordinator/inboxclient/clienttestutil_test.go +++ b/coordinator/inboxclient/clienttestutil_test.go @@ -210,6 +210,14 @@ func (m *mockConf) FilePeers() []string { return nil } +func (m *mockConf) FileV2Peers() []string { + return nil +} + +func (m *mockConf) FileV2NodeIds(spaceId string) []string { + return nil +} + func (m *mockConf) ConsensusPeers() []string { return nil } From 1693009b6c8b6efc3e35edd24e637134fbf4dde1 Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Fri, 3 Jul 2026 11:22:14 +0200 Subject: [PATCH 11/20] feat(secureservice): bump ProtoVersion to 13 for files v2 Proto version 13 aligns with the v0.13.X release line and covers the files v2 work (v2 filenodes, chash routing, space header fileprotoVersion). Version 13 is already present in defaultCompatibleVersions. Co-Authored-By: Claude Opus 4.8 --- net/secureservice/secureservice.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/net/secureservice/secureservice.go b/net/secureservice/secureservice.go index ab85933d2..c4b332bab 100644 --- a/net/secureservice/secureservice.go +++ b/net/secureservice/secureservice.go @@ -38,8 +38,9 @@ var ( // ProtoVersion 9 - nested derived objects, delete restrictions (bridge release v0.11.Y) // ProtoVersion 10 - reserved (not used, version alignment gap) // ProtoVersion 11 - reserved (not used, version alignment gap) - // ProtoVersion 12 will align with v0.12.X - ProtoVersion = uint32(12) + // ProtoVersion 12 aligns with v0.12.X + // ProtoVersion 13 - files v2 (v2 filenodes, chash routing, space header fileprotoVersion), aligns with v0.13.X + ProtoVersion = uint32(13) ) var ( From 7979c1309a30a970f693ead4ec34337c25a92a9e Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Fri, 3 Jul 2026 11:23:36 +0200 Subject: [PATCH 12/20] feat(secureservice): set compatibleVersions to [12,13,14] for v0.13 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per docs/proto-versioning.md the default compatibility range is self ±1. v0.13.X accepts V-1=12, V=13, V+1=14. Drops bridge v9, which is no longer needed now that v0.12 has shipped. Co-Authored-By: Claude Opus 4.8 --- net/secureservice/secureservice.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/net/secureservice/secureservice.go b/net/secureservice/secureservice.go index c4b332bab..fb1a393e7 100644 --- a/net/secureservice/secureservice.go +++ b/net/secureservice/secureservice.go @@ -44,7 +44,8 @@ var ( ) var ( - defaultCompatibleVersions = []uint32{9, 12, 13} + // v0.13.X: accept self ±1 (V-1, V, V+1). Bridge v9 is dropped now that v0.12 has shipped. + defaultCompatibleVersions = []uint32{12, 13, 14} ) func New() SecureService { From 2cdccb36592f5d6bdf733ec31a6441c5efe6fbc4 Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Fri, 3 Jul 2026 20:46:36 +0200 Subject: [PATCH 13/20] =?UTF-8?q?feat(nodeconf):=20fileNetworkId=20?= =?UTF-8?q?=E2=80=94=20the=20fileV2=20fleet's=20receipt-signing=20identity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One shared account (signing) key signs fileV2 durability receipts for the whole NodeTypeFileV2 fleet (per-node signing was rejected: receipts live in CRDT rows for years and must survive fleet churn). Clients need its public identity — the fileNetworkId — from the network config, so: - nodeconf.Configuration gains FileNetworkId (yaml fileNetworkId, network string encoding, decode with crypto.DecodeNetworkId) - coordinatorproto NetworkConfigurationResponse gains fileNetworkId=5 - nodeconfsource maps it off the proto, so coordinator-refreshed configs carry it Co-Authored-By: Claude Fable 5 --- .../coordinatorproto/coordinator.pb.go | 19 ++++++-- .../coordinator_vtproto.pb.go | 43 +++++++++++++++++++ .../coordinatorproto/protos/coordinator.proto | 3 ++ coordinator/nodeconfsource/nodeconfsource.go | 9 ++-- nodeconf/config.go | 13 ++++-- 5 files changed, 75 insertions(+), 12 deletions(-) diff --git a/coordinator/coordinatorproto/coordinator.pb.go b/coordinator/coordinatorproto/coordinator.pb.go index 02575d311..a03908f0d 100644 --- a/coordinator/coordinatorproto/coordinator.pb.go +++ b/coordinator/coordinatorproto/coordinator.pb.go @@ -1583,8 +1583,11 @@ type NetworkConfigurationResponse struct { Nodes []*Node `protobuf:"bytes,3,rep,name=nodes,proto3" json:"nodes,omitempty"` // unix timestamp of the creation time of configuration CreationTimeUnix uint64 `protobuf:"varint,4,opt,name=creationTimeUnix,proto3" json:"creationTimeUnix,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // identity of the fileV2 fleet's shared receipt-signing key (network + // string encoding); empty if the network has no fileV2 fleet + FileNetworkId string `protobuf:"bytes,5,opt,name=fileNetworkId,proto3" json:"fileNetworkId,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *NetworkConfigurationResponse) Reset() { @@ -1645,6 +1648,13 @@ func (x *NetworkConfigurationResponse) GetCreationTimeUnix() uint64 { return 0 } +func (x *NetworkConfigurationResponse) GetFileNetworkId() string { + if x != nil { + return x.FileNetworkId + } + return "" +} + // Node describes one node in the network type Node struct { state protoimpl.MessageState `protogen:"open.v1"` @@ -3539,12 +3549,13 @@ const file_coordinator_coordinatorproto_protos_coordinator_proto_rawDesc = "" + "\aaclHead\x18\x02 \x01(\tR\aaclHead\"\x1e\n" + "\x1cSpaceMakeUnshareableResponse\";\n" + "\x1bNetworkConfigurationRequest\x12\x1c\n" + - "\tcurrentId\x18\x01 \x01(\tR\tcurrentId\"\xbb\x01\n" + + "\tcurrentId\x18\x01 \x01(\tR\tcurrentId\"\xe1\x01\n" + "\x1cNetworkConfigurationResponse\x12(\n" + "\x0fconfigurationId\x18\x01 \x01(\tR\x0fconfigurationId\x12\x1c\n" + "\tnetworkId\x18\x02 \x01(\tR\tnetworkId\x12'\n" + "\x05nodes\x18\x03 \x03(\v2\x11.coordinator.NodeR\x05nodes\x12*\n" + - "\x10creationTimeUnix\x18\x04 \x01(\x04R\x10creationTimeUnix\"i\n" + + "\x10creationTimeUnix\x18\x04 \x01(\x04R\x10creationTimeUnix\x12$\n" + + "\rfileNetworkId\x18\x05 \x01(\tR\rfileNetworkId\"i\n" + "\x04Node\x12\x16\n" + "\x06peerId\x18\x01 \x01(\tR\x06peerId\x12\x1c\n" + "\taddresses\x18\x02 \x03(\tR\taddresses\x12+\n" + diff --git a/coordinator/coordinatorproto/coordinator_vtproto.pb.go b/coordinator/coordinatorproto/coordinator_vtproto.pb.go index 1c09090ad..0c5b8020f 100644 --- a/coordinator/coordinatorproto/coordinator_vtproto.pb.go +++ b/coordinator/coordinatorproto/coordinator_vtproto.pb.go @@ -899,6 +899,13 @@ func (m *NetworkConfigurationResponse) MarshalToSizedBufferVT(dAtA []byte) (int, i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if len(m.FileNetworkId) > 0 { + i -= len(m.FileNetworkId) + copy(dAtA[i:], m.FileNetworkId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.FileNetworkId))) + i-- + dAtA[i] = 0x2a + } if m.CreationTimeUnix != 0 { i = protohelpers.EncodeVarint(dAtA, i, uint64(m.CreationTimeUnix)) i-- @@ -2893,6 +2900,10 @@ func (m *NetworkConfigurationResponse) SizeVT() (n int) { if m.CreationTimeUnix != 0 { n += 1 + protohelpers.SizeOfVarint(uint64(m.CreationTimeUnix)) } + l = len(m.FileNetworkId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } n += len(m.unknownFields) return n } @@ -5608,6 +5619,38 @@ func (m *NetworkConfigurationResponse) UnmarshalVT(dAtA []byte) error { break } } + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field FileNetworkId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.FileNetworkId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) diff --git a/coordinator/coordinatorproto/protos/coordinator.proto b/coordinator/coordinatorproto/protos/coordinator.proto index 2ddb801d5..372a79072 100644 --- a/coordinator/coordinatorproto/protos/coordinator.proto +++ b/coordinator/coordinatorproto/protos/coordinator.proto @@ -210,6 +210,9 @@ message NetworkConfigurationResponse { repeated Node nodes = 3; // unix timestamp of the creation time of configuration uint64 creationTimeUnix = 4; + // identity of the fileV2 fleet's shared receipt-signing key (network + // string encoding); empty if the network has no fileV2 fleet + string fileNetworkId = 5; } // NodeType determines the type of API that a node supports diff --git a/coordinator/nodeconfsource/nodeconfsource.go b/coordinator/nodeconfsource/nodeconfsource.go index 536c4b8d9..90543e91d 100644 --- a/coordinator/nodeconfsource/nodeconfsource.go +++ b/coordinator/nodeconfsource/nodeconfsource.go @@ -70,9 +70,10 @@ func (n *nodeConfSource) GetLast(ctx context.Context, currentId string) (c nodec } return nodeconf.Configuration{ - Id: res.ConfigurationId, - NetworkId: res.NetworkId, - Nodes: nodes, - CreationTime: time.Unix(int64(res.CreationTimeUnix), 0), + Id: res.ConfigurationId, + NetworkId: res.NetworkId, + FileNetworkId: res.FileNetworkId, + Nodes: nodes, + CreationTime: time.Unix(int64(res.CreationTimeUnix), 0), }, nil } diff --git a/nodeconf/config.go b/nodeconf/config.go index bcf9df455..d0d6cefbb 100644 --- a/nodeconf/config.go +++ b/nodeconf/config.go @@ -54,8 +54,13 @@ func (n Node) HasType(t NodeType) bool { } type Configuration struct { - Id string `yaml:"id"` - NetworkId string `yaml:"networkId"` - Nodes []Node `yaml:"nodes"` - CreationTime time.Time `yaml:"creationTime"` + Id string `yaml:"id"` + NetworkId string `yaml:"networkId"` + // FileNetworkId is the identity of the NodeTypeFileV2 fleet's shared + // signing key (network string encoding, crypto.DecodeNetworkId) — + // the key fileV2 durability receipts (networkSign) verify against. + // Empty on networks without a fileV2 fleet. + FileNetworkId string `yaml:"fileNetworkId,omitempty"` + Nodes []Node `yaml:"nodes"` + CreationTime time.Time `yaml:"creationTime"` } From 71a5fdb71f3a03206b4c6163390356bb20c363ea Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Sat, 4 Jul 2026 18:03:11 +0200 Subject: [PATCH 14/20] feat(filep2p): peer-to-peer file transfer protocol (SYN-48) New commonfile/fileproto/filep2p service filesyncp2p.FileP2P, served read-only by clients to LAN peers for direct file fetch: - FileCheck(spaceId, rootCids) -> None/Partial/Full availability - ObjectRead(spaceId, rootCid, offset, length) -> (data, totalSize) A client stores one immutable CARv2 object per file, byte-identical to the S3/public-read object, so a peer is just an ALTERNATE SOURCE of that same object: ObjectRead returns object byte ranges exactly like an HTTP Range GET, and the fetcher's existing seed/verify path is reused. Only full holders serve ObjectRead; FileCheck reports which. Requests are root-scoped (no global block index). Blocks are ciphertext; the fetcher verifies hash==cid. Whole-RPC errors in filep2perr at rpcerr offset 900 (v1=200, v2=800). Route prefix /filesyncp2p.FileP2P/ coexists on one mux. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_017Hq5ube4stoD1XM5QAU931 --- Makefile | 1 + commonfile/fileproto/filep2p/filep2p.pb.go | 498 ++++++++++ .../fileproto/filep2p/filep2p_drpc.pb.go | 154 +++ .../fileproto/filep2p/filep2p_vtproto.pb.go | 918 ++++++++++++++++++ .../filep2p/filep2perr/filep2perr.go | 35 + .../fileproto/filep2p/protos/filep2p.proto | 92 ++ 6 files changed, 1698 insertions(+) create mode 100644 commonfile/fileproto/filep2p/filep2p.pb.go create mode 100644 commonfile/fileproto/filep2p/filep2p_drpc.pb.go create mode 100644 commonfile/fileproto/filep2p/filep2p_vtproto.pb.go create mode 100644 commonfile/fileproto/filep2p/filep2perr/filep2perr.go create mode 100644 commonfile/fileproto/filep2p/protos/filep2p.proto diff --git a/Makefile b/Makefile index 91b23fa6b..7cc62d8d0 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,7 @@ proto: $(call generate_drpc,$(PKGMAP),commonspace/clientspaceproto/protos) $(call generate_drpc,$(PKGMAP),commonfile/fileproto/protos) $(call generate_drpc,,commonfile/fileproto/fileprotov2/protos) + $(call generate_drpc,,commonfile/fileproto/filep2p/protos) $(call generate_drpc,$(PKGMAP),net/streampool/testservice/protos) $(call generate_drpc,$(PKGMAP),net/endtoendtest/testpeer/testproto/protos) diff --git a/commonfile/fileproto/filep2p/filep2p.pb.go b/commonfile/fileproto/filep2p/filep2p.pb.go new file mode 100644 index 000000000..b65a36c25 --- /dev/null +++ b/commonfile/fileproto/filep2p/filep2p.pb.go @@ -0,0 +1,498 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v3.14.0 +// source: commonfile/fileproto/filep2p/protos/filep2p.proto + +package filep2p + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// ErrCodes are WHOLE-RPC (drpc-level) errors, mirroring v1 filesync.ErrCodes +// and v2's fileprotov2.ErrCodes. Wired through rpcerr.ErrGroup(ErrorOffset) in +// the filep2perr package. Offset 900 is the next free rpcerr block (v1=200, +// v2=800). Zero value Unexpected=0 registers at offset+0 so an uncoded fault +// reads as Unexpected. +type ErrCodes int32 + +const ( + ErrCodes_Unexpected ErrCodes = 0 // unclassified server fault (registry fallback) + ErrCodes_Forbidden ErrCodes = 1 // connection identity may not read this space + ErrCodes_InvalidRequest ErrCodes = 2 // malformed: empty spaceId, bad rootCid, bad range + ErrCodes_FileNotFound ErrCodes = 3 // server does not hold this (spaceId, rootCid) in full + ErrCodes_ErrorOffset ErrCodes = 900 // rpcerr.ErrGroup base (constant, never an error) +) + +// Enum value maps for ErrCodes. +var ( + ErrCodes_name = map[int32]string{ + 0: "Unexpected", + 1: "Forbidden", + 2: "InvalidRequest", + 3: "FileNotFound", + 900: "ErrorOffset", + } + ErrCodes_value = map[string]int32{ + "Unexpected": 0, + "Forbidden": 1, + "InvalidRequest": 2, + "FileNotFound": 3, + "ErrorOffset": 900, + } +) + +func (x ErrCodes) Enum() *ErrCodes { + p := new(ErrCodes) + *p = x + return p +} + +func (x ErrCodes) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ErrCodes) Descriptor() protoreflect.EnumDescriptor { + return file_commonfile_fileproto_filep2p_protos_filep2p_proto_enumTypes[0].Descriptor() +} + +func (ErrCodes) Type() protoreflect.EnumType { + return &file_commonfile_fileproto_filep2p_protos_filep2p_proto_enumTypes[0] +} + +func (x ErrCodes) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ErrCodes.Descriptor instead. +func (ErrCodes) EnumDescriptor() ([]byte, []int) { + return file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDescGZIP(), []int{0} +} + +type Availability int32 + +const ( + Availability_None Availability = 0 // server has no block of this file + Availability_Partial Availability = 1 // server has some but not all blocks (cannot ObjectRead yet) + Availability_Full Availability = 2 // server has every block; ObjectRead will serve +) + +// Enum value maps for Availability. +var ( + Availability_name = map[int32]string{ + 0: "None", + 1: "Partial", + 2: "Full", + } + Availability_value = map[string]int32{ + "None": 0, + "Partial": 1, + "Full": 2, + } +) + +func (x Availability) Enum() *Availability { + p := new(Availability) + *p = x + return p +} + +func (x Availability) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (Availability) Descriptor() protoreflect.EnumDescriptor { + return file_commonfile_fileproto_filep2p_protos_filep2p_proto_enumTypes[1].Descriptor() +} + +func (Availability) Type() protoreflect.EnumType { + return &file_commonfile_fileproto_filep2p_protos_filep2p_proto_enumTypes[1] +} + +func (x Availability) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use Availability.Descriptor instead. +func (Availability) EnumDescriptor() ([]byte, []int) { + return file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDescGZIP(), []int{1} +} + +type FileCheckRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + SpaceId string `protobuf:"bytes,1,opt,name=spaceId,proto3" json:"spaceId,omitempty"` // auth scope + RootCids [][]byte `protobuf:"bytes,2,rep,name=rootCids,proto3" json:"rootCids,omitempty"` // files to report on (result match key) + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileCheckRequest) Reset() { + *x = FileCheckRequest{} + mi := &file_commonfile_fileproto_filep2p_protos_filep2p_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileCheckRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileCheckRequest) ProtoMessage() {} + +func (x *FileCheckRequest) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_filep2p_protos_filep2p_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileCheckRequest.ProtoReflect.Descriptor instead. +func (*FileCheckRequest) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDescGZIP(), []int{0} +} + +func (x *FileCheckRequest) GetSpaceId() string { + if x != nil { + return x.SpaceId + } + return "" +} + +func (x *FileCheckRequest) GetRootCids() [][]byte { + if x != nil { + return x.RootCids + } + return nil +} + +type FileCheckResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Files []*FileAvailability `protobuf:"bytes,1,rep,name=files,proto3" json:"files,omitempty"` // one per requested rootCid, keyed by rootCid + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileCheckResponse) Reset() { + *x = FileCheckResponse{} + mi := &file_commonfile_fileproto_filep2p_protos_filep2p_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileCheckResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileCheckResponse) ProtoMessage() {} + +func (x *FileCheckResponse) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_filep2p_protos_filep2p_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileCheckResponse.ProtoReflect.Descriptor instead. +func (*FileCheckResponse) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDescGZIP(), []int{1} +} + +func (x *FileCheckResponse) GetFiles() []*FileAvailability { + if x != nil { + return x.Files + } + return nil +} + +type FileAvailability struct { + state protoimpl.MessageState `protogen:"open.v1"` + RootCid []byte `protobuf:"bytes,1,opt,name=rootCid,proto3" json:"rootCid,omitempty"` // echoes the requested rootCid (match key) + Have Availability `protobuf:"varint,2,opt,name=have,proto3,enum=filesyncp2p.Availability" json:"have,omitempty"` // how much of the file the server holds + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileAvailability) Reset() { + *x = FileAvailability{} + mi := &file_commonfile_fileproto_filep2p_protos_filep2p_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileAvailability) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileAvailability) ProtoMessage() {} + +func (x *FileAvailability) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_filep2p_protos_filep2p_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileAvailability.ProtoReflect.Descriptor instead. +func (*FileAvailability) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDescGZIP(), []int{2} +} + +func (x *FileAvailability) GetRootCid() []byte { + if x != nil { + return x.RootCid + } + return nil +} + +func (x *FileAvailability) GetHave() Availability { + if x != nil { + return x.Have + } + return Availability_None +} + +type ObjectReadRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + SpaceId string `protobuf:"bytes,1,opt,name=spaceId,proto3" json:"spaceId,omitempty"` // auth scope + RootCid []byte `protobuf:"bytes,2,opt,name=rootCid,proto3" json:"rootCid,omitempty"` // file root — selects the CAR object to read from + Offset uint64 `protobuf:"varint,3,opt,name=offset,proto3" json:"offset,omitempty"` // start byte within the object + Length uint32 `protobuf:"varint,4,opt,name=length,proto3" json:"length,omitempty"` // number of bytes requested from offset + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ObjectReadRequest) Reset() { + *x = ObjectReadRequest{} + mi := &file_commonfile_fileproto_filep2p_protos_filep2p_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ObjectReadRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ObjectReadRequest) ProtoMessage() {} + +func (x *ObjectReadRequest) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_filep2p_protos_filep2p_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ObjectReadRequest.ProtoReflect.Descriptor instead. +func (*ObjectReadRequest) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDescGZIP(), []int{3} +} + +func (x *ObjectReadRequest) GetSpaceId() string { + if x != nil { + return x.SpaceId + } + return "" +} + +func (x *ObjectReadRequest) GetRootCid() []byte { + if x != nil { + return x.RootCid + } + return nil +} + +func (x *ObjectReadRequest) GetOffset() uint64 { + if x != nil { + return x.Offset + } + return 0 +} + +func (x *ObjectReadRequest) GetLength() uint32 { + if x != nil { + return x.Length + } + return 0 +} + +type ObjectReadResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` // the requested bytes (short only at object end) + TotalSize uint64 `protobuf:"varint,2,opt,name=totalSize,proto3" json:"totalSize,omitempty"` // whole-object size (like HTTP Content-Range total) + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ObjectReadResponse) Reset() { + *x = ObjectReadResponse{} + mi := &file_commonfile_fileproto_filep2p_protos_filep2p_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ObjectReadResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ObjectReadResponse) ProtoMessage() {} + +func (x *ObjectReadResponse) ProtoReflect() protoreflect.Message { + mi := &file_commonfile_fileproto_filep2p_protos_filep2p_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ObjectReadResponse.ProtoReflect.Descriptor instead. +func (*ObjectReadResponse) Descriptor() ([]byte, []int) { + return file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDescGZIP(), []int{4} +} + +func (x *ObjectReadResponse) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +func (x *ObjectReadResponse) GetTotalSize() uint64 { + if x != nil { + return x.TotalSize + } + return 0 +} + +var File_commonfile_fileproto_filep2p_protos_filep2p_proto protoreflect.FileDescriptor + +const file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDesc = "" + + "\n" + + "1commonfile/fileproto/filep2p/protos/filep2p.proto\x12\vfilesyncp2p\"H\n" + + "\x10FileCheckRequest\x12\x18\n" + + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x12\x1a\n" + + "\brootCids\x18\x02 \x03(\fR\brootCids\"H\n" + + "\x11FileCheckResponse\x123\n" + + "\x05files\x18\x01 \x03(\v2\x1d.filesyncp2p.FileAvailabilityR\x05files\"[\n" + + "\x10FileAvailability\x12\x18\n" + + "\arootCid\x18\x01 \x01(\fR\arootCid\x12-\n" + + "\x04have\x18\x02 \x01(\x0e2\x19.filesyncp2p.AvailabilityR\x04have\"w\n" + + "\x11ObjectReadRequest\x12\x18\n" + + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x12\x18\n" + + "\arootCid\x18\x02 \x01(\fR\arootCid\x12\x16\n" + + "\x06offset\x18\x03 \x01(\x04R\x06offset\x12\x16\n" + + "\x06length\x18\x04 \x01(\rR\x06length\"F\n" + + "\x12ObjectReadResponse\x12\x12\n" + + "\x04data\x18\x01 \x01(\fR\x04data\x12\x1c\n" + + "\ttotalSize\x18\x02 \x01(\x04R\ttotalSize*a\n" + + "\bErrCodes\x12\x0e\n" + + "\n" + + "Unexpected\x10\x00\x12\r\n" + + "\tForbidden\x10\x01\x12\x12\n" + + "\x0eInvalidRequest\x10\x02\x12\x10\n" + + "\fFileNotFound\x10\x03\x12\x10\n" + + "\vErrorOffset\x10\x84\a*/\n" + + "\fAvailability\x12\b\n" + + "\x04None\x10\x00\x12\v\n" + + "\aPartial\x10\x01\x12\b\n" + + "\x04Full\x10\x022\xa4\x01\n" + + "\aFileP2P\x12J\n" + + "\tFileCheck\x12\x1d.filesyncp2p.FileCheckRequest\x1a\x1e.filesyncp2p.FileCheckResponse\x12M\n" + + "\n" + + "ObjectRead\x12\x1e.filesyncp2p.ObjectReadRequest\x1a\x1f.filesyncp2p.ObjectReadResponseB\x1eZ\x1ccommonfile/fileproto/filep2pb\x06proto3" + +var ( + file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDescOnce sync.Once + file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDescData []byte +) + +func file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDescGZIP() []byte { + file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDescOnce.Do(func() { + file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDesc), len(file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDesc))) + }) + return file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDescData +} + +var file_commonfile_fileproto_filep2p_protos_filep2p_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_commonfile_fileproto_filep2p_protos_filep2p_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_commonfile_fileproto_filep2p_protos_filep2p_proto_goTypes = []any{ + (ErrCodes)(0), // 0: filesyncp2p.ErrCodes + (Availability)(0), // 1: filesyncp2p.Availability + (*FileCheckRequest)(nil), // 2: filesyncp2p.FileCheckRequest + (*FileCheckResponse)(nil), // 3: filesyncp2p.FileCheckResponse + (*FileAvailability)(nil), // 4: filesyncp2p.FileAvailability + (*ObjectReadRequest)(nil), // 5: filesyncp2p.ObjectReadRequest + (*ObjectReadResponse)(nil), // 6: filesyncp2p.ObjectReadResponse +} +var file_commonfile_fileproto_filep2p_protos_filep2p_proto_depIdxs = []int32{ + 4, // 0: filesyncp2p.FileCheckResponse.files:type_name -> filesyncp2p.FileAvailability + 1, // 1: filesyncp2p.FileAvailability.have:type_name -> filesyncp2p.Availability + 2, // 2: filesyncp2p.FileP2P.FileCheck:input_type -> filesyncp2p.FileCheckRequest + 5, // 3: filesyncp2p.FileP2P.ObjectRead:input_type -> filesyncp2p.ObjectReadRequest + 3, // 4: filesyncp2p.FileP2P.FileCheck:output_type -> filesyncp2p.FileCheckResponse + 6, // 5: filesyncp2p.FileP2P.ObjectRead:output_type -> filesyncp2p.ObjectReadResponse + 4, // [4:6] is the sub-list for method output_type + 2, // [2:4] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_commonfile_fileproto_filep2p_protos_filep2p_proto_init() } +func file_commonfile_fileproto_filep2p_protos_filep2p_proto_init() { + if File_commonfile_fileproto_filep2p_protos_filep2p_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDesc), len(file_commonfile_fileproto_filep2p_protos_filep2p_proto_rawDesc)), + NumEnums: 2, + NumMessages: 5, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_commonfile_fileproto_filep2p_protos_filep2p_proto_goTypes, + DependencyIndexes: file_commonfile_fileproto_filep2p_protos_filep2p_proto_depIdxs, + EnumInfos: file_commonfile_fileproto_filep2p_protos_filep2p_proto_enumTypes, + MessageInfos: file_commonfile_fileproto_filep2p_protos_filep2p_proto_msgTypes, + }.Build() + File_commonfile_fileproto_filep2p_protos_filep2p_proto = out.File + file_commonfile_fileproto_filep2p_protos_filep2p_proto_goTypes = nil + file_commonfile_fileproto_filep2p_protos_filep2p_proto_depIdxs = nil +} diff --git a/commonfile/fileproto/filep2p/filep2p_drpc.pb.go b/commonfile/fileproto/filep2p/filep2p_drpc.pb.go new file mode 100644 index 000000000..b85a10985 --- /dev/null +++ b/commonfile/fileproto/filep2p/filep2p_drpc.pb.go @@ -0,0 +1,154 @@ +// Code generated by protoc-gen-go-drpc. DO NOT EDIT. +// protoc-gen-go-drpc version: v1.0.0 +// source: commonfile/fileproto/filep2p/protos/filep2p.proto + +package filep2p + +import ( + context "context" + errors "errors" + drpc1 "github.com/planetscale/vtprotobuf/codec/drpc" + drpc "storj.io/drpc" + drpcerr "storj.io/drpc/drpcerr" +) + +type drpcEncoding_File_commonfile_fileproto_filep2p_protos_filep2p_proto struct{} + +func (drpcEncoding_File_commonfile_fileproto_filep2p_protos_filep2p_proto) Marshal(msg drpc.Message) ([]byte, error) { + return drpc1.Marshal(msg) +} + +func (drpcEncoding_File_commonfile_fileproto_filep2p_protos_filep2p_proto) Unmarshal(buf []byte, msg drpc.Message) error { + return drpc1.Unmarshal(buf, msg) +} + +func (drpcEncoding_File_commonfile_fileproto_filep2p_protos_filep2p_proto) JSONMarshal(msg drpc.Message) ([]byte, error) { + return drpc1.JSONMarshal(msg) +} + +func (drpcEncoding_File_commonfile_fileproto_filep2p_protos_filep2p_proto) JSONUnmarshal(buf []byte, msg drpc.Message) error { + return drpc1.JSONUnmarshal(buf, msg) +} + +type DRPCFileP2PClient interface { + DRPCConn() drpc.Conn + + FileCheck(ctx context.Context, in *FileCheckRequest) (*FileCheckResponse, error) + ObjectRead(ctx context.Context, in *ObjectReadRequest) (*ObjectReadResponse, error) +} + +type drpcFileP2PClient struct { + cc drpc.Conn +} + +func NewDRPCFileP2PClient(cc drpc.Conn) DRPCFileP2PClient { + return &drpcFileP2PClient{cc} +} + +func (c *drpcFileP2PClient) DRPCConn() drpc.Conn { return c.cc } + +func (c *drpcFileP2PClient) FileCheck(ctx context.Context, in *FileCheckRequest) (*FileCheckResponse, error) { + out := new(FileCheckResponse) + err := c.cc.Invoke(ctx, "/filesyncp2p.FileP2P/FileCheck", drpcEncoding_File_commonfile_fileproto_filep2p_protos_filep2p_proto{}, in, out) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *drpcFileP2PClient) ObjectRead(ctx context.Context, in *ObjectReadRequest) (*ObjectReadResponse, error) { + out := new(ObjectReadResponse) + err := c.cc.Invoke(ctx, "/filesyncp2p.FileP2P/ObjectRead", drpcEncoding_File_commonfile_fileproto_filep2p_protos_filep2p_proto{}, in, out) + if err != nil { + return nil, err + } + return out, nil +} + +type DRPCFileP2PServer interface { + FileCheck(context.Context, *FileCheckRequest) (*FileCheckResponse, error) + ObjectRead(context.Context, *ObjectReadRequest) (*ObjectReadResponse, error) +} + +type DRPCFileP2PUnimplementedServer struct{} + +func (s *DRPCFileP2PUnimplementedServer) FileCheck(context.Context, *FileCheckRequest) (*FileCheckResponse, error) { + return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) +} + +func (s *DRPCFileP2PUnimplementedServer) ObjectRead(context.Context, *ObjectReadRequest) (*ObjectReadResponse, error) { + return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) +} + +type DRPCFileP2PDescription struct{} + +func (DRPCFileP2PDescription) NumMethods() int { return 2 } + +func (DRPCFileP2PDescription) Method(n int) (string, drpc.Encoding, drpc.Receiver, interface{}, bool) { + switch n { + case 0: + return "/filesyncp2p.FileP2P/FileCheck", drpcEncoding_File_commonfile_fileproto_filep2p_protos_filep2p_proto{}, + func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) { + return srv.(DRPCFileP2PServer). + FileCheck( + ctx, + in1.(*FileCheckRequest), + ) + }, DRPCFileP2PServer.FileCheck, true + case 1: + return "/filesyncp2p.FileP2P/ObjectRead", drpcEncoding_File_commonfile_fileproto_filep2p_protos_filep2p_proto{}, + func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) { + return srv.(DRPCFileP2PServer). + ObjectRead( + ctx, + in1.(*ObjectReadRequest), + ) + }, DRPCFileP2PServer.ObjectRead, true + default: + return "", nil, nil, nil, false + } +} + +func DRPCRegisterFileP2P(mux drpc.Mux, impl DRPCFileP2PServer) error { + return mux.Register(impl, DRPCFileP2PDescription{}) +} + +type DRPCFileP2P_FileCheckStream interface { + drpc.Stream + SendAndClose(*FileCheckResponse) error +} + +type drpcFileP2P_FileCheckStream struct { + drpc.Stream +} + +func (x *drpcFileP2P_FileCheckStream) GetStream() drpc.Stream { + return x.Stream +} + +func (x *drpcFileP2P_FileCheckStream) SendAndClose(m *FileCheckResponse) error { + if err := x.MsgSend(m, drpcEncoding_File_commonfile_fileproto_filep2p_protos_filep2p_proto{}); err != nil { + return err + } + return x.CloseSend() +} + +type DRPCFileP2P_ObjectReadStream interface { + drpc.Stream + SendAndClose(*ObjectReadResponse) error +} + +type drpcFileP2P_ObjectReadStream struct { + drpc.Stream +} + +func (x *drpcFileP2P_ObjectReadStream) GetStream() drpc.Stream { + return x.Stream +} + +func (x *drpcFileP2P_ObjectReadStream) SendAndClose(m *ObjectReadResponse) error { + if err := x.MsgSend(m, drpcEncoding_File_commonfile_fileproto_filep2p_protos_filep2p_proto{}); err != nil { + return err + } + return x.CloseSend() +} diff --git a/commonfile/fileproto/filep2p/filep2p_vtproto.pb.go b/commonfile/fileproto/filep2p/filep2p_vtproto.pb.go new file mode 100644 index 000000000..0ebf30138 --- /dev/null +++ b/commonfile/fileproto/filep2p/filep2p_vtproto.pb.go @@ -0,0 +1,918 @@ +// Code generated by protoc-gen-go-vtproto. DO NOT EDIT. +// protoc-gen-go-vtproto version: v0.6.0 +// source: commonfile/fileproto/filep2p/protos/filep2p.proto + +package filep2p + +import ( + fmt "fmt" + protohelpers "github.com/planetscale/vtprotobuf/protohelpers" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + io "io" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +func (m *FileCheckRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FileCheckRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *FileCheckRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.RootCids) > 0 { + for iNdEx := len(m.RootCids) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.RootCids[iNdEx]) + copy(dAtA[i:], m.RootCids[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.RootCids[iNdEx]))) + i-- + dAtA[i] = 0x12 + } + } + if len(m.SpaceId) > 0 { + i -= len(m.SpaceId) + copy(dAtA[i:], m.SpaceId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *FileCheckResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FileCheckResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *FileCheckResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Files) > 0 { + for iNdEx := len(m.Files) - 1; iNdEx >= 0; iNdEx-- { + size, err := m.Files[iNdEx].MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *FileAvailability) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FileAvailability) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *FileAvailability) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.Have != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Have)) + i-- + dAtA[i] = 0x10 + } + if len(m.RootCid) > 0 { + i -= len(m.RootCid) + copy(dAtA[i:], m.RootCid) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.RootCid))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *ObjectReadRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ObjectReadRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *ObjectReadRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.Length != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Length)) + i-- + dAtA[i] = 0x20 + } + if m.Offset != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.Offset)) + i-- + dAtA[i] = 0x18 + } + if len(m.RootCid) > 0 { + i -= len(m.RootCid) + copy(dAtA[i:], m.RootCid) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.RootCid))) + i-- + dAtA[i] = 0x12 + } + if len(m.SpaceId) > 0 { + i -= len(m.SpaceId) + copy(dAtA[i:], m.SpaceId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *ObjectReadResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ObjectReadResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *ObjectReadResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.TotalSize != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.TotalSize)) + i-- + dAtA[i] = 0x10 + } + if len(m.Data) > 0 { + i -= len(m.Data) + copy(dAtA[i:], m.Data) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Data))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *FileCheckRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.SpaceId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if len(m.RootCids) > 0 { + for _, b := range m.RootCids { + l = len(b) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *FileCheckResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Files) > 0 { + for _, e := range m.Files { + l = e.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *FileAvailability) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.RootCid) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.Have != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Have)) + } + n += len(m.unknownFields) + return n +} + +func (m *ObjectReadRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.SpaceId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.RootCid) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.Offset != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Offset)) + } + if m.Length != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.Length)) + } + n += len(m.unknownFields) + return n +} + +func (m *ObjectReadResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Data) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.TotalSize != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.TotalSize)) + } + n += len(m.unknownFields) + return n +} + +func (m *FileCheckRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FileCheckRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FileCheckRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RootCids", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RootCids = append(m.RootCids, make([]byte, postIndex-iNdEx)) + copy(m.RootCids[len(m.RootCids)-1], dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *FileCheckResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FileCheckResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FileCheckResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Files", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Files = append(m.Files, &FileAvailability{}) + if err := m.Files[len(m.Files)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *FileAvailability) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FileAvailability: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FileAvailability: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RootCid", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RootCid = append(m.RootCid[:0], dAtA[iNdEx:postIndex]...) + if m.RootCid == nil { + m.RootCid = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Have", wireType) + } + m.Have = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Have |= Availability(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ObjectReadRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ObjectReadRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ObjectReadRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RootCid", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RootCid = append(m.RootCid[:0], dAtA[iNdEx:postIndex]...) + if m.RootCid == nil { + m.RootCid = []byte{} + } + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Offset", wireType) + } + m.Offset = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Offset |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Length", wireType) + } + m.Length = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Length |= uint32(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ObjectReadResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ObjectReadResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ObjectReadResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Data", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Data = append(m.Data[:0], dAtA[iNdEx:postIndex]...) + if m.Data == nil { + m.Data = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field TotalSize", wireType) + } + m.TotalSize = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.TotalSize |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} diff --git a/commonfile/fileproto/filep2p/filep2perr/filep2perr.go b/commonfile/fileproto/filep2p/filep2perr/filep2perr.go new file mode 100644 index 000000000..c6e8bf10a --- /dev/null +++ b/commonfile/fileproto/filep2p/filep2perr/filep2perr.go @@ -0,0 +1,35 @@ +package filep2perr + +import ( + "fmt" + + "github.com/anyproto/any-sync/commonfile/fileproto/filep2p" + "github.com/anyproto/any-sync/net/rpc/rpcerr" +) + +// errGroup registers the peer-to-peer file WHOLE-RPC (drpc-level) errors in the +// offset-900 block, distinct from v1 filesync's 200 and v2's 800. Mirrors the +// commonfile/fileproto/fileprotoerr and fileprotov2err packages. +var ( + errGroup = rpcerr.ErrGroup(filep2p.ErrCodes_ErrorOffset) + ErrUnexpected = errGroup.Register(fmt.Errorf("unexpected p2p file error"), uint64(filep2p.ErrCodes_Unexpected)) + ErrForbidden = errGroup.Register(fmt.Errorf("forbidden"), uint64(filep2p.ErrCodes_Forbidden)) + ErrInvalidRequest = errGroup.Register(fmt.Errorf("invalid request"), uint64(filep2p.ErrCodes_InvalidRequest)) + ErrFileNotFound = errGroup.Register(fmt.Errorf("file not found"), uint64(filep2p.ErrCodes_FileNotFound)) +) + +// FromCode maps a whole-RPC ErrCodes value to its registered drpc error so +// handlers can uniformly `return filep2perr.FromCode(code)` at the RPC +// boundary. Unknown/zero codes fall back to ErrUnexpected. +func FromCode(code filep2p.ErrCodes) error { + switch code { + case filep2p.ErrCodes_Forbidden: + return ErrForbidden + case filep2p.ErrCodes_InvalidRequest: + return ErrInvalidRequest + case filep2p.ErrCodes_FileNotFound: + return ErrFileNotFound + default: + return ErrUnexpected + } +} diff --git a/commonfile/fileproto/filep2p/protos/filep2p.proto b/commonfile/fileproto/filep2p/protos/filep2p.proto new file mode 100644 index 000000000..4d79fd0f4 --- /dev/null +++ b/commonfile/fileproto/filep2p/protos/filep2p.proto @@ -0,0 +1,92 @@ +syntax = "proto3"; +package filesyncp2p; + +option go_package = "commonfile/fileproto/filep2p"; + +// FileP2P is the peer-to-peer file transfer service. It is served +// READ-ONLY by a client to other LAN peers so a device can fetch a file's +// encrypted CAR object directly from a nearby peer that already holds it, +// instead of the file-node / object-storage path — enabling local-network +// file sync, offline (no reachable node) reads, and durability takeover +// (a peer re-uploading a file whose creator never made it durable). +// +// drpc route prefix is "/filesyncp2p.FileP2P/..." — DISTINCT from v1 +// "/filesync.File/..." and v2 "/filesyncv2.FileV2/...", so all three can be +// mounted on one drpc mux. Names are load-bearing: method paths derive from +// `package filesyncp2p` + `service FileP2P`. +// +// Model: a client stores one immutable, content-addressed CARv2 object per +// file, byte-identical to the object the S3/public-read path serves. So a peer +// is simply an ALTERNATE SOURCE of that same object: ObjectRead returns object +// byte RANGES exactly like an HTTP Range GET, and the fetcher's existing +// seed/verify path (parse header+index, then verify every block against its +// cid) is reused unchanged. Because a partial local copy has holes, ObjectRead +// is served only by peers that hold the file in FULL; FileCheck reports which. +// +// Conventions: +// * requests are ROOT-SCOPED (rootCid) — the store is one CAR per root, keyed +// by rootCid, with no global block-CID index; +// * identity is NEVER a request field — it is the authenticated connection; +// authorization (may this peer read this space) is the server's job; +// * object bytes are CIPHERTEXT (the file layer is encrypted before the DAG +// is built), so serving them discloses no plaintext; the FETCHER verifies +// every block against its cid (nothing on this wire does). +service FileP2P { + // FileCheck reports, per requested rootCid, whether the server holds the + // file fully, partially, or not at all — so a client can pick a FULL peer + // to ObjectRead from without first knowing the object geometry. + rpc FileCheck(FileCheckRequest) returns (FileCheckResponse); + // ObjectRead returns a byte range of the file's CARv2 object. Mirrors an + // HTTP Range GET: totalSize echoes the whole-object size (for the head + // probe / index-tail read). Served only when the file is held in full. + rpc ObjectRead(ObjectReadRequest) returns (ObjectReadResponse); +} + +// ErrCodes are WHOLE-RPC (drpc-level) errors, mirroring v1 filesync.ErrCodes +// and v2's fileprotov2.ErrCodes. Wired through rpcerr.ErrGroup(ErrorOffset) in +// the filep2perr package. Offset 900 is the next free rpcerr block (v1=200, +// v2=800). Zero value Unexpected=0 registers at offset+0 so an uncoded fault +// reads as Unexpected. +enum ErrCodes { + Unexpected = 0; // unclassified server fault (registry fallback) + Forbidden = 1; // connection identity may not read this space + InvalidRequest = 2; // malformed: empty spaceId, bad rootCid, bad range + FileNotFound = 3; // server does not hold this (spaceId, rootCid) in full + ErrorOffset = 900; // rpcerr.ErrGroup base (constant, never an error) +} + +// ---- FileCheck (availability) --------------------------------------------- + +message FileCheckRequest { + string spaceId = 1; // auth scope + repeated bytes rootCids = 2; // files to report on (result match key) +} + +message FileCheckResponse { + repeated FileAvailability files = 1; // one per requested rootCid, keyed by rootCid +} + +message FileAvailability { + bytes rootCid = 1; // echoes the requested rootCid (match key) + Availability have = 2; // how much of the file the server holds +} + +enum Availability { + None = 0; // server has no block of this file + Partial = 1; // server has some but not all blocks (cannot ObjectRead yet) + Full = 2; // server has every block; ObjectRead will serve +} + +// ---- ObjectRead (ranged CAR bytes) ---------------------------------------- + +message ObjectReadRequest { + string spaceId = 1; // auth scope + bytes rootCid = 2; // file root — selects the CAR object to read from + uint64 offset = 3; // start byte within the object + uint32 length = 4; // number of bytes requested from offset +} + +message ObjectReadResponse { + bytes data = 1; // the requested bytes (short only at object end) + uint64 totalSize = 2; // whole-object size (like HTTP Content-Range total) +} From 49876c95f8d78a9f349228d8698ba1d31d6fb976 Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Sat, 4 Jul 2026 19:21:43 +0200 Subject: [PATCH 15/20] docs(filep2p): claim rpc error offset 900 in the registry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit filesyncp2p.FileP2P already declares ErrorOffset=900 (the next free core band); record it in docs/rpc-error-offsets.md, the source of truth for who owns which offset. Adds a filep2perr test that co-registers the neighbouring v1/v2 error groups so a colliding offset would panic at init — a green run proves the 900 band is clear. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_017Hq5ube4stoD1XM5QAU931 --- .../filep2p/filep2perr/filep2perr_test.go | 32 +++++++++++++++++++ docs/rpc-error-offsets.md | 2 +- 2 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 commonfile/fileproto/filep2p/filep2perr/filep2perr_test.go diff --git a/commonfile/fileproto/filep2p/filep2perr/filep2perr_test.go b/commonfile/fileproto/filep2p/filep2perr/filep2perr_test.go new file mode 100644 index 000000000..30605323a --- /dev/null +++ b/commonfile/fileproto/filep2p/filep2perr/filep2perr_test.go @@ -0,0 +1,32 @@ +package filep2perr + +import ( + "testing" + + // Import the neighbouring error groups so this test binary registers + // all of them together: rpcerr panics at init if two groups claim the + // same offset+code, so a green run proves the 900 band does not + // collide with v1 (200) / v2 (800) or anything else linked here. + _ "github.com/anyproto/any-sync/commonfile/fileproto/fileprotoerr" + _ "github.com/anyproto/any-sync/commonfile/fileproto/fileprotov2/fileprotov2err" + + "github.com/anyproto/any-sync/commonfile/fileproto/filep2p" +) + +func TestFromCodeMapsEveryCode(t *testing.T) { + cases := map[filep2p.ErrCodes]error{ + filep2p.ErrCodes_Unexpected: ErrUnexpected, + filep2p.ErrCodes_Forbidden: ErrForbidden, + filep2p.ErrCodes_InvalidRequest: ErrInvalidRequest, + filep2p.ErrCodes_FileNotFound: ErrFileNotFound, + } + for code, want := range cases { + if got := FromCode(code); got != want { + t.Fatalf("FromCode(%v) = %v, want %v", code, got, want) + } + } + // Unknown codes fall back to Unexpected. + if got := FromCode(filep2p.ErrCodes(999)); got != ErrUnexpected { + t.Fatalf("FromCode(unknown) = %v, want ErrUnexpected", got) + } +} diff --git a/docs/rpc-error-offsets.md b/docs/rpc-error-offsets.md index cd06944f6..e52695a69 100644 --- a/docs/rpc-error-offsets.md +++ b/docs/rpc-error-offsets.md @@ -75,7 +75,7 @@ may use offset `0`**: | 600 | 600–699 | `payment` | `paymentservice/paymentserviceproto` | | 700 | 700–799 | `limiter` | `net/rpc/limiter/limiterproto` | | 800 | 800–899 | `filesyncv2` (FileV2, v2 broker) | `commonfile/fileproto/fileprotov2` | -| 900 | 900–999 | _free_ | — | +| 900 | 900–999 | `filesyncp2p` (FileP2P, p2p file transfer) | `commonfile/fileproto/filep2p` | ### Downstream repos (`>= 1000`) From e8413ccf48ca2c51000dc6939a0e98fa1815e17c Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Sat, 4 Jul 2026 20:55:24 +0200 Subject: [PATCH 16/20] feat(coordinatorproto): FileLimitsGet + FileUsageReport for files-v2 storage limits (SYN-19) The v2 file-limits API: brokers pull absolute account-pool bounds per (spaceId, uploader identity) and push per-(space, identity) durable usage reports; the coordinator aggregates the identity total. In-flight never reaches the coordinator; values are absolute and idempotent. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01AoLHZmQFxyhxDkkwJ7Y1jX --- .../coordinatorclient/coordinatorclient.go | 26 + .../mock_coordinatorclient.go | 29 + .../coordinatorproto/coordinator.pb.go | 389 +++++++- .../coordinatorproto/coordinator_drpc.pb.go | 82 +- .../coordinator_vtproto.pb.go | 937 +++++++++++++++++- .../coordinatorproto/protos/coordinator.proto | 36 + 6 files changed, 1396 insertions(+), 103 deletions(-) diff --git a/coordinator/coordinatorclient/coordinatorclient.go b/coordinator/coordinatorclient/coordinatorclient.go index e242afefc..feaa3a2f8 100644 --- a/coordinator/coordinatorclient/coordinatorclient.go +++ b/coordinator/coordinatorclient/coordinatorclient.go @@ -56,6 +56,9 @@ type CoordinatorClient interface { AclUploadInvite(ctx context.Context, block blocks.Block) (err error) + FileLimitsGet(ctx context.Context, spaceId, identity string) (resp *coordinatorproto.FileLimitsGetResponse, err error) + FileUsageReport(ctx context.Context, rows []*coordinatorproto.FileUsageRow) (err error) + app.Component } @@ -296,6 +299,29 @@ func (c *coordinatorClient) AccountLimitsSet(ctx context.Context, req *coordinat }) } +func (c *coordinatorClient) FileLimitsGet(ctx context.Context, spaceId, identity string) (resp *coordinatorproto.FileLimitsGetResponse, err error) { + err = c.doClient(ctx, func(cl coordinatorproto.DRPCCoordinatorClient) error { + resp, err = cl.FileLimitsGet(ctx, &coordinatorproto.FileLimitsGetRequest{ + SpaceId: spaceId, + Identity: identity, + }) + if err != nil { + return rpcerr.Unwrap(err) + } + return nil + }) + return +} + +func (c *coordinatorClient) FileUsageReport(ctx context.Context, rows []*coordinatorproto.FileUsageRow) (err error) { + return c.doClient(ctx, func(cl coordinatorproto.DRPCCoordinatorClient) error { + if _, err := cl.FileUsageReport(ctx, &coordinatorproto.FileUsageReportRequest{Rows: rows}); err != nil { + return rpcerr.Unwrap(err) + } + return nil + }) +} + func (c *coordinatorClient) SpaceMakeShareable(ctx context.Context, spaceId string) (err error) { return c.doClient(ctx, func(cl coordinatorproto.DRPCCoordinatorClient) error { if _, err := cl.SpaceMakeShareable(ctx, &coordinatorproto.SpaceMakeShareableRequest{ diff --git a/coordinator/coordinatorclient/mock_coordinatorclient/mock_coordinatorclient.go b/coordinator/coordinatorclient/mock_coordinatorclient/mock_coordinatorclient.go index a22ef0a66..a6adbfd9e 100644 --- a/coordinator/coordinatorclient/mock_coordinatorclient/mock_coordinatorclient.go +++ b/coordinator/coordinatorclient/mock_coordinatorclient/mock_coordinatorclient.go @@ -163,6 +163,35 @@ func (mr *MockCoordinatorClientMockRecorder) DeletionLog(ctx, lastRecordId, limi return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeletionLog", reflect.TypeOf((*MockCoordinatorClient)(nil).DeletionLog), ctx, lastRecordId, limit) } +// FileLimitsGet mocks base method. +func (m *MockCoordinatorClient) FileLimitsGet(ctx context.Context, spaceId, identity string) (*coordinatorproto.FileLimitsGetResponse, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FileLimitsGet", ctx, spaceId, identity) + ret0, _ := ret[0].(*coordinatorproto.FileLimitsGetResponse) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// FileLimitsGet indicates an expected call of FileLimitsGet. +func (mr *MockCoordinatorClientMockRecorder) FileLimitsGet(ctx, spaceId, identity any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FileLimitsGet", reflect.TypeOf((*MockCoordinatorClient)(nil).FileLimitsGet), ctx, spaceId, identity) +} + +// FileUsageReport mocks base method. +func (m *MockCoordinatorClient) FileUsageReport(ctx context.Context, rows []*coordinatorproto.FileUsageRow) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FileUsageReport", ctx, rows) + ret0, _ := ret[0].(error) + return ret0 +} + +// FileUsageReport indicates an expected call of FileUsageReport. +func (mr *MockCoordinatorClientMockRecorder) FileUsageReport(ctx, rows any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FileUsageReport", reflect.TypeOf((*MockCoordinatorClient)(nil).FileUsageReport), ctx, rows) +} + // IdentityRepoGet mocks base method. func (m *MockCoordinatorClient) IdentityRepoGet(ctx context.Context, identities, kinds []string) ([]*identityrepoproto.DataWithIdentity, error) { m.ctrl.T.Helper() diff --git a/coordinator/coordinatorproto/coordinator.pb.go b/coordinator/coordinatorproto/coordinator.pb.go index a03908f0d..b844c8dbd 100644 --- a/coordinator/coordinatorproto/coordinator.pb.go +++ b/coordinator/coordinatorproto/coordinator.pb.go @@ -3490,6 +3490,272 @@ func (*AclUploadInviteResponse) Descriptor() ([]byte, []int) { return file_coordinator_coordinatorproto_protos_coordinator_proto_rawDescGZIP(), []int{51} } +type FileLimitsGetRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + SpaceId string `protobuf:"bytes,1,opt,name=spaceId,proto3" json:"spaceId,omitempty"` + // Identity is the uploader account identity whose pool bounds are requested + Identity string `protobuf:"bytes,2,opt,name=identity,proto3" json:"identity,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileLimitsGetRequest) Reset() { + *x = FileLimitsGetRequest{} + mi := &file_coordinator_coordinatorproto_protos_coordinator_proto_msgTypes[52] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileLimitsGetRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileLimitsGetRequest) ProtoMessage() {} + +func (x *FileLimitsGetRequest) ProtoReflect() protoreflect.Message { + mi := &file_coordinator_coordinatorproto_protos_coordinator_proto_msgTypes[52] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileLimitsGetRequest.ProtoReflect.Descriptor instead. +func (*FileLimitsGetRequest) Descriptor() ([]byte, []int) { + return file_coordinator_coordinatorproto_protos_coordinator_proto_rawDescGZIP(), []int{52} +} + +func (x *FileLimitsGetRequest) GetSpaceId() string { + if x != nil { + return x.SpaceId + } + return "" +} + +func (x *FileLimitsGetRequest) GetIdentity() string { + if x != nil { + return x.Identity + } + return "" +} + +type FileLimitsGetResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // AccountLimitBytes is the identity's total file-storage cap (the common account pool) + AccountLimitBytes uint64 `protobuf:"varint,1,opt,name=accountLimitBytes,proto3" json:"accountLimitBytes,omitempty"` + // AccountTotalUsageBytes is the coordinator-aggregated durable usage across all the identity's spaces + AccountTotalUsageBytes uint64 `protobuf:"varint,2,opt,name=accountTotalUsageBytes,proto3" json:"accountTotalUsageBytes,omitempty"` + // SpaceLimitBytes is an optional space-scoped cap; 0 = unset + SpaceLimitBytes uint64 `protobuf:"varint,3,opt,name=spaceLimitBytes,proto3" json:"spaceLimitBytes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileLimitsGetResponse) Reset() { + *x = FileLimitsGetResponse{} + mi := &file_coordinator_coordinatorproto_protos_coordinator_proto_msgTypes[53] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileLimitsGetResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileLimitsGetResponse) ProtoMessage() {} + +func (x *FileLimitsGetResponse) ProtoReflect() protoreflect.Message { + mi := &file_coordinator_coordinatorproto_protos_coordinator_proto_msgTypes[53] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileLimitsGetResponse.ProtoReflect.Descriptor instead. +func (*FileLimitsGetResponse) Descriptor() ([]byte, []int) { + return file_coordinator_coordinatorproto_protos_coordinator_proto_rawDescGZIP(), []int{53} +} + +func (x *FileLimitsGetResponse) GetAccountLimitBytes() uint64 { + if x != nil { + return x.AccountLimitBytes + } + return 0 +} + +func (x *FileLimitsGetResponse) GetAccountTotalUsageBytes() uint64 { + if x != nil { + return x.AccountTotalUsageBytes + } + return 0 +} + +func (x *FileLimitsGetResponse) GetSpaceLimitBytes() uint64 { + if x != nil { + return x.SpaceLimitBytes + } + return 0 +} + +type FileUsageReportRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Rows []*FileUsageRow `protobuf:"bytes,1,rep,name=rows,proto3" json:"rows,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileUsageReportRequest) Reset() { + *x = FileUsageReportRequest{} + mi := &file_coordinator_coordinatorproto_protos_coordinator_proto_msgTypes[54] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileUsageReportRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileUsageReportRequest) ProtoMessage() {} + +func (x *FileUsageReportRequest) ProtoReflect() protoreflect.Message { + mi := &file_coordinator_coordinatorproto_protos_coordinator_proto_msgTypes[54] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileUsageReportRequest.ProtoReflect.Descriptor instead. +func (*FileUsageReportRequest) Descriptor() ([]byte, []int) { + return file_coordinator_coordinatorproto_protos_coordinator_proto_rawDescGZIP(), []int{54} +} + +func (x *FileUsageReportRequest) GetRows() []*FileUsageRow { + if x != nil { + return x.Rows + } + return nil +} + +type FileUsageRow struct { + state protoimpl.MessageState `protogen:"open.v1"` + SpaceId string `protobuf:"bytes,1,opt,name=spaceId,proto3" json:"spaceId,omitempty"` + // Identity is the charged uploader identity (the author of the root's earliest bind row) + Identity string `protobuf:"bytes,2,opt,name=identity,proto3" json:"identity,omitempty"` + // DurableBytes is the HEAD-measured durable usage of this (space, identity) slice; in-flight is never reported + DurableBytes uint64 `protobuf:"varint,3,opt,name=durableBytes,proto3" json:"durableBytes,omitempty"` + DurableFilesCount uint64 `protobuf:"varint,4,opt,name=durableFilesCount,proto3" json:"durableFilesCount,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileUsageRow) Reset() { + *x = FileUsageRow{} + mi := &file_coordinator_coordinatorproto_protos_coordinator_proto_msgTypes[55] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileUsageRow) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileUsageRow) ProtoMessage() {} + +func (x *FileUsageRow) ProtoReflect() protoreflect.Message { + mi := &file_coordinator_coordinatorproto_protos_coordinator_proto_msgTypes[55] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileUsageRow.ProtoReflect.Descriptor instead. +func (*FileUsageRow) Descriptor() ([]byte, []int) { + return file_coordinator_coordinatorproto_protos_coordinator_proto_rawDescGZIP(), []int{55} +} + +func (x *FileUsageRow) GetSpaceId() string { + if x != nil { + return x.SpaceId + } + return "" +} + +func (x *FileUsageRow) GetIdentity() string { + if x != nil { + return x.Identity + } + return "" +} + +func (x *FileUsageRow) GetDurableBytes() uint64 { + if x != nil { + return x.DurableBytes + } + return 0 +} + +func (x *FileUsageRow) GetDurableFilesCount() uint64 { + if x != nil { + return x.DurableFilesCount + } + return 0 +} + +type FileUsageReportResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *FileUsageReportResponse) Reset() { + *x = FileUsageReportResponse{} + mi := &file_coordinator_coordinatorproto_protos_coordinator_proto_msgTypes[56] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *FileUsageReportResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileUsageReportResponse) ProtoMessage() {} + +func (x *FileUsageReportResponse) ProtoReflect() protoreflect.Message { + mi := &file_coordinator_coordinatorproto_protos_coordinator_proto_msgTypes[56] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileUsageReportResponse.ProtoReflect.Descriptor instead. +func (*FileUsageReportResponse) Descriptor() ([]byte, []int) { + return file_coordinator_coordinatorproto_protos_coordinator_proto_rawDescGZIP(), []int{56} +} + var File_coordinator_coordinatorproto_protos_coordinator_proto protoreflect.FileDescriptor const file_coordinator_coordinatorproto_protos_coordinator_proto_rawDesc = "" + @@ -3665,7 +3931,22 @@ const file_coordinator_coordinatorproto_protos_coordinator_proto_rawDesc = "" + "\x16AclUploadInviteRequest\x12\x10\n" + "\x03cid\x18\x01 \x01(\fR\x03cid\x12\x12\n" + "\x04data\x18\x02 \x01(\fR\x04data\"\x19\n" + - "\x17AclUploadInviteResponse*\xba\x02\n" + + "\x17AclUploadInviteResponse\"L\n" + + "\x14FileLimitsGetRequest\x12\x18\n" + + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x12\x1a\n" + + "\bidentity\x18\x02 \x01(\tR\bidentity\"\xa7\x01\n" + + "\x15FileLimitsGetResponse\x12,\n" + + "\x11accountLimitBytes\x18\x01 \x01(\x04R\x11accountLimitBytes\x126\n" + + "\x16accountTotalUsageBytes\x18\x02 \x01(\x04R\x16accountTotalUsageBytes\x12(\n" + + "\x0fspaceLimitBytes\x18\x03 \x01(\x04R\x0fspaceLimitBytes\"G\n" + + "\x16FileUsageReportRequest\x12-\n" + + "\x04rows\x18\x01 \x03(\v2\x19.coordinator.FileUsageRowR\x04rows\"\x96\x01\n" + + "\fFileUsageRow\x12\x18\n" + + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x12\x1a\n" + + "\bidentity\x18\x02 \x01(\tR\bidentity\x12\"\n" + + "\fdurableBytes\x18\x03 \x01(\x04R\fdurableBytes\x12,\n" + + "\x11durableFilesCount\x18\x04 \x01(\x04R\x11durableFilesCount\"\x19\n" + + "\x17FileUsageReportResponse*\xba\x02\n" + "\n" + "ErrorCodes\x12\x0e\n" + "\n" + @@ -3726,7 +4007,7 @@ const file_coordinator_coordinatorproto_protos_coordinator_proto_rawDesc = "" + "\x0fNotifyEventType\x12\x14\n" + "\x10UnspecifiedEvent\x10\x00\x12\x18\n" + "\x14InboxNewMessageEvent\x10\x01\x12\x1d\n" + - "\x19NetworkConfigChangedEvent\x10\x022\x80\x0e\n" + + "\x19NetworkConfigChangedEvent\x10\x022\xb6\x0f\n" + "\vCoordinator\x12J\n" + "\tSpaceSign\x12\x1d.coordinator.SpaceSignRequest\x1a\x1e.coordinator.SpaceSignResponse\x12_\n" + "\x10SpaceStatusCheck\x12$.coordinator.SpaceStatusCheckRequest\x1a%.coordinator.SpaceStatusCheckResponse\x12k\n" + @@ -3747,7 +4028,9 @@ const file_coordinator_coordinatorproto_protos_coordinator_proto_rawDesc = "" + "InboxFetch\x12\x1e.coordinator.InboxFetchRequest\x1a\x1f.coordinator.InboxFetchResponse\x12\\\n" + "\x0fInboxAddMessage\x12#.coordinator.InboxAddMessageRequest\x1a$.coordinator.InboxAddMessageResponse\x12[\n" + "\x0fNotifySubscribe\x12#.coordinator.NotifySubscribeRequest\x1a!.coordinator.NotifySubscribeEvent0\x01\x12\\\n" + - "\x0fAclUploadInvite\x12#.coordinator.AclUploadInviteRequest\x1a$.coordinator.AclUploadInviteResponseB\x1eZ\x1ccoordinator/coordinatorprotob\x06proto3" + "\x0fAclUploadInvite\x12#.coordinator.AclUploadInviteRequest\x1a$.coordinator.AclUploadInviteResponse\x12V\n" + + "\rFileLimitsGet\x12!.coordinator.FileLimitsGetRequest\x1a\".coordinator.FileLimitsGetResponse\x12\\\n" + + "\x0fFileUsageReport\x12#.coordinator.FileUsageReportRequest\x1a$.coordinator.FileUsageReportResponseB\x1eZ\x1ccoordinator/coordinatorprotob\x06proto3" var ( file_coordinator_coordinatorproto_protos_coordinator_proto_rawDescOnce sync.Once @@ -3762,7 +4045,7 @@ func file_coordinator_coordinatorproto_protos_coordinator_proto_rawDescGZIP() [] } var file_coordinator_coordinatorproto_protos_coordinator_proto_enumTypes = make([]protoimpl.EnumInfo, 11) -var file_coordinator_coordinatorproto_protos_coordinator_proto_msgTypes = make([]protoimpl.MessageInfo, 52) +var file_coordinator_coordinatorproto_protos_coordinator_proto_msgTypes = make([]protoimpl.MessageInfo, 57) var file_coordinator_coordinatorproto_protos_coordinator_proto_goTypes = []any{ (ErrorCodes)(0), // 0: coordinator.ErrorCodes (SpaceStatus)(0), // 1: coordinator.SpaceStatus @@ -3827,6 +4110,11 @@ var file_coordinator_coordinatorproto_protos_coordinator_proto_goTypes = []any{ (*NotifySubscribeEvent)(nil), // 60: coordinator.NotifySubscribeEvent (*AclUploadInviteRequest)(nil), // 61: coordinator.AclUploadInviteRequest (*AclUploadInviteResponse)(nil), // 62: coordinator.AclUploadInviteResponse + (*FileLimitsGetRequest)(nil), // 63: coordinator.FileLimitsGetRequest + (*FileLimitsGetResponse)(nil), // 64: coordinator.FileLimitsGetResponse + (*FileUsageReportRequest)(nil), // 65: coordinator.FileUsageReportRequest + (*FileUsageRow)(nil), // 66: coordinator.FileUsageRow + (*FileUsageReportResponse)(nil), // 67: coordinator.FileUsageReportResponse } var file_coordinator_coordinatorproto_protos_coordinator_proto_depIdxs = []int32{ 1, // 0: coordinator.SpaceStatusPayload.status:type_name -> coordinator.SpaceStatus @@ -3853,49 +4141,54 @@ var file_coordinator_coordinatorproto_protos_coordinator_proto_depIdxs = []int32 52, // 21: coordinator.InboxAddMessageRequest.message:type_name -> coordinator.InboxMessage 10, // 22: coordinator.NotifySubscribeRequest.eventType:type_name -> coordinator.NotifyEventType 10, // 23: coordinator.NotifySubscribeEvent.eventType:type_name -> coordinator.NotifyEventType - 11, // 24: coordinator.Coordinator.SpaceSign:input_type -> coordinator.SpaceSignRequest - 17, // 25: coordinator.Coordinator.SpaceStatusCheck:input_type -> coordinator.SpaceStatusCheckRequest - 19, // 26: coordinator.Coordinator.SpaceStatusCheckMany:input_type -> coordinator.SpaceStatusCheckManyRequest - 22, // 27: coordinator.Coordinator.SpaceStatusChange:input_type -> coordinator.SpaceStatusChangeRequest - 24, // 28: coordinator.Coordinator.SpaceMakeShareable:input_type -> coordinator.SpaceMakeShareableRequest - 26, // 29: coordinator.Coordinator.SpaceMakeUnshareable:input_type -> coordinator.SpaceMakeUnshareableRequest - 28, // 30: coordinator.Coordinator.NetworkConfiguration:input_type -> coordinator.NetworkConfigurationRequest - 33, // 31: coordinator.Coordinator.DeletionLog:input_type -> coordinator.DeletionLogRequest - 36, // 32: coordinator.Coordinator.SpaceDelete:input_type -> coordinator.SpaceDeleteRequest - 38, // 33: coordinator.Coordinator.AccountDelete:input_type -> coordinator.AccountDeleteRequest - 41, // 34: coordinator.Coordinator.AccountRevertDeletion:input_type -> coordinator.AccountRevertDeletionRequest - 43, // 35: coordinator.Coordinator.AclAddRecord:input_type -> coordinator.AclAddRecordRequest - 45, // 36: coordinator.Coordinator.AclGetRecords:input_type -> coordinator.AclGetRecordsRequest - 47, // 37: coordinator.Coordinator.AccountLimitsSet:input_type -> coordinator.AccountLimitsSetRequest - 49, // 38: coordinator.Coordinator.AclEventLog:input_type -> coordinator.AclEventLogRequest - 55, // 39: coordinator.Coordinator.InboxFetch:input_type -> coordinator.InboxFetchRequest - 57, // 40: coordinator.Coordinator.InboxAddMessage:input_type -> coordinator.InboxAddMessageRequest - 59, // 41: coordinator.Coordinator.NotifySubscribe:input_type -> coordinator.NotifySubscribeRequest - 61, // 42: coordinator.Coordinator.AclUploadInvite:input_type -> coordinator.AclUploadInviteRequest - 14, // 43: coordinator.Coordinator.SpaceSign:output_type -> coordinator.SpaceSignResponse - 18, // 44: coordinator.Coordinator.SpaceStatusCheck:output_type -> coordinator.SpaceStatusCheckResponse - 20, // 45: coordinator.Coordinator.SpaceStatusCheckMany:output_type -> coordinator.SpaceStatusCheckManyResponse - 23, // 46: coordinator.Coordinator.SpaceStatusChange:output_type -> coordinator.SpaceStatusChangeResponse - 25, // 47: coordinator.Coordinator.SpaceMakeShareable:output_type -> coordinator.SpaceMakeShareableResponse - 27, // 48: coordinator.Coordinator.SpaceMakeUnshareable:output_type -> coordinator.SpaceMakeUnshareableResponse - 29, // 49: coordinator.Coordinator.NetworkConfiguration:output_type -> coordinator.NetworkConfigurationResponse - 34, // 50: coordinator.Coordinator.DeletionLog:output_type -> coordinator.DeletionLogResponse - 37, // 51: coordinator.Coordinator.SpaceDelete:output_type -> coordinator.SpaceDeleteResponse - 40, // 52: coordinator.Coordinator.AccountDelete:output_type -> coordinator.AccountDeleteResponse - 42, // 53: coordinator.Coordinator.AccountRevertDeletion:output_type -> coordinator.AccountRevertDeletionResponse - 44, // 54: coordinator.Coordinator.AclAddRecord:output_type -> coordinator.AclAddRecordResponse - 46, // 55: coordinator.Coordinator.AclGetRecords:output_type -> coordinator.AclGetRecordsResponse - 48, // 56: coordinator.Coordinator.AccountLimitsSet:output_type -> coordinator.AccountLimitsSetResponse - 50, // 57: coordinator.Coordinator.AclEventLog:output_type -> coordinator.AclEventLogResponse - 56, // 58: coordinator.Coordinator.InboxFetch:output_type -> coordinator.InboxFetchResponse - 58, // 59: coordinator.Coordinator.InboxAddMessage:output_type -> coordinator.InboxAddMessageResponse - 60, // 60: coordinator.Coordinator.NotifySubscribe:output_type -> coordinator.NotifySubscribeEvent - 62, // 61: coordinator.Coordinator.AclUploadInvite:output_type -> coordinator.AclUploadInviteResponse - 43, // [43:62] is the sub-list for method output_type - 24, // [24:43] is the sub-list for method input_type - 24, // [24:24] is the sub-list for extension type_name - 24, // [24:24] is the sub-list for extension extendee - 0, // [0:24] is the sub-list for field type_name + 66, // 24: coordinator.FileUsageReportRequest.rows:type_name -> coordinator.FileUsageRow + 11, // 25: coordinator.Coordinator.SpaceSign:input_type -> coordinator.SpaceSignRequest + 17, // 26: coordinator.Coordinator.SpaceStatusCheck:input_type -> coordinator.SpaceStatusCheckRequest + 19, // 27: coordinator.Coordinator.SpaceStatusCheckMany:input_type -> coordinator.SpaceStatusCheckManyRequest + 22, // 28: coordinator.Coordinator.SpaceStatusChange:input_type -> coordinator.SpaceStatusChangeRequest + 24, // 29: coordinator.Coordinator.SpaceMakeShareable:input_type -> coordinator.SpaceMakeShareableRequest + 26, // 30: coordinator.Coordinator.SpaceMakeUnshareable:input_type -> coordinator.SpaceMakeUnshareableRequest + 28, // 31: coordinator.Coordinator.NetworkConfiguration:input_type -> coordinator.NetworkConfigurationRequest + 33, // 32: coordinator.Coordinator.DeletionLog:input_type -> coordinator.DeletionLogRequest + 36, // 33: coordinator.Coordinator.SpaceDelete:input_type -> coordinator.SpaceDeleteRequest + 38, // 34: coordinator.Coordinator.AccountDelete:input_type -> coordinator.AccountDeleteRequest + 41, // 35: coordinator.Coordinator.AccountRevertDeletion:input_type -> coordinator.AccountRevertDeletionRequest + 43, // 36: coordinator.Coordinator.AclAddRecord:input_type -> coordinator.AclAddRecordRequest + 45, // 37: coordinator.Coordinator.AclGetRecords:input_type -> coordinator.AclGetRecordsRequest + 47, // 38: coordinator.Coordinator.AccountLimitsSet:input_type -> coordinator.AccountLimitsSetRequest + 49, // 39: coordinator.Coordinator.AclEventLog:input_type -> coordinator.AclEventLogRequest + 55, // 40: coordinator.Coordinator.InboxFetch:input_type -> coordinator.InboxFetchRequest + 57, // 41: coordinator.Coordinator.InboxAddMessage:input_type -> coordinator.InboxAddMessageRequest + 59, // 42: coordinator.Coordinator.NotifySubscribe:input_type -> coordinator.NotifySubscribeRequest + 61, // 43: coordinator.Coordinator.AclUploadInvite:input_type -> coordinator.AclUploadInviteRequest + 63, // 44: coordinator.Coordinator.FileLimitsGet:input_type -> coordinator.FileLimitsGetRequest + 65, // 45: coordinator.Coordinator.FileUsageReport:input_type -> coordinator.FileUsageReportRequest + 14, // 46: coordinator.Coordinator.SpaceSign:output_type -> coordinator.SpaceSignResponse + 18, // 47: coordinator.Coordinator.SpaceStatusCheck:output_type -> coordinator.SpaceStatusCheckResponse + 20, // 48: coordinator.Coordinator.SpaceStatusCheckMany:output_type -> coordinator.SpaceStatusCheckManyResponse + 23, // 49: coordinator.Coordinator.SpaceStatusChange:output_type -> coordinator.SpaceStatusChangeResponse + 25, // 50: coordinator.Coordinator.SpaceMakeShareable:output_type -> coordinator.SpaceMakeShareableResponse + 27, // 51: coordinator.Coordinator.SpaceMakeUnshareable:output_type -> coordinator.SpaceMakeUnshareableResponse + 29, // 52: coordinator.Coordinator.NetworkConfiguration:output_type -> coordinator.NetworkConfigurationResponse + 34, // 53: coordinator.Coordinator.DeletionLog:output_type -> coordinator.DeletionLogResponse + 37, // 54: coordinator.Coordinator.SpaceDelete:output_type -> coordinator.SpaceDeleteResponse + 40, // 55: coordinator.Coordinator.AccountDelete:output_type -> coordinator.AccountDeleteResponse + 42, // 56: coordinator.Coordinator.AccountRevertDeletion:output_type -> coordinator.AccountRevertDeletionResponse + 44, // 57: coordinator.Coordinator.AclAddRecord:output_type -> coordinator.AclAddRecordResponse + 46, // 58: coordinator.Coordinator.AclGetRecords:output_type -> coordinator.AclGetRecordsResponse + 48, // 59: coordinator.Coordinator.AccountLimitsSet:output_type -> coordinator.AccountLimitsSetResponse + 50, // 60: coordinator.Coordinator.AclEventLog:output_type -> coordinator.AclEventLogResponse + 56, // 61: coordinator.Coordinator.InboxFetch:output_type -> coordinator.InboxFetchResponse + 58, // 62: coordinator.Coordinator.InboxAddMessage:output_type -> coordinator.InboxAddMessageResponse + 60, // 63: coordinator.Coordinator.NotifySubscribe:output_type -> coordinator.NotifySubscribeEvent + 62, // 64: coordinator.Coordinator.AclUploadInvite:output_type -> coordinator.AclUploadInviteResponse + 64, // 65: coordinator.Coordinator.FileLimitsGet:output_type -> coordinator.FileLimitsGetResponse + 67, // 66: coordinator.Coordinator.FileUsageReport:output_type -> coordinator.FileUsageReportResponse + 46, // [46:67] is the sub-list for method output_type + 25, // [25:46] is the sub-list for method input_type + 25, // [25:25] is the sub-list for extension type_name + 25, // [25:25] is the sub-list for extension extendee + 0, // [0:25] is the sub-list for field type_name } func init() { file_coordinator_coordinatorproto_protos_coordinator_proto_init() } @@ -3909,7 +4202,7 @@ func file_coordinator_coordinatorproto_protos_coordinator_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_coordinator_coordinatorproto_protos_coordinator_proto_rawDesc), len(file_coordinator_coordinatorproto_protos_coordinator_proto_rawDesc)), NumEnums: 11, - NumMessages: 52, + NumMessages: 57, NumExtensions: 0, NumServices: 1, }, diff --git a/coordinator/coordinatorproto/coordinator_drpc.pb.go b/coordinator/coordinatorproto/coordinator_drpc.pb.go index 302b51fa1..3916a2cf4 100644 --- a/coordinator/coordinatorproto/coordinator_drpc.pb.go +++ b/coordinator/coordinatorproto/coordinator_drpc.pb.go @@ -52,6 +52,8 @@ type DRPCCoordinatorClient interface { InboxAddMessage(ctx context.Context, in *InboxAddMessageRequest) (*InboxAddMessageResponse, error) NotifySubscribe(ctx context.Context, in *NotifySubscribeRequest) (DRPCCoordinator_NotifySubscribeClient, error) AclUploadInvite(ctx context.Context, in *AclUploadInviteRequest) (*AclUploadInviteResponse, error) + FileLimitsGet(ctx context.Context, in *FileLimitsGetRequest) (*FileLimitsGetResponse, error) + FileUsageReport(ctx context.Context, in *FileUsageReportRequest) (*FileUsageReportResponse, error) } type drpcCoordinatorClient struct { @@ -266,6 +268,24 @@ func (c *drpcCoordinatorClient) AclUploadInvite(ctx context.Context, in *AclUplo return out, nil } +func (c *drpcCoordinatorClient) FileLimitsGet(ctx context.Context, in *FileLimitsGetRequest) (*FileLimitsGetResponse, error) { + out := new(FileLimitsGetResponse) + err := c.cc.Invoke(ctx, "/coordinator.Coordinator/FileLimitsGet", drpcEncoding_File_coordinator_coordinatorproto_protos_coordinator_proto{}, in, out) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *drpcCoordinatorClient) FileUsageReport(ctx context.Context, in *FileUsageReportRequest) (*FileUsageReportResponse, error) { + out := new(FileUsageReportResponse) + err := c.cc.Invoke(ctx, "/coordinator.Coordinator/FileUsageReport", drpcEncoding_File_coordinator_coordinatorproto_protos_coordinator_proto{}, in, out) + if err != nil { + return nil, err + } + return out, nil +} + type DRPCCoordinatorServer interface { SpaceSign(context.Context, *SpaceSignRequest) (*SpaceSignResponse, error) SpaceStatusCheck(context.Context, *SpaceStatusCheckRequest) (*SpaceStatusCheckResponse, error) @@ -286,6 +306,8 @@ type DRPCCoordinatorServer interface { InboxAddMessage(context.Context, *InboxAddMessageRequest) (*InboxAddMessageResponse, error) NotifySubscribe(*NotifySubscribeRequest, DRPCCoordinator_NotifySubscribeStream) error AclUploadInvite(context.Context, *AclUploadInviteRequest) (*AclUploadInviteResponse, error) + FileLimitsGet(context.Context, *FileLimitsGetRequest) (*FileLimitsGetResponse, error) + FileUsageReport(context.Context, *FileUsageReportRequest) (*FileUsageReportResponse, error) } type DRPCCoordinatorUnimplementedServer struct{} @@ -366,9 +388,17 @@ func (s *DRPCCoordinatorUnimplementedServer) AclUploadInvite(context.Context, *A return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) } +func (s *DRPCCoordinatorUnimplementedServer) FileLimitsGet(context.Context, *FileLimitsGetRequest) (*FileLimitsGetResponse, error) { + return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) +} + +func (s *DRPCCoordinatorUnimplementedServer) FileUsageReport(context.Context, *FileUsageReportRequest) (*FileUsageReportResponse, error) { + return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) +} + type DRPCCoordinatorDescription struct{} -func (DRPCCoordinatorDescription) NumMethods() int { return 19 } +func (DRPCCoordinatorDescription) NumMethods() int { return 21 } func (DRPCCoordinatorDescription) Method(n int) (string, drpc.Encoding, drpc.Receiver, interface{}, bool) { switch n { @@ -543,6 +573,24 @@ func (DRPCCoordinatorDescription) Method(n int) (string, drpc.Encoding, drpc.Rec in1.(*AclUploadInviteRequest), ) }, DRPCCoordinatorServer.AclUploadInvite, true + case 19: + return "/coordinator.Coordinator/FileLimitsGet", drpcEncoding_File_coordinator_coordinatorproto_protos_coordinator_proto{}, + func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) { + return srv.(DRPCCoordinatorServer). + FileLimitsGet( + ctx, + in1.(*FileLimitsGetRequest), + ) + }, DRPCCoordinatorServer.FileLimitsGet, true + case 20: + return "/coordinator.Coordinator/FileUsageReport", drpcEncoding_File_coordinator_coordinatorproto_protos_coordinator_proto{}, + func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) { + return srv.(DRPCCoordinatorServer). + FileUsageReport( + ctx, + in1.(*FileUsageReportRequest), + ) + }, DRPCCoordinatorServer.FileUsageReport, true default: return "", nil, nil, nil, false } @@ -852,3 +900,35 @@ func (x *drpcCoordinator_AclUploadInviteStream) SendAndClose(m *AclUploadInviteR } return x.CloseSend() } + +type DRPCCoordinator_FileLimitsGetStream interface { + drpc.Stream + SendAndClose(*FileLimitsGetResponse) error +} + +type drpcCoordinator_FileLimitsGetStream struct { + drpc.Stream +} + +func (x *drpcCoordinator_FileLimitsGetStream) SendAndClose(m *FileLimitsGetResponse) error { + if err := x.MsgSend(m, drpcEncoding_File_coordinator_coordinatorproto_protos_coordinator_proto{}); err != nil { + return err + } + return x.CloseSend() +} + +type DRPCCoordinator_FileUsageReportStream interface { + drpc.Stream + SendAndClose(*FileUsageReportResponse) error +} + +type drpcCoordinator_FileUsageReportStream struct { + drpc.Stream +} + +func (x *drpcCoordinator_FileUsageReportStream) SendAndClose(m *FileUsageReportResponse) error { + if err := x.MsgSend(m, drpcEncoding_File_coordinator_coordinatorproto_protos_coordinator_proto{}); err != nil { + return err + } + return x.CloseSend() +} diff --git a/coordinator/coordinatorproto/coordinator_vtproto.pb.go b/coordinator/coordinatorproto/coordinator_vtproto.pb.go index 0c5b8020f..d227c9458 100644 --- a/coordinator/coordinatorproto/coordinator_vtproto.pb.go +++ b/coordinator/coordinatorproto/coordinator_vtproto.pb.go @@ -2563,6 +2563,236 @@ func (m *AclUploadInviteResponse) MarshalToSizedBufferVT(dAtA []byte) (int, erro return len(dAtA) - i, nil } +func (m *FileLimitsGetRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FileLimitsGetRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *FileLimitsGetRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Identity) > 0 { + i -= len(m.Identity) + copy(dAtA[i:], m.Identity) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Identity))) + i-- + dAtA[i] = 0x12 + } + if len(m.SpaceId) > 0 { + i -= len(m.SpaceId) + copy(dAtA[i:], m.SpaceId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *FileLimitsGetResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FileLimitsGetResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *FileLimitsGetResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.SpaceLimitBytes != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.SpaceLimitBytes)) + i-- + dAtA[i] = 0x18 + } + if m.AccountTotalUsageBytes != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.AccountTotalUsageBytes)) + i-- + dAtA[i] = 0x10 + } + if m.AccountLimitBytes != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.AccountLimitBytes)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + +func (m *FileUsageReportRequest) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FileUsageReportRequest) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *FileUsageReportRequest) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.Rows) > 0 { + for iNdEx := len(m.Rows) - 1; iNdEx >= 0; iNdEx-- { + size, err := m.Rows[iNdEx].MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *FileUsageRow) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FileUsageRow) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *FileUsageRow) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.DurableFilesCount != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.DurableFilesCount)) + i-- + dAtA[i] = 0x20 + } + if m.DurableBytes != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.DurableBytes)) + i-- + dAtA[i] = 0x18 + } + if len(m.Identity) > 0 { + i -= len(m.Identity) + copy(dAtA[i:], m.Identity) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Identity))) + i-- + dAtA[i] = 0x12 + } + if len(m.SpaceId) > 0 { + i -= len(m.SpaceId) + copy(dAtA[i:], m.SpaceId) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceId))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *FileUsageReportResponse) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FileUsageReportResponse) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *FileUsageReportResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + return len(dAtA) - i, nil +} + func (m *SpaceSignRequest) SizeVT() (n int) { if m == nil { return 0 @@ -3527,60 +3757,147 @@ func (m *AclUploadInviteResponse) SizeVT() (n int) { return n } -func (m *SpaceSignRequest) UnmarshalVT(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: SpaceSignRequest: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: SpaceSignRequest: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field SpaceId", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return protohelpers.ErrIntOverflow - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLen := int(stringLen) - if intStringLen < 0 { - return protohelpers.ErrInvalidLength - } - postIndex := iNdEx + intStringLen - if postIndex < 0 { +func (m *FileLimitsGetRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.SpaceId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Identity) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + +func (m *FileLimitsGetResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.AccountLimitBytes != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.AccountLimitBytes)) + } + if m.AccountTotalUsageBytes != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.AccountTotalUsageBytes)) + } + if m.SpaceLimitBytes != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.SpaceLimitBytes)) + } + n += len(m.unknownFields) + return n +} + +func (m *FileUsageReportRequest) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Rows) > 0 { + for _, e := range m.Rows { + l = e.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + +func (m *FileUsageRow) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.SpaceId) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + l = len(m.Identity) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if m.DurableBytes != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.DurableBytes)) + } + if m.DurableFilesCount != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.DurableFilesCount)) + } + n += len(m.unknownFields) + return n +} + +func (m *FileUsageReportResponse) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + n += len(m.unknownFields) + return n +} + +func (m *SpaceSignRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SpaceSignRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SpaceSignRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { return protohelpers.ErrInvalidLength } if postIndex > l { @@ -9528,3 +9845,515 @@ func (m *AclUploadInviteResponse) UnmarshalVT(dAtA []byte) error { } return nil } +func (m *FileLimitsGetRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FileLimitsGetRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FileLimitsGetRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Identity", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Identity = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *FileLimitsGetResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FileLimitsGetResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FileLimitsGetResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AccountLimitBytes", wireType) + } + m.AccountLimitBytes = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.AccountLimitBytes |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AccountTotalUsageBytes", wireType) + } + m.AccountTotalUsageBytes = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.AccountTotalUsageBytes |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceLimitBytes", wireType) + } + m.SpaceLimitBytes = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.SpaceLimitBytes |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *FileUsageReportRequest) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FileUsageReportRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FileUsageReportRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Rows", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Rows = append(m.Rows, &FileUsageRow{}) + if err := m.Rows[len(m.Rows)-1].UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *FileUsageRow) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FileUsageRow: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FileUsageRow: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Identity", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Identity = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field DurableBytes", wireType) + } + m.DurableBytes = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.DurableBytes |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field DurableFilesCount", wireType) + } + m.DurableFilesCount = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.DurableFilesCount |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *FileUsageReportResponse) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FileUsageReportResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FileUsageReportResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} diff --git a/coordinator/coordinatorproto/protos/coordinator.proto b/coordinator/coordinatorproto/protos/coordinator.proto index 372a79072..2b0bff0a2 100644 --- a/coordinator/coordinatorproto/protos/coordinator.proto +++ b/coordinator/coordinatorproto/protos/coordinator.proto @@ -57,6 +57,12 @@ service Coordinator { rpc NotifySubscribe(NotifySubscribeRequest) returns (stream NotifySubscribeEvent); // AclUploadInvite uploads invite binary data to the filenodes rpc AclUploadInvite(AclUploadInviteRequest) returns (AclUploadInviteResponse); + // FileLimitsGet returns the file-storage pool bounds for an uploader identity in a space (files v2; fileV2 nodes only). + // The response carries absolute bounds — the caller computes effective headroom locally. + rpc FileLimitsGet(FileLimitsGetRequest) returns (FileLimitsGetResponse); + // FileUsageReport accepts per-(space, identity) durable file usage reports from fileV2 nodes. + // The reporting node is taken from the authenticated peer context. Values are absolute (idempotent re-reports). + rpc FileUsageReport(FileUsageReportRequest) returns (FileUsageReportResponse); } @@ -500,3 +506,33 @@ message AclUploadInviteRequest { } message AclUploadInviteResponse {} + +message FileLimitsGetRequest { + string spaceId = 1; + // Identity is the uploader account identity whose pool bounds are requested + string identity = 2; +} + +message FileLimitsGetResponse { + // AccountLimitBytes is the identity's total file-storage cap (the common account pool) + uint64 accountLimitBytes = 1; + // AccountTotalUsageBytes is the coordinator-aggregated durable usage across all the identity's spaces + uint64 accountTotalUsageBytes = 2; + // SpaceLimitBytes is an optional space-scoped cap; 0 = unset + uint64 spaceLimitBytes = 3; +} + +message FileUsageReportRequest { + repeated FileUsageRow rows = 1; +} + +message FileUsageRow { + string spaceId = 1; + // Identity is the charged uploader identity (the author of the root's earliest bind row) + string identity = 2; + // DurableBytes is the HEAD-measured durable usage of this (space, identity) slice; in-flight is never reported + uint64 durableBytes = 3; + uint64 durableFilesCount = 4; +} + +message FileUsageReportResponse {} From 07c232258bc4c90f33e74464718dccce77e63188 Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Sat, 4 Jul 2026 21:10:23 +0200 Subject: [PATCH 17/20] feat(fileprotov2): account-pool fields in SpaceQuota (SYN-19) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SpaceInfo now surfaces the requesting identity's shared account pool (accountLimitBytes / accountUsageBytes) next to the per-space view — files are charged to their uploader, all spaces share one pool. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01AoLHZmQFxyhxDkkwJ7Y1jX --- commonfile/fileproto/fileprotov2/filev2.pb.go | 31 +++++++++-- .../fileprotov2/filev2_vtproto.pb.go | 54 +++++++++++++++++++ .../fileproto/fileprotov2/protos/filev2.proto | 7 ++- 3 files changed, 86 insertions(+), 6 deletions(-) diff --git a/commonfile/fileproto/fileprotov2/filev2.pb.go b/commonfile/fileproto/fileprotov2/filev2.pb.go index 64f747220..fd252132d 100644 --- a/commonfile/fileproto/fileprotov2/filev2.pb.go +++ b/commonfile/fileproto/fileprotov2/filev2.pb.go @@ -1061,13 +1061,18 @@ func (x *SpaceInfoResult) GetQuota() *SpaceQuota { type SpaceQuota struct { state protoimpl.MessageState `protogen:"open.v1"` - LimitBytes uint64 `protobuf:"varint,1,opt,name=limitBytes,proto3" json:"limitBytes,omitempty"` // effective storage limit for this space + LimitBytes uint64 `protobuf:"varint,1,opt,name=limitBytes,proto3" json:"limitBytes,omitempty"` // effective storage limit for the requester in this space TotalUsageBytes uint64 `protobuf:"varint,2,opt,name=totalUsageBytes,proto3" json:"totalUsageBytes,omitempty"` // counted usage == durableUsageBytes + inflightUsageBytes DurableUsageBytes uint64 `protobuf:"varint,3,opt,name=durableUsageBytes,proto3" json:"durableUsageBytes,omitempty"` // bytes fully committed to durable storage InflightUsageBytes uint64 `protobuf:"varint,4,opt,name=inflightUsageBytes,proto3" json:"inflightUsageBytes,omitempty"` // bytes reserved by presigned/not-yet-committed uploads FilesCount uint64 `protobuf:"varint,5,opt,name=filesCount,proto3" json:"filesCount,omitempty"` // number of file roots stored in the space - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // account-pool view for the requesting identity (files are charged to + // their uploader; all the identity's spaces share one pool). Zero when + // the node has no coordinator-backed limits (self-hosted fallback). + AccountLimitBytes uint64 `protobuf:"varint,6,opt,name=accountLimitBytes,proto3" json:"accountLimitBytes,omitempty"` // the identity's total file-storage cap + AccountUsageBytes uint64 `protobuf:"varint,7,opt,name=accountUsageBytes,proto3" json:"accountUsageBytes,omitempty"` // aggregated durable usage across the identity's spaces + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SpaceQuota) Reset() { @@ -1135,6 +1140,20 @@ func (x *SpaceQuota) GetFilesCount() uint64 { return 0 } +func (x *SpaceQuota) GetAccountLimitBytes() uint64 { + if x != nil { + return x.AccountLimitBytes + } + return 0 +} + +func (x *SpaceQuota) GetAccountUsageBytes() uint64 { + if x != nil { + return x.AccountUsageBytes + } + return 0 +} + // InfoRequest is deliberately empty: Info is static node metadata, // extensible later (capabilities, limits) without a wire break. type InfoRequest struct { @@ -1277,7 +1296,7 @@ const file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDesc = "" + "\x0fSpaceInfoResult\x12\x18\n" + "\aspaceId\x18\x01 \x01(\tR\aspaceId\x12'\n" + "\x04code\x18\x02 \x01(\x0e2\x13.filesyncv2.ErrCodeR\x04code\x12,\n" + - "\x05quota\x18\x03 \x01(\v2\x16.filesyncv2.SpaceQuotaR\x05quota\"\xd4\x01\n" + + "\x05quota\x18\x03 \x01(\v2\x16.filesyncv2.SpaceQuotaR\x05quota\"\xb0\x02\n" + "\n" + "SpaceQuota\x12\x1e\n" + "\n" + @@ -1288,7 +1307,9 @@ const file_commonfile_fileproto_fileprotov2_protos_filev2_proto_rawDesc = "" + "\x12inflightUsageBytes\x18\x04 \x01(\x04R\x12inflightUsageBytes\x12\x1e\n" + "\n" + "filesCount\x18\x05 \x01(\x04R\n" + - "filesCount\"\r\n" + + "filesCount\x12,\n" + + "\x11accountLimitBytes\x18\x06 \x01(\x04R\x11accountLimitBytes\x12,\n" + + "\x11accountUsageBytes\x18\a \x01(\x04R\x11accountUsageBytes\"\r\n" + "\vInfoRequest\"<\n" + "\fInfoResponse\x12,\n" + "\x11publicReadBaseUrl\x18\x01 \x01(\tR\x11publicReadBaseUrl*z\n" + diff --git a/commonfile/fileproto/fileprotov2/filev2_vtproto.pb.go b/commonfile/fileproto/fileprotov2/filev2_vtproto.pb.go index c31a407d7..ad91628c1 100644 --- a/commonfile/fileproto/fileprotov2/filev2_vtproto.pb.go +++ b/commonfile/fileproto/fileprotov2/filev2_vtproto.pb.go @@ -886,6 +886,16 @@ func (m *SpaceQuota) MarshalToSizedBufferVT(dAtA []byte) (int, error) { i -= len(m.unknownFields) copy(dAtA[i:], m.unknownFields) } + if m.AccountUsageBytes != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.AccountUsageBytes)) + i-- + dAtA[i] = 0x38 + } + if m.AccountLimitBytes != 0 { + i = protohelpers.EncodeVarint(dAtA, i, uint64(m.AccountLimitBytes)) + i-- + dAtA[i] = 0x30 + } if m.FilesCount != 0 { i = protohelpers.EncodeVarint(dAtA, i, uint64(m.FilesCount)) i-- @@ -1328,6 +1338,12 @@ func (m *SpaceQuota) SizeVT() (n int) { if m.FilesCount != 0 { n += 1 + protohelpers.SizeOfVarint(uint64(m.FilesCount)) } + if m.AccountLimitBytes != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.AccountLimitBytes)) + } + if m.AccountUsageBytes != 0 { + n += 1 + protohelpers.SizeOfVarint(uint64(m.AccountUsageBytes)) + } n += len(m.unknownFields) return n } @@ -3403,6 +3419,44 @@ func (m *SpaceQuota) UnmarshalVT(dAtA []byte) error { break } } + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AccountLimitBytes", wireType) + } + m.AccountLimitBytes = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.AccountLimitBytes |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 7: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field AccountUsageBytes", wireType) + } + m.AccountUsageBytes = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.AccountUsageBytes |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := protohelpers.Skip(dAtA[iNdEx:]) diff --git a/commonfile/fileproto/fileprotov2/protos/filev2.proto b/commonfile/fileproto/fileprotov2/protos/filev2.proto index 0fd7f2cfe..67dcf3d1a 100644 --- a/commonfile/fileproto/fileprotov2/protos/filev2.proto +++ b/commonfile/fileproto/fileprotov2/protos/filev2.proto @@ -186,11 +186,16 @@ message SpaceInfoResult { } message SpaceQuota { - uint64 limitBytes = 1; // effective storage limit for this space + uint64 limitBytes = 1; // effective storage limit for the requester in this space uint64 totalUsageBytes = 2; // counted usage == durableUsageBytes + inflightUsageBytes uint64 durableUsageBytes = 3; // bytes fully committed to durable storage uint64 inflightUsageBytes = 4; // bytes reserved by presigned/not-yet-committed uploads uint64 filesCount = 5; // number of file roots stored in the space + // account-pool view for the requesting identity (files are charged to + // their uploader; all the identity's spaces share one pool). Zero when + // the node has no coordinator-backed limits (self-hosted fallback). + uint64 accountLimitBytes = 6; // the identity's total file-storage cap + uint64 accountUsageBytes = 7; // aggregated durable usage across the identity's spaces } // ---- Info (node-level metadata) --------------------------------------------- From b98dfbeb1aa46ad37c291f41f01e98ca52c802ca Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Mon, 6 Jul 2026 14:07:58 +0200 Subject: [PATCH 18/20] feat(clientspaceproto): SpaceExchangeV2 token-based LAN space discovery Replaces plaintext spaceId lists in LAN peer discovery with per-space HMAC tokens keyed by a discovery key derived (HKDF) from the space's first ACL read key. Peers learn only the intersection of their space sets; a valid token simultaneously hides the space from non-members and proves the sender's membership, so strangers on the LAN can no longer enumerate spaces, correlate devices across sessions, or poison the local peer table with spaces they don't hold. Request tokens are padded with random fillers to fixed buckets and sorted to hide the true space count; response tokens are keyed by the caller's nonce (challenge-response). Gated by ProtoVersion 13; v1 fallback must be version-gated, never error-triggered. Bench (1000 spaces/peer, 500 shared): ~1.5ms CPU for a full round trip, plus 0.66ms one-time discovery-key derivation. Co-Authored-By: Claude Fable 5 --- .../clientspaceproto/clientspace.pb.go | 159 +++++++- .../clientspaceproto/clientspace_drpc.pb.go | 42 +- .../clientspace_vtproto.pb.go | 377 ++++++++++++++++++ commonspace/clientspaceproto/exchange2.go | 144 +++++++ .../clientspaceproto/exchange2_bench_test.go | 135 +++++++ .../clientspaceproto/exchange2_test.go | 165 ++++++++ .../clientspaceproto/protos/clientspace.proto | 23 ++ net/secureservice/secureservice.go | 2 +- 8 files changed, 1026 insertions(+), 21 deletions(-) create mode 100644 commonspace/clientspaceproto/exchange2.go create mode 100644 commonspace/clientspaceproto/exchange2_bench_test.go create mode 100644 commonspace/clientspaceproto/exchange2_test.go diff --git a/commonspace/clientspaceproto/clientspace.pb.go b/commonspace/clientspaceproto/clientspace.pb.go index e38e3206d..2dfd4a434 100644 --- a/commonspace/clientspaceproto/clientspace.pb.go +++ b/commonspace/clientspaceproto/clientspace.pb.go @@ -117,6 +117,115 @@ func (x *SpaceExchangeResponse) GetSpaceIds() []string { return nil } +type SpaceExchangeV2Request struct { + state protoimpl.MessageState `protogen:"open.v1"` + // nonce is 32 random bytes chosen by the caller; it challenges both request and response tokens + Nonce []byte `protobuf:"bytes,1,opt,name=nonce,proto3" json:"nonce,omitempty"` + // spaceTokens holds one 32-byte request token per caller space, + // padded with random tokens up to a bucket size and sorted to hide the true count + SpaceTokens [][]byte `protobuf:"bytes,2,rep,name=spaceTokens,proto3" json:"spaceTokens,omitempty"` + LocalServer *LocalServer `protobuf:"bytes,3,opt,name=localServer,proto3" json:"localServer,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SpaceExchangeV2Request) Reset() { + *x = SpaceExchangeV2Request{} + mi := &file_commonspace_clientspaceproto_protos_clientspace_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SpaceExchangeV2Request) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SpaceExchangeV2Request) ProtoMessage() {} + +func (x *SpaceExchangeV2Request) ProtoReflect() protoreflect.Message { + mi := &file_commonspace_clientspaceproto_protos_clientspace_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SpaceExchangeV2Request.ProtoReflect.Descriptor instead. +func (*SpaceExchangeV2Request) Descriptor() ([]byte, []int) { + return file_commonspace_clientspaceproto_protos_clientspace_proto_rawDescGZIP(), []int{2} +} + +func (x *SpaceExchangeV2Request) GetNonce() []byte { + if x != nil { + return x.Nonce + } + return nil +} + +func (x *SpaceExchangeV2Request) GetSpaceTokens() [][]byte { + if x != nil { + return x.SpaceTokens + } + return nil +} + +func (x *SpaceExchangeV2Request) GetLocalServer() *LocalServer { + if x != nil { + return x.LocalServer + } + return nil +} + +type SpaceExchangeV2Response struct { + state protoimpl.MessageState `protogen:"open.v1"` + // spaceTokens holds one 32-byte response token per intersecting space — + // the responder's proof of membership, keyed by the caller's nonce + SpaceTokens [][]byte `protobuf:"bytes,1,rep,name=spaceTokens,proto3" json:"spaceTokens,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SpaceExchangeV2Response) Reset() { + *x = SpaceExchangeV2Response{} + mi := &file_commonspace_clientspaceproto_protos_clientspace_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SpaceExchangeV2Response) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SpaceExchangeV2Response) ProtoMessage() {} + +func (x *SpaceExchangeV2Response) ProtoReflect() protoreflect.Message { + mi := &file_commonspace_clientspaceproto_protos_clientspace_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SpaceExchangeV2Response.ProtoReflect.Descriptor instead. +func (*SpaceExchangeV2Response) Descriptor() ([]byte, []int) { + return file_commonspace_clientspaceproto_protos_clientspace_proto_rawDescGZIP(), []int{3} +} + +func (x *SpaceExchangeV2Response) GetSpaceTokens() [][]byte { + if x != nil { + return x.SpaceTokens + } + return nil +} + type LocalServer struct { state protoimpl.MessageState `protogen:"open.v1"` Ips []string `protobuf:"bytes,1,rep,name=Ips,proto3" json:"Ips,omitempty"` @@ -127,7 +236,7 @@ type LocalServer struct { func (x *LocalServer) Reset() { *x = LocalServer{} - mi := &file_commonspace_clientspaceproto_protos_clientspace_proto_msgTypes[2] + mi := &file_commonspace_clientspaceproto_protos_clientspace_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -139,7 +248,7 @@ func (x *LocalServer) String() string { func (*LocalServer) ProtoMessage() {} func (x *LocalServer) ProtoReflect() protoreflect.Message { - mi := &file_commonspace_clientspaceproto_protos_clientspace_proto_msgTypes[2] + mi := &file_commonspace_clientspaceproto_protos_clientspace_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -152,7 +261,7 @@ func (x *LocalServer) ProtoReflect() protoreflect.Message { // Deprecated: Use LocalServer.ProtoReflect.Descriptor instead. func (*LocalServer) Descriptor() ([]byte, []int) { - return file_commonspace_clientspaceproto_protos_clientspace_proto_rawDescGZIP(), []int{2} + return file_commonspace_clientspaceproto_protos_clientspace_proto_rawDescGZIP(), []int{4} } func (x *LocalServer) GetIps() []string { @@ -178,12 +287,19 @@ const file_commonspace_clientspaceproto_protos_clientspace_proto_rawDesc = "" + "\bspaceIds\x18\x01 \x03(\tR\bspaceIds\x12:\n" + "\vlocalServer\x18\x02 \x01(\v2\x18.clientspace.LocalServerR\vlocalServer\"3\n" + "\x15SpaceExchangeResponse\x12\x1a\n" + - "\bspaceIds\x18\x01 \x03(\tR\bspaceIds\"3\n" + + "\bspaceIds\x18\x01 \x03(\tR\bspaceIds\"\x8c\x01\n" + + "\x16SpaceExchangeV2Request\x12\x14\n" + + "\x05nonce\x18\x01 \x01(\fR\x05nonce\x12 \n" + + "\vspaceTokens\x18\x02 \x03(\fR\vspaceTokens\x12:\n" + + "\vlocalServer\x18\x03 \x01(\v2\x18.clientspace.LocalServerR\vlocalServer\";\n" + + "\x17SpaceExchangeV2Response\x12 \n" + + "\vspaceTokens\x18\x01 \x03(\fR\vspaceTokens\"3\n" + "\vLocalServer\x12\x10\n" + "\x03Ips\x18\x01 \x03(\tR\x03Ips\x12\x12\n" + - "\x04port\x18\x02 \x01(\x05R\x04port2e\n" + + "\x04port\x18\x02 \x01(\x05R\x04port2\xc3\x01\n" + "\vClientSpace\x12V\n" + - "\rSpaceExchange\x12!.clientspace.SpaceExchangeRequest\x1a\".clientspace.SpaceExchangeResponseB\x1eZ\x1ccommonspace/clientspaceprotob\x06proto3" + "\rSpaceExchange\x12!.clientspace.SpaceExchangeRequest\x1a\".clientspace.SpaceExchangeResponse\x12\\\n" + + "\x0fSpaceExchangeV2\x12#.clientspace.SpaceExchangeV2Request\x1a$.clientspace.SpaceExchangeV2ResponseB\x1eZ\x1ccommonspace/clientspaceprotob\x06proto3" var ( file_commonspace_clientspaceproto_protos_clientspace_proto_rawDescOnce sync.Once @@ -197,21 +313,26 @@ func file_commonspace_clientspaceproto_protos_clientspace_proto_rawDescGZIP() [] return file_commonspace_clientspaceproto_protos_clientspace_proto_rawDescData } -var file_commonspace_clientspaceproto_protos_clientspace_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_commonspace_clientspaceproto_protos_clientspace_proto_msgTypes = make([]protoimpl.MessageInfo, 5) var file_commonspace_clientspaceproto_protos_clientspace_proto_goTypes = []any{ - (*SpaceExchangeRequest)(nil), // 0: clientspace.SpaceExchangeRequest - (*SpaceExchangeResponse)(nil), // 1: clientspace.SpaceExchangeResponse - (*LocalServer)(nil), // 2: clientspace.LocalServer + (*SpaceExchangeRequest)(nil), // 0: clientspace.SpaceExchangeRequest + (*SpaceExchangeResponse)(nil), // 1: clientspace.SpaceExchangeResponse + (*SpaceExchangeV2Request)(nil), // 2: clientspace.SpaceExchangeV2Request + (*SpaceExchangeV2Response)(nil), // 3: clientspace.SpaceExchangeV2Response + (*LocalServer)(nil), // 4: clientspace.LocalServer } var file_commonspace_clientspaceproto_protos_clientspace_proto_depIdxs = []int32{ - 2, // 0: clientspace.SpaceExchangeRequest.localServer:type_name -> clientspace.LocalServer - 0, // 1: clientspace.ClientSpace.SpaceExchange:input_type -> clientspace.SpaceExchangeRequest - 1, // 2: clientspace.ClientSpace.SpaceExchange:output_type -> clientspace.SpaceExchangeResponse - 2, // [2:3] is the sub-list for method output_type - 1, // [1:2] is the sub-list for method input_type - 1, // [1:1] is the sub-list for extension type_name - 1, // [1:1] is the sub-list for extension extendee - 0, // [0:1] is the sub-list for field type_name + 4, // 0: clientspace.SpaceExchangeRequest.localServer:type_name -> clientspace.LocalServer + 4, // 1: clientspace.SpaceExchangeV2Request.localServer:type_name -> clientspace.LocalServer + 0, // 2: clientspace.ClientSpace.SpaceExchange:input_type -> clientspace.SpaceExchangeRequest + 2, // 3: clientspace.ClientSpace.SpaceExchangeV2:input_type -> clientspace.SpaceExchangeV2Request + 1, // 4: clientspace.ClientSpace.SpaceExchange:output_type -> clientspace.SpaceExchangeResponse + 3, // 5: clientspace.ClientSpace.SpaceExchangeV2:output_type -> clientspace.SpaceExchangeV2Response + 4, // [4:6] is the sub-list for method output_type + 2, // [2:4] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name } func init() { file_commonspace_clientspaceproto_protos_clientspace_proto_init() } @@ -225,7 +346,7 @@ func file_commonspace_clientspaceproto_protos_clientspace_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_commonspace_clientspaceproto_protos_clientspace_proto_rawDesc), len(file_commonspace_clientspaceproto_protos_clientspace_proto_rawDesc)), NumEnums: 0, - NumMessages: 3, + NumMessages: 5, NumExtensions: 0, NumServices: 1, }, diff --git a/commonspace/clientspaceproto/clientspace_drpc.pb.go b/commonspace/clientspaceproto/clientspace_drpc.pb.go index eea12aefa..1a707ff40 100644 --- a/commonspace/clientspaceproto/clientspace_drpc.pb.go +++ b/commonspace/clientspaceproto/clientspace_drpc.pb.go @@ -34,6 +34,7 @@ type DRPCClientSpaceClient interface { DRPCConn() drpc.Conn SpaceExchange(ctx context.Context, in *SpaceExchangeRequest) (*SpaceExchangeResponse, error) + SpaceExchangeV2(ctx context.Context, in *SpaceExchangeV2Request) (*SpaceExchangeV2Response, error) } type drpcClientSpaceClient struct { @@ -55,8 +56,18 @@ func (c *drpcClientSpaceClient) SpaceExchange(ctx context.Context, in *SpaceExch return out, nil } +func (c *drpcClientSpaceClient) SpaceExchangeV2(ctx context.Context, in *SpaceExchangeV2Request) (*SpaceExchangeV2Response, error) { + out := new(SpaceExchangeV2Response) + err := c.cc.Invoke(ctx, "/clientspace.ClientSpace/SpaceExchangeV2", drpcEncoding_File_commonspace_clientspaceproto_protos_clientspace_proto{}, in, out) + if err != nil { + return nil, err + } + return out, nil +} + type DRPCClientSpaceServer interface { SpaceExchange(context.Context, *SpaceExchangeRequest) (*SpaceExchangeResponse, error) + SpaceExchangeV2(context.Context, *SpaceExchangeV2Request) (*SpaceExchangeV2Response, error) } type DRPCClientSpaceUnimplementedServer struct{} @@ -65,9 +76,13 @@ func (s *DRPCClientSpaceUnimplementedServer) SpaceExchange(context.Context, *Spa return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) } +func (s *DRPCClientSpaceUnimplementedServer) SpaceExchangeV2(context.Context, *SpaceExchangeV2Request) (*SpaceExchangeV2Response, error) { + return nil, drpcerr.WithCode(errors.New("Unimplemented"), drpcerr.Unimplemented) +} + type DRPCClientSpaceDescription struct{} -func (DRPCClientSpaceDescription) NumMethods() int { return 1 } +func (DRPCClientSpaceDescription) NumMethods() int { return 2 } func (DRPCClientSpaceDescription) Method(n int) (string, drpc.Encoding, drpc.Receiver, interface{}, bool) { switch n { @@ -80,6 +95,15 @@ func (DRPCClientSpaceDescription) Method(n int) (string, drpc.Encoding, drpc.Rec in1.(*SpaceExchangeRequest), ) }, DRPCClientSpaceServer.SpaceExchange, true + case 1: + return "/clientspace.ClientSpace/SpaceExchangeV2", drpcEncoding_File_commonspace_clientspaceproto_protos_clientspace_proto{}, + func(srv interface{}, ctx context.Context, in1, in2 interface{}) (drpc.Message, error) { + return srv.(DRPCClientSpaceServer). + SpaceExchangeV2( + ctx, + in1.(*SpaceExchangeV2Request), + ) + }, DRPCClientSpaceServer.SpaceExchangeV2, true default: return "", nil, nil, nil, false } @@ -104,3 +128,19 @@ func (x *drpcClientSpace_SpaceExchangeStream) SendAndClose(m *SpaceExchangeRespo } return x.CloseSend() } + +type DRPCClientSpace_SpaceExchangeV2Stream interface { + drpc.Stream + SendAndClose(*SpaceExchangeV2Response) error +} + +type drpcClientSpace_SpaceExchangeV2Stream struct { + drpc.Stream +} + +func (x *drpcClientSpace_SpaceExchangeV2Stream) SendAndClose(m *SpaceExchangeV2Response) error { + if err := x.MsgSend(m, drpcEncoding_File_commonspace_clientspaceproto_protos_clientspace_proto{}); err != nil { + return err + } + return x.CloseSend() +} diff --git a/commonspace/clientspaceproto/clientspace_vtproto.pb.go b/commonspace/clientspaceproto/clientspace_vtproto.pb.go index dc6f59246..02e0c9b31 100644 --- a/commonspace/clientspaceproto/clientspace_vtproto.pb.go +++ b/commonspace/clientspaceproto/clientspace_vtproto.pb.go @@ -112,6 +112,107 @@ func (m *SpaceExchangeResponse) MarshalToSizedBufferVT(dAtA []byte) (int, error) return len(dAtA) - i, nil } +func (m *SpaceExchangeV2Request) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SpaceExchangeV2Request) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *SpaceExchangeV2Request) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if m.LocalServer != nil { + size, err := m.LocalServer.MarshalToSizedBufferVT(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = protohelpers.EncodeVarint(dAtA, i, uint64(size)) + i-- + dAtA[i] = 0x1a + } + if len(m.SpaceTokens) > 0 { + for iNdEx := len(m.SpaceTokens) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.SpaceTokens[iNdEx]) + copy(dAtA[i:], m.SpaceTokens[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceTokens[iNdEx]))) + i-- + dAtA[i] = 0x12 + } + } + if len(m.Nonce) > 0 { + i -= len(m.Nonce) + copy(dAtA[i:], m.Nonce) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.Nonce))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *SpaceExchangeV2Response) MarshalVT() (dAtA []byte, err error) { + if m == nil { + return nil, nil + } + size := m.SizeVT() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBufferVT(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SpaceExchangeV2Response) MarshalToVT(dAtA []byte) (int, error) { + size := m.SizeVT() + return m.MarshalToSizedBufferVT(dAtA[:size]) +} + +func (m *SpaceExchangeV2Response) MarshalToSizedBufferVT(dAtA []byte) (int, error) { + if m == nil { + return 0, nil + } + i := len(dAtA) + _ = i + var l int + _ = l + if m.unknownFields != nil { + i -= len(m.unknownFields) + copy(dAtA[i:], m.unknownFields) + } + if len(m.SpaceTokens) > 0 { + for iNdEx := len(m.SpaceTokens) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.SpaceTokens[iNdEx]) + copy(dAtA[i:], m.SpaceTokens[iNdEx]) + i = protohelpers.EncodeVarint(dAtA, i, uint64(len(m.SpaceTokens[iNdEx]))) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + func (m *LocalServer) MarshalVT() (dAtA []byte, err error) { if m == nil { return nil, nil @@ -195,6 +296,46 @@ func (m *SpaceExchangeResponse) SizeVT() (n int) { return n } +func (m *SpaceExchangeV2Request) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Nonce) + if l > 0 { + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + if len(m.SpaceTokens) > 0 { + for _, b := range m.SpaceTokens { + l = len(b) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + if m.LocalServer != nil { + l = m.LocalServer.SizeVT() + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + n += len(m.unknownFields) + return n +} + +func (m *SpaceExchangeV2Response) SizeVT() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.SpaceTokens) > 0 { + for _, b := range m.SpaceTokens { + l = len(b) + n += 1 + l + protohelpers.SizeOfVarint(uint64(l)) + } + } + n += len(m.unknownFields) + return n +} + func (m *LocalServer) SizeVT() (n int) { if m == nil { return 0 @@ -416,6 +557,242 @@ func (m *SpaceExchangeResponse) UnmarshalVT(dAtA []byte) error { } return nil } +func (m *SpaceExchangeV2Request) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SpaceExchangeV2Request: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SpaceExchangeV2Request: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Nonce", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Nonce = append(m.Nonce[:0], dAtA[iNdEx:postIndex]...) + if m.Nonce == nil { + m.Nonce = []byte{} + } + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceTokens", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceTokens = append(m.SpaceTokens, make([]byte, postIndex-iNdEx)) + copy(m.SpaceTokens[len(m.SpaceTokens)-1], dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field LocalServer", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.LocalServer == nil { + m.LocalServer = &LocalServer{} + } + if err := m.LocalServer.UnmarshalVT(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SpaceExchangeV2Response) UnmarshalVT(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SpaceExchangeV2Response: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SpaceExchangeV2Response: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field SpaceTokens", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return protohelpers.ErrIntOverflow + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return protohelpers.ErrInvalidLength + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return protohelpers.ErrInvalidLength + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.SpaceTokens = append(m.SpaceTokens, make([]byte, postIndex-iNdEx)) + copy(m.SpaceTokens[len(m.SpaceTokens)-1], dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := protohelpers.Skip(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return protohelpers.ErrInvalidLength + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.unknownFields = append(m.unknownFields, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *LocalServer) UnmarshalVT(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/commonspace/clientspaceproto/exchange2.go b/commonspace/clientspaceproto/exchange2.go new file mode 100644 index 000000000..0f8d6e314 --- /dev/null +++ b/commonspace/clientspaceproto/exchange2.go @@ -0,0 +1,144 @@ +package clientspaceproto + +import ( + "crypto/hmac" + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "errors" + "io" + "sort" + + "golang.org/x/crypto/hkdf" + + "github.com/anyproto/any-sync/util/crypto" +) + +// SpaceExchangeV2 token scheme. +// +// Peers never send space ids. Instead, for every space a peer holds it sends +// HMAC-SHA256(discoveryKey, nonce || label || callerPeerId || responderPeerId), +// where discoveryKey is derived from the space's first ACL read key — a stable +// secret available to every member. A matching token therefore simultaneously +// identifies a shared space and proves the sender's membership in it: strangers +// on the LAN see values indistinguishable from random, cannot correlate them +// across sessions (fresh nonce each run), and cannot claim spaces they are not +// members of. Spaces whose ACL state is not available locally are skipped. +// +// The responder answers only for the intersection, keying its tokens by the +// caller's nonce (challenge-response), so a response cannot be precomputed or +// replayed. Both request and response tokens are bound to the ordered pair of +// peer ids, so tokens observed on one connection are useless on another. + +const ( + // NonceSizeV2 is the required size of SpaceExchangeV2Request.nonce + NonceSizeV2 = 32 + // TokenSizeV2 is the size of every element of spaceTokens + TokenSizeV2 = sha256.Size + // MaxTokensV2 bounds spaceTokens in a request; handlers must reject bigger lists + MaxTokensV2 = 4096 +) + +// tokenBucketsV2 are the sizes request token lists are padded to, hiding the real space count +var tokenBucketsV2 = []int{16, 32, 64, 128, 256} + +var ( + labelRequestV2 = []byte("any-sync:space-exchange:v2:req") + labelResponseV2 = []byte("any-sync:space-exchange:v2:resp") + discoveryInfoV2 = []byte("any-sync:space-exchange:v2:discovery-key") +) + +var ErrInvalidNonce = errors.New("space exchange v2: invalid nonce size") + +// DeriveDiscoveryKey derives the per-space LAN discovery key from the first ACL read key. +// The first read key never rotates, so discovery keeps working for members whose local +// ACL copy is behind; former members retain it, but downstream sync still enforces the +// current ACL, so they can at most learn that a peer holds the space. +func DeriveDiscoveryKey(firstReadKey crypto.SymKey, spaceId string) ([]byte, error) { + ikm, err := firstReadKey.Raw() + if err != nil { + return nil, err + } + key := make([]byte, 32) + if _, err = io.ReadFull(hkdf.New(sha256.New, ikm, []byte(spaceId), discoveryInfoV2), key); err != nil { + return nil, err + } + return key, nil +} + +// NewNonceV2 returns a fresh random nonce for a SpaceExchangeV2Request +func NewNonceV2() ([]byte, error) { + nonce := make([]byte, NonceSizeV2) + if _, err := rand.Read(nonce); err != nil { + return nil, err + } + return nonce, nil +} + +// RequestTokenV2 computes the caller's token for one space +func RequestTokenV2(discoveryKey, nonce []byte, callerPeerId, responderPeerId string) []byte { + return tokenV2(discoveryKey, labelRequestV2, nonce, callerPeerId, responderPeerId) +} + +// ResponseTokenV2 computes the responder's proof token for one intersecting space, +// keyed by the caller's nonce +func ResponseTokenV2(discoveryKey, nonce []byte, callerPeerId, responderPeerId string) []byte { + return tokenV2(discoveryKey, labelResponseV2, nonce, callerPeerId, responderPeerId) +} + +func tokenV2(discoveryKey, label, nonce []byte, callerPeerId, responderPeerId string) []byte { + mac := hmac.New(sha256.New, discoveryKey) + writeField(mac, label) + writeField(mac, nonce) + writeField(mac, []byte(callerPeerId)) + writeField(mac, []byte(responderPeerId)) + return mac.Sum(nil) +} + +// writeField length-prefixes every field so variable-length peer ids cannot +// produce colliding concatenations +func writeField(w io.Writer, field []byte) { + var lenBuf [4]byte + binary.BigEndian.PutUint32(lenBuf[:], uint32(len(field))) + _, _ = w.Write(lenBuf[:]) + _, _ = w.Write(field) +} + +// PadTokensV2 pads tokens with random fillers up to the next bucket size (beyond the +// largest bucket — up to the next multiple of it) and sorts the result. Sorting is +// required: without it the position of real tokens would reveal the true space count +// the padding is meant to hide. +func PadTokensV2(tokens [][]byte) ([][]byte, error) { + maxBucket := tokenBucketsV2[len(tokenBucketsV2)-1] + target := (len(tokens) + maxBucket - 1) / maxBucket * maxBucket + for _, bucket := range tokenBucketsV2 { + if len(tokens) <= bucket { + target = bucket + break + } + } + padded := make([][]byte, 0, target) + padded = append(padded, tokens...) + for len(padded) < target { + filler := make([]byte, TokenSizeV2) + if _, err := rand.Read(filler); err != nil { + return nil, err + } + padded = append(padded, filler) + } + sort.Slice(padded, func(i, j int) bool { + return string(padded[i]) < string(padded[j]) + }) + return padded, nil +} + +// ContainsTokenV2 reports whether token is present in tokens, comparing in constant time +func ContainsTokenV2(tokens [][]byte, token []byte) bool { + var found bool + for _, t := range tokens { + if hmac.Equal(t, token) { + found = true + } + } + return found +} diff --git a/commonspace/clientspaceproto/exchange2_bench_test.go b/commonspace/clientspaceproto/exchange2_bench_test.go new file mode 100644 index 000000000..5bf86760e --- /dev/null +++ b/commonspace/clientspaceproto/exchange2_bench_test.go @@ -0,0 +1,135 @@ +package clientspaceproto + +import ( + "testing" + + "github.com/anyproto/any-sync/util/crypto" +) + +// BenchmarkExchangeV2 measures a full exchange where both peers hold 1000 spaces, +// 500 of them shared. Discovery keys are precomputed once outside the loops — +// they are stable per space and meant to be cached; DeriveKeys measures that +// one-time cost separately. Matching uses a hash set, the right approach at this +// scale (ContainsTokenV2's linear scan is for small sets). +func BenchmarkExchangeV2(b *testing.B) { + const ( + spacesPerPeer = 1000 + shared = 500 + ) + caller, responder := "peerA", "peerB" + + newKeys := func(n int) []crypto.SymKey { + keys := make([]crypto.SymKey, n) + for i := range keys { + key, err := crypto.NewRandomAES() + if err != nil { + b.Fatal(err) + } + keys[i] = key + } + return keys + } + deriveAll := func(keys []crypto.SymKey, ids []string) [][]byte { + dks := make([][]byte, len(keys)) + for i := range keys { + dk, err := DeriveDiscoveryKey(keys[i], ids[i]) + if err != nil { + b.Fatal(err) + } + dks[i] = dk + } + return dks + } + + sharedKeys := newKeys(shared) + callerKeys := append(append([]crypto.SymKey{}, sharedKeys...), newKeys(spacesPerPeer-shared)...) + responderKeys := append(append([]crypto.SymKey{}, sharedKeys...), newKeys(spacesPerPeer-shared)...) + callerIds := make([]string, spacesPerPeer) + responderIds := make([]string, spacesPerPeer) + for i := range spacesPerPeer { + if i < shared { + callerIds[i] = "shared" + string(rune('0'+i%10)) + string(rune('a'+i/10%26)) + string(rune('a'+i/260)) + responderIds[i] = callerIds[i] + } else { + callerIds[i] = "caller-only-" + string(rune('a'+i%26)) + string(rune('a'+i/26%26)) + string(rune('a'+i/676)) + responderIds[i] = "responder-only-" + string(rune('a'+i%26)) + string(rune('a'+i/26%26)) + string(rune('a'+i/676)) + } + } + callerDks := deriveAll(callerKeys, callerIds) + responderDks := deriveAll(responderKeys, responderIds) + + nonce, err := NewNonceV2() + if err != nil { + b.Fatal(err) + } + + buildRequest := func() [][]byte { + tokens := make([][]byte, len(callerDks)) + for i, dk := range callerDks { + tokens[i] = RequestTokenV2(dk, nonce, caller, responder) + } + padded, err := PadTokensV2(tokens) + if err != nil { + b.Fatal(err) + } + return padded + } + handleRequest := func(reqTokens [][]byte) [][]byte { + received := make(map[string]struct{}, len(reqTokens)) + for _, t := range reqTokens { + received[string(t)] = struct{}{} + } + var respTokens [][]byte + for _, dk := range responderDks { + if _, ok := received[string(RequestTokenV2(dk, nonce, caller, responder))]; ok { + respTokens = append(respTokens, ResponseTokenV2(dk, nonce, caller, responder)) + } + } + return respTokens + } + verifyResponse := func(respTokens [][]byte) int { + received := make(map[string]struct{}, len(respTokens)) + for _, t := range respTokens { + received[string(t)] = struct{}{} + } + var matched int + for _, dk := range callerDks { + if _, ok := received[string(ResponseTokenV2(dk, nonce, caller, responder))]; ok { + matched++ + } + } + return matched + } + + reqTokens := buildRequest() + respTokens := handleRequest(reqTokens) + if got := verifyResponse(respTokens); got != shared { + b.Fatalf("expected %d shared spaces, got %d", shared, got) + } + + b.Run("DeriveKeys", func(b *testing.B) { + for b.Loop() { + deriveAll(callerKeys, callerIds) + } + }) + b.Run("CallerBuildRequest", func(b *testing.B) { + for b.Loop() { + buildRequest() + } + }) + b.Run("ResponderHandle", func(b *testing.B) { + for b.Loop() { + handleRequest(reqTokens) + } + }) + b.Run("CallerVerifyResponse", func(b *testing.B) { + for b.Loop() { + verifyResponse(respTokens) + } + }) + b.Run("FullRoundTrip", func(b *testing.B) { + for b.Loop() { + verifyResponse(handleRequest(buildRequest())) + } + }) +} diff --git a/commonspace/clientspaceproto/exchange2_test.go b/commonspace/clientspaceproto/exchange2_test.go new file mode 100644 index 000000000..38caec71e --- /dev/null +++ b/commonspace/clientspaceproto/exchange2_test.go @@ -0,0 +1,165 @@ +package clientspaceproto + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/anyproto/any-sync/util/crypto" +) + +func TestDeriveDiscoveryKey(t *testing.T) { + readKey, err := crypto.NewRandomAES() + require.NoError(t, err) + + key1, err := DeriveDiscoveryKey(readKey, "space1") + require.NoError(t, err) + require.Len(t, key1, 32) + + // deterministic for the same inputs + key1Again, err := DeriveDiscoveryKey(readKey, "space1") + require.NoError(t, err) + require.Equal(t, key1, key1Again) + + // different per space even with the same read key + key2, err := DeriveDiscoveryKey(readKey, "space2") + require.NoError(t, err) + require.NotEqual(t, key1, key2) + + // raw read key never appears as the discovery key + raw, err := readKey.Raw() + require.NoError(t, err) + require.NotEqual(t, raw, key1) +} + +func TestTokensV2(t *testing.T) { + readKey, err := crypto.NewRandomAES() + require.NoError(t, err) + dk, err := DeriveDiscoveryKey(readKey, "space1") + require.NoError(t, err) + nonce, err := NewNonceV2() + require.NoError(t, err) + + req := RequestTokenV2(dk, nonce, "peerA", "peerB") + require.Len(t, req, TokenSizeV2) + + // request and response tokens must differ (domain separation) + resp := ResponseTokenV2(dk, nonce, "peerA", "peerB") + require.NotEqual(t, req, resp) + + // bound to the peer pair + require.NotEqual(t, req, RequestTokenV2(dk, nonce, "peerA", "peerC")) + require.NotEqual(t, req, RequestTokenV2(dk, nonce, "peerB", "peerA")) + + // bound to the nonce + nonce2, err := NewNonceV2() + require.NoError(t, err) + require.NotEqual(t, req, RequestTokenV2(dk, nonce2, "peerA", "peerB")) + + // length prefixing: shifting a boundary between peer ids must change the token + require.NotEqual(t, RequestTokenV2(dk, nonce, "peerAp", "eerB"), RequestTokenV2(dk, nonce, "peerA", "peerB")) +} + +func TestPadTokensV2(t *testing.T) { + tokens := make([][]byte, 0, 20) + for range 20 { + tok, err := NewNonceV2() + require.NoError(t, err) + tokens = append(tokens, tok) + } + + padded, err := PadTokensV2(tokens[:3]) + require.NoError(t, err) + require.Len(t, padded, 16) + + padded, err = PadTokensV2(tokens[:16]) + require.NoError(t, err) + require.Len(t, padded, 16) + + padded, err = PadTokensV2(tokens[:17]) + require.NoError(t, err) + require.Len(t, padded, 32) + + // all real tokens survive padding + for _, tok := range tokens[:17] { + require.True(t, ContainsTokenV2(padded, tok)) + } + + // sorted, so real token positions don't leak the true count + for i := 1; i < len(padded); i++ { + require.LessOrEqual(t, string(padded[i-1]), string(padded[i])) + } + + // beyond the largest bucket: next multiple of it + big := make([][]byte, 300) + for i := range big { + big[i], err = NewNonceV2() + require.NoError(t, err) + } + padded, err = PadTokensV2(big) + require.NoError(t, err) + require.Len(t, padded, 512) +} + +// TestExchangeV2RoundTrip simulates a full exchange between two peers +func TestExchangeV2RoundTrip(t *testing.T) { + shared1, err := crypto.NewRandomAES() + require.NoError(t, err) + shared2, err := crypto.NewRandomAES() + require.NoError(t, err) + callerOnly, err := crypto.NewRandomAES() + require.NoError(t, err) + responderOnly, err := crypto.NewRandomAES() + require.NoError(t, err) + + type space struct { + id string + readKey crypto.SymKey + } + callerSpaces := []space{{"shared1", shared1}, {"shared2", shared2}, {"callerOnly", callerOnly}} + responderSpaces := []space{{"shared1", shared1}, {"shared2", shared2}, {"responderOnly", responderOnly}} + const caller, responder = "peerA", "peerB" + + // caller builds the request + nonce, err := NewNonceV2() + require.NoError(t, err) + var reqTokens [][]byte + for _, sp := range callerSpaces { + dk, err := DeriveDiscoveryKey(sp.readKey, sp.id) + require.NoError(t, err) + reqTokens = append(reqTokens, RequestTokenV2(dk, nonce, caller, responder)) + } + reqTokens, err = PadTokensV2(reqTokens) + require.NoError(t, err) + + // responder matches against its own spaces and answers for the intersection + var respTokens [][]byte + var responderIntersection []string + for _, sp := range responderSpaces { + dk, err := DeriveDiscoveryKey(sp.readKey, sp.id) + require.NoError(t, err) + if ContainsTokenV2(reqTokens, RequestTokenV2(dk, nonce, caller, responder)) { + responderIntersection = append(responderIntersection, sp.id) + respTokens = append(respTokens, ResponseTokenV2(dk, nonce, caller, responder)) + } + } + require.Equal(t, []string{"shared1", "shared2"}, responderIntersection) + + // caller verifies the response + var callerIntersection []string + for _, sp := range callerSpaces { + dk, err := DeriveDiscoveryKey(sp.readKey, sp.id) + require.NoError(t, err) + if ContainsTokenV2(respTokens, ResponseTokenV2(dk, nonce, caller, responder)) { + callerIntersection = append(callerIntersection, sp.id) + } + } + require.Equal(t, []string{"shared1", "shared2"}, callerIntersection) + + // a peer that knows only the space id (no read key) gets no match + wrongKey, err := crypto.NewRandomAES() + require.NoError(t, err) + dk, err := DeriveDiscoveryKey(wrongKey, "shared1") + require.NoError(t, err) + require.False(t, ContainsTokenV2(reqTokens, RequestTokenV2(dk, nonce, caller, responder))) +} diff --git a/commonspace/clientspaceproto/protos/clientspace.proto b/commonspace/clientspaceproto/protos/clientspace.proto index 046c3b19c..d4c0aea7d 100644 --- a/commonspace/clientspaceproto/protos/clientspace.proto +++ b/commonspace/clientspaceproto/protos/clientspace.proto @@ -4,7 +4,15 @@ package clientspace; option go_package = "commonspace/clientspaceproto"; service ClientSpace { + // SpaceExchange sends plaintext space id lists to any LAN peer. + // Deprecated: leaks the full space set to unauthenticated strangers; use SpaceExchangeV2. rpc SpaceExchange(SpaceExchangeRequest) returns (SpaceExchangeResponse); + // SpaceExchangeV2 lets two LAN peers learn only the intersection of their space sets. + // Instead of space ids, peers exchange per-space HMAC tokens keyed by a discovery key + // derived from the space's first ACL read key — a secret only members hold. A valid + // token both hides the space from non-members and proves the sender's membership. + // See clientspaceproto/exchange2.go for token derivation. + rpc SpaceExchangeV2(SpaceExchangeV2Request) returns (SpaceExchangeV2Response); } message SpaceExchangeRequest { @@ -16,6 +24,21 @@ message SpaceExchangeResponse { repeated string spaceIds = 1; } +message SpaceExchangeV2Request { + // nonce is 32 random bytes chosen by the caller; it challenges both request and response tokens + bytes nonce = 1; + // spaceTokens holds one 32-byte request token per caller space, + // padded with random tokens up to a bucket size and sorted to hide the true count + repeated bytes spaceTokens = 2; + LocalServer localServer = 3; +} + +message SpaceExchangeV2Response { + // spaceTokens holds one 32-byte response token per intersecting space — + // the responder's proof of membership, keyed by the caller's nonce + repeated bytes spaceTokens = 1; +} + message LocalServer { repeated string Ips = 1; int32 port = 2; diff --git a/net/secureservice/secureservice.go b/net/secureservice/secureservice.go index fb1a393e7..632fd813c 100644 --- a/net/secureservice/secureservice.go +++ b/net/secureservice/secureservice.go @@ -39,7 +39,7 @@ var ( // ProtoVersion 10 - reserved (not used, version alignment gap) // ProtoVersion 11 - reserved (not used, version alignment gap) // ProtoVersion 12 aligns with v0.12.X - // ProtoVersion 13 - files v2 (v2 filenodes, chash routing, space header fileprotoVersion), aligns with v0.13.X + // ProtoVersion 13 - files v2 (v2 filenodes, chash routing, space header fileprotoVersion), space exchange v2 (token-based LAN discovery), aligns with v0.13.X ProtoVersion = uint32(13) ) From 954fe5496d0cf03ee077cda879b291da7b80207d Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Fri, 10 Jul 2026 15:05:17 +0200 Subject: [PATCH 19/20] refactor(headsync): remove old diffsync (DiffType_V2) support All connectable peers negotiate DiffType_V3 since ProtoVersion 7 and the minimum compatible version is 12, so the V2 paths are dead code: - drop app/olddiff and the ldiff DiffContainer (single diff now) - DiffManager keeps only the V3 diff; V2 range requests are rejected - statestorage keeps a single hash (old hash is no longer written) - keep the DiffType negotiation as the future upgrade mechanism, see the upgrade recipe comment on DiffManager --- app/ldiff/diff.go | 2 +- app/ldiff/diffcontainer.go | 72 --- app/ldiff/hasher.go | 25 + app/ldiff/mock_ldiff/mock_ldiff.go | 86 +--- app/olddiff/diff.go | 322 ------------- app/olddiff/diff_test.go | 428 ------------------ app/olddiff/hashrange.go | 223 --------- commonspace/headsync/diffmanager.go | 142 ++---- commonspace/headsync/diffmanager_test.go | 285 ++++-------- commonspace/headsync/diffsyncer_test.go | 41 +- commonspace/headsync/headsync.go | 5 +- commonspace/headsync/headsync_test.go | 35 +- commonspace/headsync/remotediff.go | 43 +- commonspace/headsync/remotediff_test.go | 35 +- .../mock_statestorage/mock_statestorage.go | 8 +- .../headsync/statestorage/statestorage.go | 27 +- .../spacestorage/migration/spacemigrator.go | 2 +- .../spacesyncproto/protos/spacesync.proto | 4 +- 18 files changed, 227 insertions(+), 1558 deletions(-) delete mode 100644 app/ldiff/diffcontainer.go create mode 100644 app/ldiff/hasher.go delete mode 100644 app/olddiff/diff.go delete mode 100644 app/olddiff/diff_test.go delete mode 100644 app/olddiff/hashrange.go diff --git a/app/ldiff/diff.go b/app/ldiff/diff.go index d5781b4c6..da719aaea 100644 --- a/app/ldiff/diff.go +++ b/app/ldiff/diff.go @@ -1,7 +1,7 @@ // Package ldiff provides a container of elements with fixed id and changeable content. // Diff can calculate the difference with another diff container (you can make it remote) with minimum hops and traffic. // -//go:generate mockgen -destination mock_ldiff/mock_ldiff.go github.com/anyproto/any-sync/app/ldiff Diff,Remote,DiffContainer +//go:generate mockgen -destination mock_ldiff/mock_ldiff.go github.com/anyproto/any-sync/app/ldiff Diff,Remote package ldiff import ( diff --git a/app/ldiff/diffcontainer.go b/app/ldiff/diffcontainer.go deleted file mode 100644 index c39fbd3ca..000000000 --- a/app/ldiff/diffcontainer.go +++ /dev/null @@ -1,72 +0,0 @@ -package ldiff - -import ( - "context" - "encoding/hex" - - "github.com/zeebo/blake3" -) - -type RemoteTypeChecker interface { - DiffTypeCheck(ctx context.Context, diffContainer DiffContainer) (needsSync bool, diff Diff, err error) -} - -type DiffContainer interface { - DiffTypeCheck(ctx context.Context, typeChecker RemoteTypeChecker) (needsSync bool, diff Diff, err error) - OldDiff() Diff - NewDiff() Diff - RemoveId(id string) error -} - -type diffContainer struct { - newDiff Diff - oldDiff Diff -} - -type Hasher struct { - hasher *blake3.Hasher -} - -func (h *Hasher) HashId(id string) string { - h.hasher.Reset() - h.hasher.WriteString(id) - return hex.EncodeToString(h.hasher.Sum(nil)) -} - -func NewHasher() *Hasher { - return &Hasher{hashersPool.Get().(*blake3.Hasher)} -} - -func ReleaseHasher(hasher *Hasher) { - hashersPool.Put(hasher.hasher) -} - -func (d *diffContainer) NewDiff() Diff { - return d.newDiff -} - -func (d *diffContainer) OldDiff() Diff { - return d.oldDiff -} - -func (d *diffContainer) Set(elements ...Element) { - d.newDiff.Set(elements...) - d.oldDiff.Set(elements...) -} - -func (d *diffContainer) RemoveId(id string) error { - _ = d.newDiff.RemoveId(id) - _ = d.oldDiff.RemoveId(id) - return nil -} - -func (d *diffContainer) DiffTypeCheck(ctx context.Context, typeChecker RemoteTypeChecker) (needsSync bool, diff Diff, err error) { - return typeChecker.DiffTypeCheck(ctx, d) -} - -func NewDiffContainer(new, old Diff) DiffContainer { - return &diffContainer{ - newDiff: new, - oldDiff: old, - } -} diff --git a/app/ldiff/hasher.go b/app/ldiff/hasher.go new file mode 100644 index 000000000..c0ec71a59 --- /dev/null +++ b/app/ldiff/hasher.go @@ -0,0 +1,25 @@ +package ldiff + +import ( + "encoding/hex" + + "github.com/zeebo/blake3" +) + +type Hasher struct { + hasher *blake3.Hasher +} + +func (h *Hasher) HashId(id string) string { + h.hasher.Reset() + h.hasher.WriteString(id) + return hex.EncodeToString(h.hasher.Sum(nil)) +} + +func NewHasher() *Hasher { + return &Hasher{hashersPool.Get().(*blake3.Hasher)} +} + +func ReleaseHasher(hasher *Hasher) { + hashersPool.Put(hasher.hasher) +} diff --git a/app/ldiff/mock_ldiff/mock_ldiff.go b/app/ldiff/mock_ldiff/mock_ldiff.go index 31e80ddb3..b4113335d 100644 --- a/app/ldiff/mock_ldiff/mock_ldiff.go +++ b/app/ldiff/mock_ldiff/mock_ldiff.go @@ -1,9 +1,9 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: github.com/anyproto/any-sync/app/ldiff (interfaces: Diff,Remote,DiffContainer) +// Source: github.com/anyproto/any-sync/app/ldiff (interfaces: Diff,Remote) // // Generated by this command: // -// mockgen -destination mock_ldiff/mock_ldiff.go github.com/anyproto/any-sync/app/ldiff Diff,Remote,DiffContainer +// mockgen -destination mock_ldiff/mock_ldiff.go github.com/anyproto/any-sync/app/ldiff Diff,Remote // // Package mock_ldiff is a generated GoMock package. @@ -227,85 +227,3 @@ func (mr *MockRemoteMockRecorder) Ranges(ctx, ranges, resBuf any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Ranges", reflect.TypeOf((*MockRemote)(nil).Ranges), ctx, ranges, resBuf) } - -// MockDiffContainer is a mock of DiffContainer interface. -type MockDiffContainer struct { - ctrl *gomock.Controller - recorder *MockDiffContainerMockRecorder - isgomock struct{} -} - -// MockDiffContainerMockRecorder is the mock recorder for MockDiffContainer. -type MockDiffContainerMockRecorder struct { - mock *MockDiffContainer -} - -// NewMockDiffContainer creates a new mock instance. -func NewMockDiffContainer(ctrl *gomock.Controller) *MockDiffContainer { - mock := &MockDiffContainer{ctrl: ctrl} - mock.recorder = &MockDiffContainerMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockDiffContainer) EXPECT() *MockDiffContainerMockRecorder { - return m.recorder -} - -// DiffTypeCheck mocks base method. -func (m *MockDiffContainer) DiffTypeCheck(ctx context.Context, typeChecker ldiff.RemoteTypeChecker) (bool, ldiff.Diff, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "DiffTypeCheck", ctx, typeChecker) - ret0, _ := ret[0].(bool) - ret1, _ := ret[1].(ldiff.Diff) - ret2, _ := ret[2].(error) - return ret0, ret1, ret2 -} - -// DiffTypeCheck indicates an expected call of DiffTypeCheck. -func (mr *MockDiffContainerMockRecorder) DiffTypeCheck(ctx, typeChecker any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiffTypeCheck", reflect.TypeOf((*MockDiffContainer)(nil).DiffTypeCheck), ctx, typeChecker) -} - -// NewDiff mocks base method. -func (m *MockDiffContainer) NewDiff() ldiff.Diff { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "NewDiff") - ret0, _ := ret[0].(ldiff.Diff) - return ret0 -} - -// NewDiff indicates an expected call of NewDiff. -func (mr *MockDiffContainerMockRecorder) NewDiff() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewDiff", reflect.TypeOf((*MockDiffContainer)(nil).NewDiff)) -} - -// OldDiff mocks base method. -func (m *MockDiffContainer) OldDiff() ldiff.Diff { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "OldDiff") - ret0, _ := ret[0].(ldiff.Diff) - return ret0 -} - -// OldDiff indicates an expected call of OldDiff. -func (mr *MockDiffContainerMockRecorder) OldDiff() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "OldDiff", reflect.TypeOf((*MockDiffContainer)(nil).OldDiff)) -} - -// RemoveId mocks base method. -func (m *MockDiffContainer) RemoveId(id string) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "RemoveId", id) - ret0, _ := ret[0].(error) - return ret0 -} - -// RemoveId indicates an expected call of RemoveId. -func (mr *MockDiffContainerMockRecorder) RemoveId(id any) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveId", reflect.TypeOf((*MockDiffContainer)(nil).RemoveId), id) -} diff --git a/app/olddiff/diff.go b/app/olddiff/diff.go deleted file mode 100644 index 11f1bb226..000000000 --- a/app/olddiff/diff.go +++ /dev/null @@ -1,322 +0,0 @@ -package olddiff - -import ( - "bytes" - "context" - "encoding/hex" - "errors" - "math" - "sync" - - "github.com/cespare/xxhash" - "github.com/huandu/skiplist" - "github.com/zeebo/blake3" - - "github.com/anyproto/any-sync/app/ldiff" - "github.com/anyproto/any-sync/commonspace/spacesyncproto" -) - -// New creates precalculated Diff container -// -// divideFactor - means how many hashes you want to ask for once -// -// it must be 2 or greater -// normal value usually between 4 and 64 -// -// compareThreshold - means the maximum count of elements remote diff will send directly -// -// if elements under range will be more - remote diff will send only hash -// it must be 1 or greater -// normal value between 8 and 64 -// -// Less threshold and divideFactor - less traffic but more requests -func New(divideFactor, compareThreshold int) ldiff.Diff { - return newDiff(divideFactor, compareThreshold) -} - -func newDiff(divideFactor, compareThreshold int) ldiff.Diff { - if divideFactor < 2 { - divideFactor = 2 - } - if compareThreshold < 1 { - compareThreshold = 1 - } - d := &diff{ - divideFactor: divideFactor, - compareThreshold: compareThreshold, - } - d.sl = skiplist.New(d) - d.ranges = newHashRanges(divideFactor, compareThreshold, d.sl) - d.ranges.dirty[d.ranges.topRange] = struct{}{} - d.ranges.recalculateHashes() - return d -} - -var hashersPool = &sync.Pool{ - New: func() any { - return blake3.New() - }, -} - -var ErrElementNotFound = errors.New("ldiff: element not found") - -type element struct { - ldiff.Element - hash uint64 -} - -// Diff contains elements and can compare it with Remote diff -type diff struct { - sl *skiplist.SkipList - divideFactor int - compareThreshold int - ranges *hashRanges - mu sync.RWMutex -} - -// Compare implements skiplist interface -func (d *diff) Compare(lhs, rhs interface{}) int { - lhe := lhs.(*element) - rhe := rhs.(*element) - if lhe.Id == rhe.Id { - return 0 - } - if lhe.hash > rhe.hash { - return 1 - } else if lhe.hash < rhe.hash { - return -1 - } - if lhe.Id > rhe.Id { - return 1 - } else { - return -1 - } -} - -// CalcScore implements skiplist interface -func (d *diff) CalcScore(key interface{}) float64 { - return 0 -} - -// Set adds or update element in container -func (d *diff) Set(elements ...ldiff.Element) { - d.mu.Lock() - defer d.mu.Unlock() - for _, e := range elements { - hash := xxhash.Sum64([]byte(e.Id)) - el := &element{Element: e, hash: hash} - d.sl.Remove(el) - d.sl.Set(el, nil) - d.ranges.addElement(hash) - } - d.ranges.recalculateHashes() -} - -func (d *diff) Ids() (ids []string) { - d.mu.RLock() - defer d.mu.RUnlock() - - ids = make([]string, 0, d.sl.Len()) - - cur := d.sl.Front() - for cur != nil { - el := cur.Key().(*element).Element - ids = append(ids, el.Id) - cur = cur.Next() - } - return -} - -func (d *diff) Len() int { - d.mu.RLock() - defer d.mu.RUnlock() - return d.sl.Len() -} - -func (d *diff) DiffType() spacesyncproto.DiffType { - return spacesyncproto.DiffType_V1 -} - -func (d *diff) Elements() (elements []ldiff.Element) { - d.mu.RLock() - defer d.mu.RUnlock() - - elements = make([]ldiff.Element, 0, d.sl.Len()) - - cur := d.sl.Front() - for cur != nil { - el := cur.Key().(*element).Element - elements = append(elements, el) - cur = cur.Next() - } - return -} - -func (d *diff) Element(id string) (ldiff.Element, error) { - d.mu.RLock() - defer d.mu.RUnlock() - el := d.sl.Get(&element{Element: ldiff.Element{Id: id}, hash: xxhash.Sum64([]byte(id))}) - if el == nil { - return ldiff.Element{}, ErrElementNotFound - } - if e, ok := el.Key().(*element); ok { - return e.Element, nil - } - return ldiff.Element{}, ErrElementNotFound -} - -func (d *diff) Hash() string { - d.mu.RLock() - defer d.mu.RUnlock() - return hex.EncodeToString(d.ranges.hash()) -} - -// RemoveId removes element by id -func (d *diff) RemoveId(id string) error { - d.mu.Lock() - defer d.mu.Unlock() - hash := xxhash.Sum64([]byte(id)) - el := &element{Element: ldiff.Element{ - Id: id, - }, hash: hash} - if d.sl.Remove(el) == nil { - return ErrElementNotFound - } - d.ranges.removeElement(hash) - d.ranges.recalculateHashes() - return nil -} - -func (d *diff) getRange(r ldiff.Range) (rr ldiff.RangeResult) { - rng := d.ranges.getRange(r.From, r.To) - // if we have the division for this range - if rng != nil { - rr.Hash = rng.hash - rr.Count = rng.elements - if !r.Elements && rng.isDivided { - return - } - } - - el := d.sl.Find(&element{hash: r.From}) - rr.Elements = make([]ldiff.Element, 0, d.divideFactor) - for el != nil && el.Key().(*element).hash <= r.To { - elem := el.Key().(*element).Element - el = el.Next() - rr.Elements = append(rr.Elements, elem) - } - rr.Count = len(rr.Elements) - return -} - -// Ranges calculates given ranges and return results -func (d *diff) Ranges(ctx context.Context, ranges []ldiff.Range, resBuf []ldiff.RangeResult) (results []ldiff.RangeResult, err error) { - d.mu.RLock() - defer d.mu.RUnlock() - - results = resBuf[:0] - for _, r := range ranges { - results = append(results, d.getRange(r)) - } - return -} - -type diffCtx struct { - newIds, changedIds, removedIds []string - - toSend, prepare []ldiff.Range - myRes, otherRes []ldiff.RangeResult -} - -var errMismatched = errors.New("query and results mismatched") - -// Diff makes diff with remote container -func (d *diff) Diff(ctx context.Context, dl ldiff.Remote) (newIds, changedIds, removedIds []string, err error) { - dctx := &diffCtx{} - dctx.toSend = append(dctx.toSend, ldiff.Range{ - From: 0, - To: math.MaxUint64, - }) - for len(dctx.toSend) > 0 { - select { - case <-ctx.Done(): - err = ctx.Err() - return - default: - } - if dctx.otherRes, err = dl.Ranges(ctx, dctx.toSend, dctx.otherRes); err != nil { - return - } - if dctx.myRes, err = d.Ranges(ctx, dctx.toSend, dctx.myRes); err != nil { - return - } - if len(dctx.otherRes) != len(dctx.toSend) || len(dctx.myRes) != len(dctx.toSend) { - err = errMismatched - return - } - for i, r := range dctx.toSend { - d.compareResults(dctx, r, dctx.myRes[i], dctx.otherRes[i]) - } - dctx.toSend, dctx.prepare = dctx.prepare, dctx.toSend - dctx.prepare = dctx.prepare[:0] - } - return dctx.newIds, dctx.changedIds, dctx.removedIds, nil -} - -func (d *diff) compareResults(dctx *diffCtx, r ldiff.Range, myRes, otherRes ldiff.RangeResult) { - // both hash equals - do nothing - if bytes.Equal(myRes.Hash, otherRes.Hash) { - return - } - - // other has elements - if len(otherRes.Elements) == otherRes.Count { - if len(myRes.Elements) == myRes.Count { - d.compareElements(dctx, myRes.Elements, otherRes.Elements) - } else { - r.Elements = true - d.compareElements(dctx, d.getRange(r).Elements, otherRes.Elements) - } - return - } - // request all elements from other, because we don't have enough - if len(myRes.Elements) == myRes.Count { - r.Elements = true - dctx.prepare = append(dctx.prepare, r) - return - } - rangeTuples := genTupleRanges(r.From, r.To, d.divideFactor) - for _, tuple := range rangeTuples { - dctx.prepare = append(dctx.prepare, ldiff.Range{From: tuple.from, To: tuple.to}) - } - return -} - -func (d *diff) compareElements(dctx *diffCtx, my, other []ldiff.Element) { - find := func(list []ldiff.Element, targetEl ldiff.Element) (has, eq bool) { - for _, el := range list { - if el.Id == targetEl.Id { - return true, el.Head == targetEl.Head - } - } - return false, false - } - - for _, el := range my { - has, eq := find(other, el) - if !has { - dctx.removedIds = append(dctx.removedIds, el.Id) - continue - } else { - if !eq { - dctx.changedIds = append(dctx.changedIds, el.Id) - } - } - } - - for _, el := range other { - if has, _ := find(my, el); !has { - dctx.newIds = append(dctx.newIds, el.Id) - } - } -} diff --git a/app/olddiff/diff_test.go b/app/olddiff/diff_test.go deleted file mode 100644 index bdc5c3671..000000000 --- a/app/olddiff/diff_test.go +++ /dev/null @@ -1,428 +0,0 @@ -package olddiff - -import ( - "context" - "fmt" - "math" - "sort" - "testing" - - "github.com/google/uuid" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "golang.org/x/exp/slices" - - "github.com/anyproto/any-sync/app/ldiff" -) - -func TestDiff_fillRange(t *testing.T) { - d := New(4, 4).(*diff) - for i := 0; i < 10; i++ { - el := ldiff.Element{ - Id: fmt.Sprint(i), - Head: fmt.Sprint("h", i), - } - d.Set(el) - } - t.Log(d.sl.Len()) - - t.Run("elements", func(t *testing.T) { - r := ldiff.Range{From: 0, To: math.MaxUint64} - res := d.getRange(r) - assert.NotNil(t, res.Hash) - assert.Equal(t, res.Count, 10) - }) -} - -func TestDiff_Diff(t *testing.T) { - ctx := context.Background() - t.Run("basic", func(t *testing.T) { - d1 := New(16, 16) - d2 := New(16, 16) - for i := 0; i < 1000; i++ { - id := fmt.Sprint(i) - head := uuid.NewString() - d1.Set(ldiff.Element{ - Id: id, - Head: head, - }) - d2.Set(ldiff.Element{ - Id: id, - Head: head, - }) - } - - newIds, changedIds, removedIds, err := d1.Diff(ctx, d2) - require.NoError(t, err) - assert.Len(t, newIds, 0) - assert.Len(t, changedIds, 0) - assert.Len(t, removedIds, 0) - - d2.Set(ldiff.Element{ - Id: "newD1", - Head: "newD1", - }) - d2.Set(ldiff.Element{ - Id: "1", - Head: "changed", - }) - require.NoError(t, d2.RemoveId("0")) - - newIds, changedIds, removedIds, err = d1.Diff(ctx, d2) - require.NoError(t, err) - assert.Len(t, newIds, 1) - assert.Len(t, changedIds, 1) - assert.Len(t, removedIds, 1) - }) - t.Run("complex", func(t *testing.T) { - d1 := New(16, 128) - d2 := New(16, 128) - length := 10000 - for i := 0; i < length; i++ { - id := fmt.Sprint(i) - head := uuid.NewString() - d1.Set(ldiff.Element{ - Id: id, - Head: head, - }) - } - - newIds, changedIds, removedIds, err := d1.Diff(ctx, d2) - require.NoError(t, err) - assert.Len(t, newIds, 0) - assert.Len(t, changedIds, 0) - assert.Len(t, removedIds, length) - - for i := 0; i < length; i++ { - id := fmt.Sprint(i) - head := uuid.NewString() - d2.Set(ldiff.Element{ - Id: id, - Head: head, - }) - } - - newIds, changedIds, removedIds, err = d1.Diff(ctx, d2) - require.NoError(t, err) - assert.Len(t, newIds, 0) - assert.Len(t, changedIds, length) - assert.Len(t, removedIds, 0) - - for i := 0; i < length; i++ { - id := fmt.Sprint(i) - head := uuid.NewString() - d2.Set(ldiff.Element{ - Id: id, - Head: head, - }) - } - - res, err := d1.Ranges( - context.Background(), - []ldiff.Range{{From: 0, To: math.MaxUint64, Elements: true}}, - nil) - require.NoError(t, err) - require.Len(t, res, 1) - for i, el := range res[0].Elements { - if i < length/2 { - continue - } - id := el.Id - head := el.Head - d2.Set(ldiff.Element{ - Id: id, - Head: head, - }) - } - - newIds, changedIds, removedIds, err = d1.Diff(ctx, d2) - require.NoError(t, err) - assert.Len(t, newIds, 0) - assert.Len(t, changedIds, length/2) - assert.Len(t, removedIds, 0) - }) - t.Run("empty", func(t *testing.T) { - d1 := New(16, 16) - d2 := New(16, 16) - newIds, changedIds, removedIds, err := d1.Diff(ctx, d2) - require.NoError(t, err) - assert.Len(t, newIds, 0) - assert.Len(t, changedIds, 0) - assert.Len(t, removedIds, 0) - }) - t.Run("one empty", func(t *testing.T) { - d1 := New(4, 4) - d2 := New(4, 4) - length := 10000 - for i := 0; i < length; i++ { - d2.Set(ldiff.Element{ - Id: fmt.Sprint(i), - Head: uuid.NewString(), - }) - } - newIds, changedIds, removedIds, err := d1.Diff(ctx, d2) - require.NoError(t, err) - assert.Len(t, newIds, length) - assert.Len(t, changedIds, 0) - assert.Len(t, removedIds, 0) - }) - t.Run("not intersecting", func(t *testing.T) { - d1 := New(16, 16) - d2 := New(16, 16) - length := 10000 - for i := 0; i < length; i++ { - d1.Set(ldiff.Element{ - Id: fmt.Sprint(i), - Head: uuid.NewString(), - }) - } - for i := length; i < length*2; i++ { - d2.Set(ldiff.Element{ - Id: fmt.Sprint(i), - Head: uuid.NewString(), - }) - } - newIds, changedIds, removedIds, err := d1.Diff(ctx, d2) - require.NoError(t, err) - assert.Len(t, newIds, length) - assert.Len(t, changedIds, 0) - assert.Len(t, removedIds, length) - }) - t.Run("context cancel", func(t *testing.T) { - d1 := New(4, 4) - d2 := New(4, 4) - for i := 0; i < 10; i++ { - d2.Set(ldiff.Element{ - Id: fmt.Sprint(i), - Head: uuid.NewString(), - }) - } - var cancel func() - ctx, cancel = context.WithCancel(ctx) - cancel() - _, _, _, err := d1.Diff(ctx, d2) - assert.ErrorIs(t, err, context.Canceled) - }) -} - -func BenchmarkDiff_Ranges(b *testing.B) { - d := New(16, 16) - for i := 0; i < 10000; i++ { - id := fmt.Sprint(i) - head := uuid.NewString() - d.Set(ldiff.Element{ - Id: id, - Head: head, - }) - } - ctx := context.Background() - b.ResetTimer() - b.ReportAllocs() - var resBuf []ldiff.RangeResult - var ranges = []ldiff.Range{{From: 0, To: math.MaxUint64}} - for i := 0; i < b.N; i++ { - d.Ranges(ctx, ranges, resBuf) - resBuf = resBuf[:0] - } -} - -func TestDiff_Hash(t *testing.T) { - d := New(16, 16) - h1 := d.Hash() - assert.NotEmpty(t, h1) - d.Set(ldiff.Element{Id: "1"}) - h2 := d.Hash() - assert.NotEmpty(t, h2) - assert.NotEqual(t, h1, h2) -} - -func TestDiff_Element(t *testing.T) { - d := New(16, 16) - for i := 0; i < 10; i++ { - d.Set(ldiff.Element{Id: fmt.Sprint("id", i), Head: fmt.Sprint("head", i)}) - } - _, err := d.Element("not found") - assert.Equal(t, ErrElementNotFound, err) - - el, err := d.Element("id5") - require.NoError(t, err) - assert.Equal(t, "head5", el.Head) - - d.Set(ldiff.Element{"id5", "otherHead"}) - el, err = d.Element("id5") - require.NoError(t, err) - assert.Equal(t, "otherHead", el.Head) -} - -func TestDiff_Ids(t *testing.T) { - d := New(16, 16) - var ids []string - for i := 0; i < 10; i++ { - id := fmt.Sprint("id", i) - d.Set(ldiff.Element{Id: id, Head: fmt.Sprint("head", i)}) - ids = append(ids, id) - } - gotIds := d.Ids() - sort.Strings(gotIds) - assert.Equal(t, ids, gotIds) - assert.Equal(t, len(ids), d.Len()) -} - -func TestDiff_Elements(t *testing.T) { - d := New(16, 16) - var els []ldiff.Element - for i := 0; i < 10; i++ { - id := fmt.Sprint("id", i) - el := ldiff.Element{Id: id, Head: fmt.Sprint("head", i)} - d.Set(el) - els = append(els, el) - } - gotEls := d.Elements() - sort.Slice(gotEls, func(i, j int) bool { - return gotEls[i].Id < gotEls[j].Id - }) - assert.Equal(t, els, gotEls) -} - -func TestRangesAddRemove(t *testing.T) { - length := 10000 - divideFactor := 4 - compareThreshold := 4 - addTwice := func() string { - d := New(divideFactor, compareThreshold) - var els []ldiff.Element - for i := 0; i < length; i++ { - if i < length/20 { - continue - } - els = append(els, ldiff.Element{ - Id: fmt.Sprint(i), - Head: fmt.Sprint("h", i), - }) - } - d.Set(els...) - els = els[:0] - for i := 0; i < length/20; i++ { - els = append(els, ldiff.Element{ - Id: fmt.Sprint(i), - Head: fmt.Sprint("h", i), - }) - } - d.Set(els...) - return d.Hash() - } - addOnce := func() string { - d := New(divideFactor, compareThreshold) - var els []ldiff.Element - for i := 0; i < length; i++ { - els = append(els, ldiff.Element{ - Id: fmt.Sprint(i), - Head: fmt.Sprint("h", i), - }) - } - d.Set(els...) - return d.Hash() - } - addRemove := func() string { - d := New(divideFactor, compareThreshold) - var els []ldiff.Element - for i := 0; i < length; i++ { - els = append(els, ldiff.Element{ - Id: fmt.Sprint(i), - Head: fmt.Sprint("h", i), - }) - } - d.Set(els...) - for i := 0; i < length/20; i++ { - err := d.RemoveId(fmt.Sprint(i)) - require.NoError(t, err) - } - els = els[:0] - for i := 0; i < length/20; i++ { - els = append(els, ldiff.Element{ - Id: fmt.Sprint(i), - Head: fmt.Sprint("h", i), - }) - } - d.Set(els...) - return d.Hash() - } - require.Equal(t, addTwice(), addOnce(), addRemove()) -} - -func printBestParams() { - numTests := 10 - length := 100000 - calcParams := func(divideFactor, compareThreshold, length int) (total, maxLevel, avgLevel, zeroEls int) { - d := New(divideFactor, compareThreshold).(*diff) - var els []ldiff.Element - for i := 0; i < length; i++ { - els = append(els, ldiff.Element{ - Id: uuid.NewString(), - Head: uuid.NewString(), - }) - } - d.Set(els...) - for _, rng := range d.ranges.ranges { - if rng.elements == 0 { - zeroEls++ - } - if rng.level > maxLevel { - maxLevel = rng.level - } - avgLevel += rng.level - } - total = len(d.ranges.ranges) - avgLevel = avgLevel / total - return - } - type result struct { - divFactor, compThreshold, numRanges, maxLevel, avgLevel, zeroEls int - } - sf := func(i, j result) int { - if i.numRanges < j.numRanges { - return -1 - } else if i.numRanges == j.numRanges { - return 0 - } else { - return 1 - } - } - var results []result - for divFactor := 0; divFactor < 6; divFactor++ { - df := 1 << divFactor - for compThreshold := 0; compThreshold < 10; compThreshold++ { - ct := 1 << compThreshold - fmt.Println("starting, df:", df, "ct:", ct) - var rngs []result - for i := 0; i < numTests; i++ { - total, maxLevel, avgLevel, zeroEls := calcParams(df, ct, length) - rngs = append(rngs, result{ - divFactor: df, - compThreshold: ct, - numRanges: total, - maxLevel: maxLevel, - avgLevel: avgLevel, - zeroEls: zeroEls, - }) - } - slices.SortFunc(rngs, sf) - ranges := rngs[len(rngs)/2] - results = append(results, ranges) - } - } - slices.SortFunc(results, sf) - fmt.Println(results) - // 100000 - [{16 512 273 2 1 0} {4 512 341 4 3 0} {2 512 511 8 7 0} {1 512 511 8 7 0} - // {8 256 585 3 2 0} {8 512 585 3 2 0} {1 256 1023 9 8 0} {2 256 1023 9 8 0} - // {32 256 1057 2 1 0} {32 512 1057 2 1 0} {32 128 1089 3 1 0} {4 256 1365 5 4 0} - // {4 128 1369 6 4 0} {2 128 2049 11 9 0} {1 128 2049 11 9 0} {1 64 4157 12 10 0} - // {2 64 4159 12 10 0} {16 128 4369 3 2 0} {16 64 4369 3 2 0} {16 256 4369 3 2 0} - // {8 64 4681 4 3 0} {8 128 4681 4 3 0} {4 64 5461 6 5 0} {4 32 6389 7 5 0} - // {8 32 6505 5 4 17} {16 32 8033 4 3 374} {2 32 8619 13 11 0} {1 32 8621 13 11 0} - // {2 16 17837 15 12 0} {1 16 17847 15 12 0} {4 16 21081 8 6 22} {32 64 33825 3 2 1578} - // {32 32 33825 3 2 1559} {32 16 33825 3 2 1518} {8 16 35881 5 4 1313} {16 16 66737 4 3 13022}] - // 1000000 - [{8 256 11753 5 4 0}] - // 1000000 - [{16 128 69905 4 3 0}] - // 1000000 - [{32 256 33825 3 2 0}] -} diff --git a/app/olddiff/hashrange.go b/app/olddiff/hashrange.go deleted file mode 100644 index cfc58976f..000000000 --- a/app/olddiff/hashrange.go +++ /dev/null @@ -1,223 +0,0 @@ -package olddiff - -import ( - "math" - - "github.com/huandu/skiplist" - "github.com/zeebo/blake3" - "golang.org/x/exp/slices" -) - -type hashRange struct { - from, to uint64 - parent *hashRange - isDivided bool - elements int - level int - hash []byte -} - -type rangeTuple struct { - from, to uint64 -} - -type hashRanges struct { - ranges map[rangeTuple]*hashRange - topRange *hashRange - sl *skiplist.SkipList - dirty map[*hashRange]struct{} - divideFactor int - compareThreshold int -} - -func newHashRanges(divideFactor, compareThreshold int, sl *skiplist.SkipList) *hashRanges { - h := &hashRanges{ - ranges: make(map[rangeTuple]*hashRange), - dirty: make(map[*hashRange]struct{}), - divideFactor: divideFactor, - compareThreshold: compareThreshold, - sl: sl, - } - h.topRange = &hashRange{ - from: 0, - to: math.MaxUint64, - isDivided: true, - level: 0, - } - h.ranges[rangeTuple{from: 0, to: math.MaxUint64}] = h.topRange - h.makeBottomRanges(h.topRange) - return h -} - -func (h *hashRanges) hash() []byte { - return h.topRange.hash -} - -func (h *hashRanges) addElement(elHash uint64) { - rng := h.topRange - rng.elements++ - for rng.isDivided { - rng = h.getBottomRange(rng, elHash) - rng.elements++ - } - h.dirty[rng] = struct{}{} - if rng.elements > h.compareThreshold { - rng.isDivided = true - h.makeBottomRanges(rng) - } - if rng.parent != nil { - if _, ok := h.dirty[rng.parent]; ok { - delete(h.dirty, rng.parent) - } - } -} - -func (h *hashRanges) removeElement(elHash uint64) { - rng := h.topRange - rng.elements-- - for rng.isDivided { - rng = h.getBottomRange(rng, elHash) - rng.elements-- - } - parent := rng.parent - if parent.elements <= h.compareThreshold && parent != h.topRange { - ranges := genTupleRanges(parent.from, parent.to, h.divideFactor) - for _, tuple := range ranges { - child := h.ranges[tuple] - delete(h.ranges, tuple) - delete(h.dirty, child) - } - parent.isDivided = false - h.dirty[parent] = struct{}{} - } else { - h.dirty[rng] = struct{}{} - } -} - -func (h *hashRanges) recalculateHashes() { - for len(h.dirty) > 0 { - var slDirty []*hashRange - for rng := range h.dirty { - slDirty = append(slDirty, rng) - } - slices.SortFunc(slDirty, func(a, b *hashRange) int { - if a.level < b.level { - return -1 - } else if a.level > b.level { - return 1 - } else { - return 0 - } - }) - for _, rng := range slDirty { - if rng.isDivided { - rng.hash = h.calcDividedHash(rng) - } else { - rng.hash, rng.elements = h.calcElementsHash(rng.from, rng.to) - } - delete(h.dirty, rng) - if rng.parent != nil { - h.dirty[rng.parent] = struct{}{} - } - } - } -} - -func (h *hashRanges) getRange(from, to uint64) *hashRange { - return h.ranges[rangeTuple{from: from, to: to}] -} - -func (h *hashRanges) getBottomRange(rng *hashRange, elHash uint64) *hashRange { - df := uint64(h.divideFactor) - perRange := (rng.to - rng.from) / df - align := ((rng.to-rng.from)%df + 1) % df - if align == 0 { - perRange++ - } - bucket := (elHash - rng.from) / perRange - tuple := rangeTuple{from: rng.from + bucket*perRange, to: rng.from - 1 + (bucket+1)*perRange} - if bucket == df-1 { - tuple.to += align - } - return h.ranges[tuple] -} - -func (h *hashRanges) makeBottomRanges(rng *hashRange) { - ranges := genTupleRanges(rng.from, rng.to, h.divideFactor) - for _, tuple := range ranges { - newRange := h.makeRange(tuple, rng) - h.ranges[tuple] = newRange - if newRange.elements > h.compareThreshold { - if _, ok := h.dirty[rng]; ok { - delete(h.dirty, rng) - } - h.dirty[newRange] = struct{}{} - newRange.isDivided = true - h.makeBottomRanges(newRange) - } - } -} - -func (h *hashRanges) makeRange(tuple rangeTuple, parent *hashRange) *hashRange { - newRange := &hashRange{ - from: tuple.from, - to: tuple.to, - parent: parent, - } - hash, els := h.calcElementsHash(tuple.from, tuple.to) - newRange.hash = hash - newRange.level = parent.level + 1 - newRange.elements = els - return newRange -} - -func (h *hashRanges) calcDividedHash(rng *hashRange) (hash []byte) { - hasher := hashersPool.Get().(*blake3.Hasher) - defer hashersPool.Put(hasher) - hasher.Reset() - ranges := genTupleRanges(rng.from, rng.to, h.divideFactor) - for _, tuple := range ranges { - child := h.ranges[tuple] - hasher.Write(child.hash) - } - hash = hasher.Sum(nil) - return -} - -func genTupleRanges(from, to uint64, divideFactor int) (prepare []rangeTuple) { - df := uint64(divideFactor) - perRange := (to - from) / df - align := ((to-from)%df + 1) % df - if align == 0 { - perRange++ - } - var j = from - for i := 0; i < divideFactor; i++ { - if i == divideFactor-1 { - perRange += align - } - prepare = append(prepare, rangeTuple{from: j, to: j + perRange - 1}) - j += perRange - } - return -} - -func (h *hashRanges) calcElementsHash(from, to uint64) (hash []byte, els int) { - hasher := hashersPool.Get().(*blake3.Hasher) - defer hashersPool.Put(hasher) - hasher.Reset() - - el := h.sl.Find(&element{hash: from}) - for el != nil && el.Key().(*element).hash <= to { - elem := el.Key().(*element).Element - el = el.Next() - - hasher.WriteString(elem.Id) - hasher.WriteString(elem.Head) - els++ - } - if els != 0 { - hash = hasher.Sum(nil) - } - return -} diff --git a/commonspace/headsync/diffmanager.go b/commonspace/headsync/diffmanager.go index 9c534b264..067b4c9b6 100644 --- a/commonspace/headsync/diffmanager.go +++ b/commonspace/headsync/diffmanager.go @@ -10,72 +10,58 @@ import ( "github.com/anyproto/any-sync/commonspace/deletionstate" "github.com/anyproto/any-sync/commonspace/headsync/headstorage" "github.com/anyproto/any-sync/commonspace/object/acl/syncacl" - "github.com/anyproto/any-sync/commonspace/object/keyvalue/kvinterfaces" "github.com/anyproto/any-sync/commonspace/spacestorage" "github.com/anyproto/any-sync/commonspace/spacesyncproto" "github.com/anyproto/any-sync/net/rpc/rpcerr" ) +// DiffManager holds the current diff (DiffType_V3). +// +// Upgrading the diff algorithm to a next DiffType requires a coexistence +// period, see how V2/V3 lived together before V2 removal (git history up to +// this commit): a DiffContainer with old/new diffs filled by DiffManager, +// dual hashes in statestorage, a switch by req.DiffType in HandleRangeRequest +// and by resp.DiffType in remote.DiffTypeCheck, so that the newest common +// type wins. The old type can be dropped once the secureservice +// compatibleVersions exclude all peers that don't support the new one. type DiffManager struct { - diffContainer ldiff.DiffContainer + diff ldiff.Diff storage spacestorage.SpaceStorage syncAcl syncacl.SyncAcl log logger.CtxLogger ctx context.Context deletionState deletionstate.ObjectDeletionState - keyValue kvinterfaces.KeyValueService } func NewDiffManager( - diffContainer ldiff.DiffContainer, + diff ldiff.Diff, storage spacestorage.SpaceStorage, syncAcl syncacl.SyncAcl, log logger.CtxLogger, ctx context.Context, deletionState deletionstate.ObjectDeletionState, - keyValue kvinterfaces.KeyValueService, ) *DiffManager { return &DiffManager{ - diffContainer: diffContainer, + diff: diff, storage: storage, syncAcl: syncAcl, log: log, ctx: ctx, deletionState: deletionState, - keyValue: keyValue, } } func (dm *DiffManager) FillDiff(ctx context.Context) error { - var ( - commonEls = make([]ldiff.Element, 0, 100) - onlyOldEls = make([]ldiff.Element, 0, 100) - noCommonSnapshotEls = make([]ldiff.Element, 0, 2) - ) + els := make([]ldiff.Element, 0, 100) err := dm.storage.HeadStorage().IterateEntries(ctx, headstorage.IterOpts{}, func(entry headstorage.HeadsEntry) (bool, error) { - // empty derived roots shouldn't be set in all hashes - if entry.IsDerived && entry.Heads[0] == entry.Id { + // empty roots shouldn't be set in the diff + if entry.Heads[0] == entry.Id && (entry.IsDerived || entry.CommonSnapshot != "") { return true, nil } - if entry.CommonSnapshot != "" { - // empty roots shouldn't be set in new hashe - if entry.Heads[0] == entry.Id { - onlyOldEls = append(onlyOldEls, ldiff.Element{ - Id: entry.Id, - Head: concatStrings(entry.Heads), - }) - return true, nil - } - commonEls = append(commonEls, ldiff.Element{ - Id: entry.Id, - Head: concatStrings(entry.Heads), - }) - } else { - noCommonSnapshotEls = append(noCommonSnapshotEls, ldiff.Element{ - Id: entry.Id, - Head: concatStrings(entry.Heads), - }) - } + els = append(els, ldiff.Element{ + Id: entry.Id, + Head: concatStrings(entry.Heads), + }) return true, nil }) if err != nil { @@ -83,49 +69,28 @@ func (dm *DiffManager) FillDiff(ctx context.Context) error { } log.Debug("setting acl", zap.String("aclId", dm.syncAcl.Id()), zap.String("headId", dm.syncAcl.Head().Id)) hasher := ldiff.NewHasher() - dm.setHeadsForNewDiff(commonEls, noCommonSnapshotEls, hasher) - dm.setHeadsForOldDiff(commonEls, onlyOldEls, noCommonSnapshotEls, hasher) + for _, el := range els { + dm.diff.Set(ldiff.Element{ + Id: el.Id, + Head: hasher.HashId(el.Head), + }) + } ldiff.ReleaseHasher(hasher) - oldHash := dm.diffContainer.OldDiff().Hash() - newHash := dm.diffContainer.NewDiff().Hash() - if err := dm.storage.StateStorage().SetHash(ctx, oldHash, newHash); err != nil { + if err := dm.storage.StateStorage().SetHash(ctx, dm.diff.Hash()); err != nil { dm.log.Error("can't write space hash", zap.Error(err)) return err } return nil } -func (dm *DiffManager) setHeadsForNewDiff(commonEls, noCommonSnapshotEls []ldiff.Element, hasher *ldiff.Hasher) { - for _, el := range append(commonEls, noCommonSnapshotEls...) { - hash := hasher.HashId(el.Head) - dm.diffContainer.NewDiff().Set(ldiff.Element{ - Id: el.Id, - Head: hash, - }) - } -} - -func (dm *DiffManager) setHeadsForOldDiff(commonEls, onlyOldEls, noCommonSnapshotEls []ldiff.Element, hasher *ldiff.Hasher) { - for _, el := range append(commonEls, onlyOldEls...) { - hash := hasher.HashId(el.Head) - dm.diffContainer.OldDiff().Set(ldiff.Element{ - Id: el.Id, - Head: hash, - }) - } - for _, el := range noCommonSnapshotEls { - dm.diffContainer.OldDiff().Set(el) - } -} - func (dm *DiffManager) TryDiff(ctx context.Context, rdiff RemoteDiff) (newIds, changedIds, removedIds []string, err error) { - needsSync, diff, err := dm.diffContainer.DiffTypeCheck(ctx, rdiff) + needsSync, err := rdiff.DiffTypeCheck(ctx, dm.diff) err = rpcerr.Unwrap(err) if err != nil { return nil, nil, nil, err } if needsSync { - newIds, changedIds, removedIds, err = diff.Diff(ctx, rdiff) + newIds, changedIds, removedIds, err = dm.diff.Diff(ctx, rdiff) err = rpcerr.Unwrap(err) if err != nil { return nil, nil, nil, err @@ -136,64 +101,35 @@ func (dm *DiffManager) TryDiff(ctx context.Context, rdiff RemoteDiff) (newIds, c func (dm *DiffManager) UpdateHeads(update headstorage.HeadsEntry) { if update.DeletedStatus != headstorage.DeletedStatusNotDeleted { - _ = dm.diffContainer.RemoveId(update.Id) + _ = dm.diff.RemoveId(update.Id) } else { if dm.deletionState.Exists(update.Id) { return } - // don't update for derived in both cases - if update.IsDerived && len(update.Heads) == 1 && update.Heads[0] == update.Id { + // empty roots shouldn't be set in the diff + if len(update.Heads) == 1 && update.Heads[0] == update.Id { return } hasher := ldiff.NewHasher() defer ldiff.ReleaseHasher(hasher) - if len(update.Heads) == 1 && update.Heads[0] == update.Id { - // empty roots should be updated only for old - dm.diffContainer.OldDiff().Set(ldiff.Element{ - Id: update.Id, - Head: hasher.HashId(update.Heads[0]), - }) - return - } - concatHeads := concatStrings(update.Heads) - if update.Id == dm.keyValue.DefaultStore().Id() { - dm.diffContainer.NewDiff().Set(ldiff.Element{ - Id: update.Id, - Head: hasher.HashId(concatHeads), - }) - // this happens due to old bug, so we should update only the heads as it is - dm.diffContainer.OldDiff().Set(ldiff.Element{ - Id: update.Id, - Head: concatHeads, - }) - } else { - for _, diff := range []ldiff.Diff{dm.diffContainer.OldDiff(), dm.diffContainer.NewDiff()} { - diff.Set(ldiff.Element{ - Id: update.Id, - Head: hasher.HashId(concatHeads), - }) - } - } + dm.diff.Set(ldiff.Element{ + Id: update.Id, + Head: hasher.HashId(concatStrings(update.Heads)), + }) } - oldHash := dm.diffContainer.OldDiff().Hash() - newHash := dm.diffContainer.NewDiff().Hash() - err := dm.storage.StateStorage().SetHash(dm.ctx, oldHash, newHash) + err := dm.storage.StateStorage().SetHash(dm.ctx, dm.diff.Hash()) if err != nil { dm.log.Warn("can't write space hash", zap.Error(err)) } } func (dm *DiffManager) HandleRangeRequest(ctx context.Context, req *spacesyncproto.HeadSyncRequest) (resp *spacesyncproto.HeadSyncResponse, err error) { - switch req.DiffType { - case spacesyncproto.DiffType_V3: - return HandleRangeRequest(ctx, dm.diffContainer.NewDiff(), req) - case spacesyncproto.DiffType_V2: - return HandleRangeRequest(ctx, dm.diffContainer.OldDiff(), req) - default: + if req.DiffType != spacesyncproto.DiffType_V3 { return nil, spacesyncproto.ErrUnexpected } + return HandleRangeRequest(ctx, dm.diff, req) } func (dm *DiffManager) AllIds() []string { - return dm.diffContainer.NewDiff().Ids() + return dm.diff.Ids() } diff --git a/commonspace/headsync/diffmanager_test.go b/commonspace/headsync/diffmanager_test.go index 334d6c1a4..c90b02144 100644 --- a/commonspace/headsync/diffmanager_test.go +++ b/commonspace/headsync/diffmanager_test.go @@ -17,20 +17,15 @@ import ( "github.com/anyproto/any-sync/commonspace/headsync/statestorage/mock_statestorage" "github.com/anyproto/any-sync/commonspace/object/acl/list" "github.com/anyproto/any-sync/commonspace/object/acl/syncacl/mock_syncacl" - "github.com/anyproto/any-sync/commonspace/object/keyvalue/keyvaluestorage/mock_keyvaluestorage" - "github.com/anyproto/any-sync/commonspace/object/keyvalue/kvinterfaces/mock_kvinterfaces" "github.com/anyproto/any-sync/commonspace/spacestorage/mock_spacestorage" "github.com/anyproto/any-sync/commonspace/spacesyncproto" ) type diffManagerFixture struct { ctrl *gomock.Controller - diffContainerMock *mock_ldiff.MockDiffContainer storageMock *mock_spacestorage.MockSpaceStorage aclMock *mock_syncacl.MockSyncAcl deletionStateMock *mock_deletionstate.MockObjectDeletionState - kvMock *mock_kvinterfaces.MockKeyValueService - defStoreMock *mock_keyvaluestorage.MockStorage headStorage *mock_headstorage.MockHeadStorage stateStorage *mock_statestorage.MockStateStorage diffMock *mock_ldiff.MockDiff @@ -39,40 +34,31 @@ type diffManagerFixture struct { func newDiffManagerFixture(t *testing.T) *diffManagerFixture { ctrl := gomock.NewController(t) - diffContainerMock := mock_ldiff.NewMockDiffContainer(ctrl) storageMock := mock_spacestorage.NewMockSpaceStorage(ctrl) aclMock := mock_syncacl.NewMockSyncAcl(ctrl) deletionStateMock := mock_deletionstate.NewMockObjectDeletionState(ctrl) - kvMock := mock_kvinterfaces.NewMockKeyValueService(ctrl) - defStoreMock := mock_keyvaluestorage.NewMockStorage(ctrl) headStorage := mock_headstorage.NewMockHeadStorage(ctrl) stateStorage := mock_statestorage.NewMockStateStorage(ctrl) diffMock := mock_ldiff.NewMockDiff(ctrl) - kvMock.EXPECT().DefaultStore().Return(defStoreMock).AnyTimes() - defStoreMock.EXPECT().Id().Return("store").AnyTimes() storageMock.EXPECT().HeadStorage().Return(headStorage).AnyTimes() storageMock.EXPECT().StateStorage().Return(stateStorage).AnyTimes() log := logger.NewNamed("test") diffManager := NewDiffManager( - diffContainerMock, + diffMock, storageMock, aclMock, log, context.Background(), deletionStateMock, - kvMock, ) return &diffManagerFixture{ ctrl: ctrl, - diffContainerMock: diffContainerMock, storageMock: storageMock, aclMock: aclMock, deletionStateMock: deletionStateMock, - kvMock: kvMock, - defStoreMock: defStoreMock, headStorage: headStorage, stateStorage: stateStorage, diffMock: diffMock, @@ -84,6 +70,18 @@ func (fx *diffManagerFixture) stop() { fx.ctrl.Finish() } +func (fx *diffManagerFixture) expectIterateEntries(headEntries []headstorage.HeadsEntry) { + fx.headStorage.EXPECT().IterateEntries(gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx context.Context, opts headstorage.IterOpts, entryIter headstorage.EntryIterator) error { + for _, entry := range headEntries { + if res, err := entryIter(entry); err != nil || !res { + return err + } + } + return nil + }) +} + func TestDiffManager_FillDiff(t *testing.T) { ctx := context.Background() @@ -91,7 +89,7 @@ func TestDiffManager_FillDiff(t *testing.T) { fx := newDiffManagerFixture(t) defer fx.stop() - headEntries := []headstorage.HeadsEntry{ + fx.expectIterateEntries([]headstorage.HeadsEntry{ { Id: "id1", Heads: []string{"h1", "h2"}, @@ -104,29 +102,16 @@ func TestDiffManager_FillDiff(t *testing.T) { CommonSnapshot: "", IsDerived: false, }, - } - - fx.headStorage.EXPECT().IterateEntries(gomock.Any(), gomock.Any(), gomock.Any()). - DoAndReturn(func(ctx context.Context, opts headstorage.IterOpts, entryIter headstorage.EntryIterator) error { - for _, entry := range headEntries { - if res, err := entryIter(entry); err != nil || !res { - return err - } - } - return nil - }) + }) fx.aclMock.EXPECT().Id().Return("aclId").Times(1) fx.aclMock.EXPECT().Head().Return(&list.AclRecord{Id: "headId"}).Times(1) - // Test expects hashed values now hasher := ldiff.NewHasher() hash1 := hasher.HashId("h1h2") hash2 := hasher.HashId("h3") ldiff.ReleaseHasher(hasher) - // NewDiff gets both common and no common snapshot elements - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock).Times(2) fx.diffMock.EXPECT().Set(ldiff.Element{ Id: "id1", Head: hash1, @@ -136,22 +121,8 @@ func TestDiffManager_FillDiff(t *testing.T) { Head: hash2, }) - // OldDiff gets common elements with hash and no common snapshot elements without hash - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock).Times(2) - fx.diffMock.EXPECT().Set(ldiff.Element{ - Id: "id1", - Head: hash1, - }) - fx.diffMock.EXPECT().Set(ldiff.Element{ - Id: "id2", - Head: "h3", - }) - - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Hash().Return("oldHash") - fx.diffMock.EXPECT().Hash().Return("newHash") - fx.stateStorage.EXPECT().SetHash(ctx, "oldHash", "newHash").Return(nil) + fx.diffMock.EXPECT().Hash().Return("hash") + fx.stateStorage.EXPECT().SetHash(ctx, "hash").Return(nil) err := fx.diffManager.FillDiff(ctx) require.NoError(t, err) @@ -161,44 +132,31 @@ func TestDiffManager_FillDiff(t *testing.T) { fx := newDiffManagerFixture(t) defer fx.stop() - headEntries := []headstorage.HeadsEntry{ + fx.expectIterateEntries([]headstorage.HeadsEntry{ { Id: "id1", Heads: []string{"id1"}, CommonSnapshot: "snapshot1", IsDerived: true, }, - } - - fx.headStorage.EXPECT().IterateEntries(gomock.Any(), gomock.Any(), gomock.Any()). - DoAndReturn(func(ctx context.Context, opts headstorage.IterOpts, entryIter headstorage.EntryIterator) error { - for _, entry := range headEntries { - if res, err := entryIter(entry); err != nil || !res { - return err - } - } - return nil - }) + }) fx.aclMock.EXPECT().Id().Return("aclId").Times(1) fx.aclMock.EXPECT().Head().Return(&list.AclRecord{Id: "headId"}).Times(1) - // No elements should be set for derived entries - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Hash().Return("oldHash") - fx.diffMock.EXPECT().Hash().Return("newHash") - fx.stateStorage.EXPECT().SetHash(ctx, "oldHash", "newHash").Return(nil) + // no elements should be set for derived entries + fx.diffMock.EXPECT().Hash().Return("hash") + fx.stateStorage.EXPECT().SetHash(ctx, "hash").Return(nil) err := fx.diffManager.FillDiff(ctx) require.NoError(t, err) }) - t.Run("handle singular roots", func(t *testing.T) { + t.Run("skip singular roots", func(t *testing.T) { fx := newDiffManagerFixture(t) defer fx.stop() - headEntries := []headstorage.HeadsEntry{ + fx.expectIterateEntries([]headstorage.HeadsEntry{ { Id: "id1", Heads: []string{"id1"}, // Singular root @@ -211,49 +169,55 @@ func TestDiffManager_FillDiff(t *testing.T) { CommonSnapshot: "snapshot2", IsDerived: false, }, - } - - fx.headStorage.EXPECT().IterateEntries(gomock.Any(), gomock.Any(), gomock.Any()). - DoAndReturn(func(ctx context.Context, opts headstorage.IterOpts, entryIter headstorage.EntryIterator) error { - for _, entry := range headEntries { - if res, err := entryIter(entry); err != nil || !res { - return err - } - } - return nil - }) + }) fx.aclMock.EXPECT().Id().Return("aclId").Times(1) fx.aclMock.EXPECT().Head().Return(&list.AclRecord{Id: "headId"}).Times(1) hasher := ldiff.NewHasher() - hash1 := hasher.HashId("id1") hash2 := hasher.HashId("h1h2") ldiff.ReleaseHasher(hasher) - // NewDiff should only get id2 - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) + // only id2 should be set fx.diffMock.EXPECT().Set(ldiff.Element{ Id: "id2", Head: hash2, }) - // OldDiff should get both id1 and id2 - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock).Times(2) - fx.diffMock.EXPECT().Set(ldiff.Element{ - Id: "id2", - Head: hash2, + fx.diffMock.EXPECT().Hash().Return("hash") + fx.stateStorage.EXPECT().SetHash(ctx, "hash").Return(nil) + + err := fx.diffManager.FillDiff(ctx) + require.NoError(t, err) + }) + + t.Run("include singular root without common snapshot", func(t *testing.T) { + fx := newDiffManagerFixture(t) + defer fx.stop() + + fx.expectIterateEntries([]headstorage.HeadsEntry{ + { + Id: "id1", + Heads: []string{"id1"}, + CommonSnapshot: "", + IsDerived: false, + }, }) + + fx.aclMock.EXPECT().Id().Return("aclId").Times(1) + fx.aclMock.EXPECT().Head().Return(&list.AclRecord{Id: "headId"}).Times(1) + + hasher := ldiff.NewHasher() + hash1 := hasher.HashId("id1") + ldiff.ReleaseHasher(hasher) + fx.diffMock.EXPECT().Set(ldiff.Element{ Id: "id1", Head: hash1, }) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Hash().Return("oldHash") - fx.diffMock.EXPECT().Hash().Return("newHash") - fx.stateStorage.EXPECT().SetHash(ctx, "oldHash", "newHash").Return(nil) + fx.diffMock.EXPECT().Hash().Return("hash") + fx.stateStorage.EXPECT().SetHash(ctx, "hash").Return(nil) err := fx.diffManager.FillDiff(ctx) require.NoError(t, err) @@ -278,19 +242,29 @@ func TestDiffManager_FillDiff(t *testing.T) { fx.aclMock.EXPECT().Id().Return("aclId").Times(1) fx.aclMock.EXPECT().Head().Return(&list.AclRecord{Id: "headId"}).Times(1) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Hash().Return("oldHash") - fx.diffMock.EXPECT().Hash().Return("newHash") + fx.diffMock.EXPECT().Hash().Return("hash") expectedErr := fmt.Errorf("hash error") - fx.stateStorage.EXPECT().SetHash(ctx, "oldHash", "newHash").Return(expectedErr) + fx.stateStorage.EXPECT().SetHash(ctx, "hash").Return(expectedErr) err := fx.diffManager.FillDiff(ctx) require.ErrorIs(t, err, expectedErr) }) } +type fakeRemoteDiff struct { + needsSync bool + err error +} + +func (f *fakeRemoteDiff) Ranges(ctx context.Context, ranges []ldiff.Range, resBuf []ldiff.RangeResult) ([]ldiff.RangeResult, error) { + return nil, nil +} + +func (f *fakeRemoteDiff) DiffTypeCheck(ctx context.Context, diff ldiff.Diff) (bool, error) { + return f.needsSync, f.err +} + func TestDiffManager_TryDiff(t *testing.T) { ctx := context.Background() @@ -298,12 +272,11 @@ func TestDiffManager_TryDiff(t *testing.T) { fx := newDiffManagerFixture(t) defer fx.stop() - remoteDiff := &remote{spaceId: "space1"} + remoteDiff := &fakeRemoteDiff{needsSync: true} expectedNewIds := []string{"new1", "new2"} expectedChangedIds := []string{"changed1"} expectedRemovedIds := []string{"removed1"} - fx.diffContainerMock.EXPECT().DiffTypeCheck(ctx, remoteDiff).Return(true, fx.diffMock, nil) fx.diffMock.EXPECT().Diff(ctx, remoteDiff).Return(expectedNewIds, expectedChangedIds, expectedRemovedIds, nil) newIds, changedIds, removedIds, err := fx.diffManager.TryDiff(ctx, remoteDiff) @@ -317,9 +290,7 @@ func TestDiffManager_TryDiff(t *testing.T) { fx := newDiffManagerFixture(t) defer fx.stop() - remoteDiff := &remote{spaceId: "space1"} - - fx.diffContainerMock.EXPECT().DiffTypeCheck(ctx, remoteDiff).Return(false, nil, nil) + remoteDiff := &fakeRemoteDiff{needsSync: false} newIds, changedIds, removedIds, err := fx.diffManager.TryDiff(ctx, remoteDiff) require.NoError(t, err) @@ -327,6 +298,17 @@ func TestDiffManager_TryDiff(t *testing.T) { require.Nil(t, changedIds) require.Nil(t, removedIds) }) + + t.Run("diff type check fails", func(t *testing.T) { + fx := newDiffManagerFixture(t) + defer fx.stop() + + expectedErr := fmt.Errorf("check error") + remoteDiff := &fakeRemoteDiff{err: expectedErr} + + _, _, _, err := fx.diffManager.TryDiff(ctx, remoteDiff) + require.ErrorIs(t, err, expectedErr) + }) } func TestDiffManager_UpdateHeads(t *testing.T) { @@ -340,44 +322,9 @@ func TestDiffManager_UpdateHeads(t *testing.T) { DeletedStatus: deleteStatus, } - fx.diffContainerMock.EXPECT().RemoveId("id1").Return(nil) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Hash().Return("hash").Times(2) - fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash", "hash").Return(nil) - - fx.diffManager.UpdateHeads(update) - }) - - t.Run("update head for key-value store", func(t *testing.T) { - fx := newDiffManagerFixture(t) - defer fx.stop() - - update := headstorage.HeadsEntry{ - Id: "store", - Heads: []string{"head1"}, - } - - fx.deletionStateMock.EXPECT().Exists("store").Return(false) - - hasher := ldiff.NewHasher() - hash := hasher.HashId("head1") - ldiff.ReleaseHasher(hasher) - - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Set(ldiff.Element{ - Id: "store", - Head: hash, - }) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Set(ldiff.Element{ - Id: "store", - Head: "head1", - }) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Hash().Return("hash").Times(2) - fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash", "hash").Return(nil) + fx.diffMock.EXPECT().RemoveId("id1").Return(nil) + fx.diffMock.EXPECT().Hash().Return("hash") + fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash").Return(nil) fx.diffManager.UpdateHeads(update) }) @@ -397,20 +344,12 @@ func TestDiffManager_UpdateHeads(t *testing.T) { hash := hasher.HashId("head1head2") ldiff.ReleaseHasher(hasher) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) fx.diffMock.EXPECT().Set(ldiff.Element{ Id: "id1", Head: hash, }) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Set(ldiff.Element{ - Id: "id1", - Head: hash, - }) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Hash().Return("hash").Times(2) - fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash", "hash").Return(nil) + fx.diffMock.EXPECT().Hash().Return("hash") + fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash").Return(nil) fx.diffManager.UpdateHeads(update) }) @@ -444,7 +383,7 @@ func TestDiffManager_UpdateHeads(t *testing.T) { fx.diffManager.UpdateHeads(update) }) - t.Run("update singular root object", func(t *testing.T) { + t.Run("skip singular root object", func(t *testing.T) { fx := newDiffManagerFixture(t) defer fx.stop() @@ -455,18 +394,7 @@ func TestDiffManager_UpdateHeads(t *testing.T) { fx.deletionStateMock.EXPECT().Exists("id1").Return(false) - hasher := ldiff.NewHasher() - hash := hasher.HashId("id1") - ldiff.ReleaseHasher(hasher) - - // Singular roots should only be added to OldDiff and then return early - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Set(ldiff.Element{ - Id: "id1", - Head: hash, - }) - // No hash update since the function returns early for singular roots - + // no diff update and no hash update for singular roots fx.diffManager.UpdateHeads(update) }) @@ -485,20 +413,12 @@ func TestDiffManager_UpdateHeads(t *testing.T) { hash := hasher.HashId("") ldiff.ReleaseHasher(hasher) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) fx.diffMock.EXPECT().Set(ldiff.Element{ Id: "id1", Head: hash, }) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Set(ldiff.Element{ - Id: "id1", - Head: hash, - }) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Hash().Return("hash").Times(2) - fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash", "hash").Return(nil) + fx.diffMock.EXPECT().Hash().Return("hash") + fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash").Return(nil) fx.diffManager.UpdateHeads(update) }) @@ -513,26 +433,18 @@ func TestDiffManager_UpdateHeads(t *testing.T) { } fx.deletionStateMock.EXPECT().Exists("id1").Return(false) - + hasher := ldiff.NewHasher() hash := hasher.HashId("head1") ldiff.ReleaseHasher(hasher) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) fx.diffMock.EXPECT().Set(ldiff.Element{ Id: "id1", Head: hash, }) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Set(ldiff.Element{ - Id: "id1", - Head: hash, - }) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Hash().Return("hash").Times(2) + fx.diffMock.EXPECT().Hash().Return("hash") // UpdateHeads logs warning but doesn't fail - fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash", "hash").Return(fmt.Errorf("hash error")) + fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash").Return(fmt.Errorf("hash error")) // Should not panic or fail fx.diffManager.UpdateHeads(update) @@ -550,7 +462,6 @@ func TestDiffManager_HandleRangeRequest(t *testing.T) { DiffType: spacesyncproto.DiffType_V3, } - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) fx.diffMock.EXPECT().DiffType().Return(spacesyncproto.DiffType_V3) fx.diffMock.EXPECT().Ranges(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) @@ -566,12 +477,9 @@ func TestDiffManager_HandleRangeRequest(t *testing.T) { DiffType: spacesyncproto.DiffType_V2, } - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().DiffType().Return(spacesyncproto.DiffType_V2) - fx.diffMock.EXPECT().Ranges(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil, nil) - _, err := fx.diffManager.HandleRangeRequest(ctx, req) - require.NoError(t, err) + require.Error(t, err) + require.Equal(t, spacesyncproto.ErrUnexpected, err) }) t.Run("handle range request with unsupported diff type", func(t *testing.T) { @@ -594,7 +502,6 @@ func TestDiffManager_AllIds(t *testing.T) { defer fx.stop() expectedIds := []string{"id1", "id2", "id3"} - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) fx.diffMock.EXPECT().Ids().Return(expectedIds) ids := fx.diffManager.AllIds() diff --git a/commonspace/headsync/diffsyncer_test.go b/commonspace/headsync/diffsyncer_test.go index 7eb62bd63..54fea011c 100644 --- a/commonspace/headsync/diffsyncer_test.go +++ b/commonspace/headsync/diffsyncer_test.go @@ -81,7 +81,8 @@ func TestDiffSyncer(t *testing.T) { fx.peerManagerMock.EXPECT(). GetResponsiblePeers(gomock.Any()). Return([]peer.Peer{mPeer}, nil) - fx.diffContainerMock.EXPECT().DiffTypeCheck(gomock.Any(), gomock.Any()).Return(true, fx.diffMock, nil) + fx.diffMock.EXPECT().Hash().Return("aabbcc") + fx.clientMock.EXPECT().HeadSync(gomock.Any(), gomock.Any()).Return(&spacesyncproto.HeadSyncResponse{DiffType: spacesyncproto.DiffType_V3}, nil) fx.diffMock.EXPECT(). Diff(gomock.Any(), gomock.Eq(NewRemoteDiff(fx.spaceState.SpaceId, fx.clientMock))). Return([]string{"new"}, []string{"changed"}, nil, nil) @@ -105,7 +106,8 @@ func TestDiffSyncer(t *testing.T) { fx.peerManagerMock.EXPECT(). GetResponsiblePeers(gomock.Any()). Return([]peer.Peer{mPeer}, nil) - fx.diffContainerMock.EXPECT().DiffTypeCheck(gomock.Any(), gomock.Any()).Return(true, fx.diffMock, nil) + fx.diffMock.EXPECT().Hash().Return("aabbcc") + fx.clientMock.EXPECT().HeadSync(gomock.Any(), gomock.Any()).Return(&spacesyncproto.HeadSyncResponse{DiffType: spacesyncproto.DiffType_V3}, nil) fx.diffMock.EXPECT(). Diff(gomock.Any(), gomock.Eq(remDiff)). Return([]string{"new"}, []string{"changed"}, nil, nil) @@ -130,7 +132,8 @@ func TestDiffSyncer(t *testing.T) { fx.peerManagerMock.EXPECT(). GetResponsiblePeers(gomock.Any()). Return([]peer.Peer{mPeer}, nil) - fx.diffContainerMock.EXPECT().DiffTypeCheck(gomock.Any(), gomock.Any()).Return(true, fx.diffMock, nil) + fx.diffMock.EXPECT().Hash().Return("aabbcc") + fx.clientMock.EXPECT().HeadSync(gomock.Any(), gomock.Any()).Return(&spacesyncproto.HeadSyncResponse{DiffType: spacesyncproto.DiffType_V3}, nil) fx.diffMock.EXPECT(). Diff(gomock.Any(), gomock.Eq(remDiff)). Return([]string{"new"}, []string{"changed"}, nil, nil) @@ -161,11 +164,9 @@ func TestDiffSyncer(t *testing.T) { fx.initDiffSyncer(t) defer fx.stop() deletedId := "id" - fx.diffContainerMock.EXPECT().RemoveId(deletedId).Return(nil) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) + fx.diffMock.EXPECT().RemoveId(deletedId).Return(nil) fx.diffMock.EXPECT().Hash().AnyTimes().Return("hash") - fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash", "hash").Return(nil) + fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash").Return(nil) upd := headstorage.DeletedStatusDeleted fx.diffSyncer.updateHeads(headstorage.HeadsEntry{ @@ -184,21 +185,13 @@ func TestDiffSyncer(t *testing.T) { hasher := ldiff.NewHasher() hash := hasher.HashId("head") ldiff.ReleaseHasher(hasher) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Set(ldiff.Element{ - Id: updatedId, - Head: hash, - }) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) fx.diffMock.EXPECT().Set(ldiff.Element{ Id: updatedId, Head: hash, }) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Hash().Return("hash").Times(2) - fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash", "hash").Return(nil) + fx.diffMock.EXPECT().Hash().Return("hash") + fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash").Return(nil) fx.diffSyncer.updateHeads(headstorage.HeadsEntry{ Id: "id", Heads: []string{"head"}, @@ -228,7 +221,8 @@ func TestDiffSyncer(t *testing.T) { // in the same flow. Here the peer has not yet committed the space, so the // second TryDiff also returns ErrSpaceMissing and we fall back to the // periodic round (no second push, no tree sync). - fx.diffContainerMock.EXPECT().DiffTypeCheck(gomock.Any(), gomock.Any()).Return(true, fx.diffMock, nil).Times(2) + fx.diffMock.EXPECT().Hash().Return("aabbcc").Times(2) + fx.clientMock.EXPECT().HeadSync(gomock.Any(), gomock.Any()).Return(&spacesyncproto.HeadSyncResponse{DiffType: spacesyncproto.DiffType_V3}, nil).Times(2) fx.diffMock.EXPECT(). Diff(gomock.Any(), gomock.Eq(remDiff)). Return(nil, nil, nil, spacesyncproto.ErrSpaceMissing). @@ -279,7 +273,8 @@ func TestDiffSyncer(t *testing.T) { fx.peerManagerMock.EXPECT(). GetResponsiblePeers(gomock.Any()). Return([]peer.Peer{mPeer}, nil) - fx.diffContainerMock.EXPECT().DiffTypeCheck(gomock.Any(), gomock.Any()).Return(true, fx.diffMock, nil).Times(2) + fx.diffMock.EXPECT().Hash().Return("aabbcc").Times(2) + fx.clientMock.EXPECT().HeadSync(gomock.Any(), gomock.Any()).Return(&spacesyncproto.HeadSyncResponse{DiffType: spacesyncproto.DiffType_V3}, nil).Times(2) // First diff: space missing -> triggers push. Second diff (after push, same // flow): the peer now holds the (still empty) space, so the owner's tree // surfaces as a REMOVED id (owner has it, peer lacks it -> ldiff @@ -339,7 +334,8 @@ func TestDiffSyncer(t *testing.T) { fx.peerManagerMock.EXPECT(). GetResponsiblePeers(gomock.Any()). Return([]peer.Peer{rpctest.MockPeer{}}, nil) - fx.diffContainerMock.EXPECT().DiffTypeCheck(gomock.Any(), gomock.Any()).Return(true, fx.diffMock, nil).Times(2) + fx.diffMock.EXPECT().Hash().Return("aabbcc").Times(2) + fx.clientMock.EXPECT().HeadSync(gomock.Any(), gomock.Any()).Return(&spacesyncproto.HeadSyncResponse{DiffType: spacesyncproto.DiffType_V3}, nil).Times(2) // First diff: space missing -> push. Second diff (after push) fails with a // transient error that is NOT ErrSpaceMissing: we must not push again and // must not sync trees; the round falls back to the periodic loop and Sync @@ -385,7 +381,8 @@ func TestDiffSyncer(t *testing.T) { GetResponsiblePeers(gomock.Any()). Return([]peer.Peer{rpctest.MockPeer{}}, nil) - fx.diffContainerMock.EXPECT().DiffTypeCheck(gomock.Any(), gomock.Any()).Return(true, fx.diffMock, nil) + fx.diffMock.EXPECT().Hash().Return("aabbcc") + fx.clientMock.EXPECT().HeadSync(gomock.Any(), gomock.Any()).Return(&spacesyncproto.HeadSyncResponse{DiffType: spacesyncproto.DiffType_V3}, nil) fx.diffMock.EXPECT(). Diff(gomock.Any(), gomock.Eq(remDiff)). Return(nil, nil, nil, spacesyncproto.ErrUnexpected) @@ -404,7 +401,7 @@ func TestDiffSyncer(t *testing.T) { fx.peerManagerMock.EXPECT(). GetResponsiblePeers(gomock.Any()). Return([]peer.Peer{mPeer}, nil) - fx.diffContainerMock.EXPECT().DiffTypeCheck(gomock.Any(), gomock.Any()).Return(true, fx.diffMock, spacesyncproto.ErrSpaceIsDeleted) + fx.clientMock.EXPECT().HeadSync(gomock.Any(), gomock.Any()).Return(nil, spacesyncproto.ErrSpaceIsDeleted) fx.peerManagerMock.EXPECT().KeepAlive(gomock.Any()) require.NoError(t, fx.diffSyncer.Sync(ctx)) diff --git a/commonspace/headsync/headsync.go b/commonspace/headsync/headsync.go index 50d1220e6..f07738ca6 100644 --- a/commonspace/headsync/headsync.go +++ b/commonspace/headsync/headsync.go @@ -10,7 +10,6 @@ import ( "github.com/anyproto/any-sync/app" "github.com/anyproto/any-sync/app/ldiff" "github.com/anyproto/any-sync/app/logger" - "github.com/anyproto/any-sync/app/olddiff" "github.com/anyproto/any-sync/commonspace/config" "github.com/anyproto/any-sync/commonspace/credentialprovider" "github.com/anyproto/any-sync/commonspace/deletionstate" @@ -77,13 +76,13 @@ func (h *headSync) Init(a *app.App) (err error) { h.configuration = a.MustComponent(nodeconf.CName).(nodeconf.NodeConf) h.log = log.With(zap.String("spaceId", h.spaceId)) h.storage = a.MustComponent(spacestorage.CName).(spacestorage.SpaceStorage) - diffContainer := ldiff.NewDiffContainer(ldiff.New(32, 256), olddiff.New(32, 256)) + diff := ldiff.New(32, 256) h.peerManager = a.MustComponent(peermanager.CName).(peermanager.PeerManager) h.credentialProvider = a.MustComponent(credentialprovider.CName).(credentialprovider.CredentialProvider) h.treeSyncer = a.MustComponent(treesyncer.CName).(treesyncer.TreeSyncer) h.deletionState = a.MustComponent(deletionstate.CName).(deletionstate.ObjectDeletionState) h.keyValue = a.MustComponent(kvinterfaces.CName).(kvinterfaces.KeyValueService) - h.diffManager = NewDiffManager(diffContainer, h.storage, h.syncAcl, h.log, context.Background(), h.deletionState, h.keyValue) + h.diffManager = NewDiffManager(diff, h.storage, h.syncAcl, h.log, context.Background(), h.deletionState) h.syncer = createDiffSyncer(h) sync := func(ctx context.Context) (err error) { return h.syncer.Sync(ctx) diff --git a/commonspace/headsync/headsync_test.go b/commonspace/headsync/headsync_test.go index eaf8741fe..be36ffcf4 100644 --- a/commonspace/headsync/headsync_test.go +++ b/commonspace/headsync/headsync_test.go @@ -71,7 +71,6 @@ type headSyncFixture struct { diffSyncerMock *mock_headsync.MockDiffSyncer treeSyncerMock *mock_treesyncer.MockTreeSyncer diffMock *mock_ldiff.MockDiff - diffContainerMock *mock_ldiff.MockDiffContainer clientMock *mock_spacesyncproto.MockDRPCSpaceSyncClient aclMock *mock_syncacl.MockSyncAcl headStorage *mock_headstorage.MockHeadStorage @@ -98,7 +97,6 @@ func newHeadSyncFixture(t *testing.T) *headSyncFixture { deletionStateMock := mock_deletionstate.NewMockObjectDeletionState(ctrl) deletionStateMock.EXPECT().Name().AnyTimes().Return(deletionstate.CName) diffSyncerMock := mock_headsync.NewMockDiffSyncer(ctrl) - diffContainerMock := mock_ldiff.NewMockDiffContainer(ctrl) treeSyncerMock := mock_treesyncer.NewMockTreeSyncer(ctrl) headStorage := mock_headstorage.NewMockHeadStorage(ctrl) stateStorage := mock_statestorage.NewMockStateStorage(ctrl) @@ -137,7 +135,6 @@ func newHeadSyncFixture(t *testing.T) *headSyncFixture { defStoreMock: defStore, configurationMock: configurationMock, storageMock: storageMock, - diffContainerMock: diffContainerMock, peerManagerMock: peerManagerMock, credentialProviderMock: credentialProviderMock, treeManagerMock: treeManagerMock, @@ -161,7 +158,7 @@ func (fx *headSyncFixture) init(t *testing.T) { fx.headStorage.EXPECT().AddObserver(gomock.Any()) err := fx.headSync.Init(fx.app) require.NoError(t, err) - fx.headSync.diffManager = NewDiffManager(fx.diffContainerMock, fx.storageMock, fx.aclMock, fx.headSync.log, context.Background(), fx.deletionStateMock, fx.kvMock) + fx.headSync.diffManager = NewDiffManager(fx.diffMock, fx.storageMock, fx.aclMock, fx.headSync.log, context.Background(), fx.deletionStateMock) } func (fx *headSyncFixture) stop() { @@ -206,7 +203,6 @@ func TestHeadSync(t *testing.T) { hash1 := hasher.HashId("h1h2") hash2 := hasher.HashId("h3h4") ldiff.ReleaseHasher(hasher) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock).Times(2) fx.diffMock.EXPECT().Set(ldiff.Element{ Id: "id1", Head: hash1, @@ -216,20 +212,8 @@ func TestHeadSync(t *testing.T) { Head: hash2, }) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock).Times(2) - fx.diffMock.EXPECT().Set(ldiff.Element{ - Id: "id1", - Head: hash1, - }) - fx.diffMock.EXPECT().Set(ldiff.Element{ - Id: "id2", - Head: hash2, - }) - - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Hash().Return("hash").Times(2) - fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash", "hash").Return(nil) + fx.diffMock.EXPECT().Hash().Return("hash") + fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash").Return(nil) fx.diffSyncerMock.EXPECT().Run() fx.diffSyncerMock.EXPECT().Sync(gomock.Any()).Return(nil) fx.diffSyncerMock.EXPECT().Close() @@ -273,22 +257,13 @@ func TestHeadSync(t *testing.T) { hasher := ldiff.NewHasher() hash2 := hasher.HashId("h3h4") ldiff.ReleaseHasher(hasher) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Set(ldiff.Element{ - Id: "id2", - Head: hash2, - }) - - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) fx.diffMock.EXPECT().Set(ldiff.Element{ Id: "id2", Head: hash2, }) - fx.diffContainerMock.EXPECT().OldDiff().Return(fx.diffMock) - fx.diffContainerMock.EXPECT().NewDiff().Return(fx.diffMock) - fx.diffMock.EXPECT().Hash().Return("hash").Times(2) - fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash", "hash").Return(nil) + fx.diffMock.EXPECT().Hash().Return("hash") + fx.stateStorage.EXPECT().SetHash(gomock.Any(), "hash").Return(nil) fx.diffSyncerMock.EXPECT().Run() fx.diffSyncerMock.EXPECT().Sync(gomock.Any()).Return(nil) fx.diffSyncerMock.EXPECT().Close() diff --git a/commonspace/headsync/remotediff.go b/commonspace/headsync/remotediff.go index 7608a6a35..1da2748f1 100644 --- a/commonspace/headsync/remotediff.go +++ b/commonspace/headsync/remotediff.go @@ -15,8 +15,9 @@ type Client interface { } type RemoteDiff interface { - ldiff.RemoteTypeChecker ldiff.Remote + // DiffTypeCheck asks the remote for its total hash and compares it with the local diff + DiffTypeCheck(ctx context.Context, diff ldiff.Diff) (needsSync bool, err error) } func NewRemoteDiff(spaceId string, client Client) RemoteDiff { @@ -27,9 +28,8 @@ func NewRemoteDiff(spaceId string, client Client) RemoteDiff { } type remote struct { - spaceId string - client Client - diffType spacesyncproto.DiffType + spaceId string + client Client } func (r *remote) Ranges(ctx context.Context, ranges []ldiff.Range, resBuf []ldiff.RangeResult) (results []ldiff.RangeResult, err error) { @@ -46,7 +46,7 @@ func (r *remote) Ranges(ctx context.Context, ranges []ldiff.Range, resBuf []ldif req := &spacesyncproto.HeadSyncRequest{ SpaceId: r.spaceId, Ranges: pbRanges, - DiffType: r.diffType, + DiffType: spacesyncproto.DiffType_V3, } resp, err := r.client.HeadSync(ctx, req) if err != nil { @@ -72,7 +72,7 @@ func (r *remote) Ranges(ctx context.Context, ranges []ldiff.Range, resBuf []ldif return } -func (r *remote) DiffTypeCheck(ctx context.Context, diffContainer ldiff.DiffContainer) (needsSync bool, diff ldiff.Diff, err error) { +func (r *remote) DiffTypeCheck(ctx context.Context, diff ldiff.Diff) (needsSync bool, err error) { req := &spacesyncproto.HeadSyncRequest{ SpaceId: r.spaceId, DiffType: spacesyncproto.DiffType_V3, @@ -82,34 +82,21 @@ func (r *remote) DiffTypeCheck(ctx context.Context, diffContainer ldiff.DiffCont if err != nil { return } - needsSync = true - checkHash := func(diff ldiff.Diff) (bool, error) { - hashB, err := hex.DecodeString(diff.Hash()) - if err != nil { - return false, err - } - if len(resp.Results) != 0 && bytes.Equal(hashB, resp.Results[0].Hash) { - return false, nil - } - return true, nil + if resp.DiffType != spacesyncproto.DiffType_V3 { + return false, spacesyncproto.ErrUnexpected } - r.diffType = resp.DiffType - switch resp.DiffType { - case spacesyncproto.DiffType_V3: - diff = diffContainer.NewDiff() - needsSync, err = checkHash(diff) - case spacesyncproto.DiffType_V2: - diff = diffContainer.OldDiff() - needsSync, err = checkHash(diff) - default: - err = spacesyncproto.ErrUnexpected + hashB, err := hex.DecodeString(diff.Hash()) + if err != nil { + return false, err } - return + if len(resp.Results) != 0 && bytes.Equal(hashB, resp.Results[0].Hash) { + return false, nil + } + return true, nil } func HandleRangeRequest(ctx context.Context, d ldiff.Diff, req *spacesyncproto.HeadSyncRequest) (resp *spacesyncproto.HeadSyncResponse, err error) { ranges := make([]ldiff.Range, 0, len(req.Ranges)) - // basically we gather data applicable for both diffs for _, reqRange := range req.Ranges { ranges = append(ranges, ldiff.Range{ From: reqRange.From, diff --git a/commonspace/headsync/remotediff_test.go b/commonspace/headsync/remotediff_test.go index 2a9e5dda1..7c0f74982 100644 --- a/commonspace/headsync/remotediff_test.go +++ b/commonspace/headsync/remotediff_test.go @@ -10,7 +10,6 @@ import ( "github.com/stretchr/testify/require" "github.com/anyproto/any-sync/app/ldiff" - "github.com/anyproto/any-sync/app/olddiff" "github.com/anyproto/any-sync/commonspace/spacesyncproto" ) @@ -67,12 +66,6 @@ func TestBenchRemoteWithDifferentCounts(t *testing.T) { return ldiff.New(32, 256) }, 32) }) - //old has higher head lengths because of hashes - t.Run("OldLdiff", func(t *testing.T) { - benchmarkDifferentDiffs(t, func() ldiff.Diff { - return olddiff.New(32, 256) - }, 100) - }) } type mockClient struct { @@ -108,31 +101,24 @@ func TestRemoteDiffTypeCheck(t *testing.T) { mockClient := &mockClientWithDiffType{t: t, l: contRemote, diffType: spacesyncproto.DiffType_V3} rd := NewRemoteDiff("space1", mockClient) - container := ldiff.NewDiffContainer(contLocal, ldiff.New(32, 256)) - needsSync, diff, err := rd.DiffTypeCheck(ctx, container) + needsSync, err := rd.DiffTypeCheck(ctx, contLocal) require.NoError(t, err) require.False(t, needsSync) - require.Equal(t, contLocal, diff) }) t.Run("diff type check with V2", func(t *testing.T) { ctx := context.Background() + contLocal := ldiff.New(32, 256) contRemote := ldiff.New(32, 256) - oldDiff := ldiff.New(32, 256) - - oldDiff.Set(ldiff.Element{Id: "1", Head: "head1"}) - contRemote.Set(ldiff.Element{Id: "1", Head: "head1"}) mockClient := &mockClientWithDiffType{t: t, l: contRemote, diffType: spacesyncproto.DiffType_V2} rd := NewRemoteDiff("space1", mockClient) - container := ldiff.NewDiffContainer(ldiff.New(32, 256), oldDiff) - needsSync, diff, err := rd.DiffTypeCheck(ctx, container) + _, err := rd.DiffTypeCheck(ctx, contLocal) - require.NoError(t, err) - require.False(t, needsSync) - require.Equal(t, oldDiff, diff) + require.Error(t, err) + require.Equal(t, spacesyncproto.ErrUnexpected, err) }) t.Run("diff type check with unsupported type", func(t *testing.T) { @@ -143,8 +129,7 @@ func TestRemoteDiffTypeCheck(t *testing.T) { mockClient := &mockClientWithDiffType{t: t, l: contRemote, diffType: spacesyncproto.DiffType_V1} rd := NewRemoteDiff("space1", mockClient) - container := ldiff.NewDiffContainer(contLocal, ldiff.New(32, 256)) - _, _, err := rd.DiffTypeCheck(ctx, container) + _, err := rd.DiffTypeCheck(ctx, contLocal) require.Error(t, err) require.Equal(t, spacesyncproto.ErrUnexpected, err) @@ -161,12 +146,10 @@ func TestRemoteDiffTypeCheck(t *testing.T) { mockClient := &mockClientWithDiffType{t: t, l: contRemote, diffType: spacesyncproto.DiffType_V3} rd := NewRemoteDiff("space1", mockClient) - container := ldiff.NewDiffContainer(contLocal, ldiff.New(32, 256)) - needsSync, diff, err := rd.DiffTypeCheck(ctx, container) + needsSync, err := rd.DiffTypeCheck(ctx, contLocal) require.NoError(t, err) require.True(t, needsSync) - require.Equal(t, contLocal, diff) }) t.Run("head sync request fails", func(t *testing.T) { @@ -176,12 +159,10 @@ func TestRemoteDiffTypeCheck(t *testing.T) { mockClient := &mockClientWithError{err: fmt.Errorf("network error")} rd := NewRemoteDiff("space1", mockClient) - container := ldiff.NewDiffContainer(contLocal, ldiff.New(32, 256)) - needsSync, diff, err := rd.DiffTypeCheck(ctx, container) + needsSync, err := rd.DiffTypeCheck(ctx, contLocal) require.Error(t, err) require.False(t, needsSync) - require.Nil(t, diff) }) } diff --git a/commonspace/headsync/statestorage/mock_statestorage/mock_statestorage.go b/commonspace/headsync/statestorage/mock_statestorage/mock_statestorage.go index 0b13c6e81..b6b21c97e 100644 --- a/commonspace/headsync/statestorage/mock_statestorage/mock_statestorage.go +++ b/commonspace/headsync/statestorage/mock_statestorage/mock_statestorage.go @@ -57,17 +57,17 @@ func (mr *MockStateStorageMockRecorder) GetState(ctx any) *gomock.Call { } // SetHash mocks base method. -func (m *MockStateStorage) SetHash(ctx context.Context, oldHash, newHash string) error { +func (m *MockStateStorage) SetHash(ctx context.Context, hash string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SetHash", ctx, oldHash, newHash) + ret := m.ctrl.Call(m, "SetHash", ctx, hash) ret0, _ := ret[0].(error) return ret0 } // SetHash indicates an expected call of SetHash. -func (mr *MockStateStorageMockRecorder) SetHash(ctx, oldHash, newHash any) *gomock.Call { +func (mr *MockStateStorageMockRecorder) SetHash(ctx, hash any) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetHash", reflect.TypeOf((*MockStateStorage)(nil).SetHash), ctx, oldHash, newHash) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetHash", reflect.TypeOf((*MockStateStorage)(nil).SetHash), ctx, hash) } // SetObserver mocks base method. diff --git a/commonspace/headsync/statestorage/statestorage.go b/commonspace/headsync/statestorage/statestorage.go index 14d73e025..2fa4eabf6 100644 --- a/commonspace/headsync/statestorage/statestorage.go +++ b/commonspace/headsync/statestorage/statestorage.go @@ -10,7 +10,6 @@ import ( ) type State struct { - OldHash string NewHash string AclId string SettingsId string @@ -19,20 +18,19 @@ type State struct { } type Observer interface { - OnHashChange(oldHash, newHash string) + OnHashChange(hash string) } type StateStorage interface { GetState(ctx context.Context) (State, error) SettingsId() string - SetHash(ctx context.Context, oldHash, newHash string) error + SetHash(ctx context.Context, hash string) error SetObserver(observer Observer) } const ( stateCollectionKey = "state" idKey = "id" - oldHashKey = "oh" newHashKey = "nh" legacyHashKey = "h" headerKey = "e" @@ -62,20 +60,16 @@ func (s *stateStorage) SetObserver(observer Observer) { s.observer = observer } -func (s *stateStorage) SetHash(ctx context.Context, oldHash, newHash string) (err error) { +func (s *stateStorage) SetHash(ctx context.Context, hash string) (err error) { var modifyResult anystore.ModifyResult defer func() { if s.observer != nil && err == nil && modifyResult.Modified > 0 { - s.observer.OnHashChange(oldHash, newHash) + s.observer.OnHashChange(hash) } }() mod := query.ModifyFunc(func(a *anyenc.Arena, v *anyenc.Value) (result *anyenc.Value, modified bool, err error) { - if storeutil.ModifyKey(v, oldHashKey, a.NewString(oldHash)) { - modified = true - } - - if storeutil.ModifyKey(v, newHashKey, a.NewString(newHash)) { + if storeutil.ModifyKey(v, newHashKey, a.NewString(hash)) { modified = true } @@ -148,20 +142,15 @@ func (s *stateStorage) SettingsId() string { } func (s *stateStorage) stateFromDoc(doc anystore.Doc) State { - var ( - oldHash = doc.Value().GetString(oldHashKey) - newHash = doc.Value().GetString(newHashKey) - ) + newHash := doc.Value().GetString(newHashKey) // legacy hash is used for backward compatibility, which was due to a mistake in key names - if oldHash == "" || newHash == "" { - oldHash = doc.Value().GetString(legacyHashKey) - newHash = oldHash + if newHash == "" { + newHash = doc.Value().GetString(legacyHashKey) } return State{ SpaceId: doc.Value().GetString(idKey), SettingsId: doc.Value().GetString(settingsIdKey), AclId: doc.Value().GetString(aclIdKey), - OldHash: oldHash, NewHash: newHash, SpaceHeader: doc.Value().GetBytes(headerKey), } diff --git a/commonspace/spacestorage/migration/spacemigrator.go b/commonspace/spacestorage/migration/spacemigrator.go index 182b1e8b4..8fba4ce67 100644 --- a/commonspace/spacestorage/migration/spacemigrator.go +++ b/commonspace/spacestorage/migration/spacemigrator.go @@ -160,7 +160,7 @@ func (s *spaceMigrator) migrateHash(ctx context.Context, oldStorage oldstorage.S if err != nil { return err } - return newStorage.StateStorage().SetHash(ctx, spaceHash, spaceHash) + return newStorage.StateStorage().SetHash(ctx, spaceHash) } func (s *spaceMigrator) checkMigrated(ctx context.Context, id string) (bool, spacestorage.SpaceStorage) { diff --git a/commonspace/spacesyncproto/protos/spacesync.proto b/commonspace/spacesyncproto/protos/spacesync.proto index 228fb0a55..21c9b7bd2 100644 --- a/commonspace/spacesyncproto/protos/spacesync.proto +++ b/commonspace/spacesyncproto/protos/spacesync.proto @@ -272,8 +272,8 @@ message StorageHeader { // DiffType is a type of diff enum DiffType { Initial = 0; - V1 = 1; - V2 = 2; + V1 = 1; // deprecated, not supported anymore + V2 = 2; // deprecated, not supported anymore V3 = 3; } From db67c16d3d907b1b533b88e9171f90af7942f9bd Mon Sep 17 00:00:00 2001 From: Sergey Cherepanov Date: Fri, 10 Jul 2026 15:38:24 +0200 Subject: [PATCH 20/20] fix(headsync): address diffsync-V2 removal review findings - statestorage: keep mirroring the hash into the legacy "oh" key so a previous release (rollback or mixed-fleet coldsync) doesn't misread NewHash as empty via its legacy-format fallback; transitional, remove once downgrades below this version are out of support - diffmanager: guard FillDiff against empty Heads entries, set all elements with a single variadic Set (one range recalculation instead of N), document why FillDiff and UpdateHeads deliberately use different empty-root conditions, and note the V4 fallback-on-error requirement in the upgrade recipe - add a hash-stability regression test pinned to the golden value produced by the pre-removal dual-diff DiffManager --- commonspace/headsync/diffmanager.go | 29 ++++++++------ commonspace/headsync/diffmanager_test.go | 39 ++++++++++++++++++- commonspace/headsync/headsync_test.go | 3 +- .../headsync/statestorage/statestorage.go | 17 ++++++-- 4 files changed, 69 insertions(+), 19 deletions(-) diff --git a/commonspace/headsync/diffmanager.go b/commonspace/headsync/diffmanager.go index 067b4c9b6..8c85539a2 100644 --- a/commonspace/headsync/diffmanager.go +++ b/commonspace/headsync/diffmanager.go @@ -24,6 +24,11 @@ import ( // and by resp.DiffType in remote.DiffTypeCheck, so that the newest common // type wins. The old type can be dropped once the secureservice // compatibleVersions exclude all peers that don't support the new one. +// +// Note for that future upgrade: V3-only responders (this code and the node's +// HeadSync handler) reject unknown diff types with an error instead of +// answering with their best supported type, so a V(n+1) requester must treat +// such an error as "type unsupported" and fall back to requesting V(n). type DiffManager struct { diff ldiff.Diff storage spacestorage.SpaceStorage @@ -53,14 +58,18 @@ func NewDiffManager( func (dm *DiffManager) FillDiff(ctx context.Context) error { els := make([]ldiff.Element, 0, 100) + hasher := ldiff.NewHasher() + defer ldiff.ReleaseHasher(hasher) err := dm.storage.HeadStorage().IterateEntries(ctx, headstorage.IterOpts{}, func(entry headstorage.HeadsEntry) (bool, error) { - // empty roots shouldn't be set in the diff - if entry.Heads[0] == entry.Id && (entry.IsDerived || entry.CommonSnapshot != "") { + // skip empty roots, except non-derived ones without a common snapshot: + // those have historically been part of the diff and must stay to keep + // the space hash stable (UpdateHeads deliberately differs, see there) + if len(entry.Heads) > 0 && entry.Heads[0] == entry.Id && (entry.IsDerived || entry.CommonSnapshot != "") { return true, nil } els = append(els, ldiff.Element{ Id: entry.Id, - Head: concatStrings(entry.Heads), + Head: hasher.HashId(concatStrings(entry.Heads)), }) return true, nil }) @@ -68,14 +77,9 @@ func (dm *DiffManager) FillDiff(ctx context.Context) error { return err } log.Debug("setting acl", zap.String("aclId", dm.syncAcl.Id()), zap.String("headId", dm.syncAcl.Head().Id)) - hasher := ldiff.NewHasher() - for _, el := range els { - dm.diff.Set(ldiff.Element{ - Id: el.Id, - Head: hasher.HashId(el.Head), - }) + if len(els) > 0 { + dm.diff.Set(els...) } - ldiff.ReleaseHasher(hasher) if err := dm.storage.StateStorage().SetHash(ctx, dm.diff.Hash()); err != nil { dm.log.Error("can't write space hash", zap.Error(err)) return err @@ -106,7 +110,10 @@ func (dm *DiffManager) UpdateHeads(update headstorage.HeadsEntry) { if dm.deletionState.Exists(update.Id) { return } - // empty roots shouldn't be set in the diff + // live updates never add empty roots; unlike FillDiff there is no + // common-snapshot exception here — this asymmetry predates the V2 + // removal and both conditions must stay as is, or every existing + // space hash changes if len(update.Heads) == 1 && update.Heads[0] == update.Id { return } diff --git a/commonspace/headsync/diffmanager_test.go b/commonspace/headsync/diffmanager_test.go index c90b02144..94e1984ec 100644 --- a/commonspace/headsync/diffmanager_test.go +++ b/commonspace/headsync/diffmanager_test.go @@ -115,8 +115,7 @@ func TestDiffManager_FillDiff(t *testing.T) { fx.diffMock.EXPECT().Set(ldiff.Element{ Id: "id1", Head: hash1, - }) - fx.diffMock.EXPECT().Set(ldiff.Element{ + }, ldiff.Element{ Id: "id2", Head: hash2, }) @@ -508,3 +507,39 @@ func TestDiffManager_AllIds(t *testing.T) { require.Equal(t, expectedIds, ids) }) } + +// TestDiffManager_HashStability pins the V3 space hash for a scenario covering +// every head-entry class. The golden value was produced by the pre-V2-removal +// DiffManager (dual old/new diffs) on the same input: if this test fails, the +// change breaks hash compatibility with deployed peers and will force a +// network-wide resync — see the skip conditions in FillDiff and UpdateHeads. +func TestDiffManager_HashStability(t *testing.T) { + fx := newDiffManagerFixture(t) + defer fx.stop() + + fx.stateStorage.EXPECT().SetHash(gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + fx.aclMock.EXPECT().Id().Return("aclId").AnyTimes() + fx.aclMock.EXPECT().Head().Return(&list.AclRecord{Id: "headId"}).AnyTimes() + fx.deletionStateMock.EXPECT().Exists(gomock.Any()).Return(false).AnyTimes() + + fx.expectIterateEntries([]headstorage.HeadsEntry{ + {Id: "A", Heads: []string{"h1", "h2"}, CommonSnapshot: "s"}, + {Id: "B", Heads: []string{"h3"}, CommonSnapshot: ""}, + {Id: "C", Heads: []string{"C"}, CommonSnapshot: "s"}, + {Id: "D", Heads: []string{"D"}, CommonSnapshot: "s", IsDerived: true}, + {Id: "E", Heads: []string{"E"}, CommonSnapshot: ""}, + }) + + diff := ldiff.New(32, 256) + dm := NewDiffManager(diff, fx.storageMock, fx.aclMock, logger.NewNamed("test"), context.Background(), fx.deletionStateMock) + require.NoError(t, dm.FillDiff(context.Background())) + for _, u := range []headstorage.HeadsEntry{ + {Id: "A", Heads: []string{"h1", "h2", "h5"}}, + {Id: "G", Heads: []string{"G"}}, + {Id: "store", Heads: []string{"k1"}}, + {Id: "B", DeletedStatus: headstorage.DeletedStatusDeleted}, + } { + dm.UpdateHeads(u) + } + require.Equal(t, "9d2101bbe6775d7cd4f1d322d0f227e48ccc27ae3bc745dbda5d05bd4ba61fc1", diff.Hash()) +} diff --git a/commonspace/headsync/headsync_test.go b/commonspace/headsync/headsync_test.go index be36ffcf4..4314e9023 100644 --- a/commonspace/headsync/headsync_test.go +++ b/commonspace/headsync/headsync_test.go @@ -206,8 +206,7 @@ func TestHeadSync(t *testing.T) { fx.diffMock.EXPECT().Set(ldiff.Element{ Id: "id1", Head: hash1, - }) - fx.diffMock.EXPECT().Set(ldiff.Element{ + }, ldiff.Element{ Id: "id2", Head: hash2, }) diff --git a/commonspace/headsync/statestorage/statestorage.go b/commonspace/headsync/statestorage/statestorage.go index 2fa4eabf6..286dfba84 100644 --- a/commonspace/headsync/statestorage/statestorage.go +++ b/commonspace/headsync/statestorage/statestorage.go @@ -32,10 +32,15 @@ const ( stateCollectionKey = "state" idKey = "id" newHashKey = "nh" - legacyHashKey = "h" - headerKey = "e" - aclIdKey = "a" - settingsIdKey = "s" + // oldHashKey held the removed V2-diff hash; it is still mirrored on write because + // releases before the V2 removal treat a doc with an empty "oh" as legacy format + // and misread NewHash as empty (see their stateFromDoc). Drop the mirror once + // downgrades/coldsync to those releases are out of support. + oldHashKey = "oh" + legacyHashKey = "h" + headerKey = "e" + aclIdKey = "a" + settingsIdKey = "s" ) type stateStorage struct { @@ -73,6 +78,10 @@ func (s *stateStorage) SetHash(ctx context.Context, hash string) (err error) { modified = true } + if storeutil.ModifyKey(v, oldHashKey, a.NewString(hash)) { + modified = true + } + return v, modified, nil }) modifyResult, err = s.stateColl.UpsertId(ctx, s.spaceId, mod)