From ec740450ba57e6a04ac7e74b9b37b8f2ada9a758 Mon Sep 17 00:00:00 2001 From: Will Baker Date: Wed, 15 Jul 2026 20:42:42 +0000 Subject: [PATCH] broker: add opt-in delete-permission probe to FragmentStoreHealth The storage-mapping connection test never exercises a delete, so a bucket policy missing the delete action passes setup, and the problem only surfaces later during operations that require deletion, such as recovery log pruning. Add check_delete and check_delete_prefix to FragmentStoreHealthRequest. When check_delete is set, the broker writes and removes a throwaway probe object under the prefix (empty probes the store root) to verify delete permission at configuration time. The probe is on-demand and excluded from the periodic background check, and is intended to be invoked as a user-initiated check when storage mappings change. --- broker/client/fragment_store_health.go | 11 +- broker/client/fragment_store_health_test.go | 36 +- broker/fragment_store_health_api.go | 9 + broker/fragment_store_health_api_test.go | 57 +++ broker/protocol/protocol.pb.go | 457 +++++++++++------- broker/protocol/protocol.proto | 6 + broker/protocol/rpc_extensions.go | 14 + broker/protocol/rpc_extensions_test.go | 40 ++ broker/stores/health_check.go | 20 + broker/stores/health_check_test.go | 61 +++ cmd/gazctl/gazctlcmd/journals_store_health.go | 8 +- cmd/gazctl/gazctlcmd/shards_prune.go | 5 +- 12 files changed, 528 insertions(+), 196 deletions(-) diff --git a/broker/client/fragment_store_health.go b/broker/client/fragment_store_health.go index 18f9cb40..c17b8dc4 100644 --- a/broker/client/fragment_store_health.go +++ b/broker/client/fragment_store_health.go @@ -10,9 +10,14 @@ import ( // FragmentStoreHealth queries for the latest health status on the specified fragment store. // It returns the health check response or an error if the RPC fails with a status other // than FRAGMENT_STORE_UNHEALTHY. -func FragmentStoreHealth(ctx context.Context, client pb.JournalClient, store pb.FragmentStore) (*pb.FragmentStoreHealthResponse, error) { - var req = pb.FragmentStoreHealthRequest{ - FragmentStore: store, +// +// A non-nil checkDeletePrefix also probes delete permission under that prefix; +// a pointer to the empty string probes the store root. +func FragmentStoreHealth(ctx context.Context, client pb.JournalClient, store pb.FragmentStore, checkDeletePrefix *string) (*pb.FragmentStoreHealthResponse, error) { + var req = pb.FragmentStoreHealthRequest{FragmentStore: store} + if checkDeletePrefix != nil { + req.CheckDelete = true + req.CheckDeletePrefix = *checkDeletePrefix } if resp, err := client.FragmentStoreHealth(pb.WithDispatchDefault(ctx), &req); err != nil { diff --git a/broker/client/fragment_store_health_test.go b/broker/client/fragment_store_health_test.go index 53794064..705f5979 100644 --- a/broker/client/fragment_store_health_test.go +++ b/broker/client/fragment_store_health_test.go @@ -24,7 +24,7 @@ func TestCheckFragmentStoreHealth(t *testing.T) { }, nil } - resp, err := FragmentStoreHealth(ctx, broker.Client(), "s3://my-bucket/") + resp, err := FragmentStoreHealth(ctx, broker.Client(), "s3://my-bucket/", nil) require.NoError(t, err) require.Equal(t, pb.Status_OK, resp.Status) require.Empty(t, resp.StoreHealthError) @@ -38,7 +38,7 @@ func TestCheckFragmentStoreHealth(t *testing.T) { }, nil } - resp, err = FragmentStoreHealth(ctx, broker.Client(), "s3://my-bucket/") + resp, err = FragmentStoreHealth(ctx, broker.Client(), "s3://my-bucket/", nil) require.NoError(t, err) require.Equal(t, pb.Status_FRAGMENT_STORE_UNHEALTHY, resp.Status) require.Equal(t, "store is unhealthy: connection timeout", resp.StoreHealthError) @@ -48,7 +48,7 @@ func TestCheckFragmentStoreHealth(t *testing.T) { return nil, errors.New("network error") } - resp, err = FragmentStoreHealth(ctx, broker.Client(), "s3://my-bucket/") + resp, err = FragmentStoreHealth(ctx, broker.Client(), "s3://my-bucket/", nil) require.Error(t, err) require.Contains(t, err.Error(), "network error") require.Nil(t, resp) @@ -61,8 +61,36 @@ func TestCheckFragmentStoreHealth(t *testing.T) { }, nil } - resp, err = FragmentStoreHealth(ctx, broker.Client(), "s3://my-bucket/") + resp, err = FragmentStoreHealth(ctx, broker.Client(), "s3://my-bucket/", nil) require.Error(t, err) require.Equal(t, "JOURNAL_NOT_FOUND", err.Error()) require.Nil(t, resp) + + // Case 5: nil leaves check_delete unset; a non-nil prefix sets both fields. + var gotReq *pb.FragmentStoreHealthRequest + broker.FragmentStoreHealthFunc = func(ctx context.Context, req *pb.FragmentStoreHealthRequest) (*pb.FragmentStoreHealthResponse, error) { + gotReq = req + return &pb.FragmentStoreHealthResponse{ + Status: pb.Status_OK, + Header: *buildHeaderFixture(broker), + }, nil + } + + _, err = FragmentStoreHealth(ctx, broker.Client(), "s3://my-bucket/", nil) + require.NoError(t, err) + require.False(t, gotReq.CheckDelete) + require.Empty(t, gotReq.CheckDeletePrefix) + + var prefix = "recovery/" + _, err = FragmentStoreHealth(ctx, broker.Client(), "s3://my-bucket/", &prefix) + require.NoError(t, err) + require.True(t, gotReq.CheckDelete) + require.Equal(t, "recovery/", gotReq.CheckDeletePrefix) + + // Case 6: a pointer to "" probes the store root: check_delete set, prefix empty. + var root = "" + _, err = FragmentStoreHealth(ctx, broker.Client(), "s3://my-bucket/", &root) + require.NoError(t, err) + require.True(t, gotReq.CheckDelete) + require.Empty(t, gotReq.CheckDeletePrefix) } diff --git a/broker/fragment_store_health_api.go b/broker/fragment_store_health_api.go index ede7c00f..0e250cc1 100644 --- a/broker/fragment_store_health_api.go +++ b/broker/fragment_store_health_api.go @@ -3,6 +3,7 @@ package broker import ( "context" "net" + "time" log "github.com/sirupsen/logrus" pb "go.gazette.dev/core/broker/protocol" @@ -55,6 +56,14 @@ func (svc *Service) FragmentStoreHealth(ctx context.Context, claims pb.Claims, r if healthErr != nil { resp.Status = pb.Status_FRAGMENT_STORE_UNHEALTHY resp.StoreHealthError = healthErr.Error() + } else if req.CheckDelete { + var probeCtx, cancel = context.WithTimeout(ctx, time.Minute) + defer cancel() + + if deleteErr := activeStore.CheckDelete(probeCtx, req.CheckDeletePrefix); deleteErr != nil { + resp.Status = pb.Status_FRAGMENT_STORE_UNHEALTHY + resp.StoreHealthError = deleteErr.Error() + } } return resp, nil } diff --git a/broker/fragment_store_health_api_test.go b/broker/fragment_store_health_api_test.go index bb118aa1..26cc9810 100644 --- a/broker/fragment_store_health_api_test.go +++ b/broker/fragment_store_health_api_test.go @@ -55,3 +55,60 @@ func TestFragmentStoreHealthCases(t *testing.T) { require.Error(t, err) require.Contains(t, err.Error(), "context canceled") } + +func TestFragmentStoreHealthDeleteProbe(t *testing.T) { + var ctx, etcd = pb.WithDispatchDefault(context.Background()), etcdtest.TestClient() + defer etcdtest.Cleanup() + + // The "no-delete" store denies delete but delegates all else to memory, so + // the periodic health check passes and the RPC reaches the delete probe. + stores.RegisterProviders(map[string]stores.Constructor{ + "s3": func(ep *url.URL) (stores.Store, error) { + var mem = stores.NewMemoryStore(ep) + if ep.Host == "no-delete" { + return &stores.CallbackStore{ + Fallback: mem, + RemoveFunc: func(_ stores.Store, ctx context.Context, _ string) error { + if _, ok := ctx.Deadline(); !ok { + return fmt.Errorf("delete probe ran without a deadline") + } + return fmt.Errorf("simulated missing delete permission") + }, + }, nil + } + return mem, nil + }, + }) + + var broker = newTestBroker(t, etcd, pb.ProcessSpec_ID{Zone: "local", Suffix: "broker"}) + defer broker.cleanup() + + // Healthy store grants delete: passes under a sub-prefix. + var resp, err = broker.client().FragmentStoreHealth(ctx, &pb.FragmentStoreHealthRequest{ + FragmentStore: "s3://bucket/", + CheckDelete: true, + CheckDeletePrefix: "recovery/", + }) + require.NoError(t, err) + require.Equal(t, pb.Status_OK, resp.Status) + require.Empty(t, resp.StoreHealthError) + + // Store lacks delete permission: probe fails. + resp, err = broker.client().FragmentStoreHealth(ctx, &pb.FragmentStoreHealthRequest{ + FragmentStore: "s3://no-delete/", + CheckDelete: true, + CheckDeletePrefix: "recovery/", + }) + require.NoError(t, err) + require.Equal(t, pb.Status_FRAGMENT_STORE_UNHEALTHY, resp.Status) + require.Contains(t, resp.StoreHealthError, "delete-probe DELETE failed") + require.Contains(t, resp.StoreHealthError, "simulated missing delete permission") + + // check_delete false: reports healthy (background-check behavior). + resp, err = broker.client().FragmentStoreHealth(ctx, &pb.FragmentStoreHealthRequest{ + FragmentStore: "s3://no-delete/", + }) + require.NoError(t, err) + require.Equal(t, pb.Status_OK, resp.Status) + require.Empty(t, resp.StoreHealthError) +} diff --git a/broker/protocol/protocol.pb.go b/broker/protocol/protocol.pb.go index 24b19fdb..8572fc5e 100644 --- a/broker/protocol/protocol.pb.go +++ b/broker/protocol/protocol.pb.go @@ -1883,6 +1883,12 @@ var xxx_messageInfo_Header_Etcd proto.InternalMessageInfo type FragmentStoreHealthRequest struct { // Fragment store to check the health of. FragmentStore FragmentStore `protobuf:"bytes,1,opt,name=fragment_store,json=fragmentStore,proto3,casttype=FragmentStore" json:"fragment_store,omitempty"` + // If true, also verify delete permission with a throwaway probe object under + // check_delete_prefix. Defaults to false (delete is not exercised). + CheckDelete bool `protobuf:"varint,2,opt,name=check_delete,json=checkDelete,proto3" json:"check_delete,omitempty"` + // Prefix under which the delete probe is written (e.g. "recovery/"); empty + // probes the store root. Only used when check_delete is true. + CheckDeletePrefix string `protobuf:"bytes,3,opt,name=check_delete_prefix,json=checkDeletePrefix,proto3" json:"check_delete_prefix,omitempty"` } func (m *FragmentStoreHealthRequest) Reset() { *m = FragmentStoreHealthRequest{} } @@ -2042,192 +2048,195 @@ func init() { } var fileDescriptor_0c0999e5af553218 = []byte{ - // 2959 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x59, 0x3d, 0x74, 0x1b, 0xc7, - 0x11, 0xe6, 0xe1, 0xf7, 0x30, 0x00, 0xc8, 0xe3, 0xca, 0x92, 0x20, 0xc8, 0x22, 0x68, 0x48, 0x72, - 0x28, 0x59, 0x86, 0x64, 0x2a, 0xcf, 0x3f, 0xca, 0x73, 0x62, 0x80, 0x38, 0x8a, 0x90, 0x40, 0x00, - 0x6f, 0x01, 0x4a, 0x96, 0x8b, 0xdc, 0x3b, 0xe2, 0x96, 0xe0, 0x45, 0x87, 0x3b, 0xe4, 0xee, 0x20, - 0x93, 0xee, 0xdc, 0xa5, 0x70, 0x5e, 0x52, 0x3a, 0x4d, 0xe2, 0x14, 0xae, 0x53, 0xa4, 0x73, 0x1a, - 0x97, 0x4a, 0xe7, 0x32, 0xef, 0xc5, 0x61, 0x12, 0xeb, 0xbd, 0xbc, 0x14, 0xa9, 0x54, 0xba, 0xca, - 0xdb, 0x9f, 0x03, 0x8e, 0xf8, 0x11, 0xe5, 0x82, 0xdd, 0xed, 0xcc, 0xb7, 0xb3, 0xbb, 0x33, 0xb3, - 0x33, 0xb3, 0x73, 0xb0, 0xb2, 0xeb, 0x3a, 0x8f, 0x89, 0x7b, 0x73, 0xe0, 0x3a, 0xbe, 0xd3, 0x75, - 0xac, 0xd1, 0x47, 0x89, 0x7d, 0x20, 0x39, 0x18, 0xe7, 0x5f, 0xe9, 0x39, 0x3d, 0x87, 0x8d, 0x6e, - 0xd2, 0x2f, 0xce, 0xcf, 0xaf, 0xf4, 0x1c, 0xa7, 0x67, 0x11, 0x3e, 0x6d, 0x77, 0xb8, 0x77, 0xd3, - 0x18, 0xba, 0xba, 0x6f, 0x3a, 0x36, 0xe7, 0x17, 0xf7, 0x20, 0x5e, 0xd7, 0x77, 0x89, 0x85, 0x10, - 0xc4, 0x6c, 0xbd, 0x4f, 0x72, 0xd2, 0xaa, 0xb4, 0x96, 0xc2, 0xec, 0x1b, 0xbd, 0x02, 0xf1, 0x27, - 0xba, 0x35, 0x24, 0xb9, 0x08, 0x23, 0xf2, 0x01, 0x7a, 0x13, 0x12, 0x03, 0x97, 0xec, 0x99, 0x07, - 0xb9, 0xe8, 0xaa, 0xb4, 0x26, 0x57, 0xce, 0x3e, 0x3f, 0x2a, 0x2c, 0x1f, 0xea, 0x7d, 0xeb, 0x4e, - 0xf1, 0x86, 0xd3, 0x37, 0x7d, 0xd2, 0x1f, 0xf8, 0x87, 0x45, 0x2c, 0x40, 0x77, 0x62, 0xff, 0xfd, - 0xa2, 0x20, 0x15, 0x3b, 0x20, 0xb3, 0x75, 0xda, 0xc4, 0x47, 0x15, 0x48, 0x58, 0xf4, 0xdb, 0xcb, - 0x49, 0xab, 0xd1, 0xb5, 0xf4, 0xfa, 0x52, 0x69, 0x74, 0x28, 0x86, 0xa9, 0x5c, 0x78, 0x7a, 0x54, - 0x58, 0x98, 0x23, 0x95, 0xcf, 0x14, 0x52, 0x3f, 0x95, 0x20, 0x2b, 0xc4, 0x5a, 0xa4, 0xeb, 0x3b, - 0x2e, 0x5a, 0x87, 0xa4, 0x69, 0x77, 0xad, 0xa1, 0xc1, 0x4f, 0x92, 0x5e, 0x47, 0x13, 0xc2, 0xdb, - 0xc4, 0xaf, 0xc4, 0xa8, 0x7c, 0x1c, 0x00, 0xe9, 0x1c, 0x72, 0xc0, 0xe7, 0x44, 0x4e, 0x9a, 0x23, - 0x80, 0x77, 0xe4, 0xcf, 0xbf, 0x28, 0x2c, 0xb0, 0x3d, 0xfc, 0x2e, 0x0d, 0xe9, 0x7b, 0xce, 0xd0, - 0xb5, 0x75, 0xab, 0x3d, 0x20, 0x5d, 0xf4, 0xe3, 0xb0, 0x22, 0x2b, 0xab, 0x33, 0x8f, 0xf1, 0xfd, - 0x51, 0x21, 0x29, 0xe6, 0x08, 0x55, 0xbf, 0x03, 0x69, 0x97, 0x0c, 0x2c, 0xb3, 0xcb, 0x8c, 0xc3, - 0xf6, 0x11, 0x9f, 0xa7, 0xd9, 0x30, 0x12, 0xb5, 0x46, 0xca, 0x8c, 0xce, 0xdd, 0xfb, 0x15, 0xba, - 0xf7, 0x6f, 0x8e, 0x0a, 0xd2, 0xf3, 0xa3, 0x42, 0x6e, 0x52, 0xde, 0x0d, 0xd3, 0xb6, 0x4c, 0x9b, - 0x8c, 0x54, 0x8b, 0x76, 0x40, 0xde, 0x73, 0xf5, 0x5e, 0x9f, 0xd8, 0x7e, 0x2e, 0xc6, 0x64, 0xae, - 0x8c, 0x65, 0x86, 0x4e, 0x5a, 0xda, 0x14, 0xa8, 0x17, 0xd9, 0x6b, 0x24, 0x0a, 0xfd, 0x0c, 0xe2, - 0x7b, 0x96, 0xde, 0xf3, 0x72, 0x89, 0x55, 0x69, 0x2d, 0x5b, 0xb9, 0x36, 0x4f, 0x31, 0x4a, 0x68, - 0x09, 0x6d, 0xd3, 0xd2, 0x7b, 0x98, 0xcf, 0x43, 0x75, 0x58, 0xea, 0xeb, 0x07, 0x9a, 0x3e, 0x18, - 0x10, 0xdb, 0xd0, 0x5c, 0xdd, 0x27, 0xb9, 0xe4, 0xaa, 0xb4, 0x16, 0xad, 0x5c, 0x79, 0x7e, 0x54, - 0x58, 0xe5, 0xa2, 0x26, 0x00, 0xe1, 0x9d, 0x64, 0xfb, 0xfa, 0x41, 0x99, 0xb1, 0xb0, 0xee, 0x13, - 0xd4, 0x80, 0xa4, 0x37, 0xf4, 0xe8, 0x30, 0x27, 0xb3, 0x43, 0x5e, 0x9a, 0x7d, 0xc8, 0x36, 0x07, - 0xcd, 0xb3, 0x45, 0x20, 0x24, 0xff, 0x59, 0x1c, 0xe4, 0x40, 0x21, 0xf4, 0x8a, 0x58, 0xc4, 0xee, - 0xf9, 0xfb, 0xcc, 0x0b, 0xa2, 0x73, 0xaf, 0x08, 0x07, 0x21, 0x07, 0x96, 0xbb, 0x4e, 0x7f, 0xe0, - 0x12, 0xcf, 0x33, 0x1d, 0x5b, 0xeb, 0x3a, 0x06, 0xe9, 0x32, 0x17, 0x58, 0x5c, 0xcf, 0x8f, 0x77, - 0xb5, 0x31, 0x86, 0x6c, 0x50, 0x44, 0xe5, 0xf5, 0xe7, 0x47, 0x85, 0x22, 0x97, 0x3a, 0x35, 0x3d, - 0xbc, 0x8c, 0xd2, 0x9d, 0x98, 0x89, 0x7e, 0x0a, 0x09, 0xcf, 0x77, 0x5c, 0x42, 0x9d, 0x26, 0xba, - 0x96, 0x62, 0x92, 0x66, 0x1a, 0x23, 0x1b, 0x1c, 0xa9, 0x4d, 0xe1, 0x58, 0xcc, 0x42, 0x1e, 0x28, - 0x2e, 0xd9, 0x73, 0x89, 0xb7, 0xaf, 0x99, 0xb6, 0x4f, 0xdc, 0x27, 0xba, 0x25, 0x5c, 0xe5, 0x42, - 0x89, 0x07, 0x9c, 0x52, 0x10, 0x70, 0x4a, 0x55, 0x11, 0x70, 0x2a, 0x6f, 0x0a, 0x2f, 0x79, 0x8d, - 0x2f, 0x34, 0x29, 0x20, 0xb4, 0xf0, 0xe7, 0xff, 0x2c, 0x48, 0x78, 0x49, 0x00, 0x6a, 0x82, 0x8f, - 0x1e, 0x40, 0xca, 0x25, 0x3e, 0xb1, 0xd9, 0x05, 0x89, 0x9f, 0xb4, 0xda, 0xa5, 0xb9, 0x3e, 0xc9, - 0xa4, 0x8f, 0x45, 0xa1, 0x3e, 0x2c, 0xee, 0x59, 0xc3, 0xf0, 0x51, 0x12, 0x27, 0x09, 0x7f, 0x43, - 0x08, 0x2f, 0x70, 0xe1, 0xc7, 0xa7, 0x4f, 0x2e, 0x95, 0x65, 0xec, 0xd1, 0x31, 0x7e, 0x0e, 0x67, - 0x07, 0xba, 0xbf, 0xaf, 0x0d, 0x1c, 0xcf, 0xdf, 0x33, 0x0f, 0x34, 0x0a, 0xb5, 0x02, 0x67, 0x4e, - 0x55, 0xae, 0x3f, 0x3f, 0x2a, 0xbc, 0xce, 0xc5, 0xce, 0x84, 0x85, 0x0d, 0x7b, 0x86, 0x22, 0x5a, - 0x1c, 0xd0, 0x11, 0x7c, 0x1e, 0x19, 0xf3, 0x5f, 0x49, 0x90, 0x14, 0xae, 0x8b, 0x5a, 0x10, 0xb7, - 0xc8, 0x13, 0x62, 0x31, 0x67, 0x5c, 0x5c, 0xbf, 0xfc, 0x42, 0x47, 0x2f, 0xd5, 0x29, 0x74, 0x9e, - 0xc7, 0x72, 0x41, 0xd4, 0xbf, 0x9d, 0xbd, 0x3d, 0x8f, 0xf8, 0xcc, 0x4b, 0xe7, 0xfb, 0x37, 0x07, - 0x15, 0xd7, 0x20, 0xce, 0xa4, 0x22, 0x19, 0x62, 0x8d, 0x66, 0x43, 0x55, 0x16, 0x50, 0x1a, 0x92, - 0xad, 0x32, 0xee, 0xd4, 0xca, 0x75, 0x45, 0xa2, 0xe4, 0xcd, 0x9d, 0x7a, 0x5d, 0x89, 0x88, 0xb0, - 0x5e, 0x86, 0x18, 0xbd, 0xf8, 0x68, 0x19, 0xb2, 0x8d, 0x66, 0x47, 0x6b, 0xb7, 0xd4, 0x8d, 0xda, - 0x66, 0x4d, 0xad, 0x2a, 0x0b, 0x28, 0x03, 0x72, 0x53, 0xc3, 0xd5, 0x66, 0xa3, 0xfe, 0x48, 0x91, - 0xf8, 0xe8, 0x21, 0x66, 0xa3, 0x08, 0x02, 0x48, 0x50, 0xde, 0x43, 0xac, 0xc4, 0x84, 0xa0, 0x2f, - 0x25, 0x48, 0xb7, 0x5c, 0xa7, 0x4b, 0x3c, 0x8f, 0xc5, 0xe6, 0x12, 0x44, 0x4c, 0x43, 0x24, 0x86, - 0xdc, 0x58, 0x0d, 0x21, 0x48, 0xa9, 0x56, 0x15, 0xa1, 0x3e, 0x62, 0x1a, 0x68, 0x0d, 0x64, 0x62, - 0x1b, 0x03, 0xc7, 0xb4, 0xf9, 0x49, 0x53, 0x95, 0xcc, 0xf7, 0x47, 0x05, 0x59, 0x15, 0x34, 0x3c, - 0xe2, 0xe6, 0xdf, 0x86, 0x48, 0xad, 0x4a, 0x93, 0xe8, 0x27, 0x8e, 0x3d, 0x4a, 0xa2, 0xf4, 0x1b, - 0x9d, 0x83, 0x84, 0x37, 0xdc, 0xa3, 0xe9, 0x92, 0x67, 0x51, 0x31, 0xe2, 0x3b, 0xbc, 0x13, 0xfb, - 0x15, 0xdd, 0xe7, 0x1f, 0x25, 0x80, 0x0a, 0x4b, 0xf4, 0x6c, 0x9b, 0x1d, 0xc8, 0x0c, 0xf8, 0x96, - 0x34, 0x6f, 0x40, 0xba, 0x62, 0xc3, 0x67, 0x67, 0x6e, 0xb8, 0x92, 0x0f, 0x05, 0xf7, 0x45, 0x61, - 0x83, 0x20, 0xa4, 0xa7, 0x07, 0xa1, 0xc3, 0x5f, 0x86, 0xec, 0x2f, 0xb8, 0xbd, 0x35, 0xcb, 0xec, - 0x9b, 0xfc, 0x44, 0x59, 0x9c, 0x11, 0xc4, 0x3a, 0xa5, 0xa1, 0x1c, 0xcd, 0x85, 0xa6, 0x6f, 0xda, - 0x3d, 0x9e, 0xdd, 0x71, 0x30, 0x2c, 0xfe, 0x3d, 0x12, 0x0a, 0x70, 0x57, 0x21, 0x29, 0xa6, 0x89, - 0x3c, 0x97, 0x0e, 0xa7, 0xb4, 0x80, 0x87, 0x56, 0x21, 0xbe, 0x4b, 0x7a, 0xa6, 0x2d, 0xdc, 0x04, - 0xbe, 0x3f, 0x2a, 0x24, 0x9a, 0xcc, 0x27, 0x30, 0x67, 0xa0, 0x57, 0x21, 0x4a, 0x43, 0x70, 0x74, - 0x8a, 0x4f, 0xc9, 0xe8, 0x1a, 0x44, 0xbd, 0x61, 0x5f, 0x84, 0x96, 0xe5, 0xf1, 0xf9, 0xdb, 0x5b, - 0xe5, 0xb7, 0xda, 0xc3, 0xbe, 0xb0, 0x14, 0xc5, 0xa0, 0xbb, 0xb3, 0x62, 0x68, 0xfc, 0xa4, 0x18, - 0x3a, 0x23, 0x36, 0xbe, 0x0d, 0xd9, 0x5d, 0xbd, 0xfb, 0xd8, 0xb4, 0x7b, 0x1a, 0x8b, 0x76, 0x2c, - 0x1a, 0xa4, 0x2a, 0xcb, 0xd3, 0xd1, 0x30, 0x23, 0x70, 0x6c, 0x84, 0x2e, 0x80, 0xdc, 0x77, 0x0c, - 0xcd, 0x37, 0xfb, 0x22, 0x2f, 0xe1, 0x64, 0xdf, 0x31, 0x3a, 0x66, 0x9f, 0xa0, 0xd7, 0x20, 0x13, - 0xbe, 0xcb, 0x2c, 0xe1, 0xa4, 0x70, 0x3a, 0x74, 0x7b, 0x8b, 0xf7, 0x21, 0x29, 0x0e, 0x45, 0xab, - 0xae, 0x81, 0xee, 0xfa, 0x6f, 0x31, 0xcd, 0x26, 0x30, 0x1f, 0x04, 0xd4, 0x75, 0xa6, 0x4a, 0x41, - 0x5d, 0x0f, 0xa8, 0xb7, 0x99, 0x02, 0x93, 0x9c, 0x7a, 0xbb, 0xf8, 0x6d, 0x04, 0xd2, 0x98, 0xe8, - 0x06, 0x26, 0xbf, 0x1c, 0x12, 0xcf, 0x47, 0x6b, 0x90, 0xd8, 0x27, 0xba, 0x41, 0x5c, 0xe1, 0x49, - 0xca, 0x58, 0x21, 0x5b, 0x8c, 0x8e, 0x05, 0x3f, 0x6c, 0xd7, 0xc8, 0x0b, 0xec, 0x5a, 0x1c, 0xdd, - 0xff, 0x69, 0xc3, 0x09, 0x0e, 0xdd, 0xda, 0xae, 0xe5, 0x74, 0x1f, 0x33, 0xeb, 0xc9, 0x98, 0x0f, - 0xd0, 0x2a, 0x64, 0x0c, 0x47, 0xb3, 0x1d, 0x5f, 0x1b, 0xb8, 0xce, 0xc1, 0x21, 0xb3, 0x90, 0x8c, - 0xc1, 0x70, 0x1a, 0x8e, 0xdf, 0xa2, 0x14, 0xea, 0xa6, 0x7d, 0xe2, 0xeb, 0x86, 0xee, 0xeb, 0x9a, - 0x63, 0x5b, 0x87, 0x4c, 0xff, 0x32, 0xce, 0x04, 0xc4, 0xa6, 0x6d, 0x1d, 0xa2, 0x6b, 0x00, 0x34, - 0xc7, 0x8b, 0x4d, 0x24, 0xa7, 0x36, 0x91, 0x22, 0xb6, 0xc1, 0x3f, 0xd1, 0x15, 0x58, 0x64, 0xae, - 0xa6, 0x8d, 0xac, 0x23, 0x33, 0xeb, 0x64, 0x18, 0x75, 0x5b, 0x98, 0xe8, 0x3a, 0x2c, 0xf7, 0x4d, - 0x5b, 0x23, 0x7e, 0xd7, 0xd0, 0x5c, 0xf2, 0xc4, 0xa4, 0xfe, 0x90, 0x4b, 0x31, 0xe0, 0x52, 0xdf, - 0xb4, 0x55, 0xbf, 0x6b, 0x60, 0x41, 0x2e, 0xfe, 0x3e, 0x02, 0x19, 0xae, 0x5e, 0x6f, 0xe0, 0xd8, - 0x1e, 0xa1, 0xfa, 0xf5, 0x7c, 0xdd, 0x1f, 0x7a, 0x22, 0xc2, 0x86, 0xf4, 0xdb, 0x66, 0x74, 0x2c, - 0xf8, 0x21, 0x4b, 0x44, 0x4e, 0xb0, 0xc4, 0xcb, 0xa8, 0xf8, 0x1a, 0xc0, 0xc7, 0xae, 0xe9, 0x13, - 0x8d, 0xce, 0x61, 0x7a, 0x9e, 0xd0, 0x02, 0xe3, 0x52, 0xc1, 0xa8, 0x14, 0x2a, 0xea, 0xe2, 0x93, - 0x85, 0x62, 0xe0, 0xd6, 0xa1, 0x6a, 0xed, 0x35, 0xc8, 0x04, 0xdf, 0xda, 0xd0, 0xe5, 0x29, 0x31, - 0x85, 0xd3, 0x01, 0x6d, 0xc7, 0xb5, 0x68, 0xa8, 0xe8, 0x3a, 0x36, 0xcd, 0xa2, 0xcc, 0x00, 0x19, - 0x1c, 0x0c, 0x8b, 0xff, 0x8a, 0x41, 0x56, 0x94, 0x5a, 0xa7, 0xe5, 0x81, 0x93, 0x7e, 0x14, 0x9d, - 0xf2, 0xa3, 0xb1, 0x02, 0xe3, 0x73, 0x15, 0xf8, 0x01, 0x2c, 0x75, 0xf7, 0x49, 0xf7, 0xb1, 0xe6, - 0x92, 0x9e, 0xe9, 0xf9, 0xc4, 0xf5, 0x44, 0xee, 0x3f, 0x3f, 0x55, 0x45, 0xf3, 0xf7, 0x05, 0x5e, - 0x64, 0x78, 0x1c, 0xc0, 0xd1, 0x4f, 0x60, 0x69, 0x68, 0xd3, 0x80, 0x33, 0x96, 0x90, 0x9c, 0x57, - 0x87, 0xe3, 0x45, 0x06, 0x1d, 0x4f, 0x2e, 0x03, 0xf2, 0x86, 0xbb, 0xbe, 0xab, 0x77, 0xfd, 0xd0, - 0x7c, 0x79, 0xee, 0xfc, 0xe5, 0x00, 0x3d, 0x16, 0xf1, 0xde, 0xb8, 0x8c, 0x4d, 0x31, 0xdf, 0x2b, - 0x8c, 0xe7, 0x1d, 0x33, 0x41, 0x90, 0xdf, 0x47, 0x15, 0xeb, 0x6c, 0x97, 0x87, 0x99, 0x2e, 0x1f, - 0xb6, 0x75, 0xec, 0xb8, 0xad, 0x9d, 0x71, 0x9d, 0x81, 0x60, 0xb1, 0xbd, 0xd3, 0x6e, 0xa9, 0x8d, - 0xaa, 0x86, 0xd5, 0xf6, 0xce, 0x36, 0xcd, 0xf3, 0x67, 0x61, 0x39, 0xa0, 0x35, 0x9a, 0x01, 0x59, - 0x42, 0xe7, 0x00, 0x05, 0xe4, 0xda, 0xa6, 0xb6, 0x59, 0xdf, 0x69, 0x6f, 0xa9, 0x55, 0x25, 0x82, - 0x96, 0x20, 0x3d, 0x86, 0x3f, 0x54, 0xa2, 0x48, 0x81, 0x4c, 0x40, 0xb8, 0xaf, 0xaa, 0xad, 0x51, - 0x66, 0xff, 0x3a, 0x02, 0x8b, 0xc1, 0xf9, 0x7e, 0xf0, 0x2d, 0x2c, 0x9d, 0x74, 0x0b, 0x45, 0x62, - 0x09, 0x7c, 0xf2, 0x3a, 0x24, 0xba, 0x4e, 0x9f, 0xa6, 0xcc, 0xe8, 0xdc, 0xab, 0x23, 0x10, 0xe8, - 0x16, 0xad, 0x52, 0x03, 0x53, 0xc6, 0xe6, 0x9a, 0x72, 0x0c, 0xa2, 0x57, 0xcd, 0x77, 0x7c, 0xdd, - 0xd2, 0xba, 0xfb, 0x43, 0xfb, 0xb1, 0xc7, 0xdd, 0x15, 0xa7, 0x19, 0x6d, 0x83, 0x91, 0xd0, 0x55, - 0x58, 0x34, 0x88, 0xa5, 0x1f, 0x12, 0x23, 0x00, 0x25, 0x18, 0x28, 0x2b, 0xa8, 0x02, 0x76, 0x03, - 0x10, 0x4b, 0x59, 0x34, 0x1e, 0x58, 0xfe, 0xbe, 0x46, 0x5c, 0xd7, 0x71, 0x79, 0x5d, 0x89, 0x15, - 0xc6, 0xd9, 0x62, 0x0c, 0x95, 0xd2, 0x8b, 0x7f, 0x89, 0x80, 0x82, 0xc5, 0x4b, 0x92, 0xfc, 0xf0, - 0x8b, 0x5a, 0x02, 0x79, 0xe0, 0x3a, 0x03, 0xc7, 0xd3, 0xad, 0x17, 0xa8, 0x65, 0x84, 0x39, 0xae, - 0x98, 0xe4, 0xcb, 0x28, 0x66, 0x15, 0xd2, 0x7a, 0xf7, 0xb1, 0xed, 0x7c, 0x6c, 0x11, 0xa3, 0x47, - 0x44, 0x1e, 0x08, 0x93, 0xd0, 0x1d, 0x40, 0x06, 0x19, 0xb8, 0x84, 0x9e, 0xc0, 0xd0, 0x5e, 0x10, - 0x37, 0x96, 0xc7, 0x30, 0x41, 0x9a, 0xef, 0xd2, 0x34, 0x03, 0x89, 0x4f, 0xcd, 0x20, 0x96, 0xaf, - 0x0b, 0x8b, 0x64, 0x04, 0xb1, 0x4a, 0x69, 0xc5, 0xbf, 0x4a, 0xb0, 0x1c, 0xd2, 0xde, 0x29, 0x66, - 0x82, 0x70, 0xe8, 0x8e, 0xbe, 0x44, 0xe8, 0xfe, 0xc1, 0x1e, 0x58, 0xfc, 0x83, 0x04, 0xe9, 0xba, - 0xe9, 0xf9, 0x81, 0x13, 0xbc, 0x07, 0xb2, 0x27, 0x02, 0x9e, 0x70, 0x83, 0x79, 0xf1, 0x50, 0x5c, - 0x94, 0x11, 0x9c, 0x66, 0xfd, 0x8f, 0x75, 0xbf, 0xbb, 0x1f, 0x64, 0x7d, 0x36, 0x40, 0xb7, 0x21, - 0xc3, 0x3e, 0x34, 0x97, 0x78, 0xc3, 0x3e, 0x11, 0x19, 0x68, 0xfa, 0xc8, 0x69, 0x86, 0xc2, 0x0c, - 0x74, 0x2f, 0x26, 0x47, 0x94, 0xe8, 0xbd, 0x98, 0x1c, 0x55, 0x62, 0xc5, 0xff, 0x44, 0x20, 0xc3, - 0x77, 0x78, 0xea, 0x97, 0xfd, 0x03, 0x90, 0x85, 0x23, 0xf1, 0xd7, 0xf1, 0xb1, 0xf6, 0x47, 0x78, - 0x0f, 0xc1, 0xeb, 0x29, 0xd0, 0x41, 0x30, 0x2b, 0xff, 0x67, 0x09, 0x02, 0xc7, 0x43, 0x37, 0x21, - 0x36, 0xbb, 0x84, 0x0f, 0x3d, 0xbd, 0x84, 0x00, 0x06, 0xa4, 0xd1, 0x80, 0x16, 0x2a, 0xa3, 0x80, - 0x1c, 0xe1, 0xd1, 0xa0, 0xef, 0x8c, 0x83, 0xf1, 0x1b, 0x10, 0x77, 0x9d, 0xa1, 0x4f, 0x84, 0x37, - 0x84, 0xda, 0x67, 0x98, 0x92, 0x85, 0x38, 0x8e, 0x41, 0x3f, 0x82, 0xa5, 0xae, 0x4b, 0x74, 0x9f, - 0x8c, 0x45, 0xb2, 0x42, 0x01, 0x2f, 0x72, 0x72, 0x20, 0xf5, 0x5e, 0x4c, 0x8e, 0x29, 0xf1, 0xe2, - 0xb7, 0x12, 0x64, 0xca, 0x83, 0x81, 0x75, 0x18, 0xf8, 0xc2, 0xfb, 0x90, 0xec, 0xee, 0xeb, 0x76, - 0x8f, 0x04, 0xdd, 0xba, 0x4b, 0xc7, 0x12, 0xcc, 0x08, 0x58, 0xda, 0x60, 0xa8, 0xa0, 0x4f, 0x26, - 0xe6, 0xe4, 0x3f, 0x93, 0x20, 0xc1, 0x39, 0xa8, 0x04, 0x67, 0xc8, 0xc1, 0x80, 0x74, 0x7d, 0xed, - 0xd8, 0x01, 0x59, 0x87, 0x04, 0x2f, 0x73, 0xd6, 0x76, 0xe8, 0x98, 0x6f, 0x42, 0x62, 0x38, 0xf0, - 0x88, 0xeb, 0x0b, 0xc3, 0xcd, 0x56, 0x1e, 0x16, 0x20, 0x74, 0x19, 0x12, 0x06, 0xb1, 0x88, 0x50, - 0xcb, 0xc4, 0xfd, 0x17, 0xac, 0xa2, 0xc9, 0x0a, 0x13, 0xba, 0xe9, 0xd3, 0xf6, 0xa3, 0xe2, 0x3f, - 0x22, 0xa0, 0x04, 0xb7, 0xd3, 0x3b, 0xb5, 0x3a, 0x68, 0xba, 0xba, 0x8d, 0xce, 0xa8, 0x6e, 0x57, - 0x21, 0x43, 0xcb, 0xe5, 0x11, 0x86, 0x7b, 0x00, 0x2d, 0xa1, 0x03, 0xc4, 0xeb, 0xb0, 0x64, 0x93, - 0x03, 0x5f, 0x1b, 0xe8, 0x3d, 0xa2, 0xf9, 0xce, 0x63, 0x62, 0x8b, 0xa8, 0x97, 0xa5, 0xe4, 0x96, - 0xde, 0x23, 0x1d, 0x4a, 0x44, 0x97, 0x00, 0x18, 0x84, 0xbf, 0x20, 0x69, 0x48, 0x8e, 0xe3, 0x14, - 0xa5, 0xf0, 0xe7, 0xe3, 0x5d, 0xc8, 0x78, 0x66, 0xcf, 0xd6, 0xfd, 0xa1, 0x4b, 0x3a, 0x9d, 0xba, - 0x88, 0xf3, 0x2f, 0xe8, 0xa4, 0xc8, 0x4f, 0x8f, 0x0a, 0x12, 0x6b, 0x93, 0x1c, 0x9b, 0x38, 0x55, - 0xdf, 0xc9, 0x93, 0xf5, 0x5d, 0xf1, 0xab, 0x08, 0x2c, 0x87, 0xf4, 0x7b, 0xea, 0x71, 0xa1, 0x06, - 0xa9, 0x20, 0xc4, 0x06, 0x81, 0xe1, 0xea, 0x74, 0x1c, 0x1e, 0xed, 0xa4, 0xa4, 0x8d, 0xda, 0xa3, - 0x5c, 0xce, 0x78, 0xf6, 0x2c, 0x65, 0xc7, 0x66, 0x28, 0x3b, 0xff, 0x21, 0xa4, 0x46, 0x52, 0xd0, - 0x8d, 0x63, 0x91, 0x64, 0x46, 0x0a, 0x38, 0x16, 0x46, 0x2e, 0x01, 0x50, 0x7d, 0x12, 0x83, 0x55, - 0xef, 0xbc, 0xf3, 0x90, 0xe2, 0x94, 0x1d, 0xd7, 0x2a, 0xfe, 0x5a, 0x82, 0x38, 0x0b, 0x16, 0xe8, - 0x5d, 0x48, 0xf6, 0x49, 0x7f, 0x97, 0xe6, 0x0a, 0x7e, 0xbf, 0x4f, 0xea, 0x8b, 0x04, 0x70, 0x9a, - 0x40, 0x07, 0xae, 0xd9, 0xd7, 0xdd, 0x43, 0xde, 0xae, 0xc6, 0xc1, 0x10, 0x5d, 0x87, 0x54, 0xd0, - 0x18, 0x09, 0x3a, 0x8c, 0xc7, 0xfb, 0x26, 0x63, 0xb6, 0x28, 0xe7, 0xfe, 0x14, 0x81, 0x04, 0xd7, - 0x3a, 0x7a, 0x1f, 0x20, 0x68, 0x7e, 0xbc, 0x74, 0xaf, 0x26, 0x25, 0x66, 0xd4, 0x8c, 0x71, 0x70, - 0x8c, 0xbc, 0x44, 0x70, 0xbc, 0x09, 0x31, 0x5a, 0xfe, 0x8a, 0x40, 0x7a, 0x76, 0xd2, 0x03, 0x4a, - 0xb4, 0x06, 0x0e, 0xd4, 0x4a, 0x81, 0xf9, 0x4f, 0x25, 0x88, 0x51, 0x22, 0xd5, 0x6f, 0xd7, 0x1a, - 0xd2, 0xf4, 0x19, 0xec, 0x32, 0x86, 0x53, 0x82, 0x52, 0x33, 0xd0, 0x45, 0x48, 0x71, 0x35, 0x51, - 0x6e, 0x84, 0x71, 0x65, 0x4e, 0xa8, 0x19, 0x28, 0x0f, 0xf2, 0x28, 0xfa, 0xf1, 0xdb, 0x3a, 0x1a, - 0xd3, 0x89, 0xae, 0xbe, 0xe7, 0x6b, 0x3e, 0x71, 0x79, 0xdf, 0x23, 0x86, 0x65, 0x4a, 0xe8, 0x10, - 0xb7, 0x1f, 0xb4, 0x8c, 0x98, 0xc6, 0x1e, 0x40, 0xfe, 0x58, 0x37, 0x82, 0x57, 0x76, 0x41, 0x9c, - 0x79, 0x17, 0x16, 0x47, 0xcf, 0x37, 0xde, 0xc5, 0x90, 0xe6, 0x75, 0x31, 0xb2, 0x7b, 0xe1, 0x61, - 0xf1, 0x4b, 0x09, 0x2e, 0xce, 0x14, 0x7c, 0xea, 0x17, 0x6c, 0x76, 0xf5, 0x1a, 0x9d, 0x5d, 0xbd, - 0x5e, 0xff, 0x4d, 0x14, 0x12, 0x7c, 0x41, 0x94, 0x80, 0x48, 0xf3, 0x3e, 0x7f, 0x6b, 0xdc, 0x6b, - 0xee, 0xe0, 0x46, 0xb9, 0xae, 0x35, 0x9a, 0x1d, 0x6d, 0xb3, 0xb9, 0xd3, 0xa8, 0x2a, 0x12, 0xba, - 0x04, 0x17, 0x1a, 0x4d, 0x2d, 0xe0, 0xb4, 0x70, 0x6d, 0xbb, 0x8c, 0x1f, 0x69, 0x15, 0xdc, 0xbc, - 0xaf, 0x62, 0x25, 0x82, 0x56, 0x20, 0x4f, 0xd1, 0x73, 0xf8, 0x51, 0xfa, 0x54, 0x09, 0xf3, 0x05, - 0x3d, 0x8e, 0x56, 0xe1, 0xd5, 0x5a, 0xa3, 0xbd, 0xb3, 0xb9, 0x59, 0xdb, 0xa8, 0xa9, 0x8d, 0x49, - 0x40, 0x5b, 0x89, 0xa1, 0x57, 0x21, 0xd7, 0xdc, 0xdc, 0x6c, 0xab, 0x1d, 0xb6, 0x9d, 0x47, 0x6a, - 0x47, 0x2b, 0x3f, 0x28, 0xd7, 0xea, 0xe5, 0x4a, 0x5d, 0x55, 0x12, 0xf4, 0xa9, 0xf3, 0x10, 0x37, - 0x1b, 0x77, 0x35, 0xdc, 0xdc, 0xe9, 0xa8, 0x4a, 0x92, 0x6e, 0xbf, 0x85, 0x9b, 0xad, 0x66, 0xbb, - 0x5c, 0xd7, 0xb6, 0x6b, 0xed, 0xed, 0x72, 0x67, 0x63, 0x4b, 0x91, 0xd1, 0x45, 0x38, 0xaf, 0x76, - 0x36, 0xaa, 0x5a, 0x07, 0x97, 0x1b, 0xed, 0xf2, 0x46, 0xa7, 0xd6, 0x6c, 0x68, 0x9b, 0xe5, 0x5a, - 0x5d, 0xad, 0x2a, 0x29, 0x2a, 0x84, 0xca, 0x2e, 0xd7, 0xeb, 0xcd, 0x87, 0x6a, 0x55, 0x01, 0x74, - 0x1e, 0xce, 0x70, 0xa9, 0xe5, 0x16, 0x7b, 0x34, 0xf1, 0x0d, 0x28, 0x69, 0xba, 0x99, 0x5a, 0xa3, - 0xaa, 0x7e, 0xa8, 0x6d, 0x95, 0xdb, 0xda, 0x5d, 0xac, 0x96, 0x3b, 0x2a, 0x0e, 0xb8, 0x19, 0xba, - 0x36, 0x56, 0xef, 0xd6, 0xda, 0x94, 0x38, 0x5a, 0x3b, 0x8b, 0xb2, 0x90, 0x12, 0xaf, 0x2f, 0xb5, - 0xaa, 0x2c, 0x52, 0x19, 0x9b, 0xb8, 0x7c, 0x77, 0x9b, 0x1e, 0xb7, 0xdd, 0x69, 0x62, 0x55, 0xdb, - 0x69, 0x6c, 0xa9, 0xe5, 0x7a, 0x67, 0xeb, 0x91, 0xb2, 0x74, 0xdd, 0x06, 0x65, 0xb2, 0xbd, 0x86, - 0xd2, 0x90, 0xac, 0x35, 0x1e, 0x94, 0xeb, 0xb5, 0xaa, 0xb2, 0x30, 0xea, 0xfe, 0xb2, 0x86, 0xef, - 0xdd, 0x8f, 0x6a, 0x2d, 0x25, 0x42, 0x57, 0xf8, 0xa8, 0xdd, 0x29, 0x37, 0xaa, 0x65, 0x5c, 0x55, - 0xa2, 0x08, 0x20, 0xd1, 0x6e, 0x94, 0x5b, 0xad, 0x47, 0x4a, 0x8c, 0x1a, 0x86, 0x82, 0xe8, 0x26, - 0xeb, 0xcd, 0x72, 0x55, 0xab, 0xaa, 0x1b, 0xcd, 0xed, 0x16, 0x56, 0xdb, 0xed, 0x5a, 0xb3, 0xa1, - 0xc4, 0xd7, 0xff, 0x17, 0x1d, 0x97, 0x59, 0xef, 0x40, 0x8c, 0x96, 0x66, 0xe8, 0xec, 0x64, 0xa9, - 0xc6, 0xae, 0x43, 0xfe, 0xdc, 0xec, 0x0a, 0xee, 0x96, 0x84, 0xde, 0x85, 0x38, 0x2b, 0x08, 0xd0, - 0xb9, 0xd9, 0x65, 0x4d, 0xfe, 0xfc, 0x14, 0x5d, 0x5c, 0x84, 0x77, 0x20, 0x86, 0x89, 0x6e, 0x84, - 0x97, 0x0c, 0xf5, 0xdc, 0xc2, 0x4b, 0x86, 0x7b, 0x45, 0xb7, 0x24, 0xf4, 0x3e, 0x24, 0xf8, 0xcb, - 0x15, 0x9d, 0x9f, 0xf3, 0x56, 0xcf, 0xe7, 0xa6, 0x19, 0x7c, 0xfa, 0x9a, 0x84, 0xb6, 0x20, 0x35, - 0x7a, 0x77, 0xa0, 0x7c, 0x78, 0x95, 0xe3, 0x4f, 0xb9, 0xfc, 0xc5, 0x99, 0xbc, 0x40, 0xce, 0x2d, - 0x2a, 0x29, 0x4b, 0xb5, 0x31, 0x4a, 0x5d, 0x61, 0x69, 0x93, 0x95, 0x4b, 0x58, 0xda, 0x74, 0xd6, - 0xdd, 0x85, 0x33, 0x33, 0x62, 0x06, 0xba, 0x32, 0x3d, 0x67, 0x3a, 0x56, 0xe5, 0xaf, 0x9e, 0x80, - 0xe2, 0x6b, 0x54, 0xd4, 0xa7, 0xff, 0x5e, 0x59, 0x78, 0xfa, 0xdd, 0x8a, 0xf4, 0xcd, 0x77, 0x2b, - 0xd2, 0x6f, 0x9f, 0xad, 0x2c, 0x7c, 0xf1, 0x6c, 0x45, 0xfa, 0xfa, 0xd9, 0x8a, 0xf4, 0xcd, 0xb3, - 0x95, 0x85, 0xbf, 0x3d, 0x5b, 0x59, 0xf8, 0xe8, 0x72, 0xcf, 0x29, 0xf5, 0xf4, 0x4f, 0x88, 0xef, - 0x93, 0x92, 0x41, 0x9e, 0xdc, 0xec, 0x3a, 0x2e, 0xb9, 0x39, 0xf1, 0xff, 0x7c, 0x37, 0xc1, 0xbe, - 0x6e, 0xff, 0x3f, 0x00, 0x00, 0xff, 0xff, 0x36, 0x99, 0x5f, 0x69, 0x59, 0x1f, 0x00, 0x00, + // 2993 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x59, 0x4d, 0x70, 0x1b, 0xc7, + 0xb1, 0xe6, 0xe2, 0x77, 0xd1, 0x00, 0xc8, 0xe5, 0xc8, 0x92, 0x28, 0xc8, 0x22, 0x68, 0x48, 0xf2, + 0xa3, 0x64, 0x19, 0x92, 0xa9, 0x57, 0xfe, 0xd1, 0x2b, 0xbf, 0x67, 0x80, 0x58, 0x8a, 0x90, 0x40, + 0x00, 0x35, 0x00, 0x2d, 0xcb, 0x87, 0xb7, 0xb5, 0xdc, 0x1d, 0x82, 0x1b, 0x2d, 0x76, 0x91, 0xdd, + 0x85, 0x4c, 0xfa, 0xe6, 0x5b, 0x0e, 0x4e, 0x25, 0x47, 0xe7, 0x92, 0x38, 0x07, 0x1f, 0x72, 0xca, + 0x21, 0x37, 0xe7, 0xe2, 0xa3, 0x72, 0xf3, 0x31, 0x55, 0x71, 0x98, 0xc4, 0xaa, 0x4a, 0xe5, 0x90, + 0x93, 0x8e, 0x3e, 0xa5, 0xe6, 0x67, 0x81, 0x25, 0x7e, 0x44, 0xf9, 0xa0, 0xdb, 0x4e, 0xf7, 0x37, + 0x3d, 0x33, 0xdd, 0x3d, 0xdd, 0x3d, 0xbd, 0xb0, 0xba, 0xe7, 0xb9, 0x8f, 0x88, 0x77, 0x73, 0xe0, + 0xb9, 0x81, 0x6b, 0xb8, 0xf6, 0xe8, 0xa3, 0xcc, 0x3e, 0x90, 0x1c, 0x8e, 0x0b, 0xaf, 0xf4, 0xdc, + 0x9e, 0xcb, 0x46, 0x37, 0xe9, 0x17, 0xe7, 0x17, 0x56, 0x7b, 0xae, 0xdb, 0xb3, 0x09, 0x9f, 0xb6, + 0x37, 0xdc, 0xbf, 0x69, 0x0e, 0x3d, 0x3d, 0xb0, 0x5c, 0x87, 0xf3, 0x4b, 0xfb, 0x90, 0x6c, 0xe8, + 0x7b, 0xc4, 0x46, 0x08, 0x12, 0x8e, 0xde, 0x27, 0x2b, 0xd2, 0x9a, 0xb4, 0x9e, 0xc1, 0xec, 0x1b, + 0xbd, 0x02, 0xc9, 0xc7, 0xba, 0x3d, 0x24, 0x2b, 0x31, 0x46, 0xe4, 0x03, 0xf4, 0x26, 0xa4, 0x06, + 0x1e, 0xd9, 0xb7, 0x0e, 0x57, 0xe2, 0x6b, 0xd2, 0xba, 0x5c, 0x3d, 0xfb, 0xec, 0xb8, 0xb8, 0x7c, + 0xa4, 0xf7, 0xed, 0x3b, 0xa5, 0x1b, 0x6e, 0xdf, 0x0a, 0x48, 0x7f, 0x10, 0x1c, 0x95, 0xb0, 0x00, + 0xdd, 0x49, 0xfc, 0xeb, 0xcb, 0xa2, 0x54, 0xea, 0x82, 0xcc, 0xd6, 0xe9, 0x90, 0x00, 0x55, 0x21, + 0x65, 0xd3, 0x6f, 0x7f, 0x45, 0x5a, 0x8b, 0xaf, 0x67, 0x37, 0x96, 0xca, 0xa3, 0x43, 0x31, 0x4c, + 0xf5, 0xc2, 0x93, 0xe3, 0xe2, 0xc2, 0x1c, 0xa9, 0x7c, 0xa6, 0x90, 0xfa, 0x99, 0x04, 0x79, 0x21, + 0xd6, 0x26, 0x46, 0xe0, 0x7a, 0x68, 0x03, 0xd2, 0x96, 0x63, 0xd8, 0x43, 0x93, 0x9f, 0x24, 0xbb, + 0x81, 0x26, 0x84, 0x77, 0x48, 0x50, 0x4d, 0x50, 0xf9, 0x38, 0x04, 0xd2, 0x39, 0xe4, 0x90, 0xcf, + 0x89, 0x9d, 0x36, 0x47, 0x00, 0xef, 0xc8, 0x5f, 0x7c, 0x59, 0x5c, 0x60, 0x7b, 0xf8, 0x55, 0x16, + 0xb2, 0xf7, 0xdc, 0xa1, 0xe7, 0xe8, 0x76, 0x67, 0x40, 0x0c, 0xf4, 0xdf, 0x51, 0x45, 0x56, 0xd7, + 0x66, 0x1e, 0xe3, 0x87, 0xe3, 0x62, 0x5a, 0xcc, 0x11, 0xaa, 0x7e, 0x07, 0xb2, 0x1e, 0x19, 0xd8, + 0x96, 0xc1, 0x8c, 0xc3, 0xf6, 0x91, 0x9c, 0xa7, 0xd9, 0x28, 0x12, 0xb5, 0x47, 0xca, 0x8c, 0xcf, + 0xdd, 0xfb, 0x15, 0xba, 0xf7, 0x6f, 0x8f, 0x8b, 0xd2, 0xb3, 0xe3, 0xe2, 0xca, 0xa4, 0xbc, 0x1b, + 0x96, 0x63, 0x5b, 0x0e, 0x19, 0xa9, 0x16, 0xed, 0x82, 0xbc, 0xef, 0xe9, 0xbd, 0x3e, 0x71, 0x82, + 0x95, 0x04, 0x93, 0xb9, 0x3a, 0x96, 0x19, 0x39, 0x69, 0x79, 0x4b, 0xa0, 0x9e, 0x67, 0xaf, 0x91, + 0x28, 0xf4, 0x7f, 0x90, 0xdc, 0xb7, 0xf5, 0x9e, 0xbf, 0x92, 0x5a, 0x93, 0xd6, 0xf3, 0xd5, 0x6b, + 0xf3, 0x14, 0xa3, 0x44, 0x96, 0xd0, 0xb6, 0x6c, 0xbd, 0x87, 0xf9, 0x3c, 0xd4, 0x80, 0xa5, 0xbe, + 0x7e, 0xa8, 0xe9, 0x83, 0x01, 0x71, 0x4c, 0xcd, 0xd3, 0x03, 0xb2, 0x92, 0x5e, 0x93, 0xd6, 0xe3, + 0xd5, 0x2b, 0xcf, 0x8e, 0x8b, 0x6b, 0x5c, 0xd4, 0x04, 0x20, 0xba, 0x93, 0x7c, 0x5f, 0x3f, 0xac, + 0x30, 0x16, 0xd6, 0x03, 0x82, 0x9a, 0x90, 0xf6, 0x87, 0x3e, 0x1d, 0xae, 0xc8, 0xec, 0x90, 0x97, + 0x66, 0x1f, 0xb2, 0xc3, 0x41, 0xf3, 0x6c, 0x11, 0x0a, 0x29, 0x7c, 0x9e, 0x04, 0x39, 0x54, 0x08, + 0xbd, 0x22, 0x36, 0x71, 0x7a, 0xc1, 0x01, 0xf3, 0x82, 0xf8, 0xdc, 0x2b, 0xc2, 0x41, 0xc8, 0x85, + 0x65, 0xc3, 0xed, 0x0f, 0x3c, 0xe2, 0xfb, 0x96, 0xeb, 0x68, 0x86, 0x6b, 0x12, 0x83, 0xb9, 0xc0, + 0xe2, 0x46, 0x61, 0xbc, 0xab, 0xcd, 0x31, 0x64, 0x93, 0x22, 0xaa, 0xaf, 0x3f, 0x3b, 0x2e, 0x96, + 0xb8, 0xd4, 0xa9, 0xe9, 0xd1, 0x65, 0x14, 0x63, 0x62, 0x26, 0xfa, 0x5f, 0x48, 0xf9, 0x81, 0xeb, + 0x11, 0xea, 0x34, 0xf1, 0xf5, 0x0c, 0x93, 0x34, 0xd3, 0x18, 0xf9, 0xf0, 0x48, 0x1d, 0x0a, 0xc7, + 0x62, 0x16, 0xf2, 0x41, 0xf1, 0xc8, 0xbe, 0x47, 0xfc, 0x03, 0xcd, 0x72, 0x02, 0xe2, 0x3d, 0xd6, + 0x6d, 0xe1, 0x2a, 0x17, 0xca, 0x3c, 0xe0, 0x94, 0xc3, 0x80, 0x53, 0xae, 0x89, 0x80, 0x53, 0x7d, + 0x53, 0x78, 0xc9, 0x6b, 0x7c, 0xa1, 0x49, 0x01, 0x91, 0x85, 0xbf, 0xf8, 0x5b, 0x51, 0xc2, 0x4b, + 0x02, 0x50, 0x17, 0x7c, 0xf4, 0x21, 0x64, 0x3c, 0x12, 0x10, 0x87, 0x5d, 0x90, 0xe4, 0x69, 0xab, + 0x5d, 0x9a, 0xeb, 0x93, 0x4c, 0xfa, 0x58, 0x14, 0xea, 0xc3, 0xe2, 0xbe, 0x3d, 0x8c, 0x1e, 0x25, + 0x75, 0x9a, 0xf0, 0x37, 0x84, 0xf0, 0x22, 0x17, 0x7e, 0x72, 0xfa, 0xe4, 0x52, 0x79, 0xc6, 0x1e, + 0x1d, 0xe3, 0xff, 0xe1, 0xec, 0x40, 0x0f, 0x0e, 0xb4, 0x81, 0xeb, 0x07, 0xfb, 0xd6, 0xa1, 0x46, + 0xa1, 0x76, 0xe8, 0xcc, 0x99, 0xea, 0xf5, 0x67, 0xc7, 0xc5, 0xd7, 0xb9, 0xd8, 0x99, 0xb0, 0xa8, + 0x61, 0xcf, 0x50, 0x44, 0x9b, 0x03, 0xba, 0x82, 0xcf, 0x23, 0x63, 0xe1, 0x6b, 0x09, 0xd2, 0xc2, + 0x75, 0x51, 0x1b, 0x92, 0x36, 0x79, 0x4c, 0x6c, 0xe6, 0x8c, 0x8b, 0x1b, 0x97, 0x9f, 0xeb, 0xe8, + 0xe5, 0x06, 0x85, 0xce, 0xf3, 0x58, 0x2e, 0x88, 0xfa, 0xb7, 0xbb, 0xbf, 0xef, 0x93, 0x80, 0x79, + 0xe9, 0x7c, 0xff, 0xe6, 0xa0, 0xd2, 0x3a, 0x24, 0x99, 0x54, 0x24, 0x43, 0xa2, 0xd9, 0x6a, 0xaa, + 0xca, 0x02, 0xca, 0x42, 0xba, 0x5d, 0xc1, 0xdd, 0x7a, 0xa5, 0xa1, 0x48, 0x94, 0xbc, 0xb5, 0xdb, + 0x68, 0x28, 0x31, 0x11, 0xd6, 0x2b, 0x90, 0xa0, 0x17, 0x1f, 0x2d, 0x43, 0xbe, 0xd9, 0xea, 0x6a, + 0x9d, 0xb6, 0xba, 0x59, 0xdf, 0xaa, 0xab, 0x35, 0x65, 0x01, 0xe5, 0x40, 0x6e, 0x69, 0xb8, 0xd6, + 0x6a, 0x36, 0x1e, 0x2a, 0x12, 0x1f, 0x3d, 0xc0, 0x6c, 0x14, 0x43, 0x00, 0x29, 0xca, 0x7b, 0x80, + 0x95, 0x84, 0x10, 0xf4, 0x95, 0x04, 0xd9, 0xb6, 0xe7, 0x1a, 0xc4, 0xf7, 0x59, 0x6c, 0x2e, 0x43, + 0xcc, 0x32, 0x45, 0x62, 0x58, 0x19, 0xab, 0x21, 0x02, 0x29, 0xd7, 0x6b, 0x22, 0xd4, 0xc7, 0x2c, + 0x13, 0xad, 0x83, 0x4c, 0x1c, 0x73, 0xe0, 0x5a, 0x0e, 0x3f, 0x69, 0xa6, 0x9a, 0xfb, 0xe1, 0xb8, + 0x28, 0xab, 0x82, 0x86, 0x47, 0xdc, 0xc2, 0xdb, 0x10, 0xab, 0xd7, 0x68, 0x12, 0xfd, 0xd4, 0x75, + 0x46, 0x49, 0x94, 0x7e, 0xa3, 0x73, 0x90, 0xf2, 0x87, 0xfb, 0x34, 0x5d, 0xf2, 0x2c, 0x2a, 0x46, + 0x7c, 0x87, 0x77, 0x12, 0x3f, 0xa3, 0xfb, 0xfc, 0xad, 0x04, 0x50, 0x65, 0x89, 0x9e, 0x6d, 0xb3, + 0x0b, 0xb9, 0x01, 0xdf, 0x92, 0xe6, 0x0f, 0x88, 0x21, 0x36, 0x7c, 0x76, 0xe6, 0x86, 0xab, 0x85, + 0x48, 0x70, 0x5f, 0x14, 0x36, 0x08, 0x43, 0x7a, 0x76, 0x10, 0x39, 0xfc, 0x65, 0xc8, 0xff, 0x84, + 0xdb, 0x5b, 0xb3, 0xad, 0xbe, 0xc5, 0x4f, 0x94, 0xc7, 0x39, 0x41, 0x6c, 0x50, 0x1a, 0x5a, 0xa1, + 0xb9, 0xd0, 0x0a, 0x2c, 0xa7, 0xc7, 0xb3, 0x3b, 0x0e, 0x87, 0xa5, 0xbf, 0xc4, 0x22, 0x01, 0xee, + 0x2a, 0xa4, 0xc5, 0x34, 0x91, 0xe7, 0xb2, 0xd1, 0x94, 0x16, 0xf2, 0xd0, 0x1a, 0x24, 0xf7, 0x48, + 0xcf, 0x72, 0x84, 0x9b, 0xc0, 0x0f, 0xc7, 0xc5, 0x54, 0x8b, 0xf9, 0x04, 0xe6, 0x0c, 0xf4, 0x2a, + 0xc4, 0x69, 0x08, 0x8e, 0x4f, 0xf1, 0x29, 0x19, 0x5d, 0x83, 0xb8, 0x3f, 0xec, 0x8b, 0xd0, 0xb2, + 0x3c, 0x3e, 0x7f, 0x67, 0xbb, 0xf2, 0x56, 0x67, 0xd8, 0x17, 0x96, 0xa2, 0x18, 0x74, 0x77, 0x56, + 0x0c, 0x4d, 0x9e, 0x16, 0x43, 0x67, 0xc4, 0xc6, 0xb7, 0x21, 0xbf, 0xa7, 0x1b, 0x8f, 0x2c, 0xa7, + 0xa7, 0xb1, 0x68, 0xc7, 0xa2, 0x41, 0xa6, 0xba, 0x3c, 0x1d, 0x0d, 0x73, 0x02, 0xc7, 0x46, 0xe8, + 0x02, 0xc8, 0x7d, 0xd7, 0xd4, 0x02, 0xab, 0x2f, 0xf2, 0x12, 0x4e, 0xf7, 0x5d, 0xb3, 0x6b, 0xf5, + 0x09, 0x7a, 0x0d, 0x72, 0xd1, 0xbb, 0xcc, 0x12, 0x4e, 0x06, 0x67, 0x23, 0xb7, 0xb7, 0x74, 0x1f, + 0xd2, 0xe2, 0x50, 0xb4, 0xea, 0x1a, 0xe8, 0x5e, 0xf0, 0x16, 0xd3, 0x6c, 0x0a, 0xf3, 0x41, 0x48, + 0xdd, 0x60, 0xaa, 0x14, 0xd4, 0x8d, 0x90, 0x7a, 0x9b, 0x29, 0x30, 0xcd, 0xa9, 0xb7, 0x4b, 0xdf, + 0xc5, 0x20, 0x8b, 0x89, 0x6e, 0x62, 0xf2, 0xd3, 0x21, 0xf1, 0x03, 0xb4, 0x0e, 0xa9, 0x03, 0xa2, + 0x9b, 0xc4, 0x13, 0x9e, 0xa4, 0x8c, 0x15, 0xb2, 0xcd, 0xe8, 0x58, 0xf0, 0xa3, 0x76, 0x8d, 0x3d, + 0xc7, 0xae, 0xa5, 0xd1, 0xfd, 0x9f, 0x36, 0x9c, 0xe0, 0xd0, 0xad, 0xed, 0xd9, 0xae, 0xf1, 0x88, + 0x59, 0x4f, 0xc6, 0x7c, 0x80, 0xd6, 0x20, 0x67, 0xba, 0x9a, 0xe3, 0x06, 0xda, 0xc0, 0x73, 0x0f, + 0x8f, 0x98, 0x85, 0x64, 0x0c, 0xa6, 0xdb, 0x74, 0x83, 0x36, 0xa5, 0x50, 0x37, 0xed, 0x93, 0x40, + 0x37, 0xf5, 0x40, 0xd7, 0x5c, 0xc7, 0x3e, 0x62, 0xfa, 0x97, 0x71, 0x2e, 0x24, 0xb6, 0x1c, 0xfb, + 0x08, 0x5d, 0x03, 0xa0, 0x39, 0x5e, 0x6c, 0x22, 0x3d, 0xb5, 0x89, 0x0c, 0x71, 0x4c, 0xfe, 0x89, + 0xae, 0xc0, 0x22, 0x73, 0x35, 0x6d, 0x64, 0x1d, 0x99, 0x59, 0x27, 0xc7, 0xa8, 0x3b, 0xc2, 0x44, + 0xd7, 0x61, 0xb9, 0x6f, 0x39, 0x1a, 0x09, 0x0c, 0x53, 0xf3, 0xc8, 0x63, 0x8b, 0xfa, 0xc3, 0x4a, + 0x86, 0x01, 0x97, 0xfa, 0x96, 0xa3, 0x06, 0x86, 0x89, 0x05, 0xb9, 0xf4, 0xeb, 0x18, 0xe4, 0xb8, + 0x7a, 0xfd, 0x81, 0xeb, 0xf8, 0x84, 0xea, 0xd7, 0x0f, 0xf4, 0x60, 0xe8, 0x8b, 0x08, 0x1b, 0xd1, + 0x6f, 0x87, 0xd1, 0xb1, 0xe0, 0x47, 0x2c, 0x11, 0x3b, 0xc5, 0x12, 0x2f, 0xa2, 0xe2, 0x6b, 0x00, + 0x9f, 0x78, 0x56, 0x40, 0x34, 0x3a, 0x87, 0xe9, 0x79, 0x42, 0x0b, 0x8c, 0x4b, 0x05, 0xa3, 0x72, + 0xa4, 0xa8, 0x4b, 0x4e, 0x16, 0x8a, 0xa1, 0x5b, 0x47, 0xaa, 0xb5, 0xd7, 0x20, 0x17, 0x7e, 0x6b, + 0x43, 0x8f, 0xa7, 0xc4, 0x0c, 0xce, 0x86, 0xb4, 0x5d, 0xcf, 0xa6, 0xa1, 0xc2, 0x70, 0x1d, 0x9a, + 0x45, 0x99, 0x01, 0x72, 0x38, 0x1c, 0x96, 0xfe, 0x9e, 0x80, 0xbc, 0x28, 0xb5, 0x5e, 0x96, 0x07, + 0x4e, 0xfa, 0x51, 0x7c, 0xca, 0x8f, 0xc6, 0x0a, 0x4c, 0xce, 0x55, 0xe0, 0x07, 0xb0, 0x64, 0x1c, + 0x10, 0xe3, 0x91, 0xe6, 0x91, 0x9e, 0xe5, 0x07, 0xc4, 0xf3, 0x45, 0xee, 0x3f, 0x3f, 0x55, 0x45, + 0xf3, 0xf7, 0x05, 0x5e, 0x64, 0x78, 0x1c, 0xc2, 0xd1, 0xff, 0xc0, 0xd2, 0xd0, 0xa1, 0x01, 0x67, + 0x2c, 0x21, 0x3d, 0xaf, 0x0e, 0xc7, 0x8b, 0x0c, 0x3a, 0x9e, 0x5c, 0x01, 0xe4, 0x0f, 0xf7, 0x02, + 0x4f, 0x37, 0x82, 0xc8, 0x7c, 0x79, 0xee, 0xfc, 0xe5, 0x10, 0x3d, 0x16, 0xf1, 0xde, 0xb8, 0x8c, + 0xcd, 0x30, 0xdf, 0x2b, 0x8e, 0xe7, 0x9d, 0x30, 0x41, 0x98, 0xdf, 0x47, 0x15, 0xeb, 0x6c, 0x97, + 0x87, 0x99, 0x2e, 0x1f, 0xb5, 0x75, 0xe2, 0xa4, 0xad, 0xdd, 0x71, 0x9d, 0x81, 0x60, 0xb1, 0xb3, + 0xdb, 0x69, 0xab, 0xcd, 0x9a, 0x86, 0xd5, 0xce, 0xee, 0x0e, 0xcd, 0xf3, 0x67, 0x61, 0x39, 0xa4, + 0x35, 0x5b, 0x21, 0x59, 0x42, 0xe7, 0x00, 0x85, 0xe4, 0xfa, 0x96, 0xb6, 0xd5, 0xd8, 0xed, 0x6c, + 0xab, 0x35, 0x25, 0x86, 0x96, 0x20, 0x3b, 0x86, 0x3f, 0x50, 0xe2, 0x48, 0x81, 0x5c, 0x48, 0xb8, + 0xaf, 0xaa, 0xed, 0x51, 0x66, 0xff, 0x26, 0x06, 0x8b, 0xe1, 0xf9, 0x7e, 0xf4, 0x2d, 0x2c, 0x9f, + 0x76, 0x0b, 0x45, 0x62, 0x09, 0x7d, 0xf2, 0x3a, 0xa4, 0x0c, 0xb7, 0x4f, 0x53, 0x66, 0x7c, 0xee, + 0xd5, 0x11, 0x08, 0x74, 0x8b, 0x56, 0xa9, 0xa1, 0x29, 0x13, 0x73, 0x4d, 0x39, 0x06, 0xd1, 0xab, + 0x16, 0xb8, 0x81, 0x6e, 0x6b, 0xc6, 0xc1, 0xd0, 0x79, 0xe4, 0x73, 0x77, 0xc5, 0x59, 0x46, 0xdb, + 0x64, 0x24, 0x74, 0x15, 0x16, 0x4d, 0x62, 0xeb, 0x47, 0xc4, 0x0c, 0x41, 0x29, 0x06, 0xca, 0x0b, + 0xaa, 0x80, 0xdd, 0x00, 0xc4, 0x52, 0x16, 0x8d, 0x07, 0x76, 0x70, 0xa0, 0x11, 0xcf, 0x73, 0x3d, + 0x5e, 0x57, 0x62, 0x85, 0x71, 0xb6, 0x19, 0x43, 0xa5, 0xf4, 0xd2, 0x1f, 0x63, 0xa0, 0x60, 0xf1, + 0x92, 0x24, 0x3f, 0xfe, 0xa2, 0x96, 0x41, 0x1e, 0x78, 0xee, 0xc0, 0xf5, 0x75, 0xfb, 0x39, 0x6a, + 0x19, 0x61, 0x4e, 0x2a, 0x26, 0xfd, 0x22, 0x8a, 0x59, 0x83, 0xac, 0x6e, 0x3c, 0x72, 0xdc, 0x4f, + 0x6c, 0x62, 0xf6, 0x88, 0xc8, 0x03, 0x51, 0x12, 0xba, 0x03, 0xc8, 0x24, 0x03, 0x8f, 0xd0, 0x13, + 0x98, 0xda, 0x73, 0xe2, 0xc6, 0xf2, 0x18, 0x26, 0x48, 0xf3, 0x5d, 0x9a, 0x66, 0x20, 0xf1, 0xa9, + 0x99, 0xc4, 0x0e, 0x74, 0x61, 0x91, 0x9c, 0x20, 0xd6, 0x28, 0xad, 0xf4, 0x27, 0x09, 0x96, 0x23, + 0xda, 0x7b, 0x89, 0x99, 0x20, 0x1a, 0xba, 0xe3, 0x2f, 0x10, 0xba, 0x7f, 0xb4, 0x07, 0x96, 0x7e, + 0x23, 0x41, 0xb6, 0x61, 0xf9, 0x41, 0xe8, 0x04, 0xef, 0x81, 0xec, 0x8b, 0x80, 0x27, 0xdc, 0x60, + 0x5e, 0x3c, 0x14, 0x17, 0x65, 0x04, 0xa7, 0x59, 0xff, 0x13, 0x3d, 0x30, 0x0e, 0xc2, 0xac, 0xcf, + 0x06, 0xe8, 0x36, 0xe4, 0xd8, 0x87, 0xe6, 0x11, 0x7f, 0xd8, 0x27, 0x22, 0x03, 0x4d, 0x1f, 0x39, + 0xcb, 0x50, 0x98, 0x81, 0xee, 0x25, 0xe4, 0x98, 0x12, 0xbf, 0x97, 0x90, 0xe3, 0x4a, 0xa2, 0xf4, + 0xcf, 0x18, 0xe4, 0xf8, 0x0e, 0x5f, 0xfa, 0x65, 0xff, 0x00, 0x64, 0xe1, 0x48, 0xfc, 0x75, 0x7c, + 0xa2, 0xfd, 0x11, 0xdd, 0x43, 0xf8, 0x7a, 0x0a, 0x75, 0x10, 0xce, 0x2a, 0xfc, 0x41, 0x82, 0xd0, + 0xf1, 0xd0, 0x4d, 0x48, 0xcc, 0x2e, 0xe1, 0x23, 0x4f, 0x2f, 0x21, 0x80, 0x01, 0x69, 0x34, 0xa0, + 0x85, 0xca, 0x28, 0x20, 0xc7, 0x78, 0x34, 0xe8, 0xbb, 0xe3, 0x60, 0xfc, 0x06, 0x24, 0x3d, 0x77, + 0x18, 0x10, 0xe1, 0x0d, 0x91, 0xf6, 0x19, 0xa6, 0x64, 0x21, 0x8e, 0x63, 0xd0, 0x7f, 0xc1, 0x92, + 0xe1, 0x11, 0x3d, 0x20, 0x63, 0x91, 0xac, 0x50, 0xc0, 0x8b, 0x9c, 0x1c, 0x4a, 0xbd, 0x97, 0x90, + 0x13, 0x4a, 0xb2, 0xf4, 0x9d, 0x04, 0xb9, 0xca, 0x60, 0x60, 0x1f, 0x85, 0xbe, 0xf0, 0x3e, 0xa4, + 0x8d, 0x03, 0xdd, 0xe9, 0x91, 0xb0, 0x5b, 0x77, 0xe9, 0x44, 0x82, 0x19, 0x01, 0xcb, 0x9b, 0x0c, + 0x15, 0xf6, 0xc9, 0xc4, 0x9c, 0xc2, 0xe7, 0x12, 0xa4, 0x38, 0x07, 0x95, 0xe1, 0x0c, 0x39, 0x1c, + 0x10, 0x23, 0xd0, 0x4e, 0x1c, 0x90, 0x75, 0x48, 0xf0, 0x32, 0x67, 0xed, 0x44, 0x8e, 0xf9, 0x26, + 0xa4, 0x86, 0x03, 0x9f, 0x78, 0x81, 0x30, 0xdc, 0x6c, 0xe5, 0x61, 0x01, 0x42, 0x97, 0x21, 0x65, + 0x12, 0x9b, 0x08, 0xb5, 0x4c, 0xdc, 0x7f, 0xc1, 0x2a, 0x59, 0xac, 0x30, 0xa1, 0x9b, 0x7e, 0xd9, + 0x7e, 0x54, 0xfa, 0x6b, 0x0c, 0x94, 0xf0, 0x76, 0xfa, 0x2f, 0xad, 0x0e, 0x9a, 0xae, 0x6e, 0xe3, + 0x33, 0xaa, 0xdb, 0x35, 0xc8, 0xd1, 0x72, 0x79, 0x84, 0xe1, 0x1e, 0x40, 0x4b, 0xe8, 0x10, 0xf1, + 0x3a, 0x2c, 0x39, 0xe4, 0x30, 0xd0, 0x06, 0x7a, 0x8f, 0x68, 0x81, 0xfb, 0x88, 0x38, 0x22, 0xea, + 0xe5, 0x29, 0xb9, 0xad, 0xf7, 0x48, 0x97, 0x12, 0xd1, 0x25, 0x00, 0x06, 0xe1, 0x2f, 0x48, 0x1a, + 0x92, 0x93, 0x38, 0x43, 0x29, 0xfc, 0xf9, 0x78, 0x17, 0x72, 0xbe, 0xd5, 0x73, 0xf4, 0x60, 0xe8, + 0x91, 0x6e, 0xb7, 0x21, 0xe2, 0xfc, 0x73, 0x3a, 0x29, 0xf2, 0x93, 0xe3, 0xa2, 0xc4, 0xda, 0x24, + 0x27, 0x26, 0x4e, 0xd5, 0x77, 0xf2, 0x64, 0x7d, 0x57, 0xfa, 0x3a, 0x06, 0xcb, 0x11, 0xfd, 0xbe, + 0xf4, 0xb8, 0x50, 0x87, 0x4c, 0x18, 0x62, 0xc3, 0xc0, 0x70, 0x75, 0x3a, 0x0e, 0x8f, 0x76, 0x52, + 0xd6, 0x46, 0xed, 0x51, 0x2e, 0x67, 0x3c, 0x7b, 0x96, 0xb2, 0x13, 0x33, 0x94, 0x5d, 0xf8, 0x08, + 0x32, 0x23, 0x29, 0xe8, 0xc6, 0x89, 0x48, 0x32, 0x23, 0x05, 0x9c, 0x08, 0x23, 0x97, 0x00, 0xa8, + 0x3e, 0x89, 0xc9, 0xaa, 0x77, 0xde, 0x79, 0xc8, 0x70, 0xca, 0xae, 0x67, 0x97, 0x7e, 0x2e, 0x41, + 0x92, 0x05, 0x0b, 0xf4, 0x2e, 0xa4, 0xfb, 0xa4, 0xbf, 0x47, 0x73, 0x05, 0xbf, 0xdf, 0xa7, 0xf5, + 0x45, 0x42, 0x38, 0x4d, 0xa0, 0x03, 0xcf, 0xea, 0xeb, 0xde, 0x11, 0x6f, 0x57, 0xe3, 0x70, 0x88, + 0xae, 0x43, 0x26, 0x6c, 0x8c, 0x84, 0x1d, 0xc6, 0x93, 0x7d, 0x93, 0x31, 0x5b, 0x94, 0x73, 0xbf, + 0x8f, 0x41, 0x8a, 0x6b, 0x1d, 0xbd, 0x0f, 0x10, 0x36, 0x3f, 0x5e, 0xb8, 0x57, 0x93, 0x11, 0x33, + 0xea, 0xe6, 0x38, 0x38, 0xc6, 0x5e, 0x20, 0x38, 0xde, 0x84, 0x04, 0x2d, 0x7f, 0x45, 0x20, 0x3d, + 0x3b, 0xe9, 0x01, 0x65, 0x5a, 0x03, 0x87, 0x6a, 0xa5, 0xc0, 0xc2, 0x67, 0x12, 0x24, 0x28, 0x91, + 0xea, 0xd7, 0xb0, 0x87, 0x34, 0x7d, 0x86, 0xbb, 0x4c, 0xe0, 0x8c, 0xa0, 0xd4, 0x4d, 0x74, 0x11, + 0x32, 0x5c, 0x4d, 0x94, 0x1b, 0x63, 0x5c, 0x99, 0x13, 0xea, 0x26, 0x2a, 0x80, 0x3c, 0x8a, 0x7e, + 0xfc, 0xb6, 0x8e, 0xc6, 0x74, 0xa2, 0xa7, 0xef, 0x07, 0x5a, 0x40, 0x3c, 0xde, 0xf7, 0x48, 0x60, + 0x99, 0x12, 0xba, 0xc4, 0xeb, 0x87, 0x2d, 0x23, 0xa6, 0xb1, 0xdf, 0x49, 0x50, 0x38, 0xd1, 0x8e, + 0xe0, 0xa5, 0x5d, 0x18, 0x68, 0xde, 0x85, 0xc5, 0xd1, 0xfb, 0x8d, 0xb7, 0x31, 0xa4, 0x79, 0x6d, + 0x8c, 0xfc, 0x7e, 0x74, 0x48, 0x13, 0x10, 0x7f, 0x13, 0x89, 0x68, 0x1a, 0xe3, 0x65, 0x17, 0xa3, + 0xd5, 0x18, 0x89, 0x46, 0xf2, 0x28, 0x44, 0x8b, 0xfc, 0x0e, 0xca, 0xe0, 0xe5, 0x08, 0xb2, 0xcd, + 0x18, 0xa5, 0xaf, 0x24, 0xb8, 0x38, 0x73, 0xaf, 0x2f, 0xfd, 0xd2, 0xce, 0xae, 0x88, 0xe3, 0xb3, + 0x2b, 0xe2, 0xeb, 0xbf, 0x88, 0x43, 0x8a, 0x2f, 0x88, 0x52, 0x10, 0x6b, 0xdd, 0xe7, 0xef, 0x97, + 0x7b, 0xad, 0x5d, 0xdc, 0xac, 0x34, 0xb4, 0x66, 0xab, 0xab, 0x6d, 0xb5, 0x76, 0x9b, 0x35, 0x45, + 0x42, 0x97, 0xe0, 0x42, 0xb3, 0xa5, 0x85, 0x9c, 0x36, 0xae, 0xef, 0x54, 0xf0, 0x43, 0xad, 0x8a, + 0x5b, 0xf7, 0x55, 0xac, 0xc4, 0xd0, 0x2a, 0x14, 0x28, 0x7a, 0x0e, 0x3f, 0x4e, 0x9f, 0x3f, 0x51, + 0xbe, 0xa0, 0x27, 0xd1, 0x1a, 0xbc, 0x5a, 0x6f, 0x76, 0x76, 0xb7, 0xb6, 0xea, 0x9b, 0x75, 0xb5, + 0x39, 0x09, 0xe8, 0x28, 0x09, 0xf4, 0x2a, 0xac, 0xb4, 0xb6, 0xb6, 0x3a, 0x6a, 0x97, 0x6d, 0xe7, + 0xa1, 0xda, 0xd5, 0x2a, 0x1f, 0x56, 0xea, 0x8d, 0x4a, 0xb5, 0xa1, 0x2a, 0x29, 0xfa, 0x7c, 0x7a, + 0x80, 0x5b, 0xcd, 0xbb, 0x1a, 0x6e, 0xed, 0x76, 0x55, 0x25, 0x4d, 0xb7, 0xdf, 0xc6, 0xad, 0x76, + 0xab, 0x53, 0x69, 0x68, 0x3b, 0xf5, 0xce, 0x4e, 0xa5, 0xbb, 0xb9, 0xad, 0xc8, 0xe8, 0x22, 0x9c, + 0x57, 0xbb, 0x9b, 0x35, 0xad, 0x8b, 0x2b, 0xcd, 0x4e, 0x65, 0xb3, 0x5b, 0x6f, 0x35, 0xb5, 0xad, + 0x4a, 0xbd, 0xa1, 0xd6, 0x94, 0x0c, 0x15, 0x42, 0x65, 0x57, 0x1a, 0x8d, 0xd6, 0x03, 0xb5, 0xa6, + 0x00, 0x3a, 0x0f, 0x67, 0xb8, 0xd4, 0x4a, 0x9b, 0x3d, 0xc4, 0xf8, 0x06, 0x94, 0x2c, 0xdd, 0x4c, + 0xbd, 0x59, 0x53, 0x3f, 0xd2, 0xb6, 0x2b, 0x1d, 0xed, 0x2e, 0x56, 0x2b, 0x5d, 0x15, 0x87, 0xdc, + 0x1c, 0x5d, 0x1b, 0xab, 0x77, 0xeb, 0x1d, 0x4a, 0x1c, 0xad, 0x9d, 0x47, 0x79, 0xc8, 0x88, 0x17, + 0x9d, 0x5a, 0x53, 0x16, 0xa9, 0x8c, 0x2d, 0x5c, 0xb9, 0xbb, 0x43, 0x8f, 0xdb, 0xe9, 0xb6, 0xb0, + 0xaa, 0xed, 0x36, 0xb7, 0xd5, 0x4a, 0xa3, 0xbb, 0xfd, 0x50, 0x59, 0xba, 0xee, 0x80, 0x32, 0xd9, + 0xb2, 0x43, 0x59, 0x48, 0xd7, 0x9b, 0x1f, 0x56, 0x1a, 0xf5, 0x9a, 0xb2, 0x30, 0xea, 0x28, 0xb3, + 0x26, 0xf2, 0xdd, 0x8f, 0xeb, 0x6d, 0x25, 0x46, 0x57, 0xf8, 0xb8, 0xd3, 0xad, 0x34, 0x6b, 0x15, + 0x5c, 0x53, 0xe2, 0x08, 0x20, 0xd5, 0x69, 0x56, 0xda, 0xed, 0x87, 0x4a, 0x82, 0x1a, 0x86, 0x82, + 0xe8, 0x26, 0x1b, 0xad, 0x4a, 0x4d, 0xab, 0xa9, 0x9b, 0xad, 0x9d, 0x36, 0x56, 0x3b, 0x9d, 0x7a, + 0xab, 0xa9, 0x24, 0x37, 0xfe, 0x1d, 0x1f, 0x97, 0x6e, 0xef, 0x40, 0x82, 0x96, 0x7b, 0xe8, 0xec, + 0x64, 0xf9, 0xc7, 0x6e, 0x58, 0xe1, 0xdc, 0xec, 0xaa, 0xf0, 0x96, 0x84, 0xde, 0x85, 0x24, 0x2b, + 0x32, 0xd0, 0xb9, 0xd9, 0xa5, 0x52, 0xe1, 0xfc, 0x14, 0x5d, 0x5c, 0x84, 0x77, 0x20, 0x81, 0x89, + 0x6e, 0x46, 0x97, 0x8c, 0xf4, 0xf1, 0xa2, 0x4b, 0x46, 0xfb, 0x4f, 0xb7, 0x24, 0xf4, 0x3e, 0xa4, + 0xf8, 0x6b, 0x18, 0x9d, 0x9f, 0xf3, 0xfe, 0x2f, 0xac, 0x4c, 0x33, 0xf8, 0xf4, 0x75, 0x09, 0x6d, + 0x43, 0x66, 0xf4, 0x96, 0x41, 0x85, 0xe8, 0x2a, 0x27, 0x9f, 0x87, 0x85, 0x8b, 0x33, 0x79, 0xa1, + 0x9c, 0x5b, 0x54, 0x52, 0x9e, 0x6a, 0x63, 0x94, 0x0e, 0xa3, 0xd2, 0x26, 0xab, 0xa1, 0xa8, 0xb4, + 0xe9, 0x4c, 0xbe, 0x07, 0x67, 0x66, 0xc4, 0x0c, 0x74, 0x65, 0x7a, 0xce, 0x74, 0xf8, 0x2b, 0x5c, + 0x3d, 0x05, 0xc5, 0xd7, 0xa8, 0xaa, 0x4f, 0xfe, 0xb1, 0xba, 0xf0, 0xe4, 0xfb, 0x55, 0xe9, 0xdb, + 0xef, 0x57, 0xa5, 0x5f, 0x3e, 0x5d, 0x5d, 0xf8, 0xf2, 0xe9, 0xaa, 0xf4, 0xcd, 0xd3, 0x55, 0xe9, + 0xdb, 0xa7, 0xab, 0x0b, 0x7f, 0x7e, 0xba, 0xba, 0xf0, 0xf1, 0xe5, 0x9e, 0x5b, 0xee, 0xe9, 0x9f, + 0x92, 0x20, 0x20, 0x65, 0x93, 0x3c, 0xbe, 0x69, 0xb8, 0x1e, 0xb9, 0x39, 0xf1, 0x4f, 0x7e, 0x2f, + 0xc5, 0xbe, 0x6e, 0xff, 0x27, 0x00, 0x00, 0xff, 0xff, 0xa0, 0x4d, 0x98, 0xa0, 0xad, 0x1f, 0x00, + 0x00, } func (this *Label) Equal(that interface{}) bool { @@ -4736,6 +4745,23 @@ func (m *FragmentStoreHealthRequest) MarshalToSizedBuffer(dAtA []byte) (int, err _ = i var l int _ = l + if len(m.CheckDeletePrefix) > 0 { + i -= len(m.CheckDeletePrefix) + copy(dAtA[i:], m.CheckDeletePrefix) + i = encodeVarintProtocol(dAtA, i, uint64(len(m.CheckDeletePrefix))) + i-- + dAtA[i] = 0x1a + } + if m.CheckDelete { + i-- + if m.CheckDelete { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x10 + } if len(m.FragmentStore) > 0 { i -= len(m.FragmentStore) copy(dAtA[i:], m.FragmentStore) @@ -5486,6 +5512,13 @@ func (m *FragmentStoreHealthRequest) ProtoSize() (n int) { if l > 0 { n += 1 + l + sovProtocol(uint64(l)) } + if m.CheckDelete { + n += 2 + } + l = len(m.CheckDeletePrefix) + if l > 0 { + n += 1 + l + sovProtocol(uint64(l)) + } return n } @@ -10376,6 +10409,58 @@ func (m *FragmentStoreHealthRequest) Unmarshal(dAtA []byte) error { } m.FragmentStore = FragmentStore(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CheckDelete", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowProtocol + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.CheckDelete = bool(v != 0) + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field CheckDeletePrefix", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowProtocol + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthProtocol + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthProtocol + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.CheckDeletePrefix = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipProtocol(dAtA[iNdEx:]) diff --git a/broker/protocol/protocol.proto b/broker/protocol/protocol.proto index 69123094..ee4e9b79 100644 --- a/broker/protocol/protocol.proto +++ b/broker/protocol/protocol.proto @@ -831,6 +831,12 @@ message Header { message FragmentStoreHealthRequest { // Fragment store to check the health of. string fragment_store = 1 [ (gogoproto.casttype) = "FragmentStore" ]; + // If true, also verify delete permission with a throwaway probe object under + // check_delete_prefix. Defaults to false (delete is not exercised). + bool check_delete = 2; + // Prefix under which the delete probe is written (e.g. "recovery/"); empty + // probes the store root. Only used when check_delete is true. + string check_delete_prefix = 3; } // FragmentStoreHealthResponse is the unary response message of the broker FragmentStoreHealth RPC. diff --git a/broker/protocol/rpc_extensions.go b/broker/protocol/rpc_extensions.go index 8cd25d94..3e8cbdc7 100644 --- a/broker/protocol/rpc_extensions.go +++ b/broker/protocol/rpc_extensions.go @@ -440,6 +440,20 @@ func (m *FragmentStoreHealthRequest) Validate() error { if err := m.FragmentStore.Validate(); err != nil { return ExtendContext(err, "FragmentStore") } + if m.CheckDeletePrefix != "" && !m.CheckDelete { + return ExtendContext(NewValidationError("cannot be set when CheckDelete is false"), "CheckDeletePrefix") + } + if prefix := m.CheckDeletePrefix; prefix != "" { + if !strings.HasSuffix(prefix, "/") { + return ExtendContext(NewValidationError("must end in '/' (%s)", prefix), "CheckDeletePrefix") + } + var trimmed = strings.TrimSuffix(prefix, "/") + if trimmed == ".." || strings.HasPrefix(trimmed, "../") { + return ExtendContext(NewValidationError("cannot traverse outside the store (%s)", prefix), "CheckDeletePrefix") + } else if err := ValidatePathComponent(trimmed, 1, maxJournalNameLen); err != nil { + return ExtendContext(err, "CheckDeletePrefix") + } + } return nil } diff --git a/broker/protocol/rpc_extensions_test.go b/broker/protocol/rpc_extensions_test.go index 866b38c4..9a693d64 100644 --- a/broker/protocol/rpc_extensions_test.go +++ b/broker/protocol/rpc_extensions_test.go @@ -462,6 +462,46 @@ func (s *RPCSuite) TestFragmentsResponseValidationCases(c *gc.C) { c.Check(resp.Validate(), gc.IsNil) } +func (s *RPCSuite) TestFragmentStoreHealthRequestValidationCases(c *gc.C) { + var req = FragmentStoreHealthRequest{FragmentStore: "s3://bad"} + + c.Check(req.Validate(), gc.ErrorMatches, `FragmentStore: .*doesn't end in '/'.*`) + req.FragmentStore = "s3://bucket/" + c.Check(req.Validate(), gc.IsNil) // No delete probe. + + // A prefix without CheckDelete is rejected rather than silently ignored. + req.CheckDeletePrefix = "recovery/" + c.Check(req.Validate(), gc.ErrorMatches, `CheckDeletePrefix: cannot be set when CheckDelete is false`) + + // With CheckDelete, a non-empty prefix must end in '/'; empty probes the root. + req.CheckDelete = true + c.Check(req.Validate(), gc.IsNil) // "recovery/" + req.CheckDeletePrefix = "recovery/sub/" + c.Check(req.Validate(), gc.IsNil) + req.CheckDeletePrefix = "" + c.Check(req.Validate(), gc.IsNil) // CheckDelete with no prefix probes the root. + + // A missing trailing slash is strictly rejected. + req.CheckDeletePrefix = "recovery" + c.Check(req.Validate(), gc.ErrorMatches, `CheckDeletePrefix: must end in '/' \(recovery\)`) + req.CheckDeletePrefix = "recovery/sub" + c.Check(req.Validate(), gc.ErrorMatches, `CheckDeletePrefix: must end in '/' \(recovery/sub\)`) + + // Parent traversal is rejected, as it can escape the store root on file:// stores. + req.CheckDeletePrefix = "../../etc/" + c.Check(req.Validate(), gc.ErrorMatches, `CheckDeletePrefix: cannot traverse outside the store \(\.\./\.\./etc/\)`) + req.CheckDeletePrefix = "../" + c.Check(req.Validate(), gc.ErrorMatches, `CheckDeletePrefix: cannot traverse outside the store \(\.\./\)`) + + // Non-clean, rooted, and invalid-character paths are rejected. + req.CheckDeletePrefix = "recovery/../../etc/" + c.Check(req.Validate(), gc.ErrorMatches, `CheckDeletePrefix: must be a clean path .*`) + req.CheckDeletePrefix = "/rooted/" + c.Check(req.Validate(), gc.ErrorMatches, `CheckDeletePrefix: cannot begin with '/' \(/rooted\)`) + req.CheckDeletePrefix = "in valid/" + c.Check(req.Validate(), gc.ErrorMatches, `CheckDeletePrefix: not a valid token \(in valid\)`) +} + func badHeaderFixture() *Header { return &Header{ ProcessId: ProcessSpec_ID{Zone: "zone", Suffix: "name"}, diff --git a/broker/stores/health_check.go b/broker/stores/health_check.go index 89c1dd23..15b3dc1e 100644 --- a/broker/stores/health_check.go +++ b/broker/stores/health_check.go @@ -81,6 +81,26 @@ func checkLoop(stores map[pb.FragmentStore]*ActiveStore, fs pb.FragmentStore, at storesMu.RUnlock() } +// CheckDelete verifies delete permission by writing a throwaway probe object +// under prefix and removing it. It runs on-demand, not from the periodic health +// check, which never exercises delete. +func (s *ActiveStore) CheckDelete(ctx context.Context, prefix string) error { + if s.initErr != nil { + return s.initErr + } + + const probeContent = "gazette-delete-probe\n" + var probePath = prefix + ".gazette-delete-probe" + + if err := s.Store.Put(ctx, probePath, strings.NewReader(probeContent), int64(len(probeContent)), ""); err != nil { + return fmt.Errorf("delete-probe PUT failed: %w", err) + } else if err := s.Store.Remove(ctx, probePath); err != nil { + return fmt.Errorf("delete-probe DELETE failed: %w", err) + } + + return nil +} + // runCheck performs a health check of the store by verifying all critical operations. // It's designed to be tolerant of concurrent execution by multiple Gazette brokers // checking the same store simultaneously. diff --git a/broker/stores/health_check_test.go b/broker/stores/health_check_test.go index b1de244d..f6ae87d5 100644 --- a/broker/stores/health_check_test.go +++ b/broker/stores/health_check_test.go @@ -228,3 +228,64 @@ func TestHealthChecks(t *testing.T) { }) } } + +func TestCheckDelete(t *testing.T) { + const prefix = "recovery/" + + t.Run("Success", func(t *testing.T) { + // The probe removes exactly the fixed-name object it wrote under the prefix. + var putPath string + var store = &CallbackStore{ + PutFunc: func(_ Store, _ context.Context, path string, _ io.ReaderAt, _ int64, _ string) error { + require.Equal(t, prefix+".gazette-delete-probe", path) + putPath = path + return nil + }, + RemoveFunc: func(_ Store, _ context.Context, path string) error { + require.Equal(t, putPath, path) + return nil + }, + } + require.NoError(t, NewActiveStore("s3://bucket/", store, nil).CheckDelete(context.Background(), prefix)) + }) + + t.Run("Root Prefix", func(t *testing.T) { + // An empty prefix probes the store root, so the probe object has no prefix. + var store = &CallbackStore{ + PutFunc: func(_ Store, _ context.Context, path string, _ io.ReaderAt, _ int64, _ string) error { + require.Equal(t, ".gazette-delete-probe", path) + return nil + }, + RemoveFunc: func(_ Store, _ context.Context, _ string) error { return nil }, + } + require.NoError(t, NewActiveStore("s3://bucket/", store, nil).CheckDelete(context.Background(), "")) + }) + + t.Run("Delete Denied", func(t *testing.T) { + var store = &CallbackStore{ + PutFunc: func(_ Store, _ context.Context, _ string, _ io.ReaderAt, _ int64, _ string) error { return nil }, + RemoveFunc: func(_ Store, _ context.Context, _ string) error { return errors.New("access denied") }, + } + var err = NewActiveStore("s3://bucket/", store, nil).CheckDelete(context.Background(), prefix) + require.ErrorContains(t, err, "delete-probe DELETE failed") + require.ErrorContains(t, err, "access denied") + }) + + t.Run("Put Failed", func(t *testing.T) { + var store = &CallbackStore{ + PutFunc: func(_ Store, _ context.Context, _ string, _ io.ReaderAt, _ int64, _ string) error { + return errors.New("no write") + }, + RemoveFunc: func(_ Store, _ context.Context, _ string) error { + require.Fail(t, "Remove should not be called when Put fails") + return nil + }, + } + require.ErrorContains(t, NewActiveStore("s3://bucket/", store, nil).CheckDelete(context.Background(), prefix), "delete-probe PUT failed") + }) + + t.Run("Uninitialized Store", func(t *testing.T) { + var initErr = errors.New("bad store") + require.Equal(t, initErr, NewActiveStore("s3://bucket/", nil, initErr).CheckDelete(context.Background(), prefix)) + }) +} diff --git a/cmd/gazctl/gazctlcmd/journals_store_health.go b/cmd/gazctl/gazctlcmd/journals_store_health.go index 7ff512b8..78199e96 100644 --- a/cmd/gazctl/gazctlcmd/journals_store_health.go +++ b/cmd/gazctl/gazctlcmd/journals_store_health.go @@ -9,7 +9,8 @@ import ( ) type cmdJournalStoreHealth struct { - Store struct { + CheckDeletePrefix *string `long:"check-delete-prefix" description:"If set, also verify delete permission by writing and removing a throwaway probe object under this path prefix, which must end in '/' (e.g. \"recovery/\"); pass an empty value to probe the store root"` + Store struct { URL pb.FragmentStore `positional-arg-name:"STORE_URL" required:"true" description:"Fragment store URL to check health, e.g. s3://bucket/prefix/"` } `positional-args:"yes"` } @@ -31,6 +32,9 @@ Check a Google Cloud Storage store: Check a local filesystem store: > gazctl journals store-health file:///var/local/data/ + +Additionally verify delete permission under the recovery/ prefix: +> gazctl journals store-health --check-delete-prefix=recovery/ s3://my-bucket/and/prefix/ `, &cmdJournalStoreHealth{}) } @@ -40,7 +44,7 @@ func (cmd *cmdJournalStoreHealth) Execute([]string) error { var ctx = context.Background() var jc = JournalsCfg.Broker.MustJournalClient(ctx) - var resp, err = client.FragmentStoreHealth(ctx, jc, cmd.Store.URL) + var resp, err = client.FragmentStoreHealth(ctx, jc, cmd.Store.URL, cmd.CheckDeletePrefix) if err != nil { return err } diff --git a/cmd/gazctl/gazctlcmd/shards_prune.go b/cmd/gazctl/gazctlcmd/shards_prune.go index 0b98d169..f43bcc37 100644 --- a/cmd/gazctl/gazctlcmd/shards_prune.go +++ b/cmd/gazctl/gazctlcmd/shards_prune.go @@ -216,7 +216,10 @@ func checkRecoveryLogStoresHealth(ctx context.Context, jc pb.JournalClient, reco } for _, store := range journalSpec.Fragment.Stores { - var resp, err = client.FragmentStoreHealth(ctx, jc, store) + // Prunes may run regularly and autonomously, so the delete probe is + // skipped here. A failure to delete recovery fragments due to + // permissions issues is logged separately. + var resp, err = client.FragmentStoreHealth(ctx, jc, store, nil) if err != nil { log.WithFields(log.Fields{ "journal": recoveryLog,